ms2k
e4b84d9802
Resume goals after transient API failures with persisted exponential backoff while respecting pauses and interruptions.
115 lines
2.3 KiB
Go
115 lines
2.3 KiB
Go
package main
|
|
|
|
import "time"
|
|
|
|
type goalStatus string
|
|
|
|
const (
|
|
active goalStatus = "active"
|
|
paused goalStatus = "paused"
|
|
blocked goalStatus = "blocked"
|
|
)
|
|
|
|
type goal struct {
|
|
Objective string `json:"objective"`
|
|
Status goalStatus `json:"status"`
|
|
CreatedAt int64 `json:"createdAt"`
|
|
UpdatedAt int64 `json:"updatedAt"`
|
|
}
|
|
|
|
func sameGoal(a, b *goal) bool {
|
|
return a != nil && b != nil && a.Objective == b.Objective && a.CreatedAt == b.CreatedAt
|
|
}
|
|
|
|
type watch struct {
|
|
Goal *goal `json:"goal,omitempty"`
|
|
Turn turn `json:"turn"`
|
|
Armed bool `json:"armed"`
|
|
Failures int `json:"failures"`
|
|
Due time.Time `json:"due"`
|
|
revision uint64
|
|
}
|
|
|
|
func (w *watch) stop() {
|
|
w.Armed = false
|
|
w.Due = time.Time{}
|
|
}
|
|
|
|
func (w *watch) update(g *goal) {
|
|
w.revision++
|
|
if !sameGoal(w.Goal, g) {
|
|
w.stop()
|
|
w.Failures = 0
|
|
}
|
|
previous := w.Goal
|
|
w.Goal = g
|
|
if g == nil {
|
|
w.stop()
|
|
return
|
|
}
|
|
switch g.Status {
|
|
case active:
|
|
if previous != nil && previous.Status != active {
|
|
w.stop()
|
|
}
|
|
if w.Turn.Status == "inProgress" {
|
|
w.Armed = true
|
|
}
|
|
case blocked:
|
|
if !w.Due.IsZero() && previous != nil && previous.UpdatedAt != g.UpdatedAt {
|
|
w.stop()
|
|
}
|
|
default:
|
|
w.stop()
|
|
}
|
|
}
|
|
|
|
func (w *watch) observe(t turn) {
|
|
if t.ID == "" {
|
|
return
|
|
}
|
|
w.revision++
|
|
if t.ID != w.Turn.ID {
|
|
w.stop()
|
|
w.Turn = t
|
|
w.Armed = t.Status == "inProgress" && w.Goal != nil && w.Goal.Status == active
|
|
return
|
|
}
|
|
w.Turn = t
|
|
switch t.Status {
|
|
case "completed":
|
|
w.stop()
|
|
w.Failures = 0
|
|
case "interrupted":
|
|
w.stop()
|
|
case "failed":
|
|
if t.Error == nil || !t.Error.Info.retryable() {
|
|
w.stop()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *watch) eligible() bool {
|
|
return w.Armed && w.Goal != nil && w.Goal.Status == blocked &&
|
|
w.Turn.Status == "failed" && w.Turn.Error != nil && w.Turn.Error.Info.retryable()
|
|
}
|
|
|
|
func (w *watch) schedule(now time.Time, base, cap time.Duration, jitter func(time.Duration) time.Duration) bool {
|
|
if !w.eligible() || !w.Due.IsZero() {
|
|
return false
|
|
}
|
|
w.Failures++
|
|
w.Due = now.Add(min(cap, jitter(backoff(base, cap, w.Failures))))
|
|
return true
|
|
}
|
|
|
|
func backoff(base, cap time.Duration, failures int) time.Duration {
|
|
delay := base
|
|
for i := 1; i < failures && delay < cap; i++ {
|
|
if delay > cap/2 {
|
|
return cap
|
|
}
|
|
delay *= 2
|
|
}
|
|
return min(delay, cap)
|
|
}
|