Fix goal recovery
This commit is contained in:
parent
3846556035
commit
996f586bc1
1 changed files with 783 additions and 145 deletions
|
|
@ -10,6 +10,10 @@ const (
|
|||
blocked goalStatus = "blocked"
|
||||
)
|
||||
|
||||
func (s goalStatus) waiting() bool {
|
||||
return s == blocked || s == "usageLimited"
|
||||
}
|
||||
|
||||
type goal struct {
|
||||
Objective string `json:"objective"`
|
||||
Status goalStatus `json:"status"`
|
||||
|
|
@ -86,7 +90,10 @@ func (w *watch) update(g *goal) {
|
|||
} else if w.Turn.Status == "inProgress" {
|
||||
w.Armed = true
|
||||
}
|
||||
case blocked:
|
||||
case blocked, "usageLimited":
|
||||
if g.Status == "usageLimited" && (!same || previous.Status != g.Status) {
|
||||
w.Due = time.Time{}
|
||||
}
|
||||
if !same && w.Pending {
|
||||
w.Hold = g
|
||||
}
|
||||
|
|
@ -108,7 +115,7 @@ func (w *watch) notice(g *goal) {
|
|||
}
|
||||
if w.Hold != nil && sameGoal(w.Hold, g) {
|
||||
w.revision++
|
||||
if g.Status == blocked || g.Status == active {
|
||||
if g.Status.waiting() || g.Status == active {
|
||||
return
|
||||
}
|
||||
w.Goal, w.Hold = g, g
|
||||
|
|
@ -128,8 +135,7 @@ func (w *watch) observe(t turn) {
|
|||
}
|
||||
w.revision++
|
||||
fresh := t.ID != w.Turn.ID
|
||||
if w.Known && (fresh || w.Turn.Status == "inProgress") && t.Status == "failed" &&
|
||||
t.Error != nil && t.Error.Info.retryable() {
|
||||
if w.Known && (fresh || w.Turn.Status == "inProgress") && t.Status == "failed" {
|
||||
w.Armed = true
|
||||
}
|
||||
if fresh {
|
||||
|
|
@ -150,10 +156,6 @@ func (w *watch) observe(t turn) {
|
|||
desired.Status = paused
|
||||
w.Goal, w.Hold = &desired, &desired
|
||||
}
|
||||
case "failed":
|
||||
if t.Error == nil || !t.Error.Info.retryable() {
|
||||
w.stop()
|
||||
}
|
||||
}
|
||||
if w.Hold != nil {
|
||||
w.stop()
|
||||
|
|
@ -161,14 +163,18 @@ func (w *watch) observe(t turn) {
|
|||
}
|
||||
|
||||
func (w *watch) eligible() bool {
|
||||
return w.Hold == nil && w.Armed && w.Goal != nil && w.Goal.Status == blocked &&
|
||||
w.Turn.Status == "failed" && w.Turn.Error != nil && w.Turn.Error.Info.retryable()
|
||||
return w.Hold == nil && w.Armed && w.Goal != nil && w.Goal.Status.waiting() &&
|
||||
w.Turn.Status == "failed"
|
||||
}
|
||||
|
||||
func (w *watch) schedule(now time.Time, base, cap time.Duration, jitter func(time.Duration) time.Duration) bool {
|
||||
if !w.eligible() || !w.Due.IsZero() {
|
||||
return false
|
||||
}
|
||||
if w.Goal.Status == "usageLimited" {
|
||||
w.Due = now
|
||||
return true
|
||||
}
|
||||
w.Failures++
|
||||
w.Due = now.Add(min(cap, jitter(backoff(base, cap, w.Failures))))
|
||||
return true
|
||||
|
|
|
|||
Loading…
Reference in a new issue