Add daemon-attached goal recovery operator

Resume goals after transient API failures with persisted exponential backoff while respecting pauses and interruptions.

codex-keep/policy.go

@@ -0,0 +1,115 @@
+package main
+
+import "time"
+
+type goalStatus string
+
+const (
+	active  goalStatus = "active"
+	paused  goalStatus = "paused"
+	blocked goalStatus = "blocked"
+)
+
+type goal struct {
+	Objective string     `json:"objective"`
+	Status    goalStatus `json:"status"`
+	CreatedAt int64      `json:"createdAt"`
+	UpdatedAt int64      `json:"updatedAt"`
+}
+
+func sameGoal(a, b *goal) bool {
+	return a != nil && b != nil && a.Objective == b.Objective && a.CreatedAt == b.CreatedAt
+}
+
+type watch struct {
+	Goal     *goal     `json:"goal,omitempty"`
+	Turn     turn      `json:"turn"`
+	Armed    bool      `json:"armed"`
+	Failures int       `json:"failures"`
+	Due      time.Time `json:"due"`
+	revision uint64
+}
+
+func (w *watch) stop() {
+	w.Armed = false
+	w.Due = time.Time{}
+}
+
+func (w *watch) update(g *goal) {
+	w.revision++
+	if !sameGoal(w.Goal, g) {
+		w.stop()
+		w.Failures = 0
+	}
+	previous := w.Goal
+	w.Goal = g
+	if g == nil {
+		w.stop()
+		return
+	}
+	switch g.Status {
+	case active:
+		if previous != nil && previous.Status != active {
+			w.stop()
+		}
+		if w.Turn.Status == "inProgress" {
+			w.Armed = true
+		}
+	case blocked:
+		if !w.Due.IsZero() && previous != nil && previous.UpdatedAt != g.UpdatedAt {
+			w.stop()
+		}
+	default:
+		w.stop()
+	}
+}
+
+func (w *watch) observe(t turn) {
+	if t.ID == "" {
+		return
+	}
+	w.revision++
+	if t.ID != w.Turn.ID {
+		w.stop()
+		w.Turn = t
+		w.Armed = t.Status == "inProgress" && w.Goal != nil && w.Goal.Status == active
+		return
+	}
+	w.Turn = t
+	switch t.Status {
+	case "completed":
+		w.stop()
+		w.Failures = 0
+	case "interrupted":
+		w.stop()
+	case "failed":
+		if t.Error == nil || !t.Error.Info.retryable() {
+			w.stop()
+		}
+	}
+}
+
+func (w *watch) eligible() bool {
+	return w.Armed && w.Goal != nil && w.Goal.Status == blocked &&
+		w.Turn.Status == "failed" && w.Turn.Error != nil && w.Turn.Error.Info.retryable()
+}
+
+func (w *watch) schedule(now time.Time, base, cap time.Duration, jitter func(time.Duration) time.Duration) bool {
+	if !w.eligible() || !w.Due.IsZero() {
+		return false
+	}
+	w.Failures++
+	w.Due = now.Add(min(cap, jitter(backoff(base, cap, w.Failures))))
+	return true
+}
+
+func backoff(base, cap time.Duration, failures int) time.Duration {
+	delay := base
+	for i := 1; i < failures && delay < cap; i++ {
+		if delay > cap/2 {
+			return cap
+		}
+		delay *= 2
+	}
+	return min(delay, cap)
+}