package main import ( "context" "encoding/json" "io" "os" "path/filepath" "testing" "time" ) type fakeDaemon struct { op *operator t *testing.T goal *goal turn turn status string path string requests []message before func(message) lostReply bool setError *rpcError } func newFake(t *testing.T, s *store) *fakeDaemon { t.Helper() if s == nil { var err error s, err = openStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatal(err) } t.Cleanup(func() { closeStore(t, s) }) } path := filepath.Join(filepath.Dir(s.path), "rollout.jsonl") file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { t.Fatal(err) } if err := file.Close(); err != nil { t.Fatal(err) } f := &fakeDaemon{path: path, t: t, goal: testGoal(active), turn: testTurn("inProgress"), status: "systemError"} c := &connection{frames: make(chan frame, 32)} op := &operator{ opts: options{base: time.Second, cap: time.Minute}, store: s, conn: c, log: io.Discard, attached: make(map[string]bool), jitter: func(d time.Duration) time.Duration { return d }, } f.op = op c.write = f.write c.onEvent = op.event return f } func (f *fakeDaemon) enqueue(m message) { f.op.conn.frames <- frame{message: m} } func (f *fakeDaemon) notify(method string, value any) { data, err := json.Marshal(value) if err != nil { f.t.Fatal(err) } f.enqueue(message{Method: method, Params: data}) } func (f *fakeDaemon) write(data []byte) error { var m message if err := json.Unmarshal(data, &m); err != nil { return err } f.requests = append(f.requests, m) if f.before != nil { f.before(m) } var result any switch m.Method { case "thread/loaded/list": result = loadedResult{Data: []string{"thread-1"}} case "thread/goal/get": result = goalResult{f.goal} case "thread/resume": result = map[string]any{"thread": map[string]string{"path": f.path}} case "thread/turns/list": result = turnsResult{[]turn{f.turn}} case "thread/read": result = map[string]any{"thread": map[string]any{"path": f.path, "status": map[string]string{"type": f.status}}} case "thread/goal/set": var p map[string]any if err := json.Unmarshal(m.Params, &p); err != nil { return err } if len(p) != 3 || p["threadId"] != "thread-1" || p["origin"] != "automatic" { f.t.Fatalf("goal changed beyond status: %s", m.Params) } if f.setError != nil { f.enqueue(message{ID: m.ID, Error: f.setError}) return nil } if f.goal == nil { f.enqueue(message{ID: m.ID, Error: &rpcError{Code: -32600, Message: "synthetic missing goal"}}) return nil } next := *f.goal next.Status = goalStatus(p["status"].(string)) f.goal = &next if next.Status == active { f.turn = turn{ID: "turn-2", Status: "inProgress"} } if f.lostReply { f.op.conn.frames <- frame{err: io.EOF} return nil } result = goalResult{f.goal} default: f.t.Fatalf("unexpected RPC: %s", data) } raw, err := json.Marshal(result) if err != nil { return err } f.enqueue(message{ID: m.ID, Result: raw}) if m.Method == "thread/goal/set" || m.Method == "thread/resume" { f.notify("thread/goal/updated", map[string]any{"threadId": "thread-1", "goal": f.goal}) } return nil } func (f *fakeDaemon) reconcile() error { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() return f.op.reconcile(ctx) } func (f *fakeDaemon) count(method string) int { count := 0 for _, m := range f.requests { if m.Method == method { count++ } } return count } func (f *fakeDaemon) fail() { f.goal, f.turn = testGoal(blocked), testTurn("failed") } func TestRecovery(t *testing.T) { f := newFake(t, nil) if err := f.reconcile(); err != nil { t.Fatal(err) } if !f.op.store.watch("thread-1").Armed || f.count("thread/resume") != 1 { t.Fatal("did not attach active goal") } f.fail() if err := f.reconcile(); err != nil { t.Fatal(err) } w := f.op.store.watch("thread-1") if !w.eligible() || w.Failures != 1 || f.count("thread/goal/set") != 0 { t.Fatal("failed to wait for backoff") } w.Due = time.Now().Add(-time.Second) if err := f.reconcile(); err != nil { t.Fatal(err) } if f.count("thread/goal/set") != 1 || w.Failures != 1 { t.Fatal("resume count or backoff changed") } if err := f.reconcile(); err != nil { t.Fatal(err) } if f.count("thread/goal/set") != 1 { t.Fatal("resumed active work twice") } if f.count("thread/resume") != 4 { t.Fatal("missing notification barrier") } f.fail() f.turn.ID = "turn-2" if err := f.reconcile(); err != nil { t.Fatal(err) } if w.Failures != 2 || time.Until(w.Due) > 2*time.Second || time.Until(w.Due) < 1900*time.Millisecond { t.Fatalf("backoff did not double: %+v", w) } } func TestOldBlocked(t *testing.T) { f := newFake(t, nil) f.fail() if err := f.reconcile(); err != nil { t.Fatal(err) } if f.op.store.watch("thread-1").eligible() || f.count("thread/goal/set") != 0 { t.Fatal("adopted an old blocked goal") } } func TestPauseDuringRead(t *testing.T) { for _, method := range []string{"thread/goal/get", "thread/turns/list", "thread/read"} { t.Run(method, func(t *testing.T) { f := newFake(t, nil) f.fail() w := failedWatch() w.Due = time.Now().Add(-time.Second) f.op.store.Watches["thread-1"] = w f.before = func(m message) { if m.Method == method && w.Source != nil { f.notify("thread/goal/updated", map[string]any{"threadId": "thread-1", "goal": testGoal(paused)}) f.before = nil } } if err := f.reconcile(); err != nil { t.Fatal(err) } if w.Goal.Status != paused || w.eligible() || f.count("thread/goal/set") != 0 { t.Fatal("stale snapshot overrode human pause") } }) } } func TestLiveActivity(t *testing.T) { f := newFake(t, nil) f.fail() f.status = "active" w := failedWatch() w.Due = time.Now().Add(-time.Second) f.op.store.Watches["thread-1"] = w f.before = func(m message) { if m.Method == "thread/read" { f.enqueue(message{ID: json.RawMessage(`"approval-1"`), Method: "item/commandExecution/requestApproval", Params: json.RawMessage(`{"threadId":"thread-1"}`)}) } } if err := f.reconcile(); err != nil { t.Fatal(err) } if f.count("thread/goal/set") != 0 || w.eligible() { t.Fatal("competed with active turn or approval") } } func TestRestartPending(t *testing.T) { f := newFake(t, nil) w := failedWatch() w.Due = time.Now().Add(-time.Second) w.Failures = 3 f.op.store.Watches["thread-1"] = w if err := f.op.save(); err != nil { t.Fatal(err) } path := f.op.store.path closeStore(t, f.op.store) s, err := openStore(path) if err != nil { t.Fatal(err) } defer closeStore(t, s) f = newFake(t, s) f.fail() if err := f.reconcile(); err != nil { t.Fatal(err) } if f.count("thread/goal/set") != 1 || s.watch("thread-1").Failures != 3 { t.Fatal("lost pending recovery across restart") } } func TestLostSetReply(t *testing.T) { f := newFake(t, nil) f.fail() f.lostReply = true w := failedWatch() w.Due = time.Now().Add(-time.Second) w.Failures = 1 f.op.store.Watches["thread-1"] = w if err := f.reconcile(); err != io.EOF { t.Fatalf("connection error = %v", err) } if f.count("thread/goal/set") != 1 { t.Fatal("missing mutation") } path := f.op.store.path closeStore(t, f.op.store) s, err := openStore(path) if err != nil { t.Fatal(err) } defer closeStore(t, s) reconnected := newFake(t, s) reconnected.goal, reconnected.turn = f.goal, f.turn if err := reconnected.reconcile(); err != nil { t.Fatal(err) } if reconnected.count("thread/goal/set") != 0 || !s.watch("thread-1").Due.IsZero() { t.Fatal("duplicated a successful mutation after lost reply") } } func TestResetSetError(t *testing.T) { f := newFake(t, nil) f.fail() w := failedWatch() w.Failures = 1 w.Due = time.Now().Add(-time.Second) f.op.store.Watches["thread-1"] = w f.setError = &rpcError{Code: -32600, Message: "synthetic rejection"} f.before = func(m message) { if m.Method != "thread/goal/set" { return } f.notify("thread/goal/updated", map[string]any{"threadId": "thread-1", "goal": testGoal(active)}) f.notify("turn/started", map[string]any{"threadId": "thread-1", "turn": turn{ID: "turn-2", Status: "inProgress"}}) f.goal = testGoal("complete") f.notify("thread/goal/updated", map[string]any{"threadId": "thread-1", "goal": f.goal}) f.notify("turn/completed", map[string]any{"threadId": "thread-1", "turn": turn{ID: "turn-2", Status: "completed"}}) } if err := f.reconcile(); err != nil { t.Fatal(err) } if w.Failures != 0 || w.eligible() || !w.Due.IsZero() { t.Fatalf("stale activation error changed successful work: %+v", w) } } func TestNoCompetingRetry(t *testing.T) { f := newFake(t, nil) if err := f.reconcile(); err != nil { t.Fatal(err) } for _, method := range []string{"error", "item/agentMessage/delta"} { data := json.RawMessage(`{"threadId":"thread-1","willRetry":true}`) if err := f.op.event(message{Method: method, Params: data}); err != nil { t.Fatal(err) } } if err := f.reconcile(); err != nil { t.Fatal(err) } if f.count("thread/goal/set") != 0 { t.Fatal("competed with internal retry") } } func intentLine(status goalStatus) []byte { body := "\nUser set goal status: " + string(mustJSON(status)) + ".\n" return append(mustJSON(map[string]any{"type": "response_item", "payload": map[string]any{ "type": "message", "role": "user", "content": []any{map[string]string{"type": "input_text", "text": body}}, "internal_chat_message_metadata_passthrough": map[string]any{"content_item_kinds": []string{"user.goal"}}, }}), '\n') } func mustJSON(value any) []byte { data, err := json.Marshal(value) if err != nil { panic(err) } return data } func appendIntent(t *testing.T, path string, data []byte) { t.Helper() f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0600) if err != nil { t.Fatal(err) } if _, err := f.Write(data); err != nil { t.Fatal(err) } if err := f.Close(); err != nil { t.Fatal(err) } } func (f *fakeDaemon) human(g *goal) { appendIntent(f.t, f.path, intentLine(g.Status)) f.notify("thread/goal/updated", map[string]any{"threadId": "thread-1", "goal": g}) }