Fix goal recovery
This commit is contained in:
parent
3846556035
commit
996f586bc1
1 changed files with 783 additions and 145 deletions
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -9,8 +10,8 @@ import (
|
|||
)
|
||||
|
||||
func TestIntentProvenance(t *testing.T) {
|
||||
for _, status := range []goalStatus{active, paused, "complete", "budgetLimited", "usageLimited", blocked} {
|
||||
got, ok, err := parseIntent(intentLine(status))
|
||||
for _, status := range []goalStatus{active, paused, "complete", "budgetLimited", "usageLimited", blocked, "cleared"} {
|
||||
got, ok, err := (&intentLog{}).parse(intentLine(status))
|
||||
if err != nil || !ok || got != status {
|
||||
t.Fatalf("status=%s got=%s match=%v error=%v", status, got, ok, err)
|
||||
}
|
||||
|
|
@ -21,7 +22,7 @@ func TestIntentProvenance(t *testing.T) {
|
|||
strings.Replace(line, "\"role\":\"user\"", "\"role\":\"assistant\"", 1),
|
||||
strings.Replace(line, "input_text", "output_text", 1),
|
||||
} {
|
||||
if _, ok, err := parseIntent([]byte(text)); err != nil || ok {
|
||||
if _, ok, err := (&intentLog{}).parse([]byte(text)); err != nil || ok {
|
||||
t.Fatalf("accepted untrusted instruction: match=%v error=%v", ok, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -107,3 +108,106 @@ func TestIntentCut(t *testing.T) {
|
|||
t.Fatalf("lost later input: %s %v", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
func agentCall(id, name, namespace string, status goalStatus) []byte {
|
||||
return append(mustJSON(map[string]any{"type": "response_item", "payload": map[string]any{
|
||||
"type": "function_call", "name": name, "namespace": namespace, "call_id": id,
|
||||
"arguments": string(mustJSON(struct {
|
||||
Status goalStatus `json:"status"`
|
||||
}{status})),
|
||||
}}), '\n')
|
||||
}
|
||||
|
||||
func agentReply(id, output string) []byte {
|
||||
return append(mustJSON(map[string]any{"type": "response_item", "payload": map[string]any{
|
||||
"type": "function_call_output", "call_id": id, "output": output,
|
||||
}}), '\n')
|
||||
}
|
||||
|
||||
func TestAgentIntent(t *testing.T) {
|
||||
blockedReply := string(mustJSON(goalResult{Goal: testGoal(blocked)}))
|
||||
for _, tc := range []struct {
|
||||
name, tool, namespace string
|
||||
status goalStatus
|
||||
id, replyID, output string
|
||||
want goalStatus
|
||||
}{
|
||||
{"plain", "update_goal", "", blocked, "call-1", "call-1", blockedReply, blocked},
|
||||
{"functions", "update_goal", "functions", blocked, "call-1", "call-1", blockedReply, blocked},
|
||||
{"external", "update_goal", "plugin", blocked, "call-1", "call-1", blockedReply, ""},
|
||||
{"read", "get_goal", "", blocked, "call-1", "call-1", blockedReply, ""},
|
||||
{"other tool", "other", "", blocked, "call-1", "call-1", blockedReply, ""},
|
||||
{"other status", "update_goal", "", paused, "call-1", "call-1", blockedReply, ""},
|
||||
{"no id", "update_goal", "", blocked, "", "", blockedReply, ""},
|
||||
{"other id", "update_goal", "", blocked, "call-1", "call-2", blockedReply, ""},
|
||||
{"rejected", "update_goal", "", blocked, "call-1", "call-1", "failed to update goal", ""},
|
||||
{"no goal", "update_goal", "", blocked, "call-1", "call-1", string(mustJSON(goalResult{})), ""},
|
||||
{"not blocked", "update_goal", "", blocked, "call-1", "call-1", string(mustJSON(goalResult{Goal: testGoal(active)})), ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "rollout.jsonl")
|
||||
if err := os.WriteFile(path, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log, err := openIntent(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendIntent(t, path, agentCall(tc.id, tc.tool, tc.namespace, tc.status))
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != "" {
|
||||
t.Fatalf("unfinished call produced intent %q: %v", status, err)
|
||||
}
|
||||
appendIntent(t, path, agentReply(tc.replyID, tc.output))
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != tc.want {
|
||||
t.Fatalf("intent=%q want=%q error=%v", status, tc.want, err)
|
||||
}
|
||||
appendIntent(t, path, agentReply(tc.id, blockedReply))
|
||||
if tc.replyID == tc.id {
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != "" {
|
||||
t.Fatalf("replayed completed call: %q %v", status, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentPending(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "rollout.jsonl")
|
||||
if err := os.WriteFile(path, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log, err := openIntent(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendIntent(t, path, agentCall("call-1", "update_goal", "", blocked))
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != "" {
|
||||
t.Fatalf("unfinished call produced intent %q: %v", status, err)
|
||||
}
|
||||
data, err := json.Marshal(log)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restored intentLog
|
||||
if err := json.Unmarshal(data, &restored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendIntent(t, path, agentReply("call-1", string(mustJSON(goalResult{Goal: testGoal(blocked)}))))
|
||||
if status, err := restored.read(math.MaxInt64); err != nil || status != blocked {
|
||||
t.Fatalf("lost pending call across checkpoint: %q %v", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentOtherItems(t *testing.T) {
|
||||
var log intentLog
|
||||
for _, payload := range []map[string]any{
|
||||
{"type": "tool_search_call", "arguments": map[string]string{"query": "blocked"}},
|
||||
{"type": "function_call", "name": "update_goal", "call_id": "call-1", "arguments": "invalid JSON"},
|
||||
{"type": "function_call_output", "call_id": "call-1", "output": string(mustJSON(goalResult{Goal: testGoal(blocked)}))},
|
||||
} {
|
||||
line := mustJSON(map[string]any{"type": "response_item", "payload": payload})
|
||||
if status, match, err := log.parse(line); err != nil || match || status != "" {
|
||||
t.Fatalf("unrelated item produced intent=%q match=%v error=%v", status, match, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue