Fix goal recovery
This commit is contained in:
parent
3846556035
commit
996f586bc1
29 changed files with 783 additions and 145 deletions
312
codex-keep/failure_test.go
Normal file
312
codex-keep/failure_test.go
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFailureKinds(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
`{"codexErrorInfo":"cyberPolicy"}`,
|
||||
`{"codexErrorInfo":"bioPolicy"}`,
|
||||
`{"codexErrorInfo":"misalignmentPolicyViolation"}`,
|
||||
`{"codexErrorInfo":"unauthorized"}`,
|
||||
`{"codexErrorInfo":"usageLimitExceeded"}`,
|
||||
`{"codexErrorInfo":"badRequest"}`,
|
||||
`{"codexErrorInfo":"other"}`,
|
||||
`{"codexErrorInfo":{"httpConnectionFailed":{"httpStatusCode":401}}}`,
|
||||
`{"codexErrorInfo":{"futureError":{"httpStatusCode":503}}}`,
|
||||
`{"codexErrorInfo":null}`,
|
||||
`{}`,
|
||||
`null`,
|
||||
} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
var failure *turnError
|
||||
if err := json.Unmarshal([]byte(raw), &failure); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := newFake(t, nil)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := f.op.store.watch("thread-1")
|
||||
for attempt := 1; attempt <= 2; attempt++ {
|
||||
f.fail()
|
||||
if attempt == 2 {
|
||||
f.turn.ID = "turn-2"
|
||||
}
|
||||
f.turn.Error = failure
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !w.eligible() || w.Failures != attempt || f.count("thread/goal/set") != attempt-1 {
|
||||
t.Fatalf("failed turn did not enter backoff: %+v", w)
|
||||
}
|
||||
if delay := time.Until(w.Due); delay <= 0 || delay > time.Duration(attempt)*time.Second {
|
||||
t.Fatalf("unexpected backoff: %s", delay)
|
||||
}
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("thread/goal/set") != attempt-1 {
|
||||
t.Fatal("resumed before backoff elapsed")
|
||||
}
|
||||
w.Due = time.Now().Add(-time.Second)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("thread/goal/set") != attempt || f.goal.Status != active || f.turn.Status != "inProgress" {
|
||||
t.Fatal("failed turn did not resume work")
|
||||
}
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedCheckpoint(t *testing.T) {
|
||||
for _, status := range []goalStatus{blocked, "usageLimited"} {
|
||||
t.Run(string(status), func(t *testing.T) {
|
||||
f := newFake(t, nil)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.fail()
|
||||
f.goal.Status = status
|
||||
f.turn.Error.Info.Kind = "cyberPolicy"
|
||||
w := f.op.store.watch("thread-1")
|
||||
w.Goal, w.Turn, w.Armed, w.Known = f.goal, f.turn, false, true
|
||||
if status == "usageLimited" {
|
||||
w.Hold = f.goal
|
||||
}
|
||||
if err := f.op.save(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := f.op.store.path
|
||||
closeStore(t, f.op.store)
|
||||
s, err := openStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeStore(t, s)
|
||||
next := newFake(t, s)
|
||||
next.goal, next.turn = f.goal, f.turn
|
||||
if err := next.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 1
|
||||
if status == "usageLimited" {
|
||||
want = 0
|
||||
}
|
||||
if w := s.watch("thread-1"); !w.eligible() || w.Failures != want || w.Due.IsZero() {
|
||||
t.Fatal("persisted failed turn was not scheduled")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsageRecovery(t *testing.T) {
|
||||
f := newFake(t, nil)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.fail()
|
||||
f.goal.Status = "usageLimited"
|
||||
f.turn.Error.Info.Kind = "usageLimitExceeded"
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := f.op.store.watch("thread-1")
|
||||
if !w.eligible() || w.Failures != 0 || w.Due.IsZero() {
|
||||
t.Fatal("server usage stop was not scheduled")
|
||||
}
|
||||
allowed := true
|
||||
f.usage = &allowed
|
||||
w.Due = time.Now().Add(-time.Second)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("thread/goal/set") != 1 || f.goal.Status != active {
|
||||
t.Fatal("server usage stop was not resumed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailureStop(t *testing.T) {
|
||||
for _, status := range []string{"interrupted", "paused", "blocked", "usageLimited", "cleared"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
f := newFake(t, nil)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.fail()
|
||||
f.turn.Error.Info.Kind = "cyberPolicy"
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := f.op.store.watch("thread-1")
|
||||
if !w.eligible() {
|
||||
t.Fatal("failed turn was not scheduled")
|
||||
}
|
||||
w.Due = time.Now().Add(-time.Second)
|
||||
switch status {
|
||||
case "interrupted":
|
||||
f.turn.Status = status
|
||||
f.notify("turn/completed", params{ThreadID: "thread-1", Turn: f.turn})
|
||||
case "cleared":
|
||||
appendIntent(t, f.path, intentLine("cleared"))
|
||||
f.goal = nil
|
||||
default:
|
||||
f.goal = testGoal(goalStatus(status))
|
||||
f.human(f.goal)
|
||||
}
|
||||
for range 3 {
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if f.count("thread/goal/set") != 0 || w.eligible() || !w.Due.IsZero() {
|
||||
t.Fatal("explicit human interruption did not cancel recovery")
|
||||
}
|
||||
path := f.op.store.path
|
||||
closeStore(t, f.op.store)
|
||||
s, err := openStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeStore(t, s)
|
||||
if w := s.watch("thread-1"); w.eligible() || !w.Due.IsZero() {
|
||||
t.Fatal("restart revoked explicit human cancellation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockSource(t *testing.T) {
|
||||
for _, source := range []string{"agent", "human"} {
|
||||
for _, status := range []string{"completed", "failed"} {
|
||||
t.Run(source+"/"+status, func(t *testing.T) {
|
||||
f := newFake(t, nil)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.goal, f.turn = testGoal(blocked), testTurn(status)
|
||||
if source == "human" {
|
||||
f.human(f.goal)
|
||||
} else {
|
||||
f.agent(f.goal)
|
||||
}
|
||||
f.notify("turn/completed", params{ThreadID: "thread-1", Turn: f.turn})
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := f.op.store.watch("thread-1")
|
||||
if w.eligible() || f.count("thread/goal/set") != 0 || !w.Due.IsZero() {
|
||||
t.Fatal("explicit block did not cancel recovery")
|
||||
}
|
||||
for range 3 {
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if f.count("thread/goal/set") != 0 || f.goal.Status != blocked || !w.Due.IsZero() {
|
||||
t.Fatal("explicitly blocked goal was resumed")
|
||||
}
|
||||
path := f.op.store.path
|
||||
closeStore(t, f.op.store)
|
||||
s, err := openStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeStore(t, s)
|
||||
next := newFake(t, s)
|
||||
next.goal, next.turn = f.goal, f.turn
|
||||
for range 2 {
|
||||
if err := next.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if next.count("thread/goal/set") != 0 || s.watch("thread-1").eligible() {
|
||||
t.Fatal("restart revoked an explicit block")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentRace(t *testing.T) {
|
||||
for _, method := range []string{"thread/goal/set", "account/rateLimits/read"} {
|
||||
t.Run(method, func(t *testing.T) {
|
||||
f := newFake(t, nil)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.fail()
|
||||
if method == "account/rateLimits/read" {
|
||||
f.goal.Status = "usageLimited"
|
||||
ready := true
|
||||
f.usage = &ready
|
||||
}
|
||||
seen := false
|
||||
f.before = func(m message) {
|
||||
if m.Method == method {
|
||||
seen = true
|
||||
f.before = nil
|
||||
f.goal = testGoal(blocked)
|
||||
f.agent(f.goal)
|
||||
}
|
||||
}
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := f.op.store.watch("thread-1")
|
||||
if w.eligible() {
|
||||
w.Due = time.Now().Add(-time.Second)
|
||||
}
|
||||
for range 3 {
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !seen || w.eligible() || !w.Due.IsZero() || f.goal.Status != blocked {
|
||||
t.Fatal("in-flight recovery revoked an explicit agent block")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentResume(t *testing.T) {
|
||||
f := newFake(t, nil)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.fail()
|
||||
f.agent(f.goal)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.goal, f.turn = testGoal(active), testTurn("inProgress")
|
||||
f.turn.ID = "turn-2"
|
||||
f.human(f.goal)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.fail()
|
||||
f.turn.ID = "turn-2"
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := f.op.store.watch("thread-1")
|
||||
if !w.eligible() || w.Due.IsZero() {
|
||||
t.Fatal("explicit resume did not restore server-failure recovery")
|
||||
}
|
||||
w.Due = time.Now().Add(-time.Second)
|
||||
if err := f.reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.count("thread/goal/set") != 1 || f.turn.Status != "inProgress" {
|
||||
t.Fatal("server failure did not resume after explicit authorization")
|
||||
}
|
||||
}
|
||||
|
|
@ -10,11 +10,12 @@ import (
|
|||
)
|
||||
|
||||
type intentLog struct {
|
||||
Path string `json:"path"`
|
||||
Pos int64 `json:"pos"`
|
||||
Fence int64 `json:"fence"`
|
||||
Partial bool `json:"partial,omitempty"`
|
||||
Sealed bool `json:"sealed,omitempty"`
|
||||
Path string `json:"path"`
|
||||
Pos int64 `json:"pos"`
|
||||
Fence int64 `json:"fence"`
|
||||
Partial bool `json:"partial,omitempty"`
|
||||
Sealed bool `json:"sealed,omitempty"`
|
||||
Calls map[string]struct{} `json:"calls,omitempty"`
|
||||
}
|
||||
|
||||
func openIntent(path string) (*intentLog, error) {
|
||||
|
|
@ -88,7 +89,7 @@ func (l *intentLog) read(end int64) (goalStatus, error) {
|
|||
l.Pos = next
|
||||
continue
|
||||
}
|
||||
status, match, err := parseIntent(line)
|
||||
status, match, err := l.parse(line)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -99,13 +100,18 @@ func (l *intentLog) read(end int64) (goalStatus, error) {
|
|||
}
|
||||
}
|
||||
|
||||
func parseIntent(line []byte) (goalStatus, bool, error) {
|
||||
func (l *intentLog) parse(line []byte) (goalStatus, bool, error) {
|
||||
var record struct {
|
||||
Type string `json:"type"`
|
||||
Payload struct {
|
||||
Type string `json:"type"`
|
||||
Role string `json:"role"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Role string `json:"role"`
|
||||
Name string `json:"name"`
|
||||
Namespace string `json:"namespace"`
|
||||
CallID string `json:"call_id"`
|
||||
Arguments json.RawMessage `json:"arguments"`
|
||||
Output json.RawMessage `json:"output"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
|
|
@ -117,13 +123,44 @@ func parseIntent(line []byte) (goalStatus, bool, error) {
|
|||
if err := json.Unmarshal(line, &record); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if record.Type != "response_item" {
|
||||
return "", false, nil
|
||||
}
|
||||
p := record.Payload
|
||||
if record.Type != "response_item" || p.Type != "message" || p.Role != "user" || len(p.Content) != 1 ||
|
||||
switch p.Type {
|
||||
case "function_call":
|
||||
if p.Name == "update_goal" && (p.Namespace == "" || p.Namespace == "functions") && p.CallID != "" {
|
||||
var args struct {
|
||||
Status goalStatus `json:"status"`
|
||||
}
|
||||
var arguments string
|
||||
if json.Unmarshal(p.Arguments, &arguments) == nil && json.Unmarshal([]byte(arguments), &args) == nil && args.Status == blocked {
|
||||
if l.Calls == nil {
|
||||
l.Calls = make(map[string]struct{})
|
||||
}
|
||||
l.Calls[p.CallID] = struct{}{}
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
case "function_call_output":
|
||||
if _, ok := l.Calls[p.CallID]; !ok {
|
||||
return "", false, nil
|
||||
}
|
||||
delete(l.Calls, p.CallID)
|
||||
var output string
|
||||
var result goalResult
|
||||
if json.Unmarshal(p.Output, &output) == nil && json.Unmarshal([]byte(output), &result) == nil &&
|
||||
result.Goal != nil && result.Goal.Status == blocked {
|
||||
return blocked, true, nil
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
if p.Type != "message" || p.Role != "user" || len(p.Content) != 1 ||
|
||||
p.Content[0].Type != "input_text" || len(p.Meta.Kinds) != 1 ||
|
||||
p.Meta.Kinds[0] != "user.goal" && p.Meta.Kinds[0] != "user.goal.omitted" {
|
||||
return "", false, nil
|
||||
}
|
||||
for _, line := range strings.Split(p.Content[0].Text, "\n") {
|
||||
for line := range strings.SplitSeq(p.Content[0].Text, "\n") {
|
||||
if line == "User cleared the goal." {
|
||||
return "cleared", true, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -9,8 +10,8 @@ import (
|
|||
)
|
||||
|
||||
func TestIntentProvenance(t *testing.T) {
|
||||
for _, status := range []goalStatus{active, paused, "complete", "budgetLimited", "usageLimited", blocked} {
|
||||
got, ok, err := parseIntent(intentLine(status))
|
||||
for _, status := range []goalStatus{active, paused, "complete", "budgetLimited", "usageLimited", blocked, "cleared"} {
|
||||
got, ok, err := (&intentLog{}).parse(intentLine(status))
|
||||
if err != nil || !ok || got != status {
|
||||
t.Fatalf("status=%s got=%s match=%v error=%v", status, got, ok, err)
|
||||
}
|
||||
|
|
@ -21,7 +22,7 @@ func TestIntentProvenance(t *testing.T) {
|
|||
strings.Replace(line, "\"role\":\"user\"", "\"role\":\"assistant\"", 1),
|
||||
strings.Replace(line, "input_text", "output_text", 1),
|
||||
} {
|
||||
if _, ok, err := parseIntent([]byte(text)); err != nil || ok {
|
||||
if _, ok, err := (&intentLog{}).parse([]byte(text)); err != nil || ok {
|
||||
t.Fatalf("accepted untrusted instruction: match=%v error=%v", ok, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -107,3 +108,106 @@ func TestIntentCut(t *testing.T) {
|
|||
t.Fatalf("lost later input: %s %v", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
func agentCall(id, name, namespace string, status goalStatus) []byte {
|
||||
return append(mustJSON(map[string]any{"type": "response_item", "payload": map[string]any{
|
||||
"type": "function_call", "name": name, "namespace": namespace, "call_id": id,
|
||||
"arguments": string(mustJSON(struct {
|
||||
Status goalStatus `json:"status"`
|
||||
}{status})),
|
||||
}}), '\n')
|
||||
}
|
||||
|
||||
func agentReply(id, output string) []byte {
|
||||
return append(mustJSON(map[string]any{"type": "response_item", "payload": map[string]any{
|
||||
"type": "function_call_output", "call_id": id, "output": output,
|
||||
}}), '\n')
|
||||
}
|
||||
|
||||
func TestAgentIntent(t *testing.T) {
|
||||
blockedReply := string(mustJSON(goalResult{Goal: testGoal(blocked)}))
|
||||
for _, tc := range []struct {
|
||||
name, tool, namespace string
|
||||
status goalStatus
|
||||
id, replyID, output string
|
||||
want goalStatus
|
||||
}{
|
||||
{"plain", "update_goal", "", blocked, "call-1", "call-1", blockedReply, blocked},
|
||||
{"functions", "update_goal", "functions", blocked, "call-1", "call-1", blockedReply, blocked},
|
||||
{"external", "update_goal", "plugin", blocked, "call-1", "call-1", blockedReply, ""},
|
||||
{"read", "get_goal", "", blocked, "call-1", "call-1", blockedReply, ""},
|
||||
{"other tool", "other", "", blocked, "call-1", "call-1", blockedReply, ""},
|
||||
{"other status", "update_goal", "", paused, "call-1", "call-1", blockedReply, ""},
|
||||
{"no id", "update_goal", "", blocked, "", "", blockedReply, ""},
|
||||
{"other id", "update_goal", "", blocked, "call-1", "call-2", blockedReply, ""},
|
||||
{"rejected", "update_goal", "", blocked, "call-1", "call-1", "failed to update goal", ""},
|
||||
{"no goal", "update_goal", "", blocked, "call-1", "call-1", string(mustJSON(goalResult{})), ""},
|
||||
{"not blocked", "update_goal", "", blocked, "call-1", "call-1", string(mustJSON(goalResult{Goal: testGoal(active)})), ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "rollout.jsonl")
|
||||
if err := os.WriteFile(path, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log, err := openIntent(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendIntent(t, path, agentCall(tc.id, tc.tool, tc.namespace, tc.status))
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != "" {
|
||||
t.Fatalf("unfinished call produced intent %q: %v", status, err)
|
||||
}
|
||||
appendIntent(t, path, agentReply(tc.replyID, tc.output))
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != tc.want {
|
||||
t.Fatalf("intent=%q want=%q error=%v", status, tc.want, err)
|
||||
}
|
||||
appendIntent(t, path, agentReply(tc.id, blockedReply))
|
||||
if tc.replyID == tc.id {
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != "" {
|
||||
t.Fatalf("replayed completed call: %q %v", status, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentPending(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "rollout.jsonl")
|
||||
if err := os.WriteFile(path, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log, err := openIntent(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendIntent(t, path, agentCall("call-1", "update_goal", "", blocked))
|
||||
if status, err := log.read(math.MaxInt64); err != nil || status != "" {
|
||||
t.Fatalf("unfinished call produced intent %q: %v", status, err)
|
||||
}
|
||||
data, err := json.Marshal(log)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restored intentLog
|
||||
if err := json.Unmarshal(data, &restored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendIntent(t, path, agentReply("call-1", string(mustJSON(goalResult{Goal: testGoal(blocked)}))))
|
||||
if status, err := restored.read(math.MaxInt64); err != nil || status != blocked {
|
||||
t.Fatalf("lost pending call across checkpoint: %q %v", status, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentOtherItems(t *testing.T) {
|
||||
var log intentLog
|
||||
for _, payload := range []map[string]any{
|
||||
{"type": "tool_search_call", "arguments": map[string]string{"query": "blocked"}},
|
||||
{"type": "function_call", "name": "update_goal", "call_id": "call-1", "arguments": "invalid JSON"},
|
||||
{"type": "function_call_output", "call_id": "call-1", "output": string(mustJSON(goalResult{Goal: testGoal(blocked)}))},
|
||||
} {
|
||||
line := mustJSON(map[string]any{"type": "response_item", "payload": payload})
|
||||
if status, match, err := log.parse(line); err != nil || match || status != "" {
|
||||
t.Fatalf("unrelated item produced intent=%q match=%v error=%v", status, match, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
11
codex-keep/model/BlindUsage.cfg
Normal file
11
codex-keep/model/BlindUsage.cfg
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Mode = "BlindUsage"
|
||||
GoalBound = 1
|
||||
EditBound = 1
|
||||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES RecoveryProgress StopProgress HoldProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
13
codex-keep/model/Controls.cfg
Normal file
13
codex-keep/model/Controls.cfg
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
SPECIFICATION PartSpec
|
||||
CONSTANTS
|
||||
First = "paused"
|
||||
Last = "active"
|
||||
Mode = "Reconcile"
|
||||
GoalBound = 1
|
||||
EditBound = 2
|
||||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES RecoveryProgress StopProgress HoldProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
13
codex-keep/model/Controls.tla
Normal file
13
codex-keep/model/Controls.tla
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
----------------------------- MODULE Controls -----------------------------
|
||||
EXTENDS Keep
|
||||
|
||||
\* Bounded append-only controls partition all traces; fair actions never change inputs.
|
||||
CONSTANTS First, Last
|
||||
ASSUME First \in GoalStates /\ Last \in GoalStates
|
||||
Control == <<First, Last>>
|
||||
|
||||
Prefix == Len(inputs') <= Len(Control) /\
|
||||
\A i \in 1..Len(inputs') : inputs'[i] = Control[i]
|
||||
PartNext == Next /\ Prefix
|
||||
PartSpec == Init /\ [][PartNext]_vars /\ Fairness
|
||||
=============================================================================
|
||||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK ApiEvidence
|
||||
INVARIANTS TypeOK FailureEvidence
|
||||
PROPERTIES StopProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
11
codex-keep/model/Filtered.cfg
Normal file
11
codex-keep/model/Filtered.cfg
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
SPECIFICATION Spec
|
||||
CONSTANTS
|
||||
Mode = "Filtered"
|
||||
GoalBound = 1
|
||||
EditBound = 1
|
||||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES RecoveryProgress StopProgress HoldProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
@ -1,26 +1,33 @@
|
|||
------------------------------- MODULE Keep -------------------------------
|
||||
EXTENDS Naturals, Sequences
|
||||
|
||||
\* Each connection serializes RPCs; one abandoned write may finish after reconnect. Epochs audit human intent only.
|
||||
\* Host-authored controls form an append-only input stream; progress assumes eventual connectivity and fair finite work.
|
||||
\* Each connection serializes RPCs; one abandoned write may finish after reconnect. Epochs audit explicit intent only.
|
||||
\* Human controls and successful agent blocks form an append-only input stream; progress assumes eventual connectivity, usage recovery, and fair finite work.
|
||||
CONSTANTS Mode, GoalBound, EditBound, StampBound, TurnBound, FailBound
|
||||
|
||||
GoalStates == {"active", "blocked", "paused", "complete", "cleared",
|
||||
"budgetLimited", "usageLimited"}
|
||||
TurnStates == {"inProgress", "failed", "completed", "interrupted"}
|
||||
Reads == {"pollGoal", "pollTurn", "thread", "check", "attach"}
|
||||
Reads == {"pollGoal", "pollTurn", "thread", "check", "attach", "usage"}
|
||||
Events == {"goal", "start", "done"}
|
||||
Episodes == (1..GoalBound) \X (1..TurnBound)
|
||||
StopType == [id : 0..GoalBound, status : GoalStates, epoch : 0..EditBound]
|
||||
|
||||
ASSUME /\ Mode \in {"Baseline", "Local", "Observe", "Unordered", "BarrierOld", "Reconcile"}
|
||||
ASSUME /\ Mode \in {"Baseline", "Local", "Observe", "Unordered", "BarrierOld", "Filtered", "BlindUsage", "Reconcile"}
|
||||
/\ GoalBound \in Nat \ {0} /\ EditBound \in Nat \ {0}
|
||||
/\ StampBound \in Nat /\ TurnBound \in Nat \ {0}
|
||||
/\ FailBound \in Nat \ {0}
|
||||
|
||||
Recon == Mode \in {"Observe", "Unordered", "BarrierOld", "Reconcile"}
|
||||
Durable == Mode \in {"Unordered", "BarrierOld", "Reconcile"}
|
||||
Ordered == Mode \in {"BarrierOld", "Reconcile"}
|
||||
Recon == Mode \in {"Observe", "Unordered", "BarrierOld", "Filtered", "BlindUsage", "Reconcile"}
|
||||
Durable == Mode \in {"Unordered", "BarrierOld", "Filtered", "BlindUsage", "Reconcile"}
|
||||
Ordered == Mode \in {"BarrierOld", "Filtered", "BlindUsage", "Reconcile"}
|
||||
|
||||
AllFailures == Mode \in {"BlindUsage", "Reconcile"}
|
||||
Waiting == {"blocked"} \cup (IF AllFailures THEN {"usageLimited"} ELSE {})
|
||||
Running == {"active"} \cup Waiting
|
||||
|
||||
\* Error classes are observationally equivalent in the current recovery policy.
|
||||
Failures == IF AllFailures THEN {FALSE} ELSE BOOLEAN
|
||||
|
||||
GoalType == [id : 0..GoalBound, stamp : 0..StampBound,
|
||||
status : GoalStates]
|
||||
|
|
@ -29,11 +36,11 @@ LiveType == [edits : 0..EditBound, gen : 1..GoalBound, goal : GoalType, turn : T
|
|||
WatchType == [cut : 0..EditBound, cursor : 0..EditBound, fence : 0..EditBound, goal : GoalType, turn : TurnType, armed : BOOLEAN,
|
||||
fails : 0..FailBound, due : {"none", "waiting", "ready"},
|
||||
hold : GoalStates \cup {"none"}, pending : BOOLEAN, known : BOOLEAN, echo : GoalType, seed : GoalType]
|
||||
TicketType == [expired : BOOLEAN, status : GoalStates]
|
||||
TicketType == [expired : BOOLEAN, status : GoalStates, quota : BOOLEAN]
|
||||
MsgType == [kind : Events \cup Reads \cup {"set"}, goal : GoalType,
|
||||
turn : TurnType, truth : 0..EditBound, busy : BOOLEAN, ok : BOOLEAN]
|
||||
ActType == [sent : BOOLEAN, view : [goal : 0..GoalBound, turn : 0..(TurnBound + 1)], expected : TicketType,
|
||||
origin : {"user", "automatic"}, evidence : BOOLEAN, recover : BOOLEAN]
|
||||
origin : {"user", "automatic"}, evidence : BOOLEAN, recover : BOOLEAN, usage : BOOLEAN]
|
||||
|
||||
VARIABLES late, inputs, sample, attached, live, w, wire, online, phase, req, invalid,
|
||||
ticket, lastAct, known, need, resumed, cancelled, stopWant, stopNeed
|
||||
|
|
@ -49,17 +56,20 @@ Message(kind, state, ok) ==
|
|||
|
||||
Stop(x) == [x EXCEPT !.armed = FALSE, !.due = "none"]
|
||||
Increment(x) == [x EXCEPT !.fails = IF @ < FailBound THEN @ + 1 ELSE @]
|
||||
Eligible(x) == x.armed /\ x.goal.id # 0 /\ x.goal.status = "blocked"
|
||||
/\ x.turn.status = "failed" /\ x.turn.api
|
||||
Retry(t) == t.api \/ AllFailures
|
||||
Eligible(x) == x.armed /\ x.goal.id # 0 /\ x.goal.status \in Waiting
|
||||
/\ x.turn.status = "failed" /\ Retry(x.turn)
|
||||
Schedule(x) == IF Eligible(x) /\ x.due = "none"
|
||||
THEN [Increment(x) EXCEPT !.due = "waiting"] ELSE x
|
||||
THEN IF Mode = "Reconcile" /\ x.goal.status = "usageLimited"
|
||||
THEN [x EXCEPT !.due = "ready"]
|
||||
ELSE [Increment(x) EXCEPT !.due = "waiting"] ELSE x
|
||||
KnownAfter(g) == IF g.id # 0 /\ g.status = "active" THEN known \cup {g.id} ELSE known
|
||||
|
||||
BaseGoal(x, g) ==
|
||||
LET same == g.id # 0 /\ x.goal.id = g.id
|
||||
cancel == ~same \/ g.status \notin {"active", "blocked"}
|
||||
cancel == ~same \/ g.status \notin Running
|
||||
\/ (g.status = "active" /\ x.goal.status # "active")
|
||||
\/ (g.status = "blocked" /\ x.due # "none"
|
||||
\/ (g.status \in Waiting /\ x.due # "none"
|
||||
/\ x.goal.stamp # g.stamp)
|
||||
base == [x EXCEPT !.goal = g, !.fails = IF same THEN @ ELSE 0,
|
||||
!.armed = @ /\ ~cancel, !.due = IF cancel THEN "none" ELSE @]
|
||||
|
|
@ -71,17 +81,18 @@ GoalUpdate(x, g, fresh) ==
|
|||
IF ~fresh /\ g.status = "active" /\ x.hold = "none" /\ g.id = x.goal.id THEN x ELSE
|
||||
LET same == g.id # 0 /\ g.id = x.goal.id
|
||||
holding == x.hold # "none" /\ same
|
||||
stopped == g.id # 0 /\ g.status \notin {"active", "blocked"}
|
||||
stopped == g.id # 0 /\ g.status \notin Running
|
||||
base == IF ~same THEN [Stop(x) EXCEPT !.fails = 0, !.hold = "none", !.known = FALSE] ELSE x
|
||||
active == g.id # 0 /\ g.status = "active"
|
||||
transition == active /\ (~same \/ x.goal.status # "active")
|
||||
transition == (active /\ (~same \/ x.goal.status # "active"))
|
||||
\/ (Mode = "Reconcile" /\ g.status = "usageLimited" /\ (~same \/ x.goal.status # g.status))
|
||||
updated == [base EXCEPT !.goal = g, !.known = @ \/ active,
|
||||
!.armed = IF active THEN @ \/ transition ELSE IF stopped \/ g.id = 0 THEN FALSE ELSE @,
|
||||
!.due = IF transition \/ stopped \/ g.id = 0 THEN "none" ELSE @,
|
||||
!.hold = IF (stopped /\ (Durable \/ x.pending))
|
||||
\/ (~same /\ g.status = "blocked" /\ x.pending) THEN g.status ELSE @]
|
||||
\/ (~same /\ g.status \in Waiting /\ x.pending) THEN g.status ELSE @]
|
||||
IN IF holding
|
||||
THEN IF ~fresh THEN IF g.status \in {"active", "blocked"} THEN x ELSE [Stop(x) EXCEPT !.goal = g, !.hold = g.status]
|
||||
THEN IF ~fresh THEN IF g.status \in Running THEN x ELSE [Stop(x) EXCEPT !.goal = g, !.hold = g.status]
|
||||
ELSE IF g.status = x.hold /\ x.echo.id = 0
|
||||
THEN [Stop(x) EXCEPT !.goal = g, !.hold = IF Durable /\ g.status # "active" THEN @ ELSE "none",
|
||||
!.armed = g.status = "active",
|
||||
|
|
@ -96,11 +107,12 @@ TurnUpdate(x, t) ==
|
|||
started == [base EXCEPT !.armed = (Recon /\ x.armed)
|
||||
\/ (t.status = "inProgress" /\ x.goal.id # 0 /\ x.goal.status = "active")]
|
||||
terminal == t.status = "interrupted"
|
||||
\/ (t.status = "failed" /\ ~t.api)
|
||||
\/ (t.status = "failed" /\ ~Retry(t))
|
||||
\/ (t.status = "completed" /\ ~Recon)
|
||||
settled == IF terminal THEN Stop(base) ELSE base
|
||||
result == IF fresh /\ t.status = "inProgress" THEN started ELSE
|
||||
IF Recon /\ (fresh \/ x.turn.status = "inProgress") /\ t.status = "failed" /\ t.api
|
||||
IF Recon /\ (fresh \/ x.turn.status = "inProgress")
|
||||
/\ t.status = "failed" /\ Retry(t)
|
||||
/\ x.known /\ x.hold = "none"
|
||||
THEN [settled EXCEPT !.armed = TRUE] ELSE settled
|
||||
held == IF Recon /\ t.status = "interrupted" /\ x.goal.id # 0 /\ x.pending
|
||||
|
|
@ -112,7 +124,7 @@ TurnUpdate(x, t) ==
|
|||
ApplyInput(x, status) ==
|
||||
IF status = "cleared" THEN [Stop(x) EXCEPT !.hold = "none", !.known = FALSE] ELSE
|
||||
IF x.goal.id = 0 THEN x ELSE
|
||||
IF status = "active" /\ x.goal.status \in {"active", "blocked"}
|
||||
IF status = "active" /\ x.goal.status \in Running
|
||||
THEN [x EXCEPT !.hold = "none", !.known = TRUE, !.armed = TRUE, !.due = "none"]
|
||||
ELSE [Stop(x) EXCEPT !.goal.status = status, !.hold = status, !.known = status = "active"]
|
||||
|
||||
|
|
@ -136,9 +148,9 @@ Init ==
|
|||
armed |-> TRUE, fails |-> 0, due |-> "none", hold |-> "none", pending |-> FALSE, known |-> TRUE, echo |-> EmptyGoal, seed |-> EmptyGoal]
|
||||
/\ wire = <<>> /\ online = TRUE /\ phase = "idle" /\ req = "none"
|
||||
/\ invalid = FALSE
|
||||
/\ ticket = [expired |-> FALSE, status |-> "active"]
|
||||
/\ ticket = [expired |-> FALSE, status |-> "active", quota |-> FALSE]
|
||||
/\ lastAct = [sent |-> FALSE, view |-> [goal |-> w.goal.id, turn |-> w.turn.id], expected |-> ticket,
|
||||
origin |-> "automatic", evidence |-> TRUE, recover |-> TRUE]
|
||||
origin |-> "automatic", evidence |-> TRUE, recover |-> TRUE, usage |-> FALSE]
|
||||
/\ known = {1} /\ need = {} /\ resumed = {} /\ cancelled = {}
|
||||
/\ stopWant = [id |-> 0, status |-> "paused", epoch |-> 0] /\ stopNeed = FALSE
|
||||
|
||||
|
|
@ -193,10 +205,10 @@ FinishTurn(status) ==
|
|||
/\ UNCHANGED <<late, inputs, sample, attached, w, online, phase, req, invalid,
|
||||
ticket, lastAct, known, need, resumed, stopWant, stopNeed>>
|
||||
|
||||
FailTurn(isApi, stamp) ==
|
||||
FailTurn(isApi, status, stamp) ==
|
||||
/\ live.turn.status = "inProgress" /\ live.turn.id <= TurnBound
|
||||
/\ LET g == IF live.goal.status = "active"
|
||||
THEN [live.goal EXCEPT !.status = "blocked",
|
||||
THEN [live.goal EXCEPT !.status = status,
|
||||
!.stamp = stamp]
|
||||
ELSE live.goal
|
||||
state == [live EXCEPT !.goal = g, !.turn.status = "failed", !.turn.api = isApi]
|
||||
|
|
@ -204,7 +216,7 @@ FailTurn(isApi, stamp) ==
|
|||
THEN Append(wire, Message("goal", state, TRUE)) ELSE wire
|
||||
IN /\ live' = state
|
||||
/\ wire' = IF online THEN Append(events, Message("done", state, TRUE)) ELSE wire
|
||||
/\ need' = IF isApi /\ live.goal.status = "active" /\ live.goal.id \in known
|
||||
/\ need' = IF live.goal.status = "active" /\ live.goal.id \in known
|
||||
THEN need \cup {<<live.goal.id, live.turn.id>>} ELSE need
|
||||
/\ UNCHANGED <<late, inputs, sample, attached, w, online, phase, req, invalid,
|
||||
ticket, lastAct, known, resumed, cancelled, stopWant, stopNeed>>
|
||||
|
|
@ -224,7 +236,7 @@ Poll ==
|
|||
known, need, resumed, cancelled, stopWant, stopNeed>>
|
||||
|
||||
ServeRead ==
|
||||
/\ req \in Reads
|
||||
/\ req \in Reads \ {"usage"}
|
||||
/\ wire' = IF ~online THEN wire ELSE
|
||||
IF req = "attach" THEN Append(Append(wire, Message(req, live, TRUE)), Message("goal", live, TRUE))
|
||||
ELSE Append(wire, Message(req, live, TRUE))
|
||||
|
|
@ -232,6 +244,14 @@ ServeRead ==
|
|||
/\ UNCHANGED <<late, inputs, sample, attached, live, w, online, phase, invalid, ticket, lastAct,
|
||||
known, need, resumed, cancelled, stopWant, stopNeed>>
|
||||
|
||||
ServeUsage(allowed) ==
|
||||
/\ req = "usage"
|
||||
/\ wire' = IF ~online THEN wire ELSE
|
||||
Append(wire, Message(req, live, allowed))
|
||||
/\ req' = "none"
|
||||
/\ UNCHANGED <<late, inputs, sample, attached, live, w, online, phase, invalid, ticket, lastAct,
|
||||
known, need, resumed, cancelled, stopWant, stopNeed>>
|
||||
|
||||
ReadWatch(m) ==
|
||||
IF invalid /\ ~(Mode = "BarrierOld" /\ m.kind = "attach")
|
||||
THEN IF Ordered /\ m.kind = "attach"
|
||||
|
|
@ -246,6 +266,7 @@ ReadWatch(m) ==
|
|||
IN IF Recon /\ ~attached /\ m.goal.id # 0
|
||||
THEN [ready EXCEPT !.seed = IF Ordered THEN EmptyGoal ELSE m.goal, !.echo = EmptyGoal] ELSE ready
|
||||
[] m.kind = "attach" -> IF Ordered THEN [w EXCEPT !.seed = [id |-> 1, stamp |-> 0, status |-> "active"]] ELSE w
|
||||
[] m.kind = "usage" -> w
|
||||
[] m.kind = "pollTurn" -> Schedule(TurnUpdate(w, m.turn))
|
||||
[] m.kind = "thread" -> IF ~m.busy THEN w ELSE
|
||||
IF Recon THEN [w EXCEPT !.due = "none",
|
||||
|
|
@ -261,7 +282,9 @@ ReadPhase(m, updated) ==
|
|||
[] m.kind = "pollTurn" -> IF Eligible(updated) /\ updated.due = "ready" /\ updated.echo.id = 0 /\ updated.seed.id = 0
|
||||
THEN "thread" ELSE "idle"
|
||||
[] m.kind = "thread" -> IF m.busy THEN "idle" ELSE "check"
|
||||
[] OTHER -> IF Eligible(updated) THEN "send" ELSE "idle"
|
||||
[] m.kind = "usage" -> IF m.ok /\ Eligible(updated) THEN "send" ELSE "idle"
|
||||
[] OTHER -> IF ~Eligible(updated) THEN "idle" ELSE
|
||||
IF Mode = "Reconcile" /\ updated.goal.status = "usageLimited" THEN "usage" ELSE "send"
|
||||
|
||||
ReceiveRead ==
|
||||
/\ online /\ Len(wire) > 0 /\ Head(wire).kind \in Reads
|
||||
|
|
@ -272,9 +295,10 @@ ReceiveRead ==
|
|||
IN /\ w' = [updated EXCEPT !.cut = IF Ordered /\ m.kind = "pollGoal" /\ next = "attach" THEN Len(inputs) ELSE @]
|
||||
/\ phase' = next
|
||||
/\ req' = IF next \in Reads THEN next ELSE "none"
|
||||
/\ ticket' = IF next = "send"
|
||||
THEN [expired |-> w.due = "ready", status |-> "active"]
|
||||
ELSE ticket
|
||||
/\ ticket' = IF m.kind = "usage" THEN [ticket EXCEPT !.quota = m.ok]
|
||||
ELSE IF next \in {"send", "usage"}
|
||||
THEN [expired |-> w.due = "ready", status |-> "active", quota |-> FALSE]
|
||||
ELSE ticket
|
||||
/\ known' = IF ~invalid /\ m.kind \in {"pollGoal", "check"}
|
||||
THEN KnownAfter(m.goal) ELSE known
|
||||
/\ sample' = IF Head(wire).kind = "pollGoal" /\ ~invalid THEN Head(wire).goal ELSE sample
|
||||
|
|
@ -283,7 +307,7 @@ ReceiveRead ==
|
|||
/\ attached' = (attached \/ Head(wire).kind = "attach")
|
||||
/\ LET m == Head(wire)
|
||||
stopped == ~invalid /\ m.kind = "pollGoal" /\ m.goal.id # 0
|
||||
/\ m.goal.status \notin {"active", "blocked"}
|
||||
/\ m.goal.status \notin Running
|
||||
/\ (ReadWatch(m)).goal.status = m.goal.status
|
||||
IN /\ stopWant' = IF stopped THEN [id |-> m.goal.id, status |-> m.goal.status, epoch |-> m.truth] ELSE stopWant
|
||||
/\ stopNeed' = (stopNeed \/ stopped)
|
||||
|
|
@ -293,7 +317,7 @@ ReceiveRead ==
|
|||
ReadInput ==
|
||||
/\ online /\ phase = "inputs" /\ req = "none"
|
||||
/\ LET updated == SyncInput(w)
|
||||
blocked == updated.hold = "active" /\ sample.status = "blocked"
|
||||
blocked == updated.hold = "active" /\ sample.status \in Waiting
|
||||
ready == IF blocked THEN [updated EXCEPT !.hold = "none", !.goal = sample,
|
||||
!.known = TRUE, !.armed = TRUE] ELSE updated
|
||||
repair == ready.hold # "none" /\ ready.hold # sample.status
|
||||
|
|
@ -316,7 +340,7 @@ ReceiveEvent ==
|
|||
echo == Recon /\ m.kind = "goal" /\ w.echo.id # 0
|
||||
/\ m.goal.id = w.echo.id /\ m.goal.status = w.echo.status /\ m.goal.stamp = w.echo.stamp
|
||||
ignore == snapshot \/ echo \/ (m.kind = "done" /\ m.turn.id # w.turn.id)
|
||||
seal == ~ignore /\ ((~Ordered /\ m.kind = "goal" /\ m.goal.status \notin {"active", "blocked"})
|
||||
seal == ~ignore /\ ((~Ordered /\ m.kind = "goal" /\ m.goal.status \notin Running)
|
||||
\/ (m.kind = "done" /\ m.turn.status = "interrupted"))
|
||||
base == IF seal /\ Recon THEN [w EXCEPT !.fence = Len(inputs)]
|
||||
ELSE IF Ordered \/ (ignore /\ ~echo /\ ~snapshot) THEN w ELSE SyncInput(w)
|
||||
|
|
@ -327,12 +351,12 @@ ReceiveEvent ==
|
|||
ELSE IF ignore THEN w ELSE Schedule(updated)
|
||||
EXCEPT !.seed = IF seed THEN EmptyGoal ELSE @]
|
||||
/\ known' = IF m.kind = "goal" THEN KnownAfter(m.goal) ELSE known
|
||||
/\ stopWant' = IF ~echo /\ ~snapshot /\ m.kind = "goal" /\ m.goal.id # 0 /\ m.goal.status \notin {"active", "blocked"}
|
||||
/\ stopWant' = IF ~echo /\ ~snapshot /\ m.kind = "goal" /\ m.goal.id # 0 /\ m.goal.status \notin Running
|
||||
THEN [id |-> m.goal.id, status |-> m.goal.status, epoch |-> m.truth]
|
||||
ELSE IF ~ignore /\ m.kind = "done" /\ m.turn.status = "interrupted" /\ w.goal.id # 0 /\ w.pending
|
||||
THEN [id |-> w.goal.id, status |-> "paused", epoch |-> m.truth] ELSE stopWant
|
||||
/\ stopNeed' = ((stopNeed /\ ~(~seal /\ NewInput(w)))
|
||||
\/ (~echo /\ ~snapshot /\ m.kind = "goal" /\ m.goal.id # 0 /\ m.goal.status \notin {"active", "blocked"})
|
||||
\/ (~echo /\ ~snapshot /\ m.kind = "goal" /\ m.goal.id # 0 /\ m.goal.status \notin Running)
|
||||
\/ (~ignore /\ m.kind = "done" /\ m.turn.status = "interrupted" /\ w.goal.id # 0 /\ w.pending))
|
||||
/\ invalid' = (invalid \/ ((~ignore \/ (~Ordered /\ (echo \/ snapshot) /\ NewInput(w))) /\ phase # "idle"))
|
||||
/\ wire' = Tail(wire)
|
||||
|
|
@ -344,11 +368,11 @@ SendSet ==
|
|||
/\ IF phase = "send" THEN Eligible(w) ELSE w.hold # "none"
|
||||
/\ phase' = "set" /\ req' = "set" /\ invalid' = FALSE /\ w' = [w EXCEPT !.pending = TRUE]
|
||||
/\ ticket' = IF phase = "repair"
|
||||
THEN [expired |-> FALSE, status |-> w.hold] ELSE ticket
|
||||
THEN [expired |-> FALSE, status |-> w.hold, quota |-> FALSE] ELSE ticket
|
||||
/\ lastAct' = [sent |-> TRUE, view |-> [goal |-> w.goal.id, turn |-> w.turn.id],
|
||||
expected |-> IF phase = "send" THEN ticket ELSE [ticket EXCEPT !.status = w.hold],
|
||||
origin |-> IF Mode = "Baseline" THEN "user" ELSE "automatic",
|
||||
evidence |-> Eligible(w), recover |-> phase = "send"]
|
||||
evidence |-> Eligible(w), recover |-> phase = "send", usage |-> w.goal.status = "usageLimited"]
|
||||
/\ UNCHANGED <<late, inputs, sample, attached, live, wire, online, known, need, resumed, cancelled, stopWant, stopNeed>>
|
||||
|
||||
ServeSet(stamp) ==
|
||||
|
|
@ -371,7 +395,8 @@ ReceiveSet ==
|
|||
base == [w EXCEPT !.pending = FALSE, !.echo = IF m.ok /\ Recon THEN m.goal ELSE @]
|
||||
IN /\ w' = IF m.ok
|
||||
THEN IF invalid THEN base ELSE GoalUpdate(base, m.goal, TRUE)
|
||||
ELSE IF Mode = "Baseline" \/ (~invalid /\ Eligible(w))
|
||||
ELSE IF Mode = "Baseline" \/ (~invalid /\ Eligible(w)
|
||||
/\ (Mode # "Reconcile" \/ w.goal.status # "usageLimited"))
|
||||
THEN Increment(base) ELSE base
|
||||
/\ known' = IF m.ok /\ ~invalid THEN KnownAfter(m.goal) ELSE known
|
||||
/\ phase' = "idle" /\ invalid' = FALSE /\ wire' = Tail(wire)
|
||||
|
|
@ -390,7 +415,8 @@ Disconnect ==
|
|||
/\ req' = "none"
|
||||
/\ late' = IF req = "set" THEN ticket.status ELSE late
|
||||
/\ invalid' = FALSE
|
||||
/\ w' = [IF phase = "set" /\ (Mode = "Baseline" \/ (~invalid /\ Eligible(w)))
|
||||
/\ w' = [IF phase = "set" /\ (Mode = "Baseline" \/ (~invalid /\ Eligible(w)
|
||||
/\ (Mode # "Reconcile" \/ w.goal.status # "usageLimited")))
|
||||
THEN Increment(w) ELSE w EXCEPT !.echo = EmptyGoal, !.seed = EmptyGoal]
|
||||
/\ UNCHANGED <<inputs, sample, live, ticket, lastAct, known, need, resumed, cancelled, stopWant, stopNeed>>
|
||||
|
||||
|
|
@ -421,8 +447,8 @@ Next ==
|
|||
\/ \E stamp \in live.goal.stamp..StampBound : ReplaceGoal(stamp)
|
||||
\/ StartTurn
|
||||
\/ \E status \in {"completed", "interrupted"} : FinishTurn(status)
|
||||
\/ \E isApi \in BOOLEAN, stamp \in live.goal.stamp..StampBound : FailTurn(isApi, stamp)
|
||||
\/ Poll \/ ServeRead \/ Receive \/ SendSet \/ ReadInput
|
||||
\/ \E isApi \in Failures, status \in Waiting, stamp \in live.goal.stamp..StampBound : FailTurn(isApi, status, stamp)
|
||||
\/ Poll \/ ServeRead \/ ServeUsage(TRUE) \/ ServeUsage(FALSE) \/ Receive \/ SendSet \/ ReadInput
|
||||
\/ \E stamp \in live.goal.stamp..StampBound : ServeSet(stamp)
|
||||
\/ (\E stamp \in live.goal.stamp..StampBound : ServeLate(stamp))
|
||||
\/ Clock \/ Disconnect \/ Reconnect \/ HonourStop
|
||||
|
|
@ -442,12 +468,13 @@ TypeOK ==
|
|||
/\ stopWant \in StopType /\ stopNeed \in BOOLEAN
|
||||
|
||||
FreshRepair == phase # "repair" \/ ~invalid
|
||||
ApiEvidence == ~lastAct.sent \/ ~lastAct.recover \/ lastAct.evidence
|
||||
FailureEvidence == ~lastAct.sent \/ ~lastAct.recover \/ lastAct.evidence
|
||||
AutoProvenance == ~lastAct.sent \/ lastAct.origin = "automatic"
|
||||
SuccessReset == w.turn.status # "completed" \/ w.fails = 0
|
||||
UsageEvidence == ~lastAct.sent \/ ~lastAct.recover \/ ~lastAct.usage \/ lastAct.expected.quota
|
||||
NoEarlyResume == ~lastAct.sent \/ ~lastAct.recover \/ lastAct.expected.expired
|
||||
StopCancels == w.hold = "none" \/ (~w.armed /\ w.due = "none")
|
||||
HumanStops == (w.goal.status \notin {"active", "blocked"} \/ w.turn.status = "interrupted")
|
||||
HumanStops == (w.goal.status \notin Running \/ w.turn.status = "interrupted")
|
||||
=> (~Eligible(w) /\ w.due = "none")
|
||||
|
||||
|
||||
|
|
@ -466,5 +493,6 @@ FairSet == \E stamp \in live.goal.stamp..StampBound : ServeSet(stamp)
|
|||
Fairness == WF_vars(Poll) /\ WF_vars(ServeRead) /\ WF_vars(Receive)
|
||||
/\ WF_vars(SendSet) /\ WF_vars(FairSet) /\ WF_vars(Clock) /\ WF_vars(StartTurn)
|
||||
/\ WF_vars(FairLate) /\ WF_vars(HonourStop) /\ WF_vars(ReadInput)
|
||||
/\ SF_vars(ServeUsage(TRUE))
|
||||
Spec == Init /\ [][Next]_vars /\ Fairness
|
||||
=============================================================================
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 2
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset ApiEvidence NoEarlyResume
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence
|
||||
PROPERTIES RecoveryProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 1
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset ApiEvidence NoEarlyResume
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence
|
||||
PROPERTIES RecoveryProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset ApiEvidence NoEarlyResume
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence
|
||||
PROPERTIES StopProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset ApiEvidence NoEarlyResume StopCancels HumanStops
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES RecoveryProgress StopProgress HoldProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 2
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK ApiEvidence
|
||||
INVARIANTS TypeOK FailureEvidence
|
||||
PROPERTIES RecoveryProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset ApiEvidence NoEarlyResume StopCancels HumanStops
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES QueueProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ QueueInit ==
|
|||
armed |-> TRUE, fails |-> 0, due |-> "none", hold |-> "none", pending |-> FALSE, known |-> TRUE, echo |-> EmptyGoal, seed |-> EmptyGoal]
|
||||
/\ wire = <<Message("goal", [live EXCEPT !.goal.status = "paused", !.edits = 1], TRUE), Message("goal", live, TRUE)>> /\ online = TRUE /\ phase = "idle" /\ req = "none"
|
||||
/\ invalid = FALSE
|
||||
/\ ticket = [expired |-> FALSE, status |-> "active"]
|
||||
/\ ticket = [expired |-> FALSE, status |-> "active", quota |-> FALSE]
|
||||
/\ lastAct = [sent |-> FALSE, view |-> [goal |-> w.goal.id, turn |-> w.turn.id], expected |-> ticket,
|
||||
origin |-> "automatic", evidence |-> TRUE, recover |-> TRUE]
|
||||
origin |-> "automatic", evidence |-> TRUE, recover |-> TRUE, usage |-> FALSE]
|
||||
/\ known = {1} /\ need = {} /\ resumed = {} /\ cancelled = {}
|
||||
/\ stopWant = [id |-> 0, status |-> "paused", epoch |-> 0] /\ stopNeed = FALSE
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset ApiEvidence NoEarlyResume StopCancels HumanStops
|
||||
INVARIANTS TypeOK AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES QueueProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 2
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset ApiEvidence NoEarlyResume StopCancels HumanStops
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES RecoveryProgress StopProgress HoldProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 0
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset ApiEvidence NoEarlyResume StopCancels HumanStops
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES RecoveryProgress StopProgress HoldProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ CONSTANTS
|
|||
StampBound = 1
|
||||
TurnBound = 1
|
||||
FailBound = 1
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset ApiEvidence NoEarlyResume StopCancels HumanStops
|
||||
INVARIANTS TypeOK FreshRepair AutoProvenance SuccessReset FailureEvidence NoEarlyResume UsageEvidence StopCancels HumanStops
|
||||
PROPERTIES RecoveryProgress StopProgress HoldProgress
|
||||
CHECK_DEADLOCK FALSE
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue