Add Zygostates hardware qualification runner

This commit is contained in:
ms2k 2026-10-18 22:31:35 +09:00
commit 65541dc8eb
17 changed files with 2889 additions and 0 deletions

117
internal/zygostates/app.go Normal file
View file

@ -0,0 +1,117 @@
package zygostates
import (
"context"
"errors"
"fmt"
"sync/atomic"
"time"
)
func Run(ctx context.Context, configPath string) error {
config, configContents, err := loadConfig(configPath)
if err != nil {
return fmt.Errorf("load config: %w", err)
}
identity, err := readHardwareIdentity("/sys/class/dmi/id")
if err != nil {
return fmt.Errorf("read hardware identity: %w", err)
}
workloads, err := buildWorkloads(config.Workloads)
if err != nil {
return fmt.Errorf("prepare workloads: %w", err)
}
started := time.Now().UTC()
runID := started.Format("20060102T150405.000000000Z")
artifacts, err := newArtifactStore(config.Artifacts.Directory, runID, configContents, systemMetadata(identity))
if err != nil {
return fmt.Errorf("create artifacts: %w", err)
}
defer func() { _ = artifacts.Close() }()
if err := artifacts.AppendEvent(Event{Timestamp: started, Type: "run-start", Fields: map[string]any{"run_id": runID, "identity": identity.Key}}); err != nil {
return err
}
var phase atomic.Value
phase.Store("preflight")
telemetryContext, stopTelemetry := context.WithCancel(context.Background())
telemetry := startTelemetry(telemetryContext, artifacts, &phase, time.Duration(config.Artifacts.SampleIntervalSeconds)*time.Second, &TelemetryCollector{procRoot: "/proc", sysRoot: "/sys"})
var uploader *Uploader
var stopPeriodic context.CancelFunc
var periodicWait interface{ Wait() }
if config.Upload.Bucket != "" {
objectStore, err := newS3ObjectStore(ctx, config.Upload)
if err != nil {
stopTelemetry()
telemetry.Wait()
result := RunResult{RunID: runID, Identity: identity.Key, StartedAt: started, CompletedAt: time.Now().UTC(), Status: "failed", Error: err.Error()}
if writeErr := artifacts.WriteResult(result); writeErr != nil {
return writeErr
}
return err
}
uploader = newUploader(objectStore, artifacts, config.Upload, identity.Key, runID)
preflightContext, cancel := context.WithTimeout(ctx, 30*time.Second)
_, err = uploader.Upload(preflightContext)
cancel()
if err != nil {
stopTelemetry()
telemetry.Wait()
result := RunResult{RunID: runID, Identity: identity.Key, StartedAt: started, CompletedAt: time.Now().UTC(), Status: "failed", Error: "S3 preflight: " + err.Error(), UploadError: err.Error()}
_ = artifacts.WriteResult(result)
return fmt.Errorf("S3 preflight: %w", err)
}
periodicContext, cancelPeriodic := context.WithCancel(context.Background())
stopPeriodic = cancelPeriodic
periodic := &PeriodicUploader{uploader: uploader, interval: time.Duration(config.Upload.IntervalSeconds) * time.Second, store: artifacts}
periodicWait = periodic.Start(periodicContext)
}
runner := Runner{config: config, workloads: workloads, store: artifacts, phase: &phase, now: func() time.Time { return time.Now().UTC() }}
runErr := runner.Run(ctx)
if stopPeriodic != nil {
stopPeriodic()
periodicWait.Wait()
}
stopTelemetry()
telemetry.Wait()
completed := time.Now().UTC()
status := "passed"
if runErr != nil {
status = "failed"
if errors.Is(runErr, context.Canceled) {
status = "interrupted"
}
}
result := RunResult{RunID: runID, Identity: identity.Key, StartedAt: started, CompletedAt: completed, Status: status}
if runErr != nil {
result.Error = runErr.Error()
}
fields := map[string]any{"status": status, "elapsed_seconds": completed.Sub(started).Seconds()}
if runErr != nil {
fields["error"] = runErr.Error()
}
if err := artifacts.AppendEvent(Event{Timestamp: completed, Type: "run-complete", Fields: fields}); err != nil && runErr == nil {
runErr = err
result.Status = "failed"
result.Error = err.Error()
}
if err := artifacts.WriteResult(result); err != nil {
return err
}
if uploader != nil {
finalContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
_, uploadErr := uploader.Upload(finalContext)
cancel()
if uploadErr != nil {
result.UploadError = uploadErr.Error()
if err := artifacts.WriteResult(result); err != nil {
return err
}
if runErr == nil {
runErr = fmt.Errorf("final upload: %w", uploadErr)
}
}
}
return runErr
}

View file

@ -0,0 +1,165 @@
package zygostates
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"sync"
"time"
)
type Event struct {
Timestamp time.Time `json:"timestamp"`
Type string `json:"type"`
Fields map[string]any `json:"fields,omitempty"`
}
type RunResult struct {
RunID string `json:"run_id"`
Identity string `json:"identity"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
UploadError string `json:"upload_error,omitempty"`
}
type FileSnapshot struct {
Name string
Contents []byte
}
type ArtifactStore struct {
directory string
events *os.File
telemetry *os.File
mu sync.Mutex
}
func newArtifactStore(root, runID string, configContents []byte, metadata SystemMetadata) (*ArtifactStore, error) {
directory := filepath.Join(root, runID)
if err := os.MkdirAll(directory, 0o750); err != nil {
return nil, err
}
if err := os.WriteFile(filepath.Join(directory, "config.yaml"), configContents, 0o640); err != nil {
return nil, err
}
if err := writeJSONAtomic(filepath.Join(directory, "metadata.json"), metadata); err != nil {
return nil, err
}
events, err := os.OpenFile(filepath.Join(directory, "events.ndjson"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640)
if err != nil {
return nil, err
}
telemetry, err := os.OpenFile(filepath.Join(directory, "telemetry.ndjson"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640)
if err != nil {
_ = events.Close()
return nil, err
}
return &ArtifactStore{directory: directory, events: events, telemetry: telemetry}, nil
}
func (s *ArtifactStore) Directory() string {
return s.directory
}
func (s *ArtifactStore) AppendEvent(event Event) error {
return s.appendJSON(s.events, event)
}
func (s *ArtifactStore) AppendTelemetry(sample TelemetrySample) error {
return s.appendJSON(s.telemetry, sample)
}
func (s *ArtifactStore) appendJSON(file *os.File, value any) error {
contents, err := json.Marshal(value)
if err != nil {
return err
}
contents = append(contents, '\n')
s.mu.Lock()
defer s.mu.Unlock()
_, err = file.Write(contents)
return err
}
func (s *ArtifactStore) WriteResult(result RunResult) error {
s.mu.Lock()
defer s.mu.Unlock()
return writeJSONAtomic(filepath.Join(s.directory, "result.json"), result)
}
func (s *ArtifactStore) Snapshots() ([]FileSnapshot, error) {
s.mu.Lock()
defer s.mu.Unlock()
if err := s.events.Sync(); err != nil {
return nil, err
}
if err := s.telemetry.Sync(); err != nil {
return nil, err
}
entries, err := os.ReadDir(s.directory)
if err != nil {
return nil, err
}
snapshots := make([]FileSnapshot, 0, len(entries))
for _, entry := range entries {
if !entry.Type().IsRegular() {
continue
}
contents, err := os.ReadFile(filepath.Join(s.directory, entry.Name()))
if err != nil {
return nil, fmt.Errorf("read %s: %w", entry.Name(), err)
}
snapshots = append(snapshots, FileSnapshot{Name: entry.Name(), Contents: contents})
}
sort.Slice(snapshots, func(left, right int) bool { return snapshots[left].Name < snapshots[right].Name })
return snapshots, nil
}
func (s *ArtifactStore) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
var first error
for _, file := range []*os.File{s.events, s.telemetry} {
if err := file.Sync(); err != nil && first == nil {
first = err
}
if err := file.Close(); err != nil && first == nil {
first = err
}
}
return first
}
func writeJSONAtomic(path string, value any) error {
contents, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
contents = append(contents, '\n')
temporary, err := os.CreateTemp(filepath.Dir(path), ".zygostates-*")
if err != nil {
return err
}
temporaryPath := temporary.Name()
defer func() { _ = os.Remove(temporaryPath) }()
if err := temporary.Chmod(0o640); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(contents); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(temporaryPath, path)
}

View file

@ -0,0 +1,43 @@
package zygostates
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
)
func TestArtifactStoreProducesParseableDurableFiles(t *testing.T) {
store := testArtifactStore(t)
if err := store.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "test", Fields: map[string]any{"value": 1}}); err != nil {
t.Fatal(err)
}
if err := store.AppendTelemetry(TelemetrySample{Timestamp: time.Unix(2, 0), Phase: "test"}); err != nil {
t.Fatal(err)
}
result := RunResult{RunID: "run", Identity: "system", Status: "passed"}
if err := store.WriteResult(result); err != nil {
t.Fatal(err)
}
snapshots, err := store.Snapshots()
if err != nil {
t.Fatal(err)
}
files := make(map[string][]byte)
for _, snapshot := range snapshots {
files[snapshot.Name] = snapshot.Contents
}
for _, name := range []string{"config.yaml", "events.ndjson", "metadata.json", "result.json", "telemetry.ndjson"} {
if _, found := files[name]; !found {
t.Fatalf("missing %s", name)
}
}
var decoded RunResult
if err := json.Unmarshal(files["result.json"], &decoded); err != nil || decoded.Status != "passed" {
t.Fatalf("result=%#v err=%v", decoded, err)
}
if _, err := os.Stat(filepath.Join(store.Directory(), "result.json")); err != nil {
t.Fatal(err)
}
}

View file

@ -0,0 +1,158 @@
package zygostates
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
type Step struct {
Workload string
Seconds int
}
func (s *Step) UnmarshalYAML(node *yaml.Node) error {
if node.Kind != yaml.SequenceNode || len(node.Content) != 2 {
return fmt.Errorf("step must be [workload, seconds]")
}
if err := node.Content[0].Decode(&s.Workload); err != nil {
return fmt.Errorf("decode workload: %w", err)
}
if err := node.Content[1].Decode(&s.Seconds); err != nil {
return fmt.Errorf("decode seconds: %w", err)
}
return nil
}
type ArtifactConfig struct {
Directory string `yaml:"directory"`
SampleIntervalSeconds int `yaml:"sample_interval_seconds"`
}
type UploadConfig struct {
Endpoint string `yaml:"endpoint"`
Region string `yaml:"region"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
IntervalSeconds int `yaml:"interval_seconds"`
}
type WorkloadConfig struct {
MemoryFraction float64 `yaml:"memory_fraction"`
FFTSize int `yaml:"fft_size"`
MatrixSize int `yaml:"matrix_size"`
}
type Config struct {
Version int `yaml:"version"`
Repeat int `yaml:"repeat"`
Sequence []Step `yaml:"sequence"`
Artifacts ArtifactConfig `yaml:"artifacts"`
Upload UploadConfig `yaml:"upload"`
Workloads WorkloadConfig `yaml:"workloads"`
}
var workloadNames = map[string]struct{}{
"cpu": {},
"fft": {},
"idle": {},
"matrix": {},
"memory": {},
"mixed": {},
"per-core": {},
}
func loadConfig(path string) (Config, []byte, error) {
contents, err := os.ReadFile(path)
if err != nil {
return Config{}, nil, err
}
var config Config
decoder := yaml.NewDecoder(bytes.NewReader(contents))
decoder.KnownFields(true)
if err := decoder.Decode(&config); err != nil {
return Config{}, nil, err
}
if err := config.applyDefaultsAndValidate(); err != nil {
return Config{}, nil, err
}
return config, contents, nil
}
func (c *Config) applyDefaultsAndValidate() error {
if c.Version != 1 {
return fmt.Errorf("version must be 1")
}
if c.Repeat < 1 {
return fmt.Errorf("repeat must be positive")
}
if len(c.Sequence) == 0 {
return fmt.Errorf("sequence must not be empty")
}
for index, step := range c.Sequence {
if _, found := workloadNames[step.Workload]; !found {
return fmt.Errorf("sequence[%d]: unknown workload %q", index, step.Workload)
}
if step.Seconds < 1 {
return fmt.Errorf("sequence[%d]: seconds must be positive", index)
}
}
if c.Artifacts.Directory == "" {
c.Artifacts.Directory = "/var/lib/zygostates/runs"
}
if !filepath.IsAbs(c.Artifacts.Directory) {
return fmt.Errorf("artifacts.directory must be absolute")
}
if c.Artifacts.SampleIntervalSeconds == 0 {
c.Artifacts.SampleIntervalSeconds = 15
}
if c.Artifacts.SampleIntervalSeconds < 1 {
return fmt.Errorf("artifacts.sample_interval_seconds must be positive")
}
if c.Workloads.MemoryFraction == 0 {
c.Workloads.MemoryFraction = 0.5
}
if c.Workloads.MemoryFraction <= 0 || c.Workloads.MemoryFraction > 0.9 {
return fmt.Errorf("workloads.memory_fraction must be greater than 0 and at most 0.9")
}
if c.Workloads.FFTSize == 0 {
c.Workloads.FFTSize = 65536
}
if c.Workloads.FFTSize < 2 {
return fmt.Errorf("workloads.fft_size must be at least 2")
}
if c.Workloads.MatrixSize == 0 {
c.Workloads.MatrixSize = 256
}
if c.Workloads.MatrixSize < 2 {
return fmt.Errorf("workloads.matrix_size must be at least 2")
}
uploadConfigured := c.Upload.Endpoint != "" || c.Upload.Region != "" || c.Upload.Bucket != "" || c.Upload.Prefix != "" || c.Upload.IntervalSeconds != 0
if !uploadConfigured {
return nil
}
if c.Upload.Bucket == "" {
return fmt.Errorf("upload.bucket is required when upload is configured")
}
if c.Upload.Region == "" {
c.Upload.Region = "us-east-1"
}
if c.Upload.Prefix == "" {
c.Upload.Prefix = "zygostates"
}
c.Upload.Prefix = strings.Trim(c.Upload.Prefix, "/")
if c.Upload.Prefix == "" {
return fmt.Errorf("upload.prefix must not resolve to empty")
}
if c.Upload.IntervalSeconds == 0 {
c.Upload.IntervalSeconds = 15
}
if c.Upload.IntervalSeconds < 1 {
return fmt.Errorf("upload.interval_seconds must be positive")
}
return nil
}

View file

@ -0,0 +1,80 @@
package zygostates
import (
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
func writeTestFile(t *testing.T, name, contents string) string {
t.Helper()
path := filepath.Join(t.TempDir(), name)
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
return path
}
func TestLoadConfigDecodesTupleSequenceAndDefaults(t *testing.T) {
path := writeTestFile(t, "config.yaml", `
version: 1
repeat: 2
sequence:
- [mixed, 300]
- [idle, 30]
artifacts:
directory: /tmp/zygostates
upload:
endpoint: https://s3.example.test
bucket: dump
`)
config, _, err := loadConfig(path)
if err != nil {
t.Fatal(err)
}
if len(config.Sequence) != 2 || config.Sequence[0] != (Step{Workload: "mixed", Seconds: 300}) {
t.Fatalf("unexpected sequence: %#v", config.Sequence)
}
if config.Artifacts.SampleIntervalSeconds != 15 || config.Upload.IntervalSeconds != 15 || config.Upload.Region != "us-east-1" || config.Upload.Prefix != "zygostates" {
t.Fatalf("defaults not applied: %#v", config)
}
if config.Workloads.MemoryFraction != 0.5 || config.Workloads.FFTSize != 65536 || config.Workloads.MatrixSize != 256 {
t.Fatalf("workload defaults not applied: %#v", config.Workloads)
}
}
func TestLoadConfigRejectsUnknownFieldsAndWorkloads(t *testing.T) {
for _, contents := range []string{
"version: 1\nrepeat: 1\nsequence: [[idle, 1]]\nunknown: true\n",
"version: 1\nrepeat: 1\nsequence: [[unbounded, 1]]\n",
"version: 1\nrepeat: 1\nsequence: [[idle, 0]]\n",
"version: 1\nrepeat: 1\nsequence: [idle]\n",
} {
path := writeTestFile(t, "config.yaml", contents)
if _, _, err := loadConfig(path); err == nil {
t.Fatalf("accepted invalid config %q", contents)
}
}
}
func TestUploadConfigurationContainsNoCredentialFields(t *testing.T) {
typeName := strings.ToLower(strings.Join([]string{
fmtFieldNames(UploadConfig{}),
}, " "))
for _, forbidden := range []string{"credential", "accesskey", "secret", "token"} {
if strings.Contains(typeName, forbidden) {
t.Fatalf("upload config exposes %q: %s", forbidden, typeName)
}
}
}
func fmtFieldNames(value any) string {
typeOf := reflect.TypeOf(value)
var names []string
for index := 0; index < typeOf.NumField(); index++ {
names = append(names, typeOf.Field(index).Name)
}
return strings.Join(names, " ")
}

View file

@ -0,0 +1,107 @@
package zygostates
import (
"fmt"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
)
type HardwareIdentity struct {
Key string `json:"key"`
BoardVendor string `json:"board_vendor,omitempty"`
BoardName string `json:"board_name,omitempty"`
BoardVersion string `json:"board_version,omitempty"`
BoardSerial string `json:"board_serial,omitempty"`
ProductName string `json:"product_name,omitempty"`
ProductSerial string `json:"product_serial,omitempty"`
ProductUUID string `json:"product_uuid,omitempty"`
}
var uuidPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
func readHardwareIdentity(root string) (HardwareIdentity, error) {
read := func(name string) string {
contents, err := os.ReadFile(filepath.Join(root, name))
if err != nil {
return ""
}
return strings.TrimSpace(string(contents))
}
identity := HardwareIdentity{
BoardVendor: read("board_vendor"),
BoardName: read("board_name"),
BoardVersion: read("board_version"),
BoardSerial: read("board_serial"),
ProductName: read("product_name"),
ProductSerial: read("product_serial"),
ProductUUID: strings.ToLower(read("product_uuid")),
}
if validUUID(identity.ProductUUID) {
identity.Key = identity.ProductUUID
return identity, nil
}
if validIdentifier(identity.BoardSerial) {
identity.Key = sanitizeIdentifier(identity.BoardSerial)
return identity, nil
}
return HardwareIdentity{}, fmt.Errorf("no valid SMBIOS product UUID or board serial")
}
func validUUID(value string) bool {
if !uuidPattern.MatchString(value) {
return false
}
compact := strings.ReplaceAll(strings.ToLower(value), "-", "")
return compact != strings.Repeat("0", 32) && compact != strings.Repeat("f", 32)
}
func validIdentifier(value string) bool {
normalized := strings.ToLower(strings.TrimSpace(value))
if normalized == "" {
return false
}
for _, placeholder := range []string{"default string", "none", "not specified", "system serial number", "to be filled by o.e.m.", "unknown"} {
if normalized == placeholder {
return false
}
}
return sanitizeIdentifier(value) != ""
}
func sanitizeIdentifier(value string) string {
var result strings.Builder
for _, character := range strings.ToLower(strings.TrimSpace(value)) {
switch {
case character >= 'a' && character <= 'z', character >= '0' && character <= '9', character == '-', character == '_', character == '.':
result.WriteRune(character)
default:
result.WriteByte('-')
}
}
return strings.Trim(result.String(), "-.")
}
type SystemMetadata struct {
Identity HardwareIdentity `json:"identity"`
Hostname string `json:"hostname"`
Kernel string `json:"kernel"`
Architecture string `json:"architecture"`
LogicalCPUs int `json:"logical_cpus"`
GoVersion string `json:"go_version"`
}
func systemMetadata(identity HardwareIdentity) SystemMetadata {
hostname, _ := os.Hostname()
kernel, _ := os.ReadFile("/proc/sys/kernel/osrelease")
return SystemMetadata{
Identity: identity,
Hostname: hostname,
Kernel: strings.TrimSpace(string(kernel)),
Architecture: runtime.GOARCH,
LogicalCPUs: runtime.NumCPU(),
GoVersion: runtime.Version(),
}
}

View file

@ -0,0 +1,54 @@
package zygostates
import (
"os"
"path/filepath"
"testing"
)
func writeDMI(t *testing.T, values map[string]string) string {
t.Helper()
root := t.TempDir()
for name, value := range values {
if err := os.WriteFile(filepath.Join(root, name), []byte(value+"\n"), 0o600); err != nil {
t.Fatal(err)
}
}
return root
}
func TestHardwareIdentityUsesNormalizedProductUUID(t *testing.T) {
identity, err := readHardwareIdentity(writeDMI(t, map[string]string{
"product_uuid": "D5B6EC3A-DC36-4AA0-B6AE-5AF3D13D1280",
"board_serial": "board-9",
}))
if err != nil {
t.Fatal(err)
}
if identity.Key != "d5b6ec3a-dc36-4aa0-b6ae-5af3d13d1280" {
t.Fatalf("key=%q", identity.Key)
}
}
func TestHardwareIdentityFallsBackToBoardSerial(t *testing.T) {
identity, err := readHardwareIdentity(writeDMI(t, map[string]string{
"product_uuid": "00000000-0000-0000-0000-000000000000",
"board_serial": "MB 12/34",
}))
if err != nil {
t.Fatal(err)
}
if identity.Key != "mb-12-34" {
t.Fatalf("key=%q", identity.Key)
}
}
func TestHardwareIdentityRejectsPlaceholders(t *testing.T) {
_, err := readHardwareIdentity(writeDMI(t, map[string]string{
"product_uuid": "ffffffff-ffff-ffff-ffff-ffffffffffff",
"board_serial": "To Be Filled By O.E.M.",
}))
if err == nil {
t.Fatal("expected invalid identity")
}
}

View file

@ -0,0 +1,70 @@
package zygostates
import (
"context"
"fmt"
"sync/atomic"
"time"
)
type Workload interface {
Run(context.Context, time.Duration) (map[string]any, error)
}
type Runner struct {
config Config
workloads map[string]Workload
store *ArtifactStore
phase *atomic.Value
now func() time.Time
}
func (r *Runner) Run(ctx context.Context) error {
for cycle := 1; cycle <= r.config.Repeat; cycle++ {
for index, step := range r.config.Sequence {
if err := ctx.Err(); err != nil {
return err
}
workload := r.workloads[step.Workload]
phase := fmt.Sprintf("cycle-%d/%s", cycle, step.Workload)
r.phase.Store(phase)
started := r.now()
if err := r.store.AppendEvent(Event{Timestamp: started, Type: "step-start", Fields: map[string]any{
"cycle": cycle, "index": index, "workload": step.Workload, "seconds": step.Seconds,
}}); err != nil {
return err
}
metrics, stepError := workload.Run(ctx, time.Duration(step.Seconds)*time.Second)
fields := map[string]any{
"cycle": cycle, "index": index, "workload": step.Workload, "elapsed_seconds": r.now().Sub(started).Seconds(),
}
for key, value := range metrics {
fields[key] = value
}
if stepError != nil {
fields["error"] = stepError.Error()
}
if appendErr := r.store.AppendEvent(Event{Timestamp: r.now(), Type: "step-complete", Fields: fields}); appendErr != nil {
return appendErr
}
if stepError != nil {
return fmt.Errorf("cycle %d %s: %w", cycle, step.Workload, stepError)
}
}
}
r.phase.Store("complete")
return nil
}
type idleWorkload struct{}
func (idleWorkload) Run(ctx context.Context, duration time.Duration) (map[string]any, error) {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-timer.C:
return nil, nil
}
}

View file

@ -0,0 +1,70 @@
package zygostates
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
)
type fakeWorkload struct {
calls atomic.Int64
err error
}
func (w *fakeWorkload) Run(context.Context, time.Duration) (map[string]any, error) {
w.calls.Add(1)
return map[string]any{"value": 7}, w.err
}
func testArtifactStore(t *testing.T) *ArtifactStore {
t.Helper()
store, err := newArtifactStore(t.TempDir(), "run", []byte("version: 1\n"), SystemMetadata{})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := store.Close(); err != nil {
t.Error(err)
}
})
return store
}
func TestRunnerRepeatsSequence(t *testing.T) {
first := &fakeWorkload{}
second := &fakeWorkload{}
store := testArtifactStore(t)
var phase atomic.Value
phase.Store("start")
now := time.Unix(1_700_000_000, 0)
runner := Runner{
config: Config{Repeat: 3, Sequence: []Step{{Workload: "first", Seconds: 1}, {Workload: "second", Seconds: 2}}},
workloads: map[string]Workload{"first": first, "second": second}, store: store, phase: &phase,
now: func() time.Time { now = now.Add(time.Second); return now },
}
if err := runner.Run(context.Background()); err != nil {
t.Fatal(err)
}
if first.calls.Load() != 3 || second.calls.Load() != 3 || phase.Load() != "complete" {
t.Fatalf("calls=%d,%d phase=%v", first.calls.Load(), second.calls.Load(), phase.Load())
}
}
func TestRunnerStopsOnWorkloadFailure(t *testing.T) {
failure := errors.New("verification failed")
failed := &fakeWorkload{err: failure}
after := &fakeWorkload{}
store := testArtifactStore(t)
var phase atomic.Value
phase.Store("start")
runner := Runner{
config: Config{Repeat: 2, Sequence: []Step{{Workload: "failed", Seconds: 1}, {Workload: "after", Seconds: 1}}},
workloads: map[string]Workload{"failed": failed, "after": after}, store: store, phase: &phase, now: time.Now,
}
err := runner.Run(context.Background())
if !errors.Is(err, failure) || failed.calls.Load() != 1 || after.calls.Load() != 0 {
t.Fatalf("err=%v calls=%d,%d", err, failed.calls.Load(), after.calls.Load())
}
}

51
internal/zygostates/s3.go Normal file
View file

@ -0,0 +1,51 @@
package zygostates
import (
"bytes"
"context"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type s3PutClient interface {
PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error)
}
type S3ObjectStore struct {
client s3PutClient
}
func newS3ObjectStore(ctx context.Context, config UploadConfig) (*S3ObjectStore, error) {
return newS3ObjectStoreWithOptions(ctx, config)
}
func newS3ObjectStoreWithOptions(ctx context.Context, config UploadConfig, options ...func(*awsconfig.LoadOptions) error) (*S3ObjectStore, error) {
options = append([]func(*awsconfig.LoadOptions) error{awsconfig.WithRegion(config.Region)}, options...)
loaded, err := awsconfig.LoadDefaultConfig(ctx, options...)
if err != nil {
return nil, err
}
client := s3.NewFromConfig(loaded, func(options *s3.Options) {
options.UsePathStyle = config.Endpoint != ""
if config.Endpoint != "" {
options.BaseEndpoint = aws.String(config.Endpoint)
}
})
return &S3ObjectStore{client: client}, nil
}
func (s *S3ObjectStore) Put(ctx context.Context, request PutRequest) error {
input := &s3.PutObjectInput{
Bucket: aws.String(request.Bucket), Key: aws.String(request.Key), Body: bytes.NewReader(request.Body), Metadata: request.Metadata,
}
if request.ContentType != "" {
input.ContentType = aws.String(request.ContentType)
}
if request.ContentEncoding != "" {
input.ContentEncoding = aws.String(request.ContentEncoding)
}
_, err := s.client.PutObject(ctx, input)
return err
}

View file

@ -0,0 +1,59 @@
package zygostates
import (
"bytes"
"context"
"io"
"net/http"
"strings"
"sync"
"testing"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
)
type recordingHTTPClient struct {
mu sync.Mutex
request *http.Request
body []byte
}
func (c *recordingHTTPClient) Do(request *http.Request) (*http.Response, error) {
body, err := io.ReadAll(request.Body)
if err != nil {
return nil, err
}
c.mu.Lock()
c.request = request
c.body = body
c.mu.Unlock()
return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: request}, nil
}
func TestS3ObjectStoreUsesSDKCredentialsAndPathStyleEndpoint(t *testing.T) {
client := &recordingHTTPClient{}
t.Setenv("AWS_ACCESS_KEY_ID", "test-access-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key")
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
store, err := newS3ObjectStoreWithOptions(context.Background(), UploadConfig{Endpoint: "https://s3.example.test", Region: "us-east-1"}, awsconfig.WithHTTPClient(client))
if err != nil {
t.Fatal(err)
}
err = store.Put(context.Background(), PutRequest{Bucket: "dump", Key: "prefix/file", Body: []byte("contents"), ContentType: "application/octet-stream"})
if err != nil {
t.Fatal(err)
}
client.mu.Lock()
request := client.request
body := client.body
client.mu.Unlock()
if request.Method != http.MethodPut || request.URL.Path != "/dump/prefix/file" {
t.Fatalf("request=%s %s", request.Method, request.URL.Path)
}
if request.Header.Get("Authorization") == "" {
t.Fatal("request was not signed")
}
if !bytes.Contains(body, []byte("contents")) {
t.Fatalf("body=%q", body)
}
}

View file

@ -0,0 +1,244 @@
package zygostates
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
)
type TelemetrySample struct {
Timestamp time.Time `json:"timestamp"`
Phase string `json:"phase"`
BootID string `json:"boot_id,omitempty"`
Load1 float64 `json:"load_1m"`
Load5 float64 `json:"load_5m"`
Load15 float64 `json:"load_15m"`
MemoryAvailableKiB uint64 `json:"memory_available_kib"`
CPUBusyPercent *float64 `json:"cpu_busy_percent,omitempty"`
PackageWatts *float64 `json:"package_watts,omitempty"`
TemperaturesC map[string]float64 `json:"temperatures_c,omitempty"`
CPUFrequencyKHz map[string]uint64 `json:"cpu_frequency_khz,omitempty"`
}
type CPUTime struct {
total uint64
idle uint64
}
type EnergySample struct {
microjoules uint64
maximum uint64
timestamp time.Time
}
type TelemetryCollector struct {
procRoot string
sysRoot string
previousCPU *CPUTime
previousEnergy *EnergySample
}
func (c *TelemetryCollector) Sample(now time.Time, phase string) (TelemetrySample, error) {
load, err := os.ReadFile(filepath.Join(c.procRoot, "loadavg"))
if err != nil {
return TelemetrySample{}, err
}
loadFields := strings.Fields(string(load))
if len(loadFields) < 3 {
return TelemetrySample{}, fmt.Errorf("invalid loadavg")
}
load1, err := strconv.ParseFloat(loadFields[0], 64)
if err != nil {
return TelemetrySample{}, err
}
load5, err := strconv.ParseFloat(loadFields[1], 64)
if err != nil {
return TelemetrySample{}, err
}
load15, err := strconv.ParseFloat(loadFields[2], 64)
if err != nil {
return TelemetrySample{}, err
}
memory, err := availableMemoryBytes(filepath.Join(c.procRoot, "meminfo"))
if err != nil {
return TelemetrySample{}, err
}
bootID, _ := os.ReadFile(filepath.Join(c.procRoot, "sys/kernel/random/boot_id"))
sample := TelemetrySample{
Timestamp: now,
Phase: phase,
BootID: strings.TrimSpace(string(bootID)),
Load1: load1,
Load5: load5,
Load15: load15,
MemoryAvailableKiB: memory / 1024,
TemperaturesC: readTemperatures(filepath.Join(c.sysRoot, "class/hwmon")),
CPUFrequencyKHz: readCPUFrequencies(filepath.Join(c.sysRoot, "devices/system/cpu")),
}
currentCPU, err := readCPUTime(filepath.Join(c.procRoot, "stat"))
if err == nil && c.previousCPU != nil {
totalDelta := currentCPU.total - c.previousCPU.total
idleDelta := currentCPU.idle - c.previousCPU.idle
if totalDelta > 0 && idleDelta <= totalDelta {
busy := 100 * float64(totalDelta-idleDelta) / float64(totalDelta)
sample.CPUBusyPercent = &busy
}
}
if err == nil {
c.previousCPU = &currentCPU
}
currentEnergy, err := readPackageEnergy(filepath.Join(c.sysRoot, "class/powercap"), now)
if err == nil && c.previousEnergy != nil {
delta := energyDelta(*c.previousEnergy, currentEnergy)
seconds := currentEnergy.timestamp.Sub(c.previousEnergy.timestamp).Seconds()
if seconds > 0 {
watts := float64(delta) / 1e6 / seconds
sample.PackageWatts = &watts
}
}
if err == nil {
c.previousEnergy = &currentEnergy
}
return sample, nil
}
func readCPUTime(path string) (CPUTime, error) {
contents, err := os.ReadFile(path)
if err != nil {
return CPUTime{}, err
}
line, _, _ := strings.Cut(string(contents), "\n")
fields := strings.Fields(line)
if len(fields) < 5 || fields[0] != "cpu" {
return CPUTime{}, fmt.Errorf("invalid CPU stat")
}
var values []uint64
for _, field := range fields[1:] {
value, err := strconv.ParseUint(field, 10, 64)
if err != nil {
return CPUTime{}, err
}
values = append(values, value)
}
var total uint64
for _, value := range values {
total += value
}
idle := values[3]
if len(values) > 4 {
idle += values[4]
}
return CPUTime{total: total, idle: idle}, nil
}
func readTemperatures(root string) map[string]float64 {
result := make(map[string]float64)
hwmons, _ := filepath.Glob(filepath.Join(root, "hwmon*"))
for _, hwmon := range hwmons {
deviceName, _ := os.ReadFile(filepath.Join(hwmon, "name"))
inputs, _ := filepath.Glob(filepath.Join(hwmon, "temp*_input"))
for _, inputPath := range inputs {
contents, err := os.ReadFile(inputPath)
if err != nil {
continue
}
millidegrees, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64)
if err != nil {
continue
}
stem := strings.TrimSuffix(filepath.Base(inputPath), "_input")
label, _ := os.ReadFile(filepath.Join(hwmon, stem+"_label"))
labelText := strings.TrimSpace(string(label))
if labelText == "" {
labelText = stem
}
name := strings.TrimSpace(string(deviceName)) + "/" + labelText
result[name] = float64(millidegrees) / 1000
}
}
return result
}
func readCPUFrequencies(root string) map[string]uint64 {
result := make(map[string]uint64)
paths, _ := filepath.Glob(filepath.Join(root, "cpu[0-9]*/cpufreq/scaling_cur_freq"))
for _, path := range paths {
contents, err := os.ReadFile(path)
if err != nil {
continue
}
frequency, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64)
if err != nil {
continue
}
cpu := filepath.Base(filepath.Dir(filepath.Dir(path)))
result[cpu] = frequency
}
return result
}
func readPackageEnergy(root string, now time.Time) (EnergySample, error) {
paths, _ := filepath.Glob(filepath.Join(root, "*/energy_uj"))
if len(paths) == 0 {
paths, _ = filepath.Glob(filepath.Join(root, "*/*/energy_uj"))
}
if len(paths) == 0 {
return EnergySample{}, fmt.Errorf("package energy input not found")
}
energyContents, err := os.ReadFile(paths[0])
if err != nil {
return EnergySample{}, err
}
maximumContents, err := os.ReadFile(filepath.Join(filepath.Dir(paths[0]), "max_energy_range_uj"))
if err != nil {
return EnergySample{}, err
}
energy, err := strconv.ParseUint(strings.TrimSpace(string(energyContents)), 10, 64)
if err != nil {
return EnergySample{}, err
}
maximum, err := strconv.ParseUint(strings.TrimSpace(string(maximumContents)), 10, 64)
if err != nil {
return EnergySample{}, err
}
return EnergySample{microjoules: energy, maximum: maximum, timestamp: now}, nil
}
func energyDelta(previous, current EnergySample) uint64 {
if current.microjoules >= previous.microjoules {
return current.microjoules - previous.microjoules
}
return previous.maximum - previous.microjoules + current.microjoules
}
func startTelemetry(ctx context.Context, store *ArtifactStore, phase *atomic.Value, interval time.Duration, collector *TelemetryCollector) *sync.WaitGroup {
var group sync.WaitGroup
group.Add(1)
go func() {
defer group.Done()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
value := phase.Load()
phaseName, _ := value.(string)
sample, err := collector.Sample(time.Now().UTC(), phaseName)
if err != nil {
_ = store.AppendEvent(Event{Timestamp: time.Now().UTC(), Type: "telemetry-error", Fields: map[string]any{"error": err.Error()}})
} else if err := store.AppendTelemetry(sample); err != nil {
_ = store.AppendEvent(Event{Timestamp: time.Now().UTC(), Type: "telemetry-error", Fields: map[string]any{"error": err.Error()}})
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}()
return &group
}

View file

@ -0,0 +1,139 @@
package zygostates
import (
"bufio"
"bytes"
"context"
"encoding/json"
"math"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
)
func TestReadCPUTimeAndBusyDelta(t *testing.T) {
path := writeTestFile(t, "stat", "cpu 100 20 30 400 50 0 0 0 0 0\ncpu0 1 2 3 4\n")
value, err := readCPUTime(path)
if err != nil {
t.Fatal(err)
}
if value.total != 600 || value.idle != 450 {
t.Fatalf("CPU time=%#v", value)
}
}
func TestStartTelemetryWritesParseableSamples(t *testing.T) {
store := testArtifactStore(t)
var phase atomic.Value
phase.Store("test")
ctx, cancel := context.WithCancel(context.Background())
wait := startTelemetry(ctx, store, &phase, time.Millisecond, &TelemetryCollector{procRoot: "/proc", sysRoot: "/sys"})
deadline := time.Now().Add(time.Second)
collect:
for time.Now().Before(deadline) {
snapshots, err := store.Snapshots()
if err != nil {
t.Fatal(err)
}
for _, snapshot := range snapshots {
if snapshot.Name == "telemetry.ndjson" && bytes.Count(snapshot.Contents, []byte{'\n'}) >= 2 {
break collect
}
}
time.Sleep(time.Millisecond)
}
cancel()
wait.Wait()
file, err := os.Open(filepath.Join(store.Directory(), "telemetry.ndjson"))
if err != nil {
t.Fatal(err)
}
defer func() { _ = file.Close() }()
scanner := bufio.NewScanner(file)
count := 0
for scanner.Scan() {
var sample TelemetrySample
if err := json.Unmarshal(scanner.Bytes(), &sample); err != nil {
t.Fatal(err)
}
if sample.Phase != "test" {
t.Fatalf("phase=%q", sample.Phase)
}
count++
}
if err := scanner.Err(); err != nil {
t.Fatal(err)
}
if count < 2 {
t.Fatalf("samples=%d", count)
}
}
func TestEnergyDeltaHandlesCounterWrap(t *testing.T) {
previous := EnergySample{microjoules: 900, maximum: 1000}
current := EnergySample{microjoules: 100, maximum: 1000}
if delta := energyDelta(previous, current); delta != 200 {
t.Fatalf("delta=%d", delta)
}
}
func TestTelemetryCollectorSamplesLinuxInputs(t *testing.T) {
root := t.TempDir()
proc := filepath.Join(root, "proc")
sys := filepath.Join(root, "sys")
for _, directory := range []string{
filepath.Join(proc, "sys/kernel/random"),
filepath.Join(sys, "class/hwmon/hwmon0"),
filepath.Join(sys, "devices/system/cpu/cpu0/cpufreq"),
filepath.Join(sys, "class/powercap/intel-rapl:0"),
} {
if err := os.MkdirAll(directory, 0o755); err != nil {
t.Fatal(err)
}
}
files := map[string]string{
filepath.Join(proc, "loadavg"): "1.25 2.50 3.75 1/10 1\n",
filepath.Join(proc, "meminfo"): "MemTotal: 2000 kB\nMemAvailable: 1000 kB\n",
filepath.Join(proc, "stat"): "cpu 100 0 100 800 0 0 0 0 0 0\n",
filepath.Join(proc, "sys/kernel/random/boot_id"): "boot\n",
filepath.Join(sys, "class/hwmon/hwmon0/name"): "k10temp\n",
filepath.Join(sys, "class/hwmon/hwmon0/temp1_label"): "Tctl\n",
filepath.Join(sys, "class/hwmon/hwmon0/temp1_input"): "42500\n",
filepath.Join(sys, "devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"): "4100000\n",
filepath.Join(sys, "class/powercap/intel-rapl:0/energy_uj"): "1000000\n",
filepath.Join(sys, "class/powercap/intel-rapl:0/max_energy_range_uj"): "10000000\n",
}
for path, contents := range files {
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
}
collector := TelemetryCollector{procRoot: proc, sysRoot: sys}
firstTime := time.Unix(10, 0)
first, err := collector.Sample(firstTime, "idle")
if err != nil {
t.Fatal(err)
}
if first.Load1 != 1.25 || first.MemoryAvailableKiB != 1000 || first.TemperaturesC["k10temp/Tctl"] != 42.5 || first.CPUFrequencyKHz["cpu0"] != 4100000 {
t.Fatalf("sample=%#v", first)
}
if err := os.WriteFile(filepath.Join(proc, "stat"), []byte("cpu 150 0 150 900 0 0 0 0 0 0\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(sys, "class/powercap/intel-rapl:0/energy_uj"), []byte("3000000\n"), 0o600); err != nil {
t.Fatal(err)
}
second, err := collector.Sample(firstTime.Add(time.Second), "cpu")
if err != nil {
t.Fatal(err)
}
if second.CPUBusyPercent == nil || math.Abs(*second.CPUBusyPercent-50) > 1e-9 {
t.Fatalf("busy=%v", second.CPUBusyPercent)
}
if second.PackageWatts == nil || math.Abs(*second.PackageWatts-2) > 1e-9 {
t.Fatalf("watts=%v", second.PackageWatts)
}
}

View file

@ -0,0 +1,157 @@
package zygostates
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"net/url"
"path"
"strconv"
"sync"
"time"
)
type PutRequest struct {
Bucket string
Key string
Body []byte
ContentType string
ContentEncoding string
Metadata map[string]string
}
type ObjectStore interface {
Put(context.Context, PutRequest) error
}
type UploadedRange struct {
Path string `json:"path"`
Generation int `json:"generation"`
Start int64 `json:"start"`
End int64 `json:"end"`
SHA256 string `json:"sha256"`
Key string `json:"key"`
}
type UploadTick struct {
Timestamp time.Time `json:"timestamp"`
Sequence uint64 `json:"sequence"`
Uploaded []UploadedRange `json:"uploaded"`
}
type uploadPosition struct {
offset int64
generation int
}
type Uploader struct {
store ObjectStore
artifacts *ArtifactStore
bucket string
baseKey string
positions map[string]uploadPosition
sequence uint64
maxChunk int
mu sync.Mutex
}
func newUploader(store ObjectStore, artifacts *ArtifactStore, config UploadConfig, identity, runID string) *Uploader {
return &Uploader{
store: store, artifacts: artifacts, bucket: config.Bucket,
baseKey: path.Join(config.Prefix, identity, runID),
positions: make(map[string]uploadPosition), maxChunk: 8 * 1024 * 1024,
}
}
func (u *Uploader) Upload(ctx context.Context) (int, error) {
u.mu.Lock()
defer u.mu.Unlock()
snapshots, err := u.artifacts.Snapshots()
if err != nil {
return 0, err
}
staged := make(map[string]uploadPosition, len(u.positions))
for name, position := range u.positions {
staged[name] = position
}
var uploaded []UploadedRange
for _, snapshot := range snapshots {
position := staged[snapshot.Name]
if int64(len(snapshot.Contents)) < position.offset {
position.offset = 0
position.generation++
}
for position.offset < int64(len(snapshot.Contents)) {
end := min(position.offset+int64(u.maxChunk), int64(len(snapshot.Contents)))
content := snapshot.Contents[position.offset:end]
digest := sha256.Sum256(content)
digestText := hex.EncodeToString(digest[:])
key := path.Join(u.baseKey, "files", url.PathEscape(snapshot.Name), fmt.Sprintf("%04d", position.generation), fmt.Sprintf("%020d-%020d.gz", position.offset, end))
var compressed bytes.Buffer
writer, _ := gzip.NewWriterLevel(&compressed, gzip.BestSpeed)
if _, err := writer.Write(content); err != nil {
return len(uploaded), err
}
if err := writer.Close(); err != nil {
return len(uploaded), err
}
if err := u.store.Put(ctx, PutRequest{
Bucket: u.bucket, Key: key, Body: compressed.Bytes(), ContentType: "application/octet-stream", ContentEncoding: "gzip",
Metadata: map[string]string{"source-path": snapshot.Name, "generation": strconv.Itoa(position.generation), "start": strconv.FormatInt(position.offset, 10), "end": strconv.FormatInt(end, 10), "sha256": digestText},
}); err != nil {
return len(uploaded), err
}
uploaded = append(uploaded, UploadedRange{Path: snapshot.Name, Generation: position.generation, Start: position.offset, End: end, SHA256: digestText, Key: key})
position.offset = end
}
staged[snapshot.Name] = position
}
tick := UploadTick{Timestamp: time.Now().UTC(), Sequence: u.sequence, Uploaded: uploaded}
tickBody, err := json.Marshal(tick)
if err != nil {
return len(uploaded), err
}
tickKey := path.Join(u.baseKey, "ticks", fmt.Sprintf("%08d.json", u.sequence))
if err := u.store.Put(ctx, PutRequest{Bucket: u.bucket, Key: tickKey, Body: tickBody, ContentType: "application/json"}); err != nil {
return len(uploaded), err
}
u.positions = staged
u.sequence++
return len(uploaded), nil
}
type PeriodicUploader struct {
uploader *Uploader
interval time.Duration
store *ArtifactStore
}
func (u *PeriodicUploader) Start(ctx context.Context) *sync.WaitGroup {
var group sync.WaitGroup
group.Add(1)
go func() {
defer group.Done()
ticker := time.NewTicker(u.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
uploadContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
count, err := u.uploader.Upload(uploadContext)
cancel()
fields := map[string]any{"uploaded_ranges": count}
if err != nil {
fields["error"] = err.Error()
}
_ = u.store.AppendEvent(Event{Timestamp: time.Now().UTC(), Type: "upload", Fields: fields})
}
}
}()
return &group
}

View file

@ -0,0 +1,235 @@
package zygostates
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"strings"
"sync"
"testing"
"time"
)
type fakeObjectStore struct {
mu sync.Mutex
requests []PutRequest
failNextTick bool
commitAndFailNextTick bool
}
type blockingObjectStore struct {
started chan struct{}
release chan struct{}
once sync.Once
}
func (s *blockingObjectStore) Put(ctx context.Context, _ PutRequest) error {
s.once.Do(func() { close(s.started) })
select {
case <-s.release:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (s *fakeObjectStore) Put(_ context.Context, request PutRequest) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.failNextTick && strings.Contains(request.Key, "/ticks/") {
s.failNextTick = false
return fmt.Errorf("tick unavailable")
}
copyRequest := request
copyRequest.Body = bytes.Clone(request.Body)
s.requests = append(s.requests, copyRequest)
if s.commitAndFailNextTick && strings.Contains(request.Key, "/ticks/") {
s.commitAndFailNextTick = false
return fmt.Errorf("tick response unavailable")
}
return nil
}
func (s *fakeObjectStore) ticks() []PutRequest {
s.mu.Lock()
defer s.mu.Unlock()
var ticks []PutRequest
for _, request := range s.requests {
if strings.Contains(request.Key, "/ticks/") {
ticks = append(ticks, request)
}
}
return ticks
}
func decodeGzip(t *testing.T, body []byte) []byte {
t.Helper()
reader, err := gzip.NewReader(bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
defer func() { _ = reader.Close() }()
contents, err := io.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
return contents
}
func TestUploaderCommitsOnlyRangesReferencedByTick(t *testing.T) {
artifacts := testArtifactStore(t)
if err := artifacts.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "first"}); err != nil {
t.Fatal(err)
}
objects := &fakeObjectStore{failNextTick: true}
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
uploader.maxChunk = 32
if _, err := uploader.Upload(context.Background()); err == nil {
t.Fatal("expected tick failure")
}
if len(uploader.positions) != 0 || uploader.sequence != 0 {
t.Fatalf("failed upload committed state: %#v sequence=%d", uploader.positions, uploader.sequence)
}
count, err := uploader.Upload(context.Background())
if err != nil {
t.Fatal(err)
}
if count == 0 || uploader.sequence != 1 {
t.Fatalf("count=%d sequence=%d", count, uploader.sequence)
}
ticks := objects.ticks()
if len(ticks) != 1 {
t.Fatalf("ticks=%d", len(ticks))
}
var tick UploadTick
if err := json.Unmarshal(ticks[0].Body, &tick); err != nil {
t.Fatal(err)
}
if len(tick.Uploaded) != count {
t.Fatalf("tick ranges=%d count=%d", len(tick.Uploaded), count)
}
for _, request := range objects.requests {
if request.ContentEncoding == "gzip" && len(decodeGzip(t, request.Body)) == 0 {
t.Fatalf("empty range %s", request.Key)
}
}
}
func TestUploaderShipsOnlyAppendedBytesAfterCommit(t *testing.T) {
artifacts := testArtifactStore(t)
objects := &fakeObjectStore{}
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
if _, err := uploader.Upload(context.Background()); err != nil {
t.Fatal(err)
}
if err := artifacts.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "new"}); err != nil {
t.Fatal(err)
}
count, err := uploader.Upload(context.Background())
if err != nil {
t.Fatal(err)
}
if count != 1 {
t.Fatalf("uploaded ranges=%d want=1", count)
}
ticks := objects.ticks()
var tick UploadTick
if err := json.Unmarshal(ticks[len(ticks)-1].Body, &tick); err != nil {
t.Fatal(err)
}
if tick.Uploaded[0].Path != "events.ndjson" || tick.Uploaded[0].Start != 0 {
t.Fatalf("unexpected range: %#v", tick.Uploaded[0])
}
}
func TestUploaderRetriesTickAtSameKeyAfterAmbiguousFailure(t *testing.T) {
artifacts := testArtifactStore(t)
objects := &fakeObjectStore{commitAndFailNextTick: true}
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
if _, err := uploader.Upload(context.Background()); err == nil {
t.Fatal("expected tick failure")
}
if err := artifacts.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "new"}); err != nil {
t.Fatal(err)
}
if _, err := uploader.Upload(context.Background()); err != nil {
t.Fatal(err)
}
ticks := objects.ticks()
if len(ticks) != 2 {
t.Fatalf("ticks=%d", len(ticks))
}
if ticks[0].Key != ticks[1].Key {
t.Fatalf("retry keys differ: %q != %q", ticks[0].Key, ticks[1].Key)
}
for _, request := range ticks {
var tick UploadTick
if err := json.Unmarshal(request.Body, &tick); err != nil {
t.Fatal(err)
}
if tick.Sequence != 0 {
t.Fatalf("sequence=%d", tick.Sequence)
}
}
}
func TestPeriodicUploaderContinuesUntilCanceled(t *testing.T) {
artifacts := testArtifactStore(t)
objects := &fakeObjectStore{}
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
ctx, cancel := context.WithCancel(context.Background())
periodic := &PeriodicUploader{uploader: uploader, interval: time.Millisecond, store: artifacts}
wait := periodic.Start(ctx)
deadline := time.Now().Add(time.Second)
for len(objects.ticks()) < 2 && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
cancel()
wait.Wait()
if len(objects.ticks()) < 2 {
t.Fatalf("ticks=%d", len(objects.ticks()))
}
}
func TestPeriodicUploaderFinishesActiveUploadWhenCanceled(t *testing.T) {
artifacts := testArtifactStore(t)
objects := &blockingObjectStore{started: make(chan struct{}), release: make(chan struct{})}
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
ctx, cancel := context.WithCancel(context.Background())
periodic := &PeriodicUploader{uploader: uploader, interval: time.Millisecond, store: artifacts}
wait := periodic.Start(ctx)
select {
case <-objects.started:
case <-time.After(time.Second):
t.Fatal("upload did not start")
}
cancel()
done := make(chan struct{})
go func() {
wait.Wait()
close(done)
}()
select {
case <-done:
t.Fatal("uploader stopped before the active upload finished")
case <-time.After(10 * time.Millisecond):
}
close(objects.release)
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("uploader did not stop")
}
snapshots, err := artifacts.Snapshots()
if err != nil {
t.Fatal(err)
}
for _, snapshot := range snapshots {
if snapshot.Name == "events.ndjson" && bytes.Contains(snapshot.Contents, []byte(`"error"`)) {
t.Fatalf("unexpected upload error: %s", snapshot.Contents)
}
}
}

View file

@ -0,0 +1,477 @@
package zygostates
import (
"context"
"errors"
"fmt"
"math"
"os"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/sys/unix"
"gonum.org/v1/gonum/dsp/fourier"
"gonum.org/v1/gonum/mat"
)
const cpuBlockIterations = 1 << 17
type cpuBlockResult struct {
integer uint64
floating uint64
}
func cpuBlock(seed uint64) cpuBlockResult {
integer := seed | 1
floating := float64(seed%997+1) / 997
for iteration := 0; iteration < cpuBlockIterations; iteration++ {
integer ^= integer << 13
integer ^= integer >> 7
integer ^= integer << 17
integer *= 0x9e3779b97f4a7c15
floating = math.FMA(floating, 1.0000001192092896, float64(integer&0xffff)/65536)
if floating > 1e6 {
floating -= 1e6
}
}
return cpuBlockResult{integer: integer, floating: math.Float64bits(floating)}
}
type cpuWorkload struct {
workers int
CPUs []int
}
func (w cpuWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
ctx, cancel := context.WithTimeout(parent, duration)
defer cancel()
var blocks atomic.Uint64
err := runCPUWorkers(ctx, w.workers, w.CPUs, &blocks)
if errors.Is(err, context.DeadlineExceeded) {
err = nil
}
return map[string]any{"verified_blocks": blocks.Load(), "operations": blocks.Load() * cpuBlockIterations}, err
}
func runCPUWorkers(ctx context.Context, workerCount int, cpus []int, blocks *atomic.Uint64) error {
if workerCount < 1 {
return fmt.Errorf("CPU worker count must be positive")
}
workerContext, cancel := context.WithCancel(ctx)
defer cancel()
errCh := make(chan error, workerCount)
var group sync.WaitGroup
for worker := 0; worker < workerCount; worker++ {
worker := worker
group.Add(1)
go func() {
defer group.Done()
if len(cpus) > 0 {
if err := runPinned(cpus[worker%len(cpus)], func() error {
return cpuWorker(workerContext, uint64(worker+1), blocks)
}); err != nil {
errCh <- err
cancel()
}
return
}
if err := cpuWorker(workerContext, uint64(worker+1), blocks); err != nil {
errCh <- err
cancel()
}
}()
}
group.Wait()
close(errCh)
var contextErr error
for err := range errCh {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
contextErr = err
continue
}
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
if contextErr != nil {
return contextErr
}
return workerContext.Err()
}
func cpuWorker(ctx context.Context, seed uint64, blocks *atomic.Uint64) error {
expected := cpuBlock(seed)
if confirmation := cpuBlock(seed); confirmation != expected {
return fmt.Errorf("CPU preflight mismatch")
}
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if result := cpuBlock(seed); result != expected {
return fmt.Errorf("CPU verification mismatch")
}
blocks.Add(1)
}
}
func runPinned(cpu int, run func() error) error {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
var original unix.CPUSet
if err := unix.SchedGetaffinity(0, &original); err != nil {
return err
}
defer func() { _ = unix.SchedSetaffinity(0, &original) }()
var selected unix.CPUSet
selected.Zero()
selected.Set(cpu)
if err := unix.SchedSetaffinity(0, &selected); err != nil {
return err
}
return run()
}
type perCoreWorkload struct {
cpus []int
}
func (w perCoreWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
var total atomic.Uint64
completed := 0
for _, cpu := range w.cpus {
if err := parent.Err(); err != nil {
return map[string]any{"completed_cores": completed, "verified_blocks": total.Load()}, err
}
ctx, cancel := context.WithTimeout(parent, duration)
err := runCPUWorkers(ctx, 1, []int{cpu}, &total)
cancel()
if err != nil && !errors.Is(err, context.DeadlineExceeded) {
return map[string]any{"completed_cores": completed, "failed_cpu": cpu, "verified_blocks": total.Load()}, err
}
completed++
}
return map[string]any{"completed_cores": completed, "verified_blocks": total.Load(), "seconds_per_core": duration.Seconds()}, nil
}
func physicalCPUs(root string) ([]int, error) {
entries, err := os.ReadDir(root)
if err != nil {
return nil, err
}
seen := make(map[string]struct{})
var cpus []int
for _, entry := range entries {
if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "cpu") {
continue
}
cpu, err := strconv.Atoi(strings.TrimPrefix(entry.Name(), "cpu"))
if err != nil {
continue
}
core, err := os.ReadFile(root + "/" + entry.Name() + "/topology/core_id")
if err != nil {
continue
}
pack, err := os.ReadFile(root + "/" + entry.Name() + "/topology/physical_package_id")
if err != nil {
continue
}
key := strings.TrimSpace(string(pack)) + ":" + strings.TrimSpace(string(core))
if _, found := seen[key]; found {
continue
}
seen[key] = struct{}{}
cpus = append(cpus, cpu)
}
sort.Ints(cpus)
if len(cpus) == 0 {
return nil, fmt.Errorf("no physical CPUs found")
}
return cpus, nil
}
type memoryWorkload struct {
fraction float64
workers int
}
func (w memoryWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
available, err := availableMemoryBytes("/proc/meminfo")
if err != nil {
return nil, err
}
bytes := uint64(float64(available) * w.fraction)
words := int(bytes / 8)
if words < w.workers {
return nil, fmt.Errorf("available memory is too small")
}
values := make([]uint64, words)
ctx, cancel := context.WithTimeout(parent, duration)
defer cancel()
passes, err := verifyMemory(ctx, values, w.workers)
if errors.Is(err, context.DeadlineExceeded) {
err = nil
}
return map[string]any{"bytes": len(values) * 8, "verified_passes": passes}, err
}
func availableMemoryBytes(path string) (uint64, error) {
contents, err := os.ReadFile(path)
if err != nil {
return 0, err
}
for _, line := range strings.Split(string(contents), "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && fields[0] == "MemAvailable:" {
kilobytes, err := strconv.ParseUint(fields[1], 10, 64)
return kilobytes * 1024, err
}
}
return 0, fmt.Errorf("MemAvailable not found")
}
func memoryValue(seed, index uint64) uint64 {
value := seed + index*0x9e3779b97f4a7c15
value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9
value = (value ^ (value >> 27)) * 0x94d049bb133111eb
return value ^ (value >> 31)
}
func verifyMemory(ctx context.Context, values []uint64, workers int) (uint64, error) {
var pass uint64
for {
select {
case <-ctx.Done():
return pass, ctx.Err()
default:
}
seed := uint64(0x6a09e667f3bcc909) + pass
if err := parallelMemoryPass(values, workers, func(index int) error {
values[index] = memoryValue(seed, uint64(index))
return nil
}); err != nil {
return pass, err
}
if err := parallelMemoryPass(values, workers, func(index int) error {
expected := memoryValue(seed, uint64(index))
if values[index] != expected {
return fmt.Errorf("memory mismatch at byte %d: got 0x%016x want 0x%016x", index*8, values[index], expected)
}
return nil
}); err != nil {
return pass, err
}
pass++
}
}
func parallelMemoryPass(values []uint64, workers int, operation func(int) error) error {
if workers > len(values) {
workers = len(values)
}
errCh := make(chan error, workers)
var group sync.WaitGroup
for worker := 0; worker < workers; worker++ {
start := len(values) * worker / workers
end := len(values) * (worker + 1) / workers
group.Add(1)
go func() {
defer group.Done()
for index := start; index < end; index++ {
if err := operation(index); err != nil {
errCh <- err
return
}
}
}()
}
group.Wait()
close(errCh)
for err := range errCh {
return err
}
return nil
}
type fftWorkload struct {
size int
workers int
}
func (w fftWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
ctx, cancel := context.WithTimeout(parent, duration)
defer cancel()
var transforms atomic.Uint64
err := runConcurrent(ctx, w.workers, func(workerContext context.Context, worker int) error {
transform := fourier.NewFFT(w.size)
input := make([]float64, w.size)
for index := range input {
input[index] = math.Sin(float64(index+worker)/37) + math.Cos(float64(index*3+worker)/101)
}
coefficients := make([]complex128, w.size/2+1)
reconstructed := make([]float64, w.size)
for {
select {
case <-workerContext.Done():
return workerContext.Err()
default:
}
transform.Coefficients(coefficients, input)
transform.Sequence(reconstructed, coefficients)
maximumError := 0.0
for index, value := range reconstructed {
error := math.Abs(value/float64(w.size) - input[index])
maximumError = math.Max(maximumError, error)
}
if maximumError > 1e-9 {
return fmt.Errorf("FFT verification error %.17g", maximumError)
}
transforms.Add(1)
}
})
if errors.Is(err, context.DeadlineExceeded) {
err = nil
}
return map[string]any{"size": w.size, "verified_transforms": transforms.Load()}, err
}
type matrixWorkload struct {
size int
workers int
}
func (w matrixWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
ctx, cancel := context.WithTimeout(parent, duration)
defer cancel()
var products atomic.Uint64
err := runConcurrent(ctx, w.workers, func(workerContext context.Context, worker int) error {
leftData := make([]float64, w.size*w.size)
rightData := make([]float64, w.size*w.size)
for index := range leftData {
leftData[index] = float64((index+worker)%251-125) / 251
rightData[index] = float64((index*7+worker)%241-120) / 241
}
left := mat.NewDense(w.size, w.size, leftData)
right := mat.NewDense(w.size, w.size, rightData)
var expected mat.Dense
expected.Mul(left, right)
var result mat.Dense
for {
select {
case <-workerContext.Done():
return workerContext.Err()
default:
}
result.Mul(left, right)
if !mat.EqualApprox(&result, &expected, 1e-12) {
return fmt.Errorf("matrix verification mismatch")
}
products.Add(1)
}
})
if errors.Is(err, context.DeadlineExceeded) {
err = nil
}
return map[string]any{"size": w.size, "verified_products": products.Load()}, err
}
type mixedWorkload struct {
cpu cpuWorkload
memory memoryWorkload
}
func (w mixedWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
ctx, cancel := context.WithTimeout(parent, duration)
defer cancel()
type outcome struct {
name string
metrics map[string]any
err error
}
outcomes := make(chan outcome, 2)
go func() {
metrics, err := w.cpu.Run(ctx, duration)
outcomes <- outcome{name: "cpu", metrics: metrics, err: err}
}()
go func() {
metrics, err := w.memory.Run(ctx, duration)
outcomes <- outcome{name: "memory", metrics: metrics, err: err}
}()
metrics := make(map[string]any)
for range 2 {
outcome := <-outcomes
for key, value := range outcome.metrics {
metrics[outcome.name+"_"+key] = value
}
if outcome.err != nil && !errors.Is(outcome.err, context.DeadlineExceeded) {
cancel()
return metrics, outcome.err
}
}
return metrics, nil
}
func runConcurrent(ctx context.Context, count int, operation func(context.Context, int) error) error {
workerContext, cancel := context.WithCancel(ctx)
defer cancel()
errCh := make(chan error, count)
var group sync.WaitGroup
for worker := 0; worker < count; worker++ {
worker := worker
group.Add(1)
go func() {
defer group.Done()
if err := operation(workerContext, worker); err != nil {
errCh <- err
if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
cancel()
}
}
}()
}
group.Wait()
close(errCh)
var contextErr error
for err := range errCh {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
contextErr = err
continue
}
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
if contextErr != nil {
return contextErr
}
return workerContext.Err()
}
func buildWorkloads(config WorkloadConfig) (map[string]Workload, error) {
physical, err := physicalCPUs("/sys/devices/system/cpu")
if err != nil {
return nil, err
}
workers := runtime.NumCPU()
cpu := cpuWorkload{workers: workers}
memory := memoryWorkload{fraction: config.MemoryFraction, workers: workers}
return map[string]Workload{
"cpu": cpu,
"fft": fftWorkload{size: config.FFTSize, workers: workers},
"idle": idleWorkload{},
"matrix": matrixWorkload{size: config.MatrixSize, workers: workers},
"memory": memory,
"mixed": mixedWorkload{cpu: cpu, memory: memory},
"per-core": perCoreWorkload{cpus: physical},
}, nil
}

View file

@ -0,0 +1,114 @@
package zygostates
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"golang.org/x/sys/unix"
)
var errMemoryMismatch = errors.New("memory mismatch")
func TestCPUBlockIsDeterministicAndSeeded(t *testing.T) {
first := cpuBlock(1)
if first != cpuBlock(1) {
t.Fatal("CPU block is not deterministic")
}
if first == cpuBlock(2) {
t.Fatal("CPU block ignores seed")
}
}
func TestNativeComputeWorkloadsVerifyResults(t *testing.T) {
tests := map[string]Workload{
"cpu": cpuWorkload{workers: 1},
"fft": fftWorkload{size: 64, workers: 1},
"matrix": matrixWorkload{size: 8, workers: 1},
"memory": memoryWorkload{fraction: 0.000001, workers: 1},
"mixed": mixedWorkload{
cpu: cpuWorkload{workers: 1},
memory: memoryWorkload{fraction: 0.000001, workers: 1},
},
}
for name, workload := range tests {
t.Run(name, func(t *testing.T) {
metrics, err := workload.Run(context.Background(), 50*time.Millisecond)
if err != nil {
t.Fatal(err)
}
if len(metrics) == 0 {
t.Fatal("missing workload metrics")
}
})
}
}
func TestPerCoreWorkloadPinsAnAllowedCPU(t *testing.T) {
var allowed unix.CPUSet
if err := unix.SchedGetaffinity(0, &allowed); err != nil {
t.Fatal(err)
}
selected := -1
for cpu := 0; cpu < 1024; cpu++ {
if allowed.IsSet(cpu) {
selected = cpu
break
}
}
if selected < 0 {
t.Fatal("no allowed CPU")
}
metrics, err := (perCoreWorkload{cpus: []int{selected}}).Run(context.Background(), 20*time.Millisecond)
if err != nil {
t.Fatal(err)
}
if metrics["completed_cores"] != 1 {
t.Fatalf("metrics=%#v", metrics)
}
}
func TestMemoryPatternDetectsCorruption(t *testing.T) {
values := make([]uint64, 16)
seed := uint64(7)
for index := range values {
values[index] = memoryValue(seed, uint64(index))
}
values[9] ^= 1
err := parallelMemoryPass(values, 3, func(index int) error {
if values[index] != memoryValue(seed, uint64(index)) {
return errMemoryMismatch
}
return nil
})
if err != errMemoryMismatch {
t.Fatalf("err=%v", err)
}
}
func TestPhysicalCPUsSelectsOneThreadPerCore(t *testing.T) {
root := t.TempDir()
for cpu, core := range []string{"0", "0", "1", "1"} {
directory := filepath.Join(root, fmt.Sprintf("cpu%d/topology", cpu))
if err := os.MkdirAll(directory, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(directory, "core_id"), []byte(core), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(directory, "physical_package_id"), []byte("0"), 0o600); err != nil {
t.Fatal(err)
}
}
cpus, err := physicalCPUs(root)
if err != nil {
t.Fatal(err)
}
if len(cpus) != 2 || cpus[0] != 0 || cpus[1] != 2 {
t.Fatalf("cpus=%v", cpus)
}
}