Add Zygostates hardware qualification runner
This commit is contained in:
commit
65541dc8eb
17 changed files with 2889 additions and 0 deletions
117
internal/zygostates/app.go
Normal file
117
internal/zygostates/app.go
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Run(ctx context.Context, configPath string) error {
|
||||
config, configContents, err := loadConfig(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
identity, err := readHardwareIdentity("/sys/class/dmi/id")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read hardware identity: %w", err)
|
||||
}
|
||||
workloads, err := buildWorkloads(config.Workloads)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare workloads: %w", err)
|
||||
}
|
||||
started := time.Now().UTC()
|
||||
runID := started.Format("20060102T150405.000000000Z")
|
||||
artifacts, err := newArtifactStore(config.Artifacts.Directory, runID, configContents, systemMetadata(identity))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create artifacts: %w", err)
|
||||
}
|
||||
defer func() { _ = artifacts.Close() }()
|
||||
if err := artifacts.AppendEvent(Event{Timestamp: started, Type: "run-start", Fields: map[string]any{"run_id": runID, "identity": identity.Key}}); err != nil {
|
||||
return err
|
||||
}
|
||||
var phase atomic.Value
|
||||
phase.Store("preflight")
|
||||
telemetryContext, stopTelemetry := context.WithCancel(context.Background())
|
||||
telemetry := startTelemetry(telemetryContext, artifacts, &phase, time.Duration(config.Artifacts.SampleIntervalSeconds)*time.Second, &TelemetryCollector{procRoot: "/proc", sysRoot: "/sys"})
|
||||
|
||||
var uploader *Uploader
|
||||
var stopPeriodic context.CancelFunc
|
||||
var periodicWait interface{ Wait() }
|
||||
if config.Upload.Bucket != "" {
|
||||
objectStore, err := newS3ObjectStore(ctx, config.Upload)
|
||||
if err != nil {
|
||||
stopTelemetry()
|
||||
telemetry.Wait()
|
||||
result := RunResult{RunID: runID, Identity: identity.Key, StartedAt: started, CompletedAt: time.Now().UTC(), Status: "failed", Error: err.Error()}
|
||||
if writeErr := artifacts.WriteResult(result); writeErr != nil {
|
||||
return writeErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
uploader = newUploader(objectStore, artifacts, config.Upload, identity.Key, runID)
|
||||
preflightContext, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
_, err = uploader.Upload(preflightContext)
|
||||
cancel()
|
||||
if err != nil {
|
||||
stopTelemetry()
|
||||
telemetry.Wait()
|
||||
result := RunResult{RunID: runID, Identity: identity.Key, StartedAt: started, CompletedAt: time.Now().UTC(), Status: "failed", Error: "S3 preflight: " + err.Error(), UploadError: err.Error()}
|
||||
_ = artifacts.WriteResult(result)
|
||||
return fmt.Errorf("S3 preflight: %w", err)
|
||||
}
|
||||
periodicContext, cancelPeriodic := context.WithCancel(context.Background())
|
||||
stopPeriodic = cancelPeriodic
|
||||
periodic := &PeriodicUploader{uploader: uploader, interval: time.Duration(config.Upload.IntervalSeconds) * time.Second, store: artifacts}
|
||||
periodicWait = periodic.Start(periodicContext)
|
||||
}
|
||||
|
||||
runner := Runner{config: config, workloads: workloads, store: artifacts, phase: &phase, now: func() time.Time { return time.Now().UTC() }}
|
||||
runErr := runner.Run(ctx)
|
||||
if stopPeriodic != nil {
|
||||
stopPeriodic()
|
||||
periodicWait.Wait()
|
||||
}
|
||||
stopTelemetry()
|
||||
telemetry.Wait()
|
||||
completed := time.Now().UTC()
|
||||
status := "passed"
|
||||
if runErr != nil {
|
||||
status = "failed"
|
||||
if errors.Is(runErr, context.Canceled) {
|
||||
status = "interrupted"
|
||||
}
|
||||
}
|
||||
result := RunResult{RunID: runID, Identity: identity.Key, StartedAt: started, CompletedAt: completed, Status: status}
|
||||
if runErr != nil {
|
||||
result.Error = runErr.Error()
|
||||
}
|
||||
fields := map[string]any{"status": status, "elapsed_seconds": completed.Sub(started).Seconds()}
|
||||
if runErr != nil {
|
||||
fields["error"] = runErr.Error()
|
||||
}
|
||||
if err := artifacts.AppendEvent(Event{Timestamp: completed, Type: "run-complete", Fields: fields}); err != nil && runErr == nil {
|
||||
runErr = err
|
||||
result.Status = "failed"
|
||||
result.Error = err.Error()
|
||||
}
|
||||
if err := artifacts.WriteResult(result); err != nil {
|
||||
return err
|
||||
}
|
||||
if uploader != nil {
|
||||
finalContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
_, uploadErr := uploader.Upload(finalContext)
|
||||
cancel()
|
||||
if uploadErr != nil {
|
||||
result.UploadError = uploadErr.Error()
|
||||
if err := artifacts.WriteResult(result); err != nil {
|
||||
return err
|
||||
}
|
||||
if runErr == nil {
|
||||
runErr = fmt.Errorf("final upload: %w", uploadErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return runErr
|
||||
}
|
||||
165
internal/zygostates/artifacts.go
Normal file
165
internal/zygostates/artifacts.go
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Type string `json:"type"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
}
|
||||
|
||||
type RunResult struct {
|
||||
RunID string `json:"run_id"`
|
||||
Identity string `json:"identity"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
UploadError string `json:"upload_error,omitempty"`
|
||||
}
|
||||
|
||||
type FileSnapshot struct {
|
||||
Name string
|
||||
Contents []byte
|
||||
}
|
||||
|
||||
type ArtifactStore struct {
|
||||
directory string
|
||||
events *os.File
|
||||
telemetry *os.File
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newArtifactStore(root, runID string, configContents []byte, metadata SystemMetadata) (*ArtifactStore, error) {
|
||||
directory := filepath.Join(root, runID)
|
||||
if err := os.MkdirAll(directory, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, "config.yaml"), configContents, 0o640); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := writeJSONAtomic(filepath.Join(directory, "metadata.json"), metadata); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events, err := os.OpenFile(filepath.Join(directory, "events.ndjson"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
telemetry, err := os.OpenFile(filepath.Join(directory, "telemetry.ndjson"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640)
|
||||
if err != nil {
|
||||
_ = events.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &ArtifactStore{directory: directory, events: events, telemetry: telemetry}, nil
|
||||
}
|
||||
|
||||
func (s *ArtifactStore) Directory() string {
|
||||
return s.directory
|
||||
}
|
||||
|
||||
func (s *ArtifactStore) AppendEvent(event Event) error {
|
||||
return s.appendJSON(s.events, event)
|
||||
}
|
||||
|
||||
func (s *ArtifactStore) AppendTelemetry(sample TelemetrySample) error {
|
||||
return s.appendJSON(s.telemetry, sample)
|
||||
}
|
||||
|
||||
func (s *ArtifactStore) appendJSON(file *os.File, value any) error {
|
||||
contents, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
contents = append(contents, '\n')
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, err = file.Write(contents)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *ArtifactStore) WriteResult(result RunResult) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return writeJSONAtomic(filepath.Join(s.directory, "result.json"), result)
|
||||
}
|
||||
|
||||
func (s *ArtifactStore) Snapshots() ([]FileSnapshot, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := s.events.Sync(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.telemetry.Sync(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(s.directory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
snapshots := make([]FileSnapshot, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if !entry.Type().IsRegular() {
|
||||
continue
|
||||
}
|
||||
contents, err := os.ReadFile(filepath.Join(s.directory, entry.Name()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", entry.Name(), err)
|
||||
}
|
||||
snapshots = append(snapshots, FileSnapshot{Name: entry.Name(), Contents: contents})
|
||||
}
|
||||
sort.Slice(snapshots, func(left, right int) bool { return snapshots[left].Name < snapshots[right].Name })
|
||||
return snapshots, nil
|
||||
}
|
||||
|
||||
func (s *ArtifactStore) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var first error
|
||||
for _, file := range []*os.File{s.events, s.telemetry} {
|
||||
if err := file.Sync(); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
if err := file.Close(); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func writeJSONAtomic(path string, value any) error {
|
||||
contents, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
contents = append(contents, '\n')
|
||||
temporary, err := os.CreateTemp(filepath.Dir(path), ".zygostates-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryPath) }()
|
||||
if err := temporary.Chmod(0o640); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(contents); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temporaryPath, path)
|
||||
}
|
||||
43
internal/zygostates/artifacts_test.go
Normal file
43
internal/zygostates/artifacts_test.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestArtifactStoreProducesParseableDurableFiles(t *testing.T) {
|
||||
store := testArtifactStore(t)
|
||||
if err := store.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "test", Fields: map[string]any{"value": 1}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.AppendTelemetry(TelemetrySample{Timestamp: time.Unix(2, 0), Phase: "test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := RunResult{RunID: "run", Identity: "system", Status: "passed"}
|
||||
if err := store.WriteResult(result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshots, err := store.Snapshots()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files := make(map[string][]byte)
|
||||
for _, snapshot := range snapshots {
|
||||
files[snapshot.Name] = snapshot.Contents
|
||||
}
|
||||
for _, name := range []string{"config.yaml", "events.ndjson", "metadata.json", "result.json", "telemetry.ndjson"} {
|
||||
if _, found := files[name]; !found {
|
||||
t.Fatalf("missing %s", name)
|
||||
}
|
||||
}
|
||||
var decoded RunResult
|
||||
if err := json.Unmarshal(files["result.json"], &decoded); err != nil || decoded.Status != "passed" {
|
||||
t.Fatalf("result=%#v err=%v", decoded, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(store.Directory(), "result.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
158
internal/zygostates/config.go
Normal file
158
internal/zygostates/config.go
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type Step struct {
|
||||
Workload string
|
||||
Seconds int
|
||||
}
|
||||
|
||||
func (s *Step) UnmarshalYAML(node *yaml.Node) error {
|
||||
if node.Kind != yaml.SequenceNode || len(node.Content) != 2 {
|
||||
return fmt.Errorf("step must be [workload, seconds]")
|
||||
}
|
||||
if err := node.Content[0].Decode(&s.Workload); err != nil {
|
||||
return fmt.Errorf("decode workload: %w", err)
|
||||
}
|
||||
if err := node.Content[1].Decode(&s.Seconds); err != nil {
|
||||
return fmt.Errorf("decode seconds: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ArtifactConfig struct {
|
||||
Directory string `yaml:"directory"`
|
||||
SampleIntervalSeconds int `yaml:"sample_interval_seconds"`
|
||||
}
|
||||
|
||||
type UploadConfig struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Region string `yaml:"region"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
IntervalSeconds int `yaml:"interval_seconds"`
|
||||
}
|
||||
|
||||
type WorkloadConfig struct {
|
||||
MemoryFraction float64 `yaml:"memory_fraction"`
|
||||
FFTSize int `yaml:"fft_size"`
|
||||
MatrixSize int `yaml:"matrix_size"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Version int `yaml:"version"`
|
||||
Repeat int `yaml:"repeat"`
|
||||
Sequence []Step `yaml:"sequence"`
|
||||
Artifacts ArtifactConfig `yaml:"artifacts"`
|
||||
Upload UploadConfig `yaml:"upload"`
|
||||
Workloads WorkloadConfig `yaml:"workloads"`
|
||||
}
|
||||
|
||||
var workloadNames = map[string]struct{}{
|
||||
"cpu": {},
|
||||
"fft": {},
|
||||
"idle": {},
|
||||
"matrix": {},
|
||||
"memory": {},
|
||||
"mixed": {},
|
||||
"per-core": {},
|
||||
}
|
||||
|
||||
func loadConfig(path string) (Config, []byte, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Config{}, nil, err
|
||||
}
|
||||
var config Config
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(contents))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&config); err != nil {
|
||||
return Config{}, nil, err
|
||||
}
|
||||
if err := config.applyDefaultsAndValidate(); err != nil {
|
||||
return Config{}, nil, err
|
||||
}
|
||||
return config, contents, nil
|
||||
}
|
||||
|
||||
func (c *Config) applyDefaultsAndValidate() error {
|
||||
if c.Version != 1 {
|
||||
return fmt.Errorf("version must be 1")
|
||||
}
|
||||
if c.Repeat < 1 {
|
||||
return fmt.Errorf("repeat must be positive")
|
||||
}
|
||||
if len(c.Sequence) == 0 {
|
||||
return fmt.Errorf("sequence must not be empty")
|
||||
}
|
||||
for index, step := range c.Sequence {
|
||||
if _, found := workloadNames[step.Workload]; !found {
|
||||
return fmt.Errorf("sequence[%d]: unknown workload %q", index, step.Workload)
|
||||
}
|
||||
if step.Seconds < 1 {
|
||||
return fmt.Errorf("sequence[%d]: seconds must be positive", index)
|
||||
}
|
||||
}
|
||||
if c.Artifacts.Directory == "" {
|
||||
c.Artifacts.Directory = "/var/lib/zygostates/runs"
|
||||
}
|
||||
if !filepath.IsAbs(c.Artifacts.Directory) {
|
||||
return fmt.Errorf("artifacts.directory must be absolute")
|
||||
}
|
||||
if c.Artifacts.SampleIntervalSeconds == 0 {
|
||||
c.Artifacts.SampleIntervalSeconds = 15
|
||||
}
|
||||
if c.Artifacts.SampleIntervalSeconds < 1 {
|
||||
return fmt.Errorf("artifacts.sample_interval_seconds must be positive")
|
||||
}
|
||||
if c.Workloads.MemoryFraction == 0 {
|
||||
c.Workloads.MemoryFraction = 0.5
|
||||
}
|
||||
if c.Workloads.MemoryFraction <= 0 || c.Workloads.MemoryFraction > 0.9 {
|
||||
return fmt.Errorf("workloads.memory_fraction must be greater than 0 and at most 0.9")
|
||||
}
|
||||
if c.Workloads.FFTSize == 0 {
|
||||
c.Workloads.FFTSize = 65536
|
||||
}
|
||||
if c.Workloads.FFTSize < 2 {
|
||||
return fmt.Errorf("workloads.fft_size must be at least 2")
|
||||
}
|
||||
if c.Workloads.MatrixSize == 0 {
|
||||
c.Workloads.MatrixSize = 256
|
||||
}
|
||||
if c.Workloads.MatrixSize < 2 {
|
||||
return fmt.Errorf("workloads.matrix_size must be at least 2")
|
||||
}
|
||||
uploadConfigured := c.Upload.Endpoint != "" || c.Upload.Region != "" || c.Upload.Bucket != "" || c.Upload.Prefix != "" || c.Upload.IntervalSeconds != 0
|
||||
if !uploadConfigured {
|
||||
return nil
|
||||
}
|
||||
if c.Upload.Bucket == "" {
|
||||
return fmt.Errorf("upload.bucket is required when upload is configured")
|
||||
}
|
||||
if c.Upload.Region == "" {
|
||||
c.Upload.Region = "us-east-1"
|
||||
}
|
||||
if c.Upload.Prefix == "" {
|
||||
c.Upload.Prefix = "zygostates"
|
||||
}
|
||||
c.Upload.Prefix = strings.Trim(c.Upload.Prefix, "/")
|
||||
if c.Upload.Prefix == "" {
|
||||
return fmt.Errorf("upload.prefix must not resolve to empty")
|
||||
}
|
||||
if c.Upload.IntervalSeconds == 0 {
|
||||
c.Upload.IntervalSeconds = 15
|
||||
}
|
||||
if c.Upload.IntervalSeconds < 1 {
|
||||
return fmt.Errorf("upload.interval_seconds must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
80
internal/zygostates/config_test.go
Normal file
80
internal/zygostates/config_test.go
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeTestFile(t *testing.T, name, contents string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestLoadConfigDecodesTupleSequenceAndDefaults(t *testing.T) {
|
||||
path := writeTestFile(t, "config.yaml", `
|
||||
version: 1
|
||||
repeat: 2
|
||||
sequence:
|
||||
- [mixed, 300]
|
||||
- [idle, 30]
|
||||
artifacts:
|
||||
directory: /tmp/zygostates
|
||||
upload:
|
||||
endpoint: https://s3.example.test
|
||||
bucket: dump
|
||||
`)
|
||||
config, _, err := loadConfig(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Sequence) != 2 || config.Sequence[0] != (Step{Workload: "mixed", Seconds: 300}) {
|
||||
t.Fatalf("unexpected sequence: %#v", config.Sequence)
|
||||
}
|
||||
if config.Artifacts.SampleIntervalSeconds != 15 || config.Upload.IntervalSeconds != 15 || config.Upload.Region != "us-east-1" || config.Upload.Prefix != "zygostates" {
|
||||
t.Fatalf("defaults not applied: %#v", config)
|
||||
}
|
||||
if config.Workloads.MemoryFraction != 0.5 || config.Workloads.FFTSize != 65536 || config.Workloads.MatrixSize != 256 {
|
||||
t.Fatalf("workload defaults not applied: %#v", config.Workloads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigRejectsUnknownFieldsAndWorkloads(t *testing.T) {
|
||||
for _, contents := range []string{
|
||||
"version: 1\nrepeat: 1\nsequence: [[idle, 1]]\nunknown: true\n",
|
||||
"version: 1\nrepeat: 1\nsequence: [[unbounded, 1]]\n",
|
||||
"version: 1\nrepeat: 1\nsequence: [[idle, 0]]\n",
|
||||
"version: 1\nrepeat: 1\nsequence: [idle]\n",
|
||||
} {
|
||||
path := writeTestFile(t, "config.yaml", contents)
|
||||
if _, _, err := loadConfig(path); err == nil {
|
||||
t.Fatalf("accepted invalid config %q", contents)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadConfigurationContainsNoCredentialFields(t *testing.T) {
|
||||
typeName := strings.ToLower(strings.Join([]string{
|
||||
fmtFieldNames(UploadConfig{}),
|
||||
}, " "))
|
||||
for _, forbidden := range []string{"credential", "accesskey", "secret", "token"} {
|
||||
if strings.Contains(typeName, forbidden) {
|
||||
t.Fatalf("upload config exposes %q: %s", forbidden, typeName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func fmtFieldNames(value any) string {
|
||||
typeOf := reflect.TypeOf(value)
|
||||
var names []string
|
||||
for index := 0; index < typeOf.NumField(); index++ {
|
||||
names = append(names, typeOf.Field(index).Name)
|
||||
}
|
||||
return strings.Join(names, " ")
|
||||
}
|
||||
107
internal/zygostates/identity.go
Normal file
107
internal/zygostates/identity.go
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type HardwareIdentity struct {
|
||||
Key string `json:"key"`
|
||||
BoardVendor string `json:"board_vendor,omitempty"`
|
||||
BoardName string `json:"board_name,omitempty"`
|
||||
BoardVersion string `json:"board_version,omitempty"`
|
||||
BoardSerial string `json:"board_serial,omitempty"`
|
||||
ProductName string `json:"product_name,omitempty"`
|
||||
ProductSerial string `json:"product_serial,omitempty"`
|
||||
ProductUUID string `json:"product_uuid,omitempty"`
|
||||
}
|
||||
|
||||
var uuidPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
|
||||
|
||||
func readHardwareIdentity(root string) (HardwareIdentity, error) {
|
||||
read := func(name string) string {
|
||||
contents, err := os.ReadFile(filepath.Join(root, name))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(contents))
|
||||
}
|
||||
identity := HardwareIdentity{
|
||||
BoardVendor: read("board_vendor"),
|
||||
BoardName: read("board_name"),
|
||||
BoardVersion: read("board_version"),
|
||||
BoardSerial: read("board_serial"),
|
||||
ProductName: read("product_name"),
|
||||
ProductSerial: read("product_serial"),
|
||||
ProductUUID: strings.ToLower(read("product_uuid")),
|
||||
}
|
||||
if validUUID(identity.ProductUUID) {
|
||||
identity.Key = identity.ProductUUID
|
||||
return identity, nil
|
||||
}
|
||||
if validIdentifier(identity.BoardSerial) {
|
||||
identity.Key = sanitizeIdentifier(identity.BoardSerial)
|
||||
return identity, nil
|
||||
}
|
||||
return HardwareIdentity{}, fmt.Errorf("no valid SMBIOS product UUID or board serial")
|
||||
}
|
||||
|
||||
func validUUID(value string) bool {
|
||||
if !uuidPattern.MatchString(value) {
|
||||
return false
|
||||
}
|
||||
compact := strings.ReplaceAll(strings.ToLower(value), "-", "")
|
||||
return compact != strings.Repeat("0", 32) && compact != strings.Repeat("f", 32)
|
||||
}
|
||||
|
||||
func validIdentifier(value string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(value))
|
||||
if normalized == "" {
|
||||
return false
|
||||
}
|
||||
for _, placeholder := range []string{"default string", "none", "not specified", "system serial number", "to be filled by o.e.m.", "unknown"} {
|
||||
if normalized == placeholder {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return sanitizeIdentifier(value) != ""
|
||||
}
|
||||
|
||||
func sanitizeIdentifier(value string) string {
|
||||
var result strings.Builder
|
||||
for _, character := range strings.ToLower(strings.TrimSpace(value)) {
|
||||
switch {
|
||||
case character >= 'a' && character <= 'z', character >= '0' && character <= '9', character == '-', character == '_', character == '.':
|
||||
result.WriteRune(character)
|
||||
default:
|
||||
result.WriteByte('-')
|
||||
}
|
||||
}
|
||||
return strings.Trim(result.String(), "-.")
|
||||
}
|
||||
|
||||
type SystemMetadata struct {
|
||||
Identity HardwareIdentity `json:"identity"`
|
||||
Hostname string `json:"hostname"`
|
||||
Kernel string `json:"kernel"`
|
||||
Architecture string `json:"architecture"`
|
||||
LogicalCPUs int `json:"logical_cpus"`
|
||||
GoVersion string `json:"go_version"`
|
||||
}
|
||||
|
||||
func systemMetadata(identity HardwareIdentity) SystemMetadata {
|
||||
hostname, _ := os.Hostname()
|
||||
kernel, _ := os.ReadFile("/proc/sys/kernel/osrelease")
|
||||
return SystemMetadata{
|
||||
Identity: identity,
|
||||
Hostname: hostname,
|
||||
Kernel: strings.TrimSpace(string(kernel)),
|
||||
Architecture: runtime.GOARCH,
|
||||
LogicalCPUs: runtime.NumCPU(),
|
||||
GoVersion: runtime.Version(),
|
||||
}
|
||||
}
|
||||
54
internal/zygostates/identity_test.go
Normal file
54
internal/zygostates/identity_test.go
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeDMI(t *testing.T, values map[string]string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for name, value := range values {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(value+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestHardwareIdentityUsesNormalizedProductUUID(t *testing.T) {
|
||||
identity, err := readHardwareIdentity(writeDMI(t, map[string]string{
|
||||
"product_uuid": "D5B6EC3A-DC36-4AA0-B6AE-5AF3D13D1280",
|
||||
"board_serial": "board-9",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if identity.Key != "d5b6ec3a-dc36-4aa0-b6ae-5af3d13d1280" {
|
||||
t.Fatalf("key=%q", identity.Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardwareIdentityFallsBackToBoardSerial(t *testing.T) {
|
||||
identity, err := readHardwareIdentity(writeDMI(t, map[string]string{
|
||||
"product_uuid": "00000000-0000-0000-0000-000000000000",
|
||||
"board_serial": "MB 12/34",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if identity.Key != "mb-12-34" {
|
||||
t.Fatalf("key=%q", identity.Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardwareIdentityRejectsPlaceholders(t *testing.T) {
|
||||
_, err := readHardwareIdentity(writeDMI(t, map[string]string{
|
||||
"product_uuid": "ffffffff-ffff-ffff-ffff-ffffffffffff",
|
||||
"board_serial": "To Be Filled By O.E.M.",
|
||||
}))
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid identity")
|
||||
}
|
||||
}
|
||||
70
internal/zygostates/runner.go
Normal file
70
internal/zygostates/runner.go
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Workload interface {
|
||||
Run(context.Context, time.Duration) (map[string]any, error)
|
||||
}
|
||||
|
||||
type Runner struct {
|
||||
config Config
|
||||
workloads map[string]Workload
|
||||
store *ArtifactStore
|
||||
phase *atomic.Value
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (r *Runner) Run(ctx context.Context) error {
|
||||
for cycle := 1; cycle <= r.config.Repeat; cycle++ {
|
||||
for index, step := range r.config.Sequence {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
workload := r.workloads[step.Workload]
|
||||
phase := fmt.Sprintf("cycle-%d/%s", cycle, step.Workload)
|
||||
r.phase.Store(phase)
|
||||
started := r.now()
|
||||
if err := r.store.AppendEvent(Event{Timestamp: started, Type: "step-start", Fields: map[string]any{
|
||||
"cycle": cycle, "index": index, "workload": step.Workload, "seconds": step.Seconds,
|
||||
}}); err != nil {
|
||||
return err
|
||||
}
|
||||
metrics, stepError := workload.Run(ctx, time.Duration(step.Seconds)*time.Second)
|
||||
fields := map[string]any{
|
||||
"cycle": cycle, "index": index, "workload": step.Workload, "elapsed_seconds": r.now().Sub(started).Seconds(),
|
||||
}
|
||||
for key, value := range metrics {
|
||||
fields[key] = value
|
||||
}
|
||||
if stepError != nil {
|
||||
fields["error"] = stepError.Error()
|
||||
}
|
||||
if appendErr := r.store.AppendEvent(Event{Timestamp: r.now(), Type: "step-complete", Fields: fields}); appendErr != nil {
|
||||
return appendErr
|
||||
}
|
||||
if stepError != nil {
|
||||
return fmt.Errorf("cycle %d %s: %w", cycle, step.Workload, stepError)
|
||||
}
|
||||
}
|
||||
}
|
||||
r.phase.Store("complete")
|
||||
return nil
|
||||
}
|
||||
|
||||
type idleWorkload struct{}
|
||||
|
||||
func (idleWorkload) Run(ctx context.Context, duration time.Duration) (map[string]any, error) {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
70
internal/zygostates/runner_test.go
Normal file
70
internal/zygostates/runner_test.go
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeWorkload struct {
|
||||
calls atomic.Int64
|
||||
err error
|
||||
}
|
||||
|
||||
func (w *fakeWorkload) Run(context.Context, time.Duration) (map[string]any, error) {
|
||||
w.calls.Add(1)
|
||||
return map[string]any{"value": 7}, w.err
|
||||
}
|
||||
|
||||
func testArtifactStore(t *testing.T) *ArtifactStore {
|
||||
t.Helper()
|
||||
store, err := newArtifactStore(t.TempDir(), "run", []byte("version: 1\n"), SystemMetadata{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := store.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
func TestRunnerRepeatsSequence(t *testing.T) {
|
||||
first := &fakeWorkload{}
|
||||
second := &fakeWorkload{}
|
||||
store := testArtifactStore(t)
|
||||
var phase atomic.Value
|
||||
phase.Store("start")
|
||||
now := time.Unix(1_700_000_000, 0)
|
||||
runner := Runner{
|
||||
config: Config{Repeat: 3, Sequence: []Step{{Workload: "first", Seconds: 1}, {Workload: "second", Seconds: 2}}},
|
||||
workloads: map[string]Workload{"first": first, "second": second}, store: store, phase: &phase,
|
||||
now: func() time.Time { now = now.Add(time.Second); return now },
|
||||
}
|
||||
if err := runner.Run(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.calls.Load() != 3 || second.calls.Load() != 3 || phase.Load() != "complete" {
|
||||
t.Fatalf("calls=%d,%d phase=%v", first.calls.Load(), second.calls.Load(), phase.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerStopsOnWorkloadFailure(t *testing.T) {
|
||||
failure := errors.New("verification failed")
|
||||
failed := &fakeWorkload{err: failure}
|
||||
after := &fakeWorkload{}
|
||||
store := testArtifactStore(t)
|
||||
var phase atomic.Value
|
||||
phase.Store("start")
|
||||
runner := Runner{
|
||||
config: Config{Repeat: 2, Sequence: []Step{{Workload: "failed", Seconds: 1}, {Workload: "after", Seconds: 1}}},
|
||||
workloads: map[string]Workload{"failed": failed, "after": after}, store: store, phase: &phase, now: time.Now,
|
||||
}
|
||||
err := runner.Run(context.Background())
|
||||
if !errors.Is(err, failure) || failed.calls.Load() != 1 || after.calls.Load() != 0 {
|
||||
t.Fatalf("err=%v calls=%d,%d", err, failed.calls.Load(), after.calls.Load())
|
||||
}
|
||||
}
|
||||
51
internal/zygostates/s3.go
Normal file
51
internal/zygostates/s3.go
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type s3PutClient interface {
|
||||
PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
}
|
||||
|
||||
type S3ObjectStore struct {
|
||||
client s3PutClient
|
||||
}
|
||||
|
||||
func newS3ObjectStore(ctx context.Context, config UploadConfig) (*S3ObjectStore, error) {
|
||||
return newS3ObjectStoreWithOptions(ctx, config)
|
||||
}
|
||||
|
||||
func newS3ObjectStoreWithOptions(ctx context.Context, config UploadConfig, options ...func(*awsconfig.LoadOptions) error) (*S3ObjectStore, error) {
|
||||
options = append([]func(*awsconfig.LoadOptions) error{awsconfig.WithRegion(config.Region)}, options...)
|
||||
loaded, err := awsconfig.LoadDefaultConfig(ctx, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := s3.NewFromConfig(loaded, func(options *s3.Options) {
|
||||
options.UsePathStyle = config.Endpoint != ""
|
||||
if config.Endpoint != "" {
|
||||
options.BaseEndpoint = aws.String(config.Endpoint)
|
||||
}
|
||||
})
|
||||
return &S3ObjectStore{client: client}, nil
|
||||
}
|
||||
|
||||
func (s *S3ObjectStore) Put(ctx context.Context, request PutRequest) error {
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: aws.String(request.Bucket), Key: aws.String(request.Key), Body: bytes.NewReader(request.Body), Metadata: request.Metadata,
|
||||
}
|
||||
if request.ContentType != "" {
|
||||
input.ContentType = aws.String(request.ContentType)
|
||||
}
|
||||
if request.ContentEncoding != "" {
|
||||
input.ContentEncoding = aws.String(request.ContentEncoding)
|
||||
}
|
||||
_, err := s.client.PutObject(ctx, input)
|
||||
return err
|
||||
}
|
||||
59
internal/zygostates/s3_test.go
Normal file
59
internal/zygostates/s3_test.go
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
)
|
||||
|
||||
type recordingHTTPClient struct {
|
||||
mu sync.Mutex
|
||||
request *http.Request
|
||||
body []byte
|
||||
}
|
||||
|
||||
func (c *recordingHTTPClient) Do(request *http.Request) (*http.Response, error) {
|
||||
body, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.request = request
|
||||
c.body = body
|
||||
c.mu.Unlock()
|
||||
return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: request}, nil
|
||||
}
|
||||
|
||||
func TestS3ObjectStoreUsesSDKCredentialsAndPathStyleEndpoint(t *testing.T) {
|
||||
client := &recordingHTTPClient{}
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "test-access-key")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key")
|
||||
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
|
||||
store, err := newS3ObjectStoreWithOptions(context.Background(), UploadConfig{Endpoint: "https://s3.example.test", Region: "us-east-1"}, awsconfig.WithHTTPClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = store.Put(context.Background(), PutRequest{Bucket: "dump", Key: "prefix/file", Body: []byte("contents"), ContentType: "application/octet-stream"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client.mu.Lock()
|
||||
request := client.request
|
||||
body := client.body
|
||||
client.mu.Unlock()
|
||||
if request.Method != http.MethodPut || request.URL.Path != "/dump/prefix/file" {
|
||||
t.Fatalf("request=%s %s", request.Method, request.URL.Path)
|
||||
}
|
||||
if request.Header.Get("Authorization") == "" {
|
||||
t.Fatal("request was not signed")
|
||||
}
|
||||
if !bytes.Contains(body, []byte("contents")) {
|
||||
t.Fatalf("body=%q", body)
|
||||
}
|
||||
}
|
||||
244
internal/zygostates/telemetry.go
Normal file
244
internal/zygostates/telemetry.go
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type TelemetrySample struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Phase string `json:"phase"`
|
||||
BootID string `json:"boot_id,omitempty"`
|
||||
Load1 float64 `json:"load_1m"`
|
||||
Load5 float64 `json:"load_5m"`
|
||||
Load15 float64 `json:"load_15m"`
|
||||
MemoryAvailableKiB uint64 `json:"memory_available_kib"`
|
||||
CPUBusyPercent *float64 `json:"cpu_busy_percent,omitempty"`
|
||||
PackageWatts *float64 `json:"package_watts,omitempty"`
|
||||
TemperaturesC map[string]float64 `json:"temperatures_c,omitempty"`
|
||||
CPUFrequencyKHz map[string]uint64 `json:"cpu_frequency_khz,omitempty"`
|
||||
}
|
||||
|
||||
type CPUTime struct {
|
||||
total uint64
|
||||
idle uint64
|
||||
}
|
||||
|
||||
type EnergySample struct {
|
||||
microjoules uint64
|
||||
maximum uint64
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
type TelemetryCollector struct {
|
||||
procRoot string
|
||||
sysRoot string
|
||||
previousCPU *CPUTime
|
||||
previousEnergy *EnergySample
|
||||
}
|
||||
|
||||
func (c *TelemetryCollector) Sample(now time.Time, phase string) (TelemetrySample, error) {
|
||||
load, err := os.ReadFile(filepath.Join(c.procRoot, "loadavg"))
|
||||
if err != nil {
|
||||
return TelemetrySample{}, err
|
||||
}
|
||||
loadFields := strings.Fields(string(load))
|
||||
if len(loadFields) < 3 {
|
||||
return TelemetrySample{}, fmt.Errorf("invalid loadavg")
|
||||
}
|
||||
load1, err := strconv.ParseFloat(loadFields[0], 64)
|
||||
if err != nil {
|
||||
return TelemetrySample{}, err
|
||||
}
|
||||
load5, err := strconv.ParseFloat(loadFields[1], 64)
|
||||
if err != nil {
|
||||
return TelemetrySample{}, err
|
||||
}
|
||||
load15, err := strconv.ParseFloat(loadFields[2], 64)
|
||||
if err != nil {
|
||||
return TelemetrySample{}, err
|
||||
}
|
||||
memory, err := availableMemoryBytes(filepath.Join(c.procRoot, "meminfo"))
|
||||
if err != nil {
|
||||
return TelemetrySample{}, err
|
||||
}
|
||||
bootID, _ := os.ReadFile(filepath.Join(c.procRoot, "sys/kernel/random/boot_id"))
|
||||
sample := TelemetrySample{
|
||||
Timestamp: now,
|
||||
Phase: phase,
|
||||
BootID: strings.TrimSpace(string(bootID)),
|
||||
Load1: load1,
|
||||
Load5: load5,
|
||||
Load15: load15,
|
||||
MemoryAvailableKiB: memory / 1024,
|
||||
TemperaturesC: readTemperatures(filepath.Join(c.sysRoot, "class/hwmon")),
|
||||
CPUFrequencyKHz: readCPUFrequencies(filepath.Join(c.sysRoot, "devices/system/cpu")),
|
||||
}
|
||||
currentCPU, err := readCPUTime(filepath.Join(c.procRoot, "stat"))
|
||||
if err == nil && c.previousCPU != nil {
|
||||
totalDelta := currentCPU.total - c.previousCPU.total
|
||||
idleDelta := currentCPU.idle - c.previousCPU.idle
|
||||
if totalDelta > 0 && idleDelta <= totalDelta {
|
||||
busy := 100 * float64(totalDelta-idleDelta) / float64(totalDelta)
|
||||
sample.CPUBusyPercent = &busy
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
c.previousCPU = ¤tCPU
|
||||
}
|
||||
currentEnergy, err := readPackageEnergy(filepath.Join(c.sysRoot, "class/powercap"), now)
|
||||
if err == nil && c.previousEnergy != nil {
|
||||
delta := energyDelta(*c.previousEnergy, currentEnergy)
|
||||
seconds := currentEnergy.timestamp.Sub(c.previousEnergy.timestamp).Seconds()
|
||||
if seconds > 0 {
|
||||
watts := float64(delta) / 1e6 / seconds
|
||||
sample.PackageWatts = &watts
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
c.previousEnergy = ¤tEnergy
|
||||
}
|
||||
return sample, nil
|
||||
}
|
||||
|
||||
func readCPUTime(path string) (CPUTime, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return CPUTime{}, err
|
||||
}
|
||||
line, _, _ := strings.Cut(string(contents), "\n")
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 || fields[0] != "cpu" {
|
||||
return CPUTime{}, fmt.Errorf("invalid CPU stat")
|
||||
}
|
||||
var values []uint64
|
||||
for _, field := range fields[1:] {
|
||||
value, err := strconv.ParseUint(field, 10, 64)
|
||||
if err != nil {
|
||||
return CPUTime{}, err
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
var total uint64
|
||||
for _, value := range values {
|
||||
total += value
|
||||
}
|
||||
idle := values[3]
|
||||
if len(values) > 4 {
|
||||
idle += values[4]
|
||||
}
|
||||
return CPUTime{total: total, idle: idle}, nil
|
||||
}
|
||||
|
||||
func readTemperatures(root string) map[string]float64 {
|
||||
result := make(map[string]float64)
|
||||
hwmons, _ := filepath.Glob(filepath.Join(root, "hwmon*"))
|
||||
for _, hwmon := range hwmons {
|
||||
deviceName, _ := os.ReadFile(filepath.Join(hwmon, "name"))
|
||||
inputs, _ := filepath.Glob(filepath.Join(hwmon, "temp*_input"))
|
||||
for _, inputPath := range inputs {
|
||||
contents, err := os.ReadFile(inputPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
millidegrees, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stem := strings.TrimSuffix(filepath.Base(inputPath), "_input")
|
||||
label, _ := os.ReadFile(filepath.Join(hwmon, stem+"_label"))
|
||||
labelText := strings.TrimSpace(string(label))
|
||||
if labelText == "" {
|
||||
labelText = stem
|
||||
}
|
||||
name := strings.TrimSpace(string(deviceName)) + "/" + labelText
|
||||
result[name] = float64(millidegrees) / 1000
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func readCPUFrequencies(root string) map[string]uint64 {
|
||||
result := make(map[string]uint64)
|
||||
paths, _ := filepath.Glob(filepath.Join(root, "cpu[0-9]*/cpufreq/scaling_cur_freq"))
|
||||
for _, path := range paths {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
frequency, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
cpu := filepath.Base(filepath.Dir(filepath.Dir(path)))
|
||||
result[cpu] = frequency
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func readPackageEnergy(root string, now time.Time) (EnergySample, error) {
|
||||
paths, _ := filepath.Glob(filepath.Join(root, "*/energy_uj"))
|
||||
if len(paths) == 0 {
|
||||
paths, _ = filepath.Glob(filepath.Join(root, "*/*/energy_uj"))
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return EnergySample{}, fmt.Errorf("package energy input not found")
|
||||
}
|
||||
energyContents, err := os.ReadFile(paths[0])
|
||||
if err != nil {
|
||||
return EnergySample{}, err
|
||||
}
|
||||
maximumContents, err := os.ReadFile(filepath.Join(filepath.Dir(paths[0]), "max_energy_range_uj"))
|
||||
if err != nil {
|
||||
return EnergySample{}, err
|
||||
}
|
||||
energy, err := strconv.ParseUint(strings.TrimSpace(string(energyContents)), 10, 64)
|
||||
if err != nil {
|
||||
return EnergySample{}, err
|
||||
}
|
||||
maximum, err := strconv.ParseUint(strings.TrimSpace(string(maximumContents)), 10, 64)
|
||||
if err != nil {
|
||||
return EnergySample{}, err
|
||||
}
|
||||
return EnergySample{microjoules: energy, maximum: maximum, timestamp: now}, nil
|
||||
}
|
||||
|
||||
func energyDelta(previous, current EnergySample) uint64 {
|
||||
if current.microjoules >= previous.microjoules {
|
||||
return current.microjoules - previous.microjoules
|
||||
}
|
||||
return previous.maximum - previous.microjoules + current.microjoules
|
||||
}
|
||||
|
||||
func startTelemetry(ctx context.Context, store *ArtifactStore, phase *atomic.Value, interval time.Duration, collector *TelemetryCollector) *sync.WaitGroup {
|
||||
var group sync.WaitGroup
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
value := phase.Load()
|
||||
phaseName, _ := value.(string)
|
||||
sample, err := collector.Sample(time.Now().UTC(), phaseName)
|
||||
if err != nil {
|
||||
_ = store.AppendEvent(Event{Timestamp: time.Now().UTC(), Type: "telemetry-error", Fields: map[string]any{"error": err.Error()}})
|
||||
} else if err := store.AppendTelemetry(sample); err != nil {
|
||||
_ = store.AppendEvent(Event{Timestamp: time.Now().UTC(), Type: "telemetry-error", Fields: map[string]any{"error": err.Error()}})
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return &group
|
||||
}
|
||||
139
internal/zygostates/telemetry_test.go
Normal file
139
internal/zygostates/telemetry_test.go
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestReadCPUTimeAndBusyDelta(t *testing.T) {
|
||||
path := writeTestFile(t, "stat", "cpu 100 20 30 400 50 0 0 0 0 0\ncpu0 1 2 3 4\n")
|
||||
value, err := readCPUTime(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value.total != 600 || value.idle != 450 {
|
||||
t.Fatalf("CPU time=%#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartTelemetryWritesParseableSamples(t *testing.T) {
|
||||
store := testArtifactStore(t)
|
||||
var phase atomic.Value
|
||||
phase.Store("test")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
wait := startTelemetry(ctx, store, &phase, time.Millisecond, &TelemetryCollector{procRoot: "/proc", sysRoot: "/sys"})
|
||||
deadline := time.Now().Add(time.Second)
|
||||
|
||||
collect:
|
||||
for time.Now().Before(deadline) {
|
||||
snapshots, err := store.Snapshots()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, snapshot := range snapshots {
|
||||
if snapshot.Name == "telemetry.ndjson" && bytes.Count(snapshot.Contents, []byte{'\n'}) >= 2 {
|
||||
break collect
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
wait.Wait()
|
||||
file, err := os.Open(filepath.Join(store.Directory(), "telemetry.ndjson"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
scanner := bufio.NewScanner(file)
|
||||
count := 0
|
||||
for scanner.Scan() {
|
||||
var sample TelemetrySample
|
||||
if err := json.Unmarshal(scanner.Bytes(), &sample); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sample.Phase != "test" {
|
||||
t.Fatalf("phase=%q", sample.Phase)
|
||||
}
|
||||
count++
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count < 2 {
|
||||
t.Fatalf("samples=%d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnergyDeltaHandlesCounterWrap(t *testing.T) {
|
||||
previous := EnergySample{microjoules: 900, maximum: 1000}
|
||||
current := EnergySample{microjoules: 100, maximum: 1000}
|
||||
if delta := energyDelta(previous, current); delta != 200 {
|
||||
t.Fatalf("delta=%d", delta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryCollectorSamplesLinuxInputs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
proc := filepath.Join(root, "proc")
|
||||
sys := filepath.Join(root, "sys")
|
||||
for _, directory := range []string{
|
||||
filepath.Join(proc, "sys/kernel/random"),
|
||||
filepath.Join(sys, "class/hwmon/hwmon0"),
|
||||
filepath.Join(sys, "devices/system/cpu/cpu0/cpufreq"),
|
||||
filepath.Join(sys, "class/powercap/intel-rapl:0"),
|
||||
} {
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
files := map[string]string{
|
||||
filepath.Join(proc, "loadavg"): "1.25 2.50 3.75 1/10 1\n",
|
||||
filepath.Join(proc, "meminfo"): "MemTotal: 2000 kB\nMemAvailable: 1000 kB\n",
|
||||
filepath.Join(proc, "stat"): "cpu 100 0 100 800 0 0 0 0 0 0\n",
|
||||
filepath.Join(proc, "sys/kernel/random/boot_id"): "boot\n",
|
||||
filepath.Join(sys, "class/hwmon/hwmon0/name"): "k10temp\n",
|
||||
filepath.Join(sys, "class/hwmon/hwmon0/temp1_label"): "Tctl\n",
|
||||
filepath.Join(sys, "class/hwmon/hwmon0/temp1_input"): "42500\n",
|
||||
filepath.Join(sys, "devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"): "4100000\n",
|
||||
filepath.Join(sys, "class/powercap/intel-rapl:0/energy_uj"): "1000000\n",
|
||||
filepath.Join(sys, "class/powercap/intel-rapl:0/max_energy_range_uj"): "10000000\n",
|
||||
}
|
||||
for path, contents := range files {
|
||||
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
collector := TelemetryCollector{procRoot: proc, sysRoot: sys}
|
||||
firstTime := time.Unix(10, 0)
|
||||
first, err := collector.Sample(firstTime, "idle")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.Load1 != 1.25 || first.MemoryAvailableKiB != 1000 || first.TemperaturesC["k10temp/Tctl"] != 42.5 || first.CPUFrequencyKHz["cpu0"] != 4100000 {
|
||||
t.Fatalf("sample=%#v", first)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(proc, "stat"), []byte("cpu 150 0 150 900 0 0 0 0 0 0\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(sys, "class/powercap/intel-rapl:0/energy_uj"), []byte("3000000\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := collector.Sample(firstTime.Add(time.Second), "cpu")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.CPUBusyPercent == nil || math.Abs(*second.CPUBusyPercent-50) > 1e-9 {
|
||||
t.Fatalf("busy=%v", second.CPUBusyPercent)
|
||||
}
|
||||
if second.PackageWatts == nil || math.Abs(*second.PackageWatts-2) > 1e-9 {
|
||||
t.Fatalf("watts=%v", second.PackageWatts)
|
||||
}
|
||||
}
|
||||
157
internal/zygostates/uploader.go
Normal file
157
internal/zygostates/uploader.go
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type PutRequest struct {
|
||||
Bucket string
|
||||
Key string
|
||||
Body []byte
|
||||
ContentType string
|
||||
ContentEncoding string
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
type ObjectStore interface {
|
||||
Put(context.Context, PutRequest) error
|
||||
}
|
||||
|
||||
type UploadedRange struct {
|
||||
Path string `json:"path"`
|
||||
Generation int `json:"generation"`
|
||||
Start int64 `json:"start"`
|
||||
End int64 `json:"end"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
type UploadTick struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Uploaded []UploadedRange `json:"uploaded"`
|
||||
}
|
||||
|
||||
type uploadPosition struct {
|
||||
offset int64
|
||||
generation int
|
||||
}
|
||||
|
||||
type Uploader struct {
|
||||
store ObjectStore
|
||||
artifacts *ArtifactStore
|
||||
bucket string
|
||||
baseKey string
|
||||
positions map[string]uploadPosition
|
||||
sequence uint64
|
||||
maxChunk int
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newUploader(store ObjectStore, artifacts *ArtifactStore, config UploadConfig, identity, runID string) *Uploader {
|
||||
return &Uploader{
|
||||
store: store, artifacts: artifacts, bucket: config.Bucket,
|
||||
baseKey: path.Join(config.Prefix, identity, runID),
|
||||
positions: make(map[string]uploadPosition), maxChunk: 8 * 1024 * 1024,
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uploader) Upload(ctx context.Context) (int, error) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
snapshots, err := u.artifacts.Snapshots()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
staged := make(map[string]uploadPosition, len(u.positions))
|
||||
for name, position := range u.positions {
|
||||
staged[name] = position
|
||||
}
|
||||
var uploaded []UploadedRange
|
||||
for _, snapshot := range snapshots {
|
||||
position := staged[snapshot.Name]
|
||||
if int64(len(snapshot.Contents)) < position.offset {
|
||||
position.offset = 0
|
||||
position.generation++
|
||||
}
|
||||
for position.offset < int64(len(snapshot.Contents)) {
|
||||
end := min(position.offset+int64(u.maxChunk), int64(len(snapshot.Contents)))
|
||||
content := snapshot.Contents[position.offset:end]
|
||||
digest := sha256.Sum256(content)
|
||||
digestText := hex.EncodeToString(digest[:])
|
||||
key := path.Join(u.baseKey, "files", url.PathEscape(snapshot.Name), fmt.Sprintf("%04d", position.generation), fmt.Sprintf("%020d-%020d.gz", position.offset, end))
|
||||
var compressed bytes.Buffer
|
||||
writer, _ := gzip.NewWriterLevel(&compressed, gzip.BestSpeed)
|
||||
if _, err := writer.Write(content); err != nil {
|
||||
return len(uploaded), err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return len(uploaded), err
|
||||
}
|
||||
if err := u.store.Put(ctx, PutRequest{
|
||||
Bucket: u.bucket, Key: key, Body: compressed.Bytes(), ContentType: "application/octet-stream", ContentEncoding: "gzip",
|
||||
Metadata: map[string]string{"source-path": snapshot.Name, "generation": strconv.Itoa(position.generation), "start": strconv.FormatInt(position.offset, 10), "end": strconv.FormatInt(end, 10), "sha256": digestText},
|
||||
}); err != nil {
|
||||
return len(uploaded), err
|
||||
}
|
||||
uploaded = append(uploaded, UploadedRange{Path: snapshot.Name, Generation: position.generation, Start: position.offset, End: end, SHA256: digestText, Key: key})
|
||||
position.offset = end
|
||||
}
|
||||
staged[snapshot.Name] = position
|
||||
}
|
||||
tick := UploadTick{Timestamp: time.Now().UTC(), Sequence: u.sequence, Uploaded: uploaded}
|
||||
tickBody, err := json.Marshal(tick)
|
||||
if err != nil {
|
||||
return len(uploaded), err
|
||||
}
|
||||
tickKey := path.Join(u.baseKey, "ticks", fmt.Sprintf("%08d.json", u.sequence))
|
||||
if err := u.store.Put(ctx, PutRequest{Bucket: u.bucket, Key: tickKey, Body: tickBody, ContentType: "application/json"}); err != nil {
|
||||
return len(uploaded), err
|
||||
}
|
||||
u.positions = staged
|
||||
u.sequence++
|
||||
return len(uploaded), nil
|
||||
}
|
||||
|
||||
type PeriodicUploader struct {
|
||||
uploader *Uploader
|
||||
interval time.Duration
|
||||
store *ArtifactStore
|
||||
}
|
||||
|
||||
func (u *PeriodicUploader) Start(ctx context.Context) *sync.WaitGroup {
|
||||
var group sync.WaitGroup
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
ticker := time.NewTicker(u.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
uploadContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
count, err := u.uploader.Upload(uploadContext)
|
||||
cancel()
|
||||
fields := map[string]any{"uploaded_ranges": count}
|
||||
if err != nil {
|
||||
fields["error"] = err.Error()
|
||||
}
|
||||
_ = u.store.AppendEvent(Event{Timestamp: time.Now().UTC(), Type: "upload", Fields: fields})
|
||||
}
|
||||
}
|
||||
}()
|
||||
return &group
|
||||
}
|
||||
235
internal/zygostates/uploader_test.go
Normal file
235
internal/zygostates/uploader_test.go
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeObjectStore struct {
|
||||
mu sync.Mutex
|
||||
requests []PutRequest
|
||||
failNextTick bool
|
||||
commitAndFailNextTick bool
|
||||
}
|
||||
|
||||
type blockingObjectStore struct {
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (s *blockingObjectStore) Put(ctx context.Context, _ PutRequest) error {
|
||||
s.once.Do(func() { close(s.started) })
|
||||
select {
|
||||
case <-s.release:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeObjectStore) Put(_ context.Context, request PutRequest) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.failNextTick && strings.Contains(request.Key, "/ticks/") {
|
||||
s.failNextTick = false
|
||||
return fmt.Errorf("tick unavailable")
|
||||
}
|
||||
copyRequest := request
|
||||
copyRequest.Body = bytes.Clone(request.Body)
|
||||
s.requests = append(s.requests, copyRequest)
|
||||
if s.commitAndFailNextTick && strings.Contains(request.Key, "/ticks/") {
|
||||
s.commitAndFailNextTick = false
|
||||
return fmt.Errorf("tick response unavailable")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeObjectStore) ticks() []PutRequest {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var ticks []PutRequest
|
||||
for _, request := range s.requests {
|
||||
if strings.Contains(request.Key, "/ticks/") {
|
||||
ticks = append(ticks, request)
|
||||
}
|
||||
}
|
||||
return ticks
|
||||
}
|
||||
|
||||
func decodeGzip(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
reader, err := gzip.NewReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
contents, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return contents
|
||||
}
|
||||
|
||||
func TestUploaderCommitsOnlyRangesReferencedByTick(t *testing.T) {
|
||||
artifacts := testArtifactStore(t)
|
||||
if err := artifacts.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "first"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objects := &fakeObjectStore{failNextTick: true}
|
||||
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
|
||||
uploader.maxChunk = 32
|
||||
if _, err := uploader.Upload(context.Background()); err == nil {
|
||||
t.Fatal("expected tick failure")
|
||||
}
|
||||
if len(uploader.positions) != 0 || uploader.sequence != 0 {
|
||||
t.Fatalf("failed upload committed state: %#v sequence=%d", uploader.positions, uploader.sequence)
|
||||
}
|
||||
count, err := uploader.Upload(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count == 0 || uploader.sequence != 1 {
|
||||
t.Fatalf("count=%d sequence=%d", count, uploader.sequence)
|
||||
}
|
||||
ticks := objects.ticks()
|
||||
if len(ticks) != 1 {
|
||||
t.Fatalf("ticks=%d", len(ticks))
|
||||
}
|
||||
var tick UploadTick
|
||||
if err := json.Unmarshal(ticks[0].Body, &tick); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tick.Uploaded) != count {
|
||||
t.Fatalf("tick ranges=%d count=%d", len(tick.Uploaded), count)
|
||||
}
|
||||
for _, request := range objects.requests {
|
||||
if request.ContentEncoding == "gzip" && len(decodeGzip(t, request.Body)) == 0 {
|
||||
t.Fatalf("empty range %s", request.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploaderShipsOnlyAppendedBytesAfterCommit(t *testing.T) {
|
||||
artifacts := testArtifactStore(t)
|
||||
objects := &fakeObjectStore{}
|
||||
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
|
||||
if _, err := uploader.Upload(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := artifacts.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "new"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count, err := uploader.Upload(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("uploaded ranges=%d want=1", count)
|
||||
}
|
||||
ticks := objects.ticks()
|
||||
var tick UploadTick
|
||||
if err := json.Unmarshal(ticks[len(ticks)-1].Body, &tick); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tick.Uploaded[0].Path != "events.ndjson" || tick.Uploaded[0].Start != 0 {
|
||||
t.Fatalf("unexpected range: %#v", tick.Uploaded[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploaderRetriesTickAtSameKeyAfterAmbiguousFailure(t *testing.T) {
|
||||
artifacts := testArtifactStore(t)
|
||||
objects := &fakeObjectStore{commitAndFailNextTick: true}
|
||||
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
|
||||
if _, err := uploader.Upload(context.Background()); err == nil {
|
||||
t.Fatal("expected tick failure")
|
||||
}
|
||||
if err := artifacts.AppendEvent(Event{Timestamp: time.Unix(1, 0), Type: "new"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := uploader.Upload(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ticks := objects.ticks()
|
||||
if len(ticks) != 2 {
|
||||
t.Fatalf("ticks=%d", len(ticks))
|
||||
}
|
||||
if ticks[0].Key != ticks[1].Key {
|
||||
t.Fatalf("retry keys differ: %q != %q", ticks[0].Key, ticks[1].Key)
|
||||
}
|
||||
for _, request := range ticks {
|
||||
var tick UploadTick
|
||||
if err := json.Unmarshal(request.Body, &tick); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tick.Sequence != 0 {
|
||||
t.Fatalf("sequence=%d", tick.Sequence)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeriodicUploaderContinuesUntilCanceled(t *testing.T) {
|
||||
artifacts := testArtifactStore(t)
|
||||
objects := &fakeObjectStore{}
|
||||
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
periodic := &PeriodicUploader{uploader: uploader, interval: time.Millisecond, store: artifacts}
|
||||
wait := periodic.Start(ctx)
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for len(objects.ticks()) < 2 && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
wait.Wait()
|
||||
if len(objects.ticks()) < 2 {
|
||||
t.Fatalf("ticks=%d", len(objects.ticks()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeriodicUploaderFinishesActiveUploadWhenCanceled(t *testing.T) {
|
||||
artifacts := testArtifactStore(t)
|
||||
objects := &blockingObjectStore{started: make(chan struct{}), release: make(chan struct{})}
|
||||
uploader := newUploader(objects, artifacts, UploadConfig{Bucket: "dump", Prefix: "zygostates"}, "system", "run")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
periodic := &PeriodicUploader{uploader: uploader, interval: time.Millisecond, store: artifacts}
|
||||
wait := periodic.Start(ctx)
|
||||
select {
|
||||
case <-objects.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("upload did not start")
|
||||
}
|
||||
cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wait.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("uploader stopped before the active upload finished")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
close(objects.release)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("uploader did not stop")
|
||||
}
|
||||
snapshots, err := artifacts.Snapshots()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, snapshot := range snapshots {
|
||||
if snapshot.Name == "events.ndjson" && bytes.Contains(snapshot.Contents, []byte(`"error"`)) {
|
||||
t.Fatalf("unexpected upload error: %s", snapshot.Contents)
|
||||
}
|
||||
}
|
||||
}
|
||||
477
internal/zygostates/workloads.go
Normal file
477
internal/zygostates/workloads.go
Normal file
|
|
@ -0,0 +1,477 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
"gonum.org/v1/gonum/dsp/fourier"
|
||||
"gonum.org/v1/gonum/mat"
|
||||
)
|
||||
|
||||
const cpuBlockIterations = 1 << 17
|
||||
|
||||
type cpuBlockResult struct {
|
||||
integer uint64
|
||||
floating uint64
|
||||
}
|
||||
|
||||
func cpuBlock(seed uint64) cpuBlockResult {
|
||||
integer := seed | 1
|
||||
floating := float64(seed%997+1) / 997
|
||||
for iteration := 0; iteration < cpuBlockIterations; iteration++ {
|
||||
integer ^= integer << 13
|
||||
integer ^= integer >> 7
|
||||
integer ^= integer << 17
|
||||
integer *= 0x9e3779b97f4a7c15
|
||||
floating = math.FMA(floating, 1.0000001192092896, float64(integer&0xffff)/65536)
|
||||
if floating > 1e6 {
|
||||
floating -= 1e6
|
||||
}
|
||||
}
|
||||
return cpuBlockResult{integer: integer, floating: math.Float64bits(floating)}
|
||||
}
|
||||
|
||||
type cpuWorkload struct {
|
||||
workers int
|
||||
CPUs []int
|
||||
}
|
||||
|
||||
func (w cpuWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
|
||||
ctx, cancel := context.WithTimeout(parent, duration)
|
||||
defer cancel()
|
||||
var blocks atomic.Uint64
|
||||
err := runCPUWorkers(ctx, w.workers, w.CPUs, &blocks)
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
err = nil
|
||||
}
|
||||
return map[string]any{"verified_blocks": blocks.Load(), "operations": blocks.Load() * cpuBlockIterations}, err
|
||||
}
|
||||
|
||||
func runCPUWorkers(ctx context.Context, workerCount int, cpus []int, blocks *atomic.Uint64) error {
|
||||
if workerCount < 1 {
|
||||
return fmt.Errorf("CPU worker count must be positive")
|
||||
}
|
||||
workerContext, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
errCh := make(chan error, workerCount)
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < workerCount; worker++ {
|
||||
worker := worker
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
if len(cpus) > 0 {
|
||||
if err := runPinned(cpus[worker%len(cpus)], func() error {
|
||||
return cpuWorker(workerContext, uint64(worker+1), blocks)
|
||||
}); err != nil {
|
||||
errCh <- err
|
||||
cancel()
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := cpuWorker(workerContext, uint64(worker+1), blocks); err != nil {
|
||||
errCh <- err
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
close(errCh)
|
||||
var contextErr error
|
||||
for err := range errCh {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
contextErr = err
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if contextErr != nil {
|
||||
return contextErr
|
||||
}
|
||||
return workerContext.Err()
|
||||
}
|
||||
|
||||
func cpuWorker(ctx context.Context, seed uint64, blocks *atomic.Uint64) error {
|
||||
expected := cpuBlock(seed)
|
||||
if confirmation := cpuBlock(seed); confirmation != expected {
|
||||
return fmt.Errorf("CPU preflight mismatch")
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
if result := cpuBlock(seed); result != expected {
|
||||
return fmt.Errorf("CPU verification mismatch")
|
||||
}
|
||||
blocks.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runPinned(cpu int, run func() error) error {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
var original unix.CPUSet
|
||||
if err := unix.SchedGetaffinity(0, &original); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = unix.SchedSetaffinity(0, &original) }()
|
||||
var selected unix.CPUSet
|
||||
selected.Zero()
|
||||
selected.Set(cpu)
|
||||
if err := unix.SchedSetaffinity(0, &selected); err != nil {
|
||||
return err
|
||||
}
|
||||
return run()
|
||||
}
|
||||
|
||||
type perCoreWorkload struct {
|
||||
cpus []int
|
||||
}
|
||||
|
||||
func (w perCoreWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
|
||||
var total atomic.Uint64
|
||||
completed := 0
|
||||
for _, cpu := range w.cpus {
|
||||
if err := parent.Err(); err != nil {
|
||||
return map[string]any{"completed_cores": completed, "verified_blocks": total.Load()}, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, duration)
|
||||
err := runCPUWorkers(ctx, 1, []int{cpu}, &total)
|
||||
cancel()
|
||||
if err != nil && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return map[string]any{"completed_cores": completed, "failed_cpu": cpu, "verified_blocks": total.Load()}, err
|
||||
}
|
||||
completed++
|
||||
}
|
||||
return map[string]any{"completed_cores": completed, "verified_blocks": total.Load(), "seconds_per_core": duration.Seconds()}, nil
|
||||
}
|
||||
|
||||
func physicalCPUs(root string) ([]int, error) {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := make(map[string]struct{})
|
||||
var cpus []int
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "cpu") {
|
||||
continue
|
||||
}
|
||||
cpu, err := strconv.Atoi(strings.TrimPrefix(entry.Name(), "cpu"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
core, err := os.ReadFile(root + "/" + entry.Name() + "/topology/core_id")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
pack, err := os.ReadFile(root + "/" + entry.Name() + "/topology/physical_package_id")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(string(pack)) + ":" + strings.TrimSpace(string(core))
|
||||
if _, found := seen[key]; found {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
cpus = append(cpus, cpu)
|
||||
}
|
||||
sort.Ints(cpus)
|
||||
if len(cpus) == 0 {
|
||||
return nil, fmt.Errorf("no physical CPUs found")
|
||||
}
|
||||
return cpus, nil
|
||||
}
|
||||
|
||||
type memoryWorkload struct {
|
||||
fraction float64
|
||||
workers int
|
||||
}
|
||||
|
||||
func (w memoryWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
|
||||
available, err := availableMemoryBytes("/proc/meminfo")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bytes := uint64(float64(available) * w.fraction)
|
||||
words := int(bytes / 8)
|
||||
if words < w.workers {
|
||||
return nil, fmt.Errorf("available memory is too small")
|
||||
}
|
||||
values := make([]uint64, words)
|
||||
ctx, cancel := context.WithTimeout(parent, duration)
|
||||
defer cancel()
|
||||
passes, err := verifyMemory(ctx, values, w.workers)
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
err = nil
|
||||
}
|
||||
return map[string]any{"bytes": len(values) * 8, "verified_passes": passes}, err
|
||||
}
|
||||
|
||||
func availableMemoryBytes(path string) (uint64, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, line := range strings.Split(string(contents), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 && fields[0] == "MemAvailable:" {
|
||||
kilobytes, err := strconv.ParseUint(fields[1], 10, 64)
|
||||
return kilobytes * 1024, err
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("MemAvailable not found")
|
||||
}
|
||||
|
||||
func memoryValue(seed, index uint64) uint64 {
|
||||
value := seed + index*0x9e3779b97f4a7c15
|
||||
value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9
|
||||
value = (value ^ (value >> 27)) * 0x94d049bb133111eb
|
||||
return value ^ (value >> 31)
|
||||
}
|
||||
|
||||
func verifyMemory(ctx context.Context, values []uint64, workers int) (uint64, error) {
|
||||
var pass uint64
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return pass, ctx.Err()
|
||||
default:
|
||||
}
|
||||
seed := uint64(0x6a09e667f3bcc909) + pass
|
||||
if err := parallelMemoryPass(values, workers, func(index int) error {
|
||||
values[index] = memoryValue(seed, uint64(index))
|
||||
return nil
|
||||
}); err != nil {
|
||||
return pass, err
|
||||
}
|
||||
if err := parallelMemoryPass(values, workers, func(index int) error {
|
||||
expected := memoryValue(seed, uint64(index))
|
||||
if values[index] != expected {
|
||||
return fmt.Errorf("memory mismatch at byte %d: got 0x%016x want 0x%016x", index*8, values[index], expected)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return pass, err
|
||||
}
|
||||
pass++
|
||||
}
|
||||
}
|
||||
|
||||
func parallelMemoryPass(values []uint64, workers int, operation func(int) error) error {
|
||||
if workers > len(values) {
|
||||
workers = len(values)
|
||||
}
|
||||
errCh := make(chan error, workers)
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < workers; worker++ {
|
||||
start := len(values) * worker / workers
|
||||
end := len(values) * (worker + 1) / workers
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for index := start; index < end; index++ {
|
||||
if err := operation(index); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type fftWorkload struct {
|
||||
size int
|
||||
workers int
|
||||
}
|
||||
|
||||
func (w fftWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
|
||||
ctx, cancel := context.WithTimeout(parent, duration)
|
||||
defer cancel()
|
||||
var transforms atomic.Uint64
|
||||
err := runConcurrent(ctx, w.workers, func(workerContext context.Context, worker int) error {
|
||||
transform := fourier.NewFFT(w.size)
|
||||
input := make([]float64, w.size)
|
||||
for index := range input {
|
||||
input[index] = math.Sin(float64(index+worker)/37) + math.Cos(float64(index*3+worker)/101)
|
||||
}
|
||||
coefficients := make([]complex128, w.size/2+1)
|
||||
reconstructed := make([]float64, w.size)
|
||||
for {
|
||||
select {
|
||||
case <-workerContext.Done():
|
||||
return workerContext.Err()
|
||||
default:
|
||||
}
|
||||
transform.Coefficients(coefficients, input)
|
||||
transform.Sequence(reconstructed, coefficients)
|
||||
maximumError := 0.0
|
||||
for index, value := range reconstructed {
|
||||
error := math.Abs(value/float64(w.size) - input[index])
|
||||
maximumError = math.Max(maximumError, error)
|
||||
}
|
||||
if maximumError > 1e-9 {
|
||||
return fmt.Errorf("FFT verification error %.17g", maximumError)
|
||||
}
|
||||
transforms.Add(1)
|
||||
}
|
||||
})
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
err = nil
|
||||
}
|
||||
return map[string]any{"size": w.size, "verified_transforms": transforms.Load()}, err
|
||||
}
|
||||
|
||||
type matrixWorkload struct {
|
||||
size int
|
||||
workers int
|
||||
}
|
||||
|
||||
func (w matrixWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
|
||||
ctx, cancel := context.WithTimeout(parent, duration)
|
||||
defer cancel()
|
||||
var products atomic.Uint64
|
||||
err := runConcurrent(ctx, w.workers, func(workerContext context.Context, worker int) error {
|
||||
leftData := make([]float64, w.size*w.size)
|
||||
rightData := make([]float64, w.size*w.size)
|
||||
for index := range leftData {
|
||||
leftData[index] = float64((index+worker)%251-125) / 251
|
||||
rightData[index] = float64((index*7+worker)%241-120) / 241
|
||||
}
|
||||
left := mat.NewDense(w.size, w.size, leftData)
|
||||
right := mat.NewDense(w.size, w.size, rightData)
|
||||
var expected mat.Dense
|
||||
expected.Mul(left, right)
|
||||
var result mat.Dense
|
||||
for {
|
||||
select {
|
||||
case <-workerContext.Done():
|
||||
return workerContext.Err()
|
||||
default:
|
||||
}
|
||||
result.Mul(left, right)
|
||||
if !mat.EqualApprox(&result, &expected, 1e-12) {
|
||||
return fmt.Errorf("matrix verification mismatch")
|
||||
}
|
||||
products.Add(1)
|
||||
}
|
||||
})
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
err = nil
|
||||
}
|
||||
return map[string]any{"size": w.size, "verified_products": products.Load()}, err
|
||||
}
|
||||
|
||||
type mixedWorkload struct {
|
||||
cpu cpuWorkload
|
||||
memory memoryWorkload
|
||||
}
|
||||
|
||||
func (w mixedWorkload) Run(parent context.Context, duration time.Duration) (map[string]any, error) {
|
||||
ctx, cancel := context.WithTimeout(parent, duration)
|
||||
defer cancel()
|
||||
type outcome struct {
|
||||
name string
|
||||
metrics map[string]any
|
||||
err error
|
||||
}
|
||||
outcomes := make(chan outcome, 2)
|
||||
go func() {
|
||||
metrics, err := w.cpu.Run(ctx, duration)
|
||||
outcomes <- outcome{name: "cpu", metrics: metrics, err: err}
|
||||
}()
|
||||
go func() {
|
||||
metrics, err := w.memory.Run(ctx, duration)
|
||||
outcomes <- outcome{name: "memory", metrics: metrics, err: err}
|
||||
}()
|
||||
metrics := make(map[string]any)
|
||||
for range 2 {
|
||||
outcome := <-outcomes
|
||||
for key, value := range outcome.metrics {
|
||||
metrics[outcome.name+"_"+key] = value
|
||||
}
|
||||
if outcome.err != nil && !errors.Is(outcome.err, context.DeadlineExceeded) {
|
||||
cancel()
|
||||
return metrics, outcome.err
|
||||
}
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
func runConcurrent(ctx context.Context, count int, operation func(context.Context, int) error) error {
|
||||
workerContext, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
errCh := make(chan error, count)
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < count; worker++ {
|
||||
worker := worker
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
if err := operation(workerContext, worker); err != nil {
|
||||
errCh <- err
|
||||
if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
close(errCh)
|
||||
var contextErr error
|
||||
for err := range errCh {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
contextErr = err
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if contextErr != nil {
|
||||
return contextErr
|
||||
}
|
||||
return workerContext.Err()
|
||||
}
|
||||
|
||||
func buildWorkloads(config WorkloadConfig) (map[string]Workload, error) {
|
||||
physical, err := physicalCPUs("/sys/devices/system/cpu")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
workers := runtime.NumCPU()
|
||||
cpu := cpuWorkload{workers: workers}
|
||||
memory := memoryWorkload{fraction: config.MemoryFraction, workers: workers}
|
||||
return map[string]Workload{
|
||||
"cpu": cpu,
|
||||
"fft": fftWorkload{size: config.FFTSize, workers: workers},
|
||||
"idle": idleWorkload{},
|
||||
"matrix": matrixWorkload{size: config.MatrixSize, workers: workers},
|
||||
"memory": memory,
|
||||
"mixed": mixedWorkload{cpu: cpu, memory: memory},
|
||||
"per-core": perCoreWorkload{cpus: physical},
|
||||
}, nil
|
||||
}
|
||||
114
internal/zygostates/workloads_test.go
Normal file
114
internal/zygostates/workloads_test.go
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
package zygostates
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
var errMemoryMismatch = errors.New("memory mismatch")
|
||||
|
||||
func TestCPUBlockIsDeterministicAndSeeded(t *testing.T) {
|
||||
first := cpuBlock(1)
|
||||
if first != cpuBlock(1) {
|
||||
t.Fatal("CPU block is not deterministic")
|
||||
}
|
||||
if first == cpuBlock(2) {
|
||||
t.Fatal("CPU block ignores seed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeComputeWorkloadsVerifyResults(t *testing.T) {
|
||||
tests := map[string]Workload{
|
||||
"cpu": cpuWorkload{workers: 1},
|
||||
"fft": fftWorkload{size: 64, workers: 1},
|
||||
"matrix": matrixWorkload{size: 8, workers: 1},
|
||||
"memory": memoryWorkload{fraction: 0.000001, workers: 1},
|
||||
"mixed": mixedWorkload{
|
||||
cpu: cpuWorkload{workers: 1},
|
||||
memory: memoryWorkload{fraction: 0.000001, workers: 1},
|
||||
},
|
||||
}
|
||||
for name, workload := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
metrics, err := workload.Run(context.Background(), 50*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(metrics) == 0 {
|
||||
t.Fatal("missing workload metrics")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerCoreWorkloadPinsAnAllowedCPU(t *testing.T) {
|
||||
var allowed unix.CPUSet
|
||||
if err := unix.SchedGetaffinity(0, &allowed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected := -1
|
||||
for cpu := 0; cpu < 1024; cpu++ {
|
||||
if allowed.IsSet(cpu) {
|
||||
selected = cpu
|
||||
break
|
||||
}
|
||||
}
|
||||
if selected < 0 {
|
||||
t.Fatal("no allowed CPU")
|
||||
}
|
||||
metrics, err := (perCoreWorkload{cpus: []int{selected}}).Run(context.Background(), 20*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if metrics["completed_cores"] != 1 {
|
||||
t.Fatalf("metrics=%#v", metrics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryPatternDetectsCorruption(t *testing.T) {
|
||||
values := make([]uint64, 16)
|
||||
seed := uint64(7)
|
||||
for index := range values {
|
||||
values[index] = memoryValue(seed, uint64(index))
|
||||
}
|
||||
values[9] ^= 1
|
||||
err := parallelMemoryPass(values, 3, func(index int) error {
|
||||
if values[index] != memoryValue(seed, uint64(index)) {
|
||||
return errMemoryMismatch
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != errMemoryMismatch {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhysicalCPUsSelectsOneThreadPerCore(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for cpu, core := range []string{"0", "0", "1", "1"} {
|
||||
directory := filepath.Join(root, fmt.Sprintf("cpu%d/topology", cpu))
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, "core_id"), []byte(core), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, "physical_package_id"), []byte("0"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cpus, err := physicalCPUs(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cpus) != 2 || cpus[0] != 0 || cpus[1] != 2 {
|
||||
t.Fatalf("cpus=%v", cpus)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue