quorate/services/short/cloud/oci_test.go

package cloud

import (
	"context"
	"errors"
	"fmt"
	"slices"
	"testing"

	"github.com/oracle/oci-go-sdk/v65/common"
	"github.com/oracle/oci-go-sdk/v65/core"
	"quorate.dev/services/short/config"
)

type networkFake struct {
	Network
	Cpes                      []core.Cpe
	Connections               []core.IpSecConnection
	Heads                     map[string][]core.IpSecConnectionTunnel
	Creates, Deletes, Updates int
	RouteUpdates              int
	HoldRoutes                bool
	PendingRoutes             []string
}

func (f *networkFake) ListCpes(context.Context, core.ListCpesRequest) (core.ListCpesResponse, error) {
	return core.ListCpesResponse{Items: f.Cpes}, nil
}
func (f *networkFake) CreateCpe(_ context.Context, r core.CreateCpeRequest) (core.CreateCpeResponse, error) {
	f.Creates++
	p := core.Cpe{Id: common.String(fmt.Sprintf("cpe-%d", f.Creates)), IpAddress: r.IpAddress, FreeformTags: r.FreeformTags}
	f.Cpes = append(f.Cpes, p)
	return core.CreateCpeResponse{Cpe: p}, nil
}
func (f *networkFake) DeleteCpe(_ context.Context, r core.DeleteCpeRequest) (core.DeleteCpeResponse, error) {
	f.Deletes++
	var keep []core.Cpe
	for _, p := range f.Cpes {
		if *p.Id != *r.CpeId {
			keep = append(keep, p)
		}
	}
	f.Cpes = keep
	return core.DeleteCpeResponse{}, nil
}
func (f *networkFake) ListIPSecConnections(context.Context, core.ListIPSecConnectionsRequest) (core.ListIPSecConnectionsResponse, error) {
	return core.ListIPSecConnectionsResponse{Items: f.Connections}, nil
}
func (f *networkFake) CreateIPSecConnection(_ context.Context, r core.CreateIPSecConnectionRequest) (core.CreateIPSecConnectionResponse, error) {
	f.Creates++
	id := fmt.Sprintf("conn-%d", f.Creates)
	p := core.IpSecConnection{Id: &id, CpeId: r.CpeId, DrgId: r.DrgId, FreeformTags: r.FreeformTags, StaticRoutes: r.StaticRoutes, CpeLocalIdentifier: r.CpeLocalIdentifier, LifecycleState: core.IpSecConnectionLifecycleStateAvailable}
	f.Connections = append(f.Connections, p)
	for i, head := range r.TunnelConfiguration {
		f.Heads[id] = append(f.Heads[id], core.IpSecConnectionTunnel{Id: common.String(fmt.Sprintf("%s-%d", id, i)), VpnIp: common.String(fmt.Sprintf("8.8.%d.%d", f.Creates, i+1)), LifecycleState: core.IpSecConnectionTunnelLifecycleStateAvailable, Status: core.IpSecConnectionTunnelStatusDown, IkeVersion: core.IpSecConnectionTunnelIkeVersionEnum(head.IkeVersion), OracleCanInitiate: core.IpSecConnectionTunnelOracleCanInitiateEnum(head.OracleInitiation)})
	}
	return core.CreateIPSecConnectionResponse{IpSecConnection: p}, nil
}
func (f *networkFake) DeleteIPSecConnection(_ context.Context, r core.DeleteIPSecConnectionRequest) (core.DeleteIPSecConnectionResponse, error) {
	f.Deletes++
	var keep []core.IpSecConnection
	for _, p := range f.Connections {
		if *p.Id != *r.IpscId {
			keep = append(keep, p)
		}
	}
	f.Connections = keep
	return core.DeleteIPSecConnectionResponse{}, nil
}
func (f *networkFake) UpdateIPSecConnection(_ context.Context, r core.UpdateIPSecConnectionRequest) (core.UpdateIPSecConnectionResponse, error) {
	f.RouteUpdates++
	f.PendingRoutes = slices.Clone(r.StaticRoutes)
	for i, p := range f.Connections {
		if *p.Id == *r.IpscId {
			p.StaticRoutes = slices.Clone(r.StaticRoutes)
			if !f.HoldRoutes {
				f.Connections[i] = p
			}
			return core.UpdateIPSecConnectionResponse{IpSecConnection: p}, nil
		}
	}
	return core.UpdateIPSecConnectionResponse{}, fmt.Errorf("unknown connection")
}
func (f *networkFake) ListIPSecConnectionTunnels(_ context.Context, r core.ListIPSecConnectionTunnelsRequest) (core.ListIPSecConnectionTunnelsResponse, error) {
	return core.ListIPSecConnectionTunnelsResponse{Items: f.Heads[*r.IpscId]}, nil
}
func (f *networkFake) UpdateIPSecConnectionTunnel(_ context.Context, r core.UpdateIPSecConnectionTunnelRequest) (core.UpdateIPSecConnectionTunnelResponse, error) {
	f.Updates++
	for i := range f.Heads[*r.IpscId] {
		p := &f.Heads[*r.IpscId][i]
		if *p.Id == *r.TunnelId {
			p.OracleCanInitiate = core.IpSecConnectionTunnelOracleCanInitiateEnum(r.OracleInitiation)
		}
	}
	return core.UpdateIPSecConnectionTunnelResponse{}, nil
}
func (f *networkFake) GetIPSecConnectionTunnelSharedSecret(context.Context, core.GetIPSecConnectionTunnelSharedSecretRequest) (core.GetIPSecConnectionTunnelSharedSecretResponse, error) {
	return core.GetIPSecConnectionTunnelSharedSecretResponse{IpSecConnectionTunnelSharedSecret: core.IpSecConnectionTunnelSharedSecret{SharedSecret: common.String("synthetic-secret")}}, nil
}

func TestWANReplacement(t *testing.T) {
	f := &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}, Cpes: []core.Cpe{{Id: common.String("foreign"), IpAddress: common.String("8.8.4.4"), FreeformTags: map[string]string{"owner": "relay"}}}}
	c := Client{API: f, Config: config.Cloud{Compartment: "synthetic-compartment", DRG: "synthetic-drg", Address: "10.76.79.10"}}
	ctx := context.Background()
	first, err := c.Reconcile(ctx, "8.8.4.4")
	if err != nil {
		t.Fatal(err)
	}
	if !first.Ready || len(first.Peers) != 2 || f.Creates != 2 {
		t.Fatal("initial headend publication failed")
	}
	for _, head := range f.Heads[*f.Connections[0].Id] {
		if head.OracleCanInitiate != core.IpSecConnectionTunnelOracleCanInitiateInitiatorOrResponder {
			t.Fatal("OCI initiation disabled")
		}
	}
	if _, err = c.Reconcile(ctx, "8.8.4.4"); err != nil {
		t.Fatal(err)
	}
	if f.Creates != 2 || f.Deletes != 0 {
		t.Fatal("steady reconciliation mutated cloud resources")
	}
	next, err := c.Reconcile(ctx, "8.8.8.8")
	if err != nil {
		t.Fatal(err)
	}
	if next.Bundle.WAN != "8.8.8.8" || len(next.Peers) != 4 || f.Creates != 4 || f.Deletes != 0 {
		t.Fatal("replacement did not retain prior peers until new tunnels established")
	}
	current := f.Connections[1]
	if *current.CpeLocalIdentifier != "8.8.8.8" || len(current.StaticRoutes) != 1 || current.StaticRoutes[0] != "10.76.79.10/32" {
		t.Fatal("replacement retained old WAN identity or node routing")
	}
	heads := f.Heads[*current.Id]
	for i := range heads {
		heads[i].Status = core.IpSecConnectionTunnelStatusUp
	}
	f.Heads[*current.Id] = heads
	if _, err = c.Reconcile(ctx, "8.8.8.8"); err != nil {
		t.Fatal(err)
	}
	if f.Deletes != 1 {
		t.Fatal("old connection not retired after both new headends became healthy")
	}
	if _, err = c.Reconcile(ctx, "8.8.8.8"); err != nil {
		t.Fatal(err)
	}
	if f.Deletes != 2 || len(f.Cpes) != 2 || *f.Cpes[0].Id != "foreign" {
		t.Fatal("CPE retirement affected another ingress owner")
	}
	steady, err := c.Reconcile(ctx, "8.8.8.8")
	if err != nil {
		t.Fatal(err)
	}
	if len(steady.Peers) != 2 || f.Creates != 4 || f.Deletes != 2 {
		t.Fatal("obsolete peer allowlist survived replacement")
	}
}

func TestInitiationDrift(t *testing.T) {
	f := &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}}
	c := Client{API: f, Config: config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "10.76.79.10"}}
	ctx := context.Background()
	if _, err := c.Reconcile(ctx, "8.8.4.4"); err != nil {
		t.Fatal(err)
	}
	id := *f.Connections[0].Id
	f.Heads[id][0].OracleCanInitiate = core.IpSecConnectionTunnelOracleCanInitiateResponderOnly
	if _, err := c.Reconcile(ctx, "8.8.4.4"); err != nil {
		t.Fatal(err)
	}
	if f.Updates != 1 {
		t.Fatal("initiation drift not reconciled")
	}
	if _, err := c.Reconcile(ctx, "8.8.4.4"); err != nil {
		t.Fatal(err)
	}
	if f.Updates != 1 {
		t.Fatal("unchanged initiation rewritten")
	}
}

type apiError struct {
	Status int
	Code   string
}

func (e apiError) Error() string          { return e.Code }
func (e apiError) GetHTTPStatusCode() int { return e.Status }
func (e apiError) GetCode() string        { return e.Code }
func (e apiError) GetMessage() string     { return e.Code }
func (e apiError) GetOpcRequestID() string {
	return "synthetic-request"
}

type retryFake struct {
	*networkFake
	CpeTokens   map[string]string
	IPSecTokens map[string]string
	LoseKind    string
	LostToken   string
	HiddenCpe   string
	HiddenIPSec string
	Replayed    []string
	Invalid     int
}

func retryNetwork() *retryFake {
	return &retryFake{
		networkFake: &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}},
		CpeTokens:   map[string]string{}, IPSecTokens: map[string]string{},
	}
}

func (f *retryFake) ListCpes(context.Context, core.ListCpesRequest) (core.ListCpesResponse, error) {
	var result core.ListCpesResponse
	for _, p := range f.Cpes {
		if *p.Id != f.HiddenCpe {
			result.Items = append(result.Items, p)
		}
	}
	return result, nil
}

func (f *retryFake) ListIPSecConnections(context.Context, core.ListIPSecConnectionsRequest) (core.ListIPSecConnectionsResponse, error) {
	var result core.ListIPSecConnectionsResponse
	for _, p := range f.Connections {
		if *p.Id != f.HiddenIPSec {
			result.Items = append(result.Items, p)
		}
	}
	return result, nil
}

func (f *retryFake) CreateCpe(ctx context.Context, r core.CreateCpeRequest) (core.CreateCpeResponse, error) {
	if id, ok := f.CpeTokens[*r.OpcRetryToken]; ok {
		for _, p := range f.Cpes {
			if *p.Id == id {
				f.Replayed = append(f.Replayed, *r.OpcRetryToken)
				return core.CreateCpeResponse{Cpe: p}, nil
			}
		}
		f.Invalid++
		return core.CreateCpeResponse{}, apiError{409, "InvalidatedRetryToken"}
	}
	result, err := f.networkFake.CreateCpe(ctx, r)
	if err != nil {
		return result, err
	}
	f.CpeTokens[*r.OpcRetryToken] = *result.Id
	if f.LoseKind == "cpe" {
		f.LoseKind, f.LostToken, f.HiddenCpe = "", *r.OpcRetryToken, *result.Id
		return core.CreateCpeResponse{}, context.DeadlineExceeded
	}
	return result, nil
}

func (f *retryFake) CreateIPSecConnection(ctx context.Context, r core.CreateIPSecConnectionRequest) (core.CreateIPSecConnectionResponse, error) {
	if id, ok := f.IPSecTokens[*r.OpcRetryToken]; ok {
		for _, p := range f.Connections {
			if *p.Id == id {
				f.Replayed = append(f.Replayed, *r.OpcRetryToken)
				return core.CreateIPSecConnectionResponse{IpSecConnection: p}, nil
			}
		}
		f.Invalid++
		return core.CreateIPSecConnectionResponse{}, apiError{409, "InvalidatedRetryToken"}
	}
	result, err := f.networkFake.CreateIPSecConnection(ctx, r)
	if err != nil {
		return result, err
	}
	f.IPSecTokens[*r.OpcRetryToken] = *result.Id
	if f.LoseKind == "ipsec" {
		f.LoseKind, f.LostToken, f.HiddenIPSec = "", *r.OpcRetryToken, *result.Id
		return core.CreateIPSecConnectionResponse{}, context.DeadlineExceeded
	}
	return result, nil
}

func settleWAN(t *testing.T, c *Client, f *retryFake, wan string) {
	t.Helper()
	if state, err := c.Reconcile(context.Background(), wan); err != nil || !state.Ready {
		t.Fatalf("WAN %s: ready=%v error=%v", wan, state.Ready, err)
	}
	for _, heads := range f.Heads {
		for i := range heads {
			heads[i].Status = core.IpSecConnectionTunnelStatusUp
		}
	}
	for range 2 {
		if _, err := c.Reconcile(context.Background(), wan); err != nil {
			t.Fatal(err)
		}
	}
}

func TestWANReturn(t *testing.T) {
	f := retryNetwork()
	c := Client{API: f, Config: config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "192.0.2.10"}}
	for _, wan := range []string{"198.51.100.10", "198.51.100.20", "198.51.100.10"} {
		settleWAN(t, &c, f, wan)
	}
	if f.Creates != 6 || f.Deletes != 4 || f.Invalid != 2 || len(f.Cpes) != 1 || len(f.Connections) != 1 {
		t.Fatalf("WAN return failed to retire and replace: creates=%d deletes=%d invalid=%d CPEs=%d connections=%d", f.Creates, f.Deletes, f.Invalid, len(f.Cpes), len(f.Connections))
	}
	if len(f.CpeTokens) != 3 || len(f.IPSecTokens) != 3 {
		t.Fatal("distinct resource generations reused a creation token")
	}
}

func TestCreateTimeout(t *testing.T) {
	for _, kind := range []string{"cpe", "ipsec"} {
		t.Run(kind, func(t *testing.T) {
			f := retryNetwork()
			cfg := config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "192.0.2.10"}
			c := Client{API: f, Config: cfg}
			settleWAN(t, &c, f, "198.51.100.10")
			settleWAN(t, &c, f, "198.51.100.20")
			f.LoseKind = kind
			if _, err := c.Reconcile(context.Background(), "198.51.100.10"); !errors.Is(err, context.DeadlineExceeded) {
				t.Fatalf("create did not commit with a lost reply: %v", err)
			}
			replacement := Client{API: f, Config: cfg}
			state, err := replacement.Reconcile(context.Background(), "198.51.100.10")
			if err != nil || !state.Ready {
				t.Fatalf("replacement leader failed: ready=%v error=%v", state.Ready, err)
			}
			if !slices.Contains(f.Replayed, f.LostToken) || f.Creates != 6 || len(f.Cpes) != 2 || len(f.Connections) != 2 {
				t.Fatalf("ambiguous create was not replayed: creates=%d CPEs=%d connections=%d", f.Creates, len(f.Cpes), len(f.Connections))
			}
			f.HiddenCpe, f.HiddenIPSec = "", ""
			if _, err := replacement.Reconcile(context.Background(), "198.51.100.10"); err != nil {
				t.Fatal(err)
			}
			if f.Creates != 6 {
				t.Fatal("visibility restoration created another resource")
			}
		})
	}
}

func TestRetryErrors(t *testing.T) {
	for _, want := range []error{context.DeadlineExceeded, apiError{409, "IncorrectState"}, apiError{403, "InvalidatedRetryToken"}} {
		calls := 0
		_, got := create(common.String("synthetic-token"), func(*string) (int, error) {
			calls++
			return 0, want
		})
		if calls != 1 || !errors.Is(got, want) {
			t.Fatalf("unrelated error was retried: %v; calls=%d", got, calls)
		}
	}
}

func TestRouteDrift(t *testing.T) {
	f := &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}, HoldRoutes: true}
	c := Client{API: f, Config: config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "192.0.2.10"}}
	ctx := context.Background()
	if _, err := c.Reconcile(ctx, "198.51.100.10"); err != nil {
		t.Fatal(err)
	}
	c.Config.Address = "192.0.2.11"
	for range 2 {
		state, err := c.Reconcile(ctx, "198.51.100.10")
		if err != nil || state.Ready || len(state.Peers) != 2 {
			t.Fatalf("pending route exposed runtime or removed peers: ready=%v peers=%d error=%v", state.Ready, len(state.Peers), err)
		}
	}
	if f.RouteUpdates != 2 || !slices.Equal(f.PendingRoutes, []string{"192.0.2.11/32"}) || f.Connections[0].StaticRoutes[0] != "192.0.2.10/32" {
		t.Fatal("route update did not target the configured address")
	}
	f.Connections[0].StaticRoutes = slices.Clone(f.PendingRoutes)
	for range 2 {
		state, err := c.Reconcile(ctx, "198.51.100.10")
		if err != nil || !state.Ready {
			t.Fatalf("observed route was not published: ready=%v error=%v", state.Ready, err)
		}
	}
	if f.Creates != 2 || f.Deletes != 0 || f.RouteUpdates != 2 {
		t.Fatal("route reconciliation replaced resources or repeated a completed update")
	}
}