package cloud import ( "context" "errors" "fmt" "slices" "testing" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/core" "quorate.dev/services/short/config" ) type networkFake struct { Network Cpes []core.Cpe Connections []core.IpSecConnection Heads map[string][]core.IpSecConnectionTunnel Creates, Deletes, Updates int RouteUpdates int HoldRoutes bool PendingRoutes []string } func (f *networkFake) ListCpes(context.Context, core.ListCpesRequest) (core.ListCpesResponse, error) { return core.ListCpesResponse{Items: f.Cpes}, nil } func (f *networkFake) CreateCpe(_ context.Context, r core.CreateCpeRequest) (core.CreateCpeResponse, error) { f.Creates++ p := core.Cpe{Id: common.String(fmt.Sprintf("cpe-%d", f.Creates)), IpAddress: r.IpAddress, FreeformTags: r.FreeformTags} f.Cpes = append(f.Cpes, p) return core.CreateCpeResponse{Cpe: p}, nil } func (f *networkFake) DeleteCpe(_ context.Context, r core.DeleteCpeRequest) (core.DeleteCpeResponse, error) { f.Deletes++ var keep []core.Cpe for _, p := range f.Cpes { if *p.Id != *r.CpeId { keep = append(keep, p) } } f.Cpes = keep return core.DeleteCpeResponse{}, nil } func (f *networkFake) ListIPSecConnections(context.Context, core.ListIPSecConnectionsRequest) (core.ListIPSecConnectionsResponse, error) { return core.ListIPSecConnectionsResponse{Items: f.Connections}, nil } func (f *networkFake) CreateIPSecConnection(_ context.Context, r core.CreateIPSecConnectionRequest) (core.CreateIPSecConnectionResponse, error) { f.Creates++ id := fmt.Sprintf("conn-%d", f.Creates) p := core.IpSecConnection{Id: &id, CpeId: r.CpeId, DrgId: r.DrgId, FreeformTags: r.FreeformTags, StaticRoutes: r.StaticRoutes, CpeLocalIdentifier: r.CpeLocalIdentifier, LifecycleState: core.IpSecConnectionLifecycleStateAvailable} f.Connections = append(f.Connections, p) for i, head := range r.TunnelConfiguration { f.Heads[id] = append(f.Heads[id], core.IpSecConnectionTunnel{Id: common.String(fmt.Sprintf("%s-%d", id, i)), VpnIp: common.String(fmt.Sprintf("8.8.%d.%d", f.Creates, i+1)), LifecycleState: core.IpSecConnectionTunnelLifecycleStateAvailable, Status: core.IpSecConnectionTunnelStatusDown, IkeVersion: core.IpSecConnectionTunnelIkeVersionEnum(head.IkeVersion), OracleCanInitiate: core.IpSecConnectionTunnelOracleCanInitiateEnum(head.OracleInitiation)}) } return core.CreateIPSecConnectionResponse{IpSecConnection: p}, nil } func (f *networkFake) DeleteIPSecConnection(_ context.Context, r core.DeleteIPSecConnectionRequest) (core.DeleteIPSecConnectionResponse, error) { f.Deletes++ var keep []core.IpSecConnection for _, p := range f.Connections { if *p.Id != *r.IpscId { keep = append(keep, p) } } f.Connections = keep return core.DeleteIPSecConnectionResponse{}, nil } func (f *networkFake) UpdateIPSecConnection(_ context.Context, r core.UpdateIPSecConnectionRequest) (core.UpdateIPSecConnectionResponse, error) { f.RouteUpdates++ f.PendingRoutes = slices.Clone(r.StaticRoutes) for i, p := range f.Connections { if *p.Id == *r.IpscId { p.StaticRoutes = slices.Clone(r.StaticRoutes) if !f.HoldRoutes { f.Connections[i] = p } return core.UpdateIPSecConnectionResponse{IpSecConnection: p}, nil } } return core.UpdateIPSecConnectionResponse{}, fmt.Errorf("unknown connection") } func (f *networkFake) ListIPSecConnectionTunnels(_ context.Context, r core.ListIPSecConnectionTunnelsRequest) (core.ListIPSecConnectionTunnelsResponse, error) { return core.ListIPSecConnectionTunnelsResponse{Items: f.Heads[*r.IpscId]}, nil } func (f *networkFake) UpdateIPSecConnectionTunnel(_ context.Context, r core.UpdateIPSecConnectionTunnelRequest) (core.UpdateIPSecConnectionTunnelResponse, error) { f.Updates++ for i := range f.Heads[*r.IpscId] { p := &f.Heads[*r.IpscId][i] if *p.Id == *r.TunnelId { p.OracleCanInitiate = core.IpSecConnectionTunnelOracleCanInitiateEnum(r.OracleInitiation) } } return core.UpdateIPSecConnectionTunnelResponse{}, nil } func (f *networkFake) GetIPSecConnectionTunnelSharedSecret(context.Context, core.GetIPSecConnectionTunnelSharedSecretRequest) (core.GetIPSecConnectionTunnelSharedSecretResponse, error) { return core.GetIPSecConnectionTunnelSharedSecretResponse{IpSecConnectionTunnelSharedSecret: core.IpSecConnectionTunnelSharedSecret{SharedSecret: common.String("synthetic-secret")}}, nil } func TestWANReplacement(t *testing.T) { f := &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}, Cpes: []core.Cpe{{Id: common.String("foreign"), IpAddress: common.String("8.8.4.4"), FreeformTags: map[string]string{"owner": "relay"}}}} c := Client{API: f, Config: config.Cloud{Compartment: "synthetic-compartment", DRG: "synthetic-drg", Address: "10.76.79.10"}} ctx := context.Background() first, err := c.Reconcile(ctx, "8.8.4.4") if err != nil { t.Fatal(err) } if !first.Ready || len(first.Peers) != 2 || f.Creates != 2 { t.Fatal("initial headend publication failed") } for _, head := range f.Heads[*f.Connections[0].Id] { if head.OracleCanInitiate != core.IpSecConnectionTunnelOracleCanInitiateInitiatorOrResponder { t.Fatal("OCI initiation disabled") } } if _, err = c.Reconcile(ctx, "8.8.4.4"); err != nil { t.Fatal(err) } if f.Creates != 2 || f.Deletes != 0 { t.Fatal("steady reconciliation mutated cloud resources") } next, err := c.Reconcile(ctx, "8.8.8.8") if err != nil { t.Fatal(err) } if next.Bundle.WAN != "8.8.8.8" || len(next.Peers) != 4 || f.Creates != 4 || f.Deletes != 0 { t.Fatal("replacement did not retain prior peers until new tunnels established") } current := f.Connections[1] if *current.CpeLocalIdentifier != "8.8.8.8" || len(current.StaticRoutes) != 1 || current.StaticRoutes[0] != "10.76.79.10/32" { t.Fatal("replacement retained old WAN identity or node routing") } heads := f.Heads[*current.Id] for i := range heads { heads[i].Status = core.IpSecConnectionTunnelStatusUp } f.Heads[*current.Id] = heads if _, err = c.Reconcile(ctx, "8.8.8.8"); err != nil { t.Fatal(err) } if f.Deletes != 1 { t.Fatal("old connection not retired after both new headends became healthy") } if _, err = c.Reconcile(ctx, "8.8.8.8"); err != nil { t.Fatal(err) } if f.Deletes != 2 || len(f.Cpes) != 2 || *f.Cpes[0].Id != "foreign" { t.Fatal("CPE retirement affected another ingress owner") } steady, err := c.Reconcile(ctx, "8.8.8.8") if err != nil { t.Fatal(err) } if len(steady.Peers) != 2 || f.Creates != 4 || f.Deletes != 2 { t.Fatal("obsolete peer allowlist survived replacement") } } func TestInitiationDrift(t *testing.T) { f := &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}} c := Client{API: f, Config: config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "10.76.79.10"}} ctx := context.Background() if _, err := c.Reconcile(ctx, "8.8.4.4"); err != nil { t.Fatal(err) } id := *f.Connections[0].Id f.Heads[id][0].OracleCanInitiate = core.IpSecConnectionTunnelOracleCanInitiateResponderOnly if _, err := c.Reconcile(ctx, "8.8.4.4"); err != nil { t.Fatal(err) } if f.Updates != 1 { t.Fatal("initiation drift not reconciled") } if _, err := c.Reconcile(ctx, "8.8.4.4"); err != nil { t.Fatal(err) } if f.Updates != 1 { t.Fatal("unchanged initiation rewritten") } } type apiError struct { Status int Code string } func (e apiError) Error() string { return e.Code } func (e apiError) GetHTTPStatusCode() int { return e.Status } func (e apiError) GetCode() string { return e.Code } func (e apiError) GetMessage() string { return e.Code } func (e apiError) GetOpcRequestID() string { return "synthetic-request" } type retryFake struct { *networkFake CpeTokens map[string]string IPSecTokens map[string]string LoseKind string LostToken string HiddenCpe string HiddenIPSec string Replayed []string Invalid int } func retryNetwork() *retryFake { return &retryFake{ networkFake: &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}}, CpeTokens: map[string]string{}, IPSecTokens: map[string]string{}, } } func (f *retryFake) ListCpes(context.Context, core.ListCpesRequest) (core.ListCpesResponse, error) { var result core.ListCpesResponse for _, p := range f.Cpes { if *p.Id != f.HiddenCpe { result.Items = append(result.Items, p) } } return result, nil } func (f *retryFake) ListIPSecConnections(context.Context, core.ListIPSecConnectionsRequest) (core.ListIPSecConnectionsResponse, error) { var result core.ListIPSecConnectionsResponse for _, p := range f.Connections { if *p.Id != f.HiddenIPSec { result.Items = append(result.Items, p) } } return result, nil } func (f *retryFake) CreateCpe(ctx context.Context, r core.CreateCpeRequest) (core.CreateCpeResponse, error) { if id, ok := f.CpeTokens[*r.OpcRetryToken]; ok { for _, p := range f.Cpes { if *p.Id == id { f.Replayed = append(f.Replayed, *r.OpcRetryToken) return core.CreateCpeResponse{Cpe: p}, nil } } f.Invalid++ return core.CreateCpeResponse{}, apiError{409, "InvalidatedRetryToken"} } result, err := f.networkFake.CreateCpe(ctx, r) if err != nil { return result, err } f.CpeTokens[*r.OpcRetryToken] = *result.Id if f.LoseKind == "cpe" { f.LoseKind, f.LostToken, f.HiddenCpe = "", *r.OpcRetryToken, *result.Id return core.CreateCpeResponse{}, context.DeadlineExceeded } return result, nil } func (f *retryFake) CreateIPSecConnection(ctx context.Context, r core.CreateIPSecConnectionRequest) (core.CreateIPSecConnectionResponse, error) { if id, ok := f.IPSecTokens[*r.OpcRetryToken]; ok { for _, p := range f.Connections { if *p.Id == id { f.Replayed = append(f.Replayed, *r.OpcRetryToken) return core.CreateIPSecConnectionResponse{IpSecConnection: p}, nil } } f.Invalid++ return core.CreateIPSecConnectionResponse{}, apiError{409, "InvalidatedRetryToken"} } result, err := f.networkFake.CreateIPSecConnection(ctx, r) if err != nil { return result, err } f.IPSecTokens[*r.OpcRetryToken] = *result.Id if f.LoseKind == "ipsec" { f.LoseKind, f.LostToken, f.HiddenIPSec = "", *r.OpcRetryToken, *result.Id return core.CreateIPSecConnectionResponse{}, context.DeadlineExceeded } return result, nil } func settleWAN(t *testing.T, c *Client, f *retryFake, wan string) { t.Helper() if state, err := c.Reconcile(context.Background(), wan); err != nil || !state.Ready { t.Fatalf("WAN %s: ready=%v error=%v", wan, state.Ready, err) } for _, heads := range f.Heads { for i := range heads { heads[i].Status = core.IpSecConnectionTunnelStatusUp } } for range 2 { if _, err := c.Reconcile(context.Background(), wan); err != nil { t.Fatal(err) } } } func TestWANReturn(t *testing.T) { f := retryNetwork() c := Client{API: f, Config: config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "192.0.2.10"}} for _, wan := range []string{"198.51.100.10", "198.51.100.20", "198.51.100.10"} { settleWAN(t, &c, f, wan) } if f.Creates != 6 || f.Deletes != 4 || f.Invalid != 2 || len(f.Cpes) != 1 || len(f.Connections) != 1 { t.Fatalf("WAN return failed to retire and replace: creates=%d deletes=%d invalid=%d CPEs=%d connections=%d", f.Creates, f.Deletes, f.Invalid, len(f.Cpes), len(f.Connections)) } if len(f.CpeTokens) != 3 || len(f.IPSecTokens) != 3 { t.Fatal("distinct resource generations reused a creation token") } } func TestCreateTimeout(t *testing.T) { for _, kind := range []string{"cpe", "ipsec"} { t.Run(kind, func(t *testing.T) { f := retryNetwork() cfg := config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "192.0.2.10"} c := Client{API: f, Config: cfg} settleWAN(t, &c, f, "198.51.100.10") settleWAN(t, &c, f, "198.51.100.20") f.LoseKind = kind if _, err := c.Reconcile(context.Background(), "198.51.100.10"); !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("create did not commit with a lost reply: %v", err) } replacement := Client{API: f, Config: cfg} state, err := replacement.Reconcile(context.Background(), "198.51.100.10") if err != nil || !state.Ready { t.Fatalf("replacement leader failed: ready=%v error=%v", state.Ready, err) } if !slices.Contains(f.Replayed, f.LostToken) || f.Creates != 6 || len(f.Cpes) != 2 || len(f.Connections) != 2 { t.Fatalf("ambiguous create was not replayed: creates=%d CPEs=%d connections=%d", f.Creates, len(f.Cpes), len(f.Connections)) } f.HiddenCpe, f.HiddenIPSec = "", "" if _, err := replacement.Reconcile(context.Background(), "198.51.100.10"); err != nil { t.Fatal(err) } if f.Creates != 6 { t.Fatal("visibility restoration created another resource") } }) } } func TestRetryErrors(t *testing.T) { for _, want := range []error{context.DeadlineExceeded, apiError{409, "IncorrectState"}, apiError{403, "InvalidatedRetryToken"}} { calls := 0 _, got := create(common.String("synthetic-token"), func(*string) (int, error) { calls++ return 0, want }) if calls != 1 || !errors.Is(got, want) { t.Fatalf("unrelated error was retried: %v; calls=%d", got, calls) } } } func TestRouteDrift(t *testing.T) { f := &networkFake{Heads: map[string][]core.IpSecConnectionTunnel{}, HoldRoutes: true} c := Client{API: f, Config: config.Cloud{Compartment: "synthetic", DRG: "synthetic", Address: "192.0.2.10"}} ctx := context.Background() if _, err := c.Reconcile(ctx, "198.51.100.10"); err != nil { t.Fatal(err) } c.Config.Address = "192.0.2.11" for range 2 { state, err := c.Reconcile(ctx, "198.51.100.10") if err != nil || state.Ready || len(state.Peers) != 2 { t.Fatalf("pending route exposed runtime or removed peers: ready=%v peers=%d error=%v", state.Ready, len(state.Peers), err) } } if f.RouteUpdates != 2 || !slices.Equal(f.PendingRoutes, []string{"192.0.2.11/32"}) || f.Connections[0].StaticRoutes[0] != "192.0.2.10/32" { t.Fatal("route update did not target the configured address") } f.Connections[0].StaticRoutes = slices.Clone(f.PendingRoutes) for range 2 { state, err := c.Reconcile(ctx, "198.51.100.10") if err != nil || !state.Ready { t.Fatalf("observed route was not published: ready=%v error=%v", state.Ready, err) } } if f.Creates != 2 || f.Deletes != 0 || f.RouteUpdates != 2 { t.Fatal("route reconciliation replaced resources or repeated a completed update") } }