diff --git a/api/v1alpha1/networking_types.go b/api/v1alpha1/networking_types.go index 8bc440b..305275b 100644 --- a/api/v1alpha1/networking_types.go +++ b/api/v1alpha1/networking_types.go @@ -43,3 +43,10 @@ func (n *NetworkingConfig) HTTPEnabled() bool { } return n.TCP == nil } + +// TCPEnabled reports whether L4 NLB per-pod P2P exposure is requested. +// Mirrors HTTPEnabled but does not carry the legacy `networking: {}` +// back-compat — TCP must be explicitly opted into. +func (n *NetworkingConfig) TCPEnabled() bool { + return n != nil && n.TCP != nil +} diff --git a/api/v1alpha1/seinode_types.go b/api/v1alpha1/seinode_types.go index bc09216..07b98c3 100644 --- a/api/v1alpha1/seinode_types.go +++ b/api/v1alpha1/seinode_types.go @@ -13,7 +13,12 @@ import ( // +kubebuilder:validation:XValidation:rule="!has(self.replayer) || (has(self.peers) && size(self.peers) > 0)",message="peers is required when replayer mode is set" type SeiNodeSpec struct { // ChainID of the chain this node belongs to. + // Constrained to DNS-1123 label characters because the controller composes + // it into P2P endpoint hostnames (e.g. `-p2p..`) when + // the parent SND opts into TCP networking; the address is a one-way door + // once peers cache it. // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` ChainID string `json:"chainId"` // Image is the seid container image. @@ -68,6 +73,13 @@ type SeiNodeSpec struct { // +optional Validator *ValidatorSpec `json:"validator,omitempty"` + // ExternalAddress is the routable P2P host:port written into seid's + // `p2p.external_address`. SND-managed nodes get this stamped by the + // SND reconciler when TCP networking is enabled. Standalone SeiNodes + // can set it directly. + // +optional + ExternalAddress string `json:"externalAddress,omitempty"` + // Paused freezes reconciliation. While true, the controller does not // advance the lifecycle, start plans, or mutate derived resources // except the owned StatefulSet — which scales to Replicas=0 so pods @@ -341,14 +353,6 @@ type SeiNodeStatus struct { // +optional ResolvedPeers []string `json:"resolvedPeers,omitempty"` - // ExternalAddress is the routable P2P address for this node — bare - // host:port, no nodeId@ prefix. Populated by the SeiNode controller - // from the per-pod LoadBalancer Service when the parent SND has - // Spec.Networking.TCP set; consumed by the planner to set - // p2p.external_address in CometBFT config. - // +optional - ExternalAddress string `json:"externalAddress,omitempty"` - // StatefulSet references the StatefulSet the controller created for // this SeiNode. UID is the identity check: an STS with the expected // name but a different UID is not the one this controller created diff --git a/api/v1alpha1/seinodedeployment_types.go b/api/v1alpha1/seinodedeployment_types.go index dfe066f..2aa97c5 100644 --- a/api/v1alpha1/seinodedeployment_types.go +++ b/api/v1alpha1/seinodedeployment_types.go @@ -71,9 +71,13 @@ type UpdateStrategy struct { // GenesisCeremonyConfig configures genesis ceremony orchestration for a node group. type GenesisCeremonyConfig struct { // ChainID for the new network. + // Constrained to DNS-1123 label characters because child SeiNodes + // compose it into P2P endpoint hostnames when the SND has + // `spec.networking.tcp` set; the address is a one-way door once peers + // cache it. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=64 - // +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]*[a-z0-9]$` + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` ChainID string `json:"chainId"` // StakingAmount is the amount each validator self-delegates in its gentx. @@ -343,6 +347,17 @@ type NetworkingStatus struct { // Routes lists the HTTPRoute hostnames managed by this deployment. // +optional Routes []RouteStatus `json:"routes,omitempty"` + + // P2PEndpoints lists the per-ordinal P2P endpoint + // hostnames stamped by the SND when `spec.networking.tcp` is set. + // Each entry mirrors the value injected into the child SeiNode's + // `spec.externalAddress` (hostname:port). Hostnames are deterministic + // from the SND identity, so this slice is populated without reading + // Service status back. + // +listType=map + // +listMapKey=ordinal + // +optional + P2PEndpoints []P2PEndpoint `json:"p2pEndpoints,omitempty"` } // RouteStatus is the observed state of a single HTTPRoute hostname. @@ -355,6 +370,23 @@ type RouteStatus struct { Protocol string `json:"protocol,omitempty"` } +// P2PEndpoint is the observed state of one child's publishable +// P2P endpoint. Stamped by the SND networking reconciler. +type P2PEndpoint struct { + // Ordinal is the child's replica index within the SND + // (matches `sei.io/nodedeployment-ordinal`). + Ordinal int32 `json:"ordinal"` + + // SeiNodeName is the child SeiNode resource name (also the + // per-pod headless Service name). + SeiNodeName string `json:"seiNodeName"` + + // Hostname is the vanity host:port advertised to peers via + // `p2p.external_address`. Equals the value injected into the + // child SeiNode's `spec.externalAddress`. + Hostname string `json:"hostname"` +} + // RolloutStatus tracks an in-progress rollout. Presence on the parent // SeiNodeDeployment is the single source of truth for "rollout in // progress" — `Status.Rollout != nil` and the `RolloutInProgress` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c94106b..f9777d7 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -466,6 +466,11 @@ func (in *NetworkingStatus) DeepCopyInto(out *NetworkingStatus) { *out = make([]RouteStatus, len(*in)) copy(*out, *in) } + if in.P2PEndpoints != nil { + in, out := &in.P2PEndpoints, &out.P2PEndpoints + *out = make([]P2PEndpoint, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkingStatus. @@ -533,6 +538,21 @@ func (in *OperatorKeyringSource) DeepCopy() *OperatorKeyringSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *P2PEndpoint) DeepCopyInto(out *P2PEndpoint) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new P2PEndpoint. +func (in *P2PEndpoint) DeepCopy() *P2PEndpoint { + if in == nil { + return nil + } + out := new(P2PEndpoint) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PassphraseSecretRef) DeepCopyInto(out *PassphraseSecretRef) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index d3199fb..1d87d0d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -213,6 +213,8 @@ func main() { os.Exit(1) } + p2pEndpointDomain := os.Getenv("SEI_P2P_ENDPOINT_DOMAIN") + //nolint:staticcheck // migrating to events.EventRecorder API is a separate effort recorder := mgr.GetEventRecorderFor("seinodedeployment-controller") if err := (&nodedeploymentcontroller.SeiNodeDeploymentReconciler{ @@ -223,6 +225,7 @@ func main() { GatewayNamespace: platformCfg.GatewayNamespace, GatewayDomain: platformCfg.GatewayDomain, GatewayPublicDomain: platformCfg.GatewayPublicDomain, + P2PEndpointDomain: p2pEndpointDomain, PlanExecutor: &planner.Executor[*seiv1alpha1.SeiNodeDeployment]{ ConfigFor: func(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) task.ExecutionConfig { var assemblerNode *seiv1alpha1.SeiNode diff --git a/config/crd/sei.io_seinodedeployments.yaml b/config/crd/sei.io_seinodedeployments.yaml index b16679c..56b6d45 100644 --- a/config/crd/sei.io_seinodedeployments.yaml +++ b/config/crd/sei.io_seinodedeployments.yaml @@ -106,10 +106,15 @@ spec: type: object type: array chainId: - description: ChainID for the new network. + description: |- + ChainID for the new network. + Constrained to DNS-1123 label characters because child SeiNodes + compose it into P2P endpoint hostnames when the SND has + `spec.networking.tcp` set; the address is a one-way door once peers + cache it. maxLength: 64 minLength: 1 - pattern: ^[a-z0-9][a-z0-9-]*[a-z0-9]$ + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string maxCeremonyDuration: description: |- @@ -222,8 +227,14 @@ spec: type: object type: object chainId: - description: ChainID of the chain this node belongs to. + description: |- + ChainID of the chain this node belongs to. + Constrained to DNS-1123 label characters because the controller composes + it into P2P endpoint hostnames (e.g. `-p2p..`) when + the parent SND opts into TCP networking; the address is a one-way door + once peers cache it. minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string dataVolume: description: |- @@ -261,6 +272,13 @@ spec: x-kubernetes-validations: - message: import cannot be unset once configured rule: (!has(oldSelf.import) || has(self.import)) + externalAddress: + description: |- + ExternalAddress is the routable P2P host:port written into seid's + `p2p.external_address`. SND-managed nodes get this stamped by the + SND reconciler when TCP networking is enabled. Standalone SeiNodes + can set it directly. + type: string fullNode: description: FullNode configures a chain-following full node (absorbs the "rpc" role). @@ -1072,6 +1090,45 @@ spec: description: NetworkingStatus reports the observed state of networking resources. properties: + p2pEndpoints: + description: |- + P2PEndpoints lists the per-ordinal P2P endpoint + hostnames stamped by the SND when `spec.networking.tcp` is set. + Each entry mirrors the value injected into the child SeiNode's + `spec.externalAddress` (hostname:port). Hostnames are deterministic + from the SND identity, so this slice is populated without reading + Service status back. + items: + description: |- + P2PEndpoint is the observed state of one child's publishable + P2P endpoint. Stamped by the SND networking reconciler. + properties: + hostname: + description: |- + Hostname is the vanity host:port advertised to peers via + `p2p.external_address`. Equals the value injected into the + child SeiNode's `spec.externalAddress`. + type: string + ordinal: + description: |- + Ordinal is the child's replica index within the SND + (matches `sei.io/nodedeployment-ordinal`). + format: int32 + type: integer + seiNodeName: + description: |- + SeiNodeName is the child SeiNode resource name (also the + per-pod headless Service name). + type: string + required: + - hostname + - ordinal + - seiNodeName + type: object + type: array + x-kubernetes-list-map-keys: + - ordinal + x-kubernetes-list-type: map routes: description: Routes lists the HTTPRoute hostnames managed by this deployment. diff --git a/config/crd/sei.io_seinodes.yaml b/config/crd/sei.io_seinodes.yaml index 8110658..8b2f86e 100644 --- a/config/crd/sei.io_seinodes.yaml +++ b/config/crd/sei.io_seinodes.yaml @@ -88,8 +88,14 @@ spec: type: object type: object chainId: - description: ChainID of the chain this node belongs to. + description: |- + ChainID of the chain this node belongs to. + Constrained to DNS-1123 label characters because the controller composes + it into P2P endpoint hostnames (e.g. `-p2p..`) when + the parent SND opts into TCP networking; the address is a one-way door + once peers cache it. minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string dataVolume: description: |- @@ -127,6 +133,13 @@ spec: x-kubernetes-validations: - message: import cannot be unset once configured rule: (!has(oldSelf.import) || has(self.import)) + externalAddress: + description: |- + ExternalAddress is the routable P2P host:port written into seid's + `p2p.external_address`. SND-managed nodes get this stamped by the + SND reconciler when TCP networking is enabled. Standalone SeiNodes + can set it directly. + type: string fullNode: description: FullNode configures a chain-following full node (absorbs the "rpc" role). @@ -800,14 +813,6 @@ spec: Parent controllers compare this against spec.image to determine whether a spec change has been fully actuated. type: string - externalAddress: - description: |- - ExternalAddress is the routable P2P address for this node — bare - host:port, no nodeId@ prefix. Populated by the SeiNode controller - from the per-pod LoadBalancer Service when the parent SND has - Spec.Networking.TCP set; consumed by the planner to set - p2p.external_address in CometBFT config. - type: string phase: description: Phase is the high-level lifecycle state. enum: diff --git a/internal/controller/nodedeployment/controller.go b/internal/controller/nodedeployment/controller.go index 7590af6..799160b 100644 --- a/internal/controller/nodedeployment/controller.go +++ b/internal/controller/nodedeployment/controller.go @@ -42,6 +42,10 @@ type SeiNodeDeploymentReconciler struct { GatewayDomain string GatewayPublicDomain string + // P2PEndpointDomain is the DNS zone for per-pod P2P endpoint hostnames, + // from SEI_P2P_ENDPOINT_DOMAIN. Empty disables the P2P endpoint path. + P2PEndpointDomain string + // PlanExecutor drives group-level task plans (e.g. genesis assembly). PlanExecutor planner.PlanExecutor[*seiv1alpha1.SeiNodeDeployment] } diff --git a/internal/controller/nodedeployment/envtest/p2p_endpoint_test.go b/internal/controller/nodedeployment/envtest/p2p_endpoint_test.go new file mode 100644 index 0000000..c3e4482 --- /dev/null +++ b/internal/controller/nodedeployment/envtest/p2p_endpoint_test.go @@ -0,0 +1,406 @@ +//go:build envtest + +package envtest_test + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" + "github.com/sei-protocol/sei-k8s-controller/internal/controller/nodedeployment/envtest/fixtures" +) + +const p2pEndpointTestDomain = "test.platform.sei.io" + +func expectedP2PEndpointHost(sndName, chainID string, ordinal int) string { + return fmt.Sprintf("%s-%d-p2p.%s.%s", sndName, ordinal, chainID, p2pEndpointTestDomain) +} + +func expectedP2PEndpointAddr(sndName, chainID string, ordinal int) string { + return expectedP2PEndpointHost(sndName, chainID, ordinal) + ":26656" +} + +// withP2PEndpointDomain sets the test domain on the shared reconciler and +// restores it after the test. +func withP2PEndpointDomain(t *testing.T, domain string) { + t.Helper() + prev := testSNDReconciler.P2PEndpointDomain + testSNDReconciler.P2PEndpointDomain = domain + t.Cleanup(func() { testSNDReconciler.P2PEndpointDomain = prev }) +} + +// withTCP enables `spec.networking.tcp` on a fixtures-built SND. Goes +// here (not in fixtures/) because TCP-only is a publishable-test +// concept; the broader fixture surface stays minimal. +func withTCP() fixtures.Option { + return func(snd *seiv1alpha1.SeiNodeDeployment) { + if snd.Spec.Networking == nil { + snd.Spec.Networking = &seiv1alpha1.NetworkingConfig{} + } + snd.Spec.Networking.TCP = &seiv1alpha1.TCPConfig{} + } +} + +// Happy path: TCP-enabled SND produces a child with Spec.ExternalAddress, +// a per-pod LB Service, and a P2PEndpoints entry. +func TestP2PEndpointP2P_CreateWithTCP_ChildHasAddressAndServiceExists(t *testing.T) { + g := NewWithT(t) + withP2PEndpointDomain(t, p2pEndpointTestDomain) + ns := makeNamespace(t) + + snd := fixtures.NewSND(ns, "pub-create", + fixtures.WithReplicas(1), + withTCP(), + ) + g.Expect(testCli.Create(testCtx, snd)).To(Succeed()) + + const chainID = "pacific-1" // fixtures default + wantHost := expectedP2PEndpointHost(snd.Name, chainID, 0) + wantAddr := expectedP2PEndpointAddr(snd.Name, chainID, 0) + + // 1. Child SeiNode is created with Spec.ExternalAddress already set. + // The brief calls out "child appears in the API with the value + // already populated" — this poll asserts the no-race-window claim. + childKey := types.NamespacedName{Name: snd.Name + "-0", Namespace: ns} + waitFor(t, func() bool { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return false + } + return child.Spec.ExternalAddress == wantAddr + }, "child SeiNode "+childKey.Name+" populated with publishable ExternalAddress") + + // 2. Per-pod LB Service exists with the expected annotations. + svcKey := types.NamespacedName{Name: snd.Name + "-0-p2p", Namespace: ns} + waitFor(t, func() bool { + svc := &corev1.Service{} + return testCli.Get(testCtx, svcKey, svc) == nil + }, "P2P endpoint Service "+svcKey.Name+" created") + + svc := &corev1.Service{} + g.Expect(testCli.Get(testCtx, svcKey, svc)).To(Succeed()) + g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer)) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "external-dns.alpha.kubernetes.io/hostname", wantHost)) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-type", "external")) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-scheme", "internet-facing")) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type", "ip")) + // Selector targets the child SeiNode's pod via `sei.io/node`. + g.Expect(svc.Spec.Selector).To(HaveKeyWithValue("sei.io/node", snd.Name+"-0")) + + // 3. Owner reference is the SND (not the child) so opt-out and SND + // deletion both reap the Service through the same path. + refs := svc.GetOwnerReferences() + g.Expect(refs).NotTo(BeEmpty()) + g.Expect(refs[0].UID).To(Equal(snd.UID)) + g.Expect(refs[0].Controller).NotTo(BeNil()) + g.Expect(*refs[0].Controller).To(BeTrue()) + + // 4. P2PEndpoints surfaces the same hostname the child got. + waitForStatus(t, client.ObjectKeyFromObject(snd), func(latest *seiv1alpha1.SeiNodeDeployment) bool { + ns := latest.Status.NetworkingStatus + if ns == nil || len(ns.P2PEndpoints) != 1 { + return false + } + ep := ns.P2PEndpoints[0] + return ep.Ordinal == 0 && + ep.SeiNodeName == snd.Name+"-0" && + ep.Hostname == wantAddr + }, "P2PEndpoints stamped with deterministic vanity host") +} + +// Opt-out: removing TCP clears Spec.ExternalAddress and deletes the Service. +func TestP2PEndpointP2P_OptOut_ClearsAddressAndDeletesService(t *testing.T) { + g := NewWithT(t) + withP2PEndpointDomain(t, p2pEndpointTestDomain) + ns := makeNamespace(t) + + snd := fixtures.NewSND(ns, "pub-optout", + fixtures.WithReplicas(1), + withTCP(), + ) + g.Expect(testCli.Create(testCtx, snd)).To(Succeed()) + + const chainID = "pacific-1" + wantAddr := expectedP2PEndpointAddr(snd.Name, chainID, 0) + childKey := types.NamespacedName{Name: snd.Name + "-0", Namespace: ns} + svcKey := types.NamespacedName{Name: snd.Name + "-0-p2p", Namespace: ns} + + // Wait for converged publishable state. + waitFor(t, func() bool { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return false + } + return child.Spec.ExternalAddress == wantAddr + }, "child has publishable address before opt-out") + waitFor(t, func() bool { + return testCli.Get(testCtx, svcKey, &corev1.Service{}) == nil + }, "P2P endpoint Service exists before opt-out") + + // Opt out: clear TCP. The SND retains an empty `networking` block + // (HTTPEnabled() back-compat will treat it as HTTP-enabled, but the + // validator template has no externally-routable HTTP protocols so + // nothing else changes). To get a clean "no networking" state, set + // Networking to nil. + latest := getSND(t, client.ObjectKeyFromObject(snd)) + patch := client.MergeFrom(latest.DeepCopy()) + latest.Spec.Networking = nil + g.Expect(testCli.Patch(testCtx, latest, patch)).To(Succeed()) + + // Child Spec.ExternalAddress is cleared by ensureSeiNode's diff. + waitFor(t, func() bool { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return false + } + return child.Spec.ExternalAddress == "" + }, "child ExternalAddress cleared after opt-out") + + // Per-pod LB Service is deleted. + waitFor(t, func() bool { + err := testCli.Get(testCtx, svcKey, &corev1.Service{}) + return apierrors.IsNotFound(err) + }, "P2P endpoint Service deleted after opt-out") + + // P2PEndpoints is cleared (or NetworkingStatus is nil). + waitForStatus(t, client.ObjectKeyFromObject(snd), func(latest *seiv1alpha1.SeiNodeDeployment) bool { + return latest.Status.NetworkingStatus == nil || + len(latest.Status.NetworkingStatus.P2PEndpoints) == 0 + }, "P2PEndpoints cleared after opt-out") +} + +// Re-opt-in: TCP restored brings the address back and recreates the Service. +func TestP2PEndpointP2P_ReOptIn_RestoresAddressAndService(t *testing.T) { + g := NewWithT(t) + withP2PEndpointDomain(t, p2pEndpointTestDomain) + ns := makeNamespace(t) + + snd := fixtures.NewSND(ns, "pub-reoptin", + fixtures.WithReplicas(1), + withTCP(), + ) + g.Expect(testCli.Create(testCtx, snd)).To(Succeed()) + + const chainID = "pacific-1" + wantAddr := expectedP2PEndpointAddr(snd.Name, chainID, 0) + childKey := types.NamespacedName{Name: snd.Name + "-0", Namespace: ns} + svcKey := types.NamespacedName{Name: snd.Name + "-0-p2p", Namespace: ns} + + waitFor(t, func() bool { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return false + } + return child.Spec.ExternalAddress == wantAddr + }, "initial converge") + + // Opt out. + latest := getSND(t, client.ObjectKeyFromObject(snd)) + patch := client.MergeFrom(latest.DeepCopy()) + latest.Spec.Networking = nil + g.Expect(testCli.Patch(testCtx, latest, patch)).To(Succeed()) + waitFor(t, func() bool { + err := testCli.Get(testCtx, svcKey, &corev1.Service{}) + return apierrors.IsNotFound(err) + }, "Service gone after opt-out") + + // Re-opt-in. + latest = getSND(t, client.ObjectKeyFromObject(snd)) + patch = client.MergeFrom(latest.DeepCopy()) + latest.Spec.Networking = &seiv1alpha1.NetworkingConfig{TCP: &seiv1alpha1.TCPConfig{}} + g.Expect(testCli.Patch(testCtx, latest, patch)).To(Succeed()) + + // Child gets the same vanity address back (identity is stable). + waitFor(t, func() bool { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return false + } + return child.Spec.ExternalAddress == wantAddr + }, "child re-stamped with same vanity address after re-opt-in") + + // Service is recreated. + waitFor(t, func() bool { + return testCli.Get(testCtx, svcKey, &corev1.Service{}) == nil + }, "P2P endpoint Service recreated after re-opt-in") +} + +// Scale-down deletes the ordinal's Service before the child SeiNode. +func TestP2PEndpointP2P_ScaleDown_DeletesOrdinalServiceBeforeChild(t *testing.T) { + g := NewWithT(t) + withP2PEndpointDomain(t, p2pEndpointTestDomain) + ns := makeNamespace(t) + + snd := fixtures.NewSND(ns, "pub-scale", + fixtures.WithReplicas(2), + withTCP(), + ) + g.Expect(testCli.Create(testCtx, snd)).To(Succeed()) + + // Wait for both replicas to converge with their P2P endpoint Services. + for i := 0; i < 2; i++ { + svcKey := types.NamespacedName{Name: fmt.Sprintf("%s-%d-p2p", snd.Name, i), Namespace: ns} + waitFor(t, func() bool { + return testCli.Get(testCtx, svcKey, &corev1.Service{}) == nil + }, fmt.Sprintf("P2P endpoint Service %s created", svcKey.Name)) + } + + // Scale down to 1 replica. + latest := getSND(t, client.ObjectKeyFromObject(snd)) + patch := client.MergeFrom(latest.DeepCopy()) + latest.Spec.Replicas = 1 + g.Expect(testCli.Patch(testCtx, latest, patch)).To(Succeed()) + + // ordinal-1 Service is gone. + scaledOutKey := types.NamespacedName{Name: snd.Name + "-1-p2p", Namespace: ns} + waitFor(t, func() bool { + err := testCli.Get(testCtx, scaledOutKey, &corev1.Service{}) + return apierrors.IsNotFound(err) + }, "ordinal-1 P2P endpoint Service deleted on scale-down") + + // ordinal-0 Service survives. + survivingKey := types.NamespacedName{Name: snd.Name + "-0-p2p", Namespace: ns} + g.Consistently(func() error { + return testCli.Get(testCtx, survivingKey, &corev1.Service{}) + }, 2*time.Second, 200*time.Millisecond).Should(Succeed()) +} + +// Standalone SeiNodes own Spec.ExternalAddress directly; no SND touches it. +func TestP2PEndpointP2P_StandaloneSeiNode_PreservesUserAddress(t *testing.T) { + g := NewWithT(t) + withP2PEndpointDomain(t, p2pEndpointTestDomain) + ns := makeNamespace(t) + + addr := "custom.example.com:26656" + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standalone", + Namespace: ns, + }, + Spec: seiv1alpha1.SeiNodeSpec{ + ChainID: "pacific-1", + Image: fixtures.DefaultImage, + ExternalAddress: addr, + FullNode: &seiv1alpha1.FullNodeSpec{}, + }, + } + g.Expect(testCli.Create(testCtx, node)).To(Succeed()) + + g.Consistently(func() string { + fetched := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, client.ObjectKeyFromObject(node), fetched); err != nil { + return "" + } + return fetched.Spec.ExternalAddress + }, 2*time.Second, 200*time.Millisecond).Should(Equal(addr)) +} + +// kubectl-edit stomp: external clear of Spec.ExternalAddress reconverges via ensureSeiNode. +func TestP2PEndpointP2P_KubectlEditStomp_ReconverresViaEnsureSeiNode(t *testing.T) { + g := NewWithT(t) + withP2PEndpointDomain(t, p2pEndpointTestDomain) + ns := makeNamespace(t) + + snd := fixtures.NewSND(ns, "pub-stomp", + fixtures.WithReplicas(1), + withTCP(), + ) + g.Expect(testCli.Create(testCtx, snd)).To(Succeed()) + + const chainID = "pacific-1" + wantAddr := expectedP2PEndpointAddr(snd.Name, chainID, 0) + childKey := types.NamespacedName{Name: snd.Name + "-0", Namespace: ns} + + waitFor(t, func() bool { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return false + } + return child.Spec.ExternalAddress == wantAddr + }, "initial converge before stomp") + + // Simulate `kubectl edit seinode` clearing the field. The SND + // reconciler's ensureSeiNode pass should re-stamp it. Patch through + // any finalizer-induced 409s with a small retry — the goal is to + // land the cleared value at the apiserver, not to assert on the + // PATCH path itself. + g.Eventually(func() error { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return err + } + patch := client.MergeFrom(child.DeepCopy()) + child.Spec.ExternalAddress = "" + return testCli.Patch(testCtx, child, patch) + }, 5*time.Second, 200*time.Millisecond).Should(Succeed()) + + // Bump the SND so a fresh reconcile is queued (envtest's predicate + // gates on generation changes for SND-side spec). A no-op label edit + // is enough. + g.Eventually(func() error { + latest := getSND(t, client.ObjectKeyFromObject(snd)) + patch := client.MergeFrom(latest.DeepCopy()) + if latest.Annotations == nil { + latest.Annotations = map[string]string{} + } + latest.Annotations["test.sei.io/stomp-retick"] = time.Now().Format(time.RFC3339Nano) + return testCli.Patch(testCtx, latest, patch) + }, 5*time.Second, 200*time.Millisecond).Should(Succeed()) + + // Reconverge. + waitFor(t, func() bool { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return false + } + return child.Spec.ExternalAddress == wantAddr + }, "child reconverged via ensureSeiNode after external stomp") +} + +// TestP2PEndpointP2P_NoDomainConfigured_SkipsServices verifies the silent +// no-op path: when P2PEndpointDomain is unset, TCP-enabled SNDs get no +// P2P endpoint Service and no child ExternalAddress. +func TestP2PEndpointP2P_NoDomainConfigured_SkipsServices(t *testing.T) { + g := NewWithT(t) + withP2PEndpointDomain(t, "") + ns := makeNamespace(t) + + snd := fixtures.NewSND(ns, "pub-no-domain", + fixtures.WithReplicas(1), + withTCP(), + ) + g.Expect(testCli.Create(testCtx, snd)).To(Succeed()) + + svcKey := types.NamespacedName{Name: snd.Name + "-0-p2p", Namespace: ns} + g.Consistently(func() bool { + err := testCli.Get(testCtx, svcKey, &corev1.Service{}) + return apierrors.IsNotFound(err) + }, 2*time.Second, 200*time.Millisecond).Should(BeTrue()) + + childKey := types.NamespacedName{Name: snd.Name + "-0", Namespace: ns} + g.Eventually(func() string { + child := &seiv1alpha1.SeiNode{} + if err := testCli.Get(testCtx, childKey, child); err != nil { + return "ERR" + } + return child.Spec.ExternalAddress + }, 5*time.Second, 200*time.Millisecond).Should(BeEmpty()) + + waitForStatus(t, client.ObjectKeyFromObject(snd), func(latest *seiv1alpha1.SeiNodeDeployment) bool { + c := apimeta.FindStatusCondition(latest.Status.Conditions, seiv1alpha1.ConditionNetworkingReady) + return c != nil && c.Status == metav1.ConditionFalse && c.Reason == "NetworkingDisabled" + }, "ConditionNetworkingReady=False/NetworkingDisabled when neither tier active") +} diff --git a/internal/controller/nodedeployment/envtest/suite_test.go b/internal/controller/nodedeployment/envtest/suite_test.go index 9ac1b5a..40e4aa3 100644 --- a/internal/controller/nodedeployment/envtest/suite_test.go +++ b/internal/controller/nodedeployment/envtest/suite_test.go @@ -46,6 +46,14 @@ var ( testCtx context.Context testCncl context.CancelFunc testFaker *envtestpkg.StatusFaker + + // testSNDReconciler is the live SND reconciler the manager runs. + // Exposed so tests that need to flip controller-level state (e.g. + // the publishable-P2P capability toggle) can do so without rebuilding + // the manager. Mutation must happen before the test creates the SND + // under test; controller-runtime serializes reconciles per-key, so + // reads during reconcile are safe. + testSNDReconciler *nodedeploymentcontroller.SeiNodeDeploymentReconciler ) func TestMain(m *testing.M) { @@ -187,14 +195,19 @@ func run(m *testing.M) (int, error) { // pure kube-client tasks and never call the sidecar. Genesis // ceremony tasks would; the InPlace fixtures don't trigger those. recorder := mgr.GetEventRecorderFor("seinodedeployment-controller") //nolint:staticcheck // new events API migration is a separate effort - if err := (&nodedeploymentcontroller.SeiNodeDeploymentReconciler{ - Client: kc, - Scheme: mgr.GetScheme(), - Recorder: recorder, - GatewayName: "sei-gateway", - GatewayNamespace: "gateway", - GatewayDomain: "test.local", - GatewayPublicDomain: "", + // Default capability state: publishability enabled, but the public + // domain stays empty — the publishable-P2P tests flip both fields + // themselves before creating their SND so they don't perturb the + // HTTP-route tests' single-hostname expectations. + testSNDReconciler = &nodedeploymentcontroller.SeiNodeDeploymentReconciler{ + Client: kc, + Scheme: mgr.GetScheme(), + Recorder: recorder, + GatewayName: "sei-gateway", + GatewayNamespace: "gateway", + GatewayDomain: "test.local", + GatewayPublicDomain: "", + P2PEndpointDomain: "", PlanExecutor: &planner.Executor[*seiv1alpha1.SeiNodeDeployment]{ ConfigFor: func(_ context.Context, group *seiv1alpha1.SeiNodeDeployment) task.ExecutionConfig { return task.ExecutionConfig{ @@ -207,7 +220,8 @@ func run(m *testing.M) (int, error) { } }, }, - }).SetupWithManager(mgr); err != nil { + } + if err := testSNDReconciler.SetupWithManager(mgr); err != nil { return 1, fmt.Errorf("setting up SeiNodeDeployment controller: %w", err) } diff --git a/internal/controller/nodedeployment/networking.go b/internal/controller/nodedeployment/networking.go index 3d7d775..63d1e03 100644 --- a/internal/controller/nodedeployment/networking.go +++ b/internal/controller/nodedeployment/networking.go @@ -38,12 +38,10 @@ type effectiveRoute struct { WSPort int32 } -// routeHostnameResolvable returns true when the deployment's first public -// hostname resolves in DNS, indicating the HTTPRoute + External-DNS pipeline -// is ready. Returns true when no routes are expected (private deployments, -// validator mode). +// routeHostnameResolvable reports whether the first HTTP route hostname +// resolves in DNS. Returns true when no HTTP routes are expected. func (r *SeiNodeDeploymentReconciler) routeHostnameResolvable(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) bool { - if group.Spec.Networking == nil { + if !group.Spec.Networking.HTTPEnabled() { return true } routes := resolveEffectiveRoutes(group, r.GatewayDomain, r.GatewayPublicDomain) @@ -58,6 +56,9 @@ func (r *SeiNodeDeploymentReconciler) routeHostnameResolvable(ctx context.Contex return true } +// reconcileNetworking runs the HTTP (L7 Gateway) and TCP (per-pod L4 NLB) +// sub-reconcilers independently. Both write ConditionNetworkingReady; when +// both are enabled the TCP branch runs second and its outcome wins. func (r *SeiNodeDeploymentReconciler) reconcileNetworking(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) error { if group.Spec.Networking == nil { setCondition(group, seiv1alpha1.ConditionNetworkingReady, metav1.ConditionFalse, @@ -65,12 +66,40 @@ func (r *SeiNodeDeploymentReconciler) reconcileNetworking(ctx context.Context, g return r.deleteNetworkingResources(ctx, group) } - if err := r.reconcileExternalService(ctx, group); err != nil { - return fmt.Errorf("reconciling external service: %w", err) + httpActive := group.Spec.Networking.HTTPEnabled() + tcpActive := group.Spec.Networking.TCPEnabled() && r.P2PEndpointDomain != "" + + if httpActive { + if err := r.reconcileExternalService(ctx, group); err != nil { + return fmt.Errorf("reconciling external service: %w", err) + } + if err := r.reconcileRoute(ctx, group); err != nil { + return fmt.Errorf("reconciling route: %w", err) + } + } else { + if err := r.deleteExternalService(ctx, group); err != nil { + return fmt.Errorf("deleting external service: %w", err) + } + if err := r.deleteHTTPRoutesByLabel(ctx, group); err != nil { + return fmt.Errorf("deleting HTTPRoutes: %w", err) + } } - if err := r.reconcileRoute(ctx, group); err != nil { - return fmt.Errorf("reconciling route: %w", err) + + if tcpActive { + if err := r.reconcileP2PEndpoints(ctx, group); err != nil { + return fmt.Errorf("reconciling P2P endpoints: %w", err) + } + } else { + if err := r.deleteP2PEndpoints(ctx, group); err != nil { + return fmt.Errorf("deleting P2P endpoints: %w", err) + } } + + if !httpActive && !tcpActive { + setCondition(group, seiv1alpha1.ConditionNetworkingReady, metav1.ConditionFalse, + "NetworkingDisabled", "spec.networking has no active tier (no HTTP, and TCP requires SEI_P2P_ENDPOINT_DOMAIN)") + } + return nil } @@ -366,6 +395,9 @@ func (r *SeiNodeDeploymentReconciler) deleteNetworkingResources(ctx context.Cont if err := r.deleteHTTPRoutesByLabel(ctx, group); err != nil { return fmt.Errorf("deleting HTTPRoutes: %w", err) } + if err := r.deleteP2PEndpoints(ctx, group); err != nil { + return fmt.Errorf("deleting P2P endpoint Services: %w", err) + } return nil } @@ -393,6 +425,10 @@ func (r *SeiNodeDeploymentReconciler) orphanNetworkingResources(ctx context.Cont } } + if err := r.orphanP2PEndpoints(ctx, group); err != nil { + return err + } + return nil } diff --git a/internal/controller/nodedeployment/nodes.go b/internal/controller/nodedeployment/nodes.go index ce32567..1abcd4b 100644 --- a/internal/controller/nodedeployment/nodes.go +++ b/internal/controller/nodedeployment/nodes.go @@ -175,6 +175,7 @@ func (r *SeiNodeDeploymentReconciler) populateIncumbentNodes(ctx context.Context func (r *SeiNodeDeploymentReconciler) ensureSeiNode(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment, ordinal int) error { desired := generateSeiNode(group, ordinal) + desired.Spec.ExternalAddress = r.p2pEndpointAddressForChild(group, ordinal) if err := ctrl.SetControllerReference(group, desired, r.Scheme); err != nil { return fmt.Errorf("setting owner reference: %w", err) } @@ -218,17 +219,28 @@ func (r *SeiNodeDeploymentReconciler) ensureSeiNode(ctx context.Context, group * existing.Spec.PodLabels = desired.Spec.PodLabels updated = true } + if existing.Spec.ExternalAddress != desired.Spec.ExternalAddress { + existing.Spec.ExternalAddress = desired.Spec.ExternalAddress + updated = true + } if updated { return r.Update(ctx, existing) } return nil } +// generateSeiNode produces the desired child SeiNode for a given ordinal. +// Pure: depends only on the SND spec and ordinal. The publishable +// external address is injected by ensureSeiNode, which needs reconciler +// state. The template's ExternalAddress is dropped to prevent every +// child from sharing the same value if a user set it on the template. func generateSeiNode(group *seiv1alpha1.SeiNodeDeployment, ordinal int) *seiv1alpha1.SeiNode { labels := seiNodeLabels(group, ordinal) annotations := seiNodeAnnotations(group) spec := group.Spec.Template.Spec.DeepCopy() + spec.ExternalAddress = "" + podLabels := make(map[string]string) if group.Spec.Template.Metadata != nil { maps.Copy(podLabels, group.Spec.Template.Metadata.Labels) @@ -261,6 +273,15 @@ func generateSeiNode(group *seiv1alpha1.SeiNodeDeployment, ordinal int) *seiv1al } } +// p2pEndpointAddressForChild returns the vanity host:port when the +// SND opts into TCP networking, "" otherwise. +func (r *SeiNodeDeploymentReconciler) p2pEndpointAddressForChild(group *seiv1alpha1.SeiNodeDeployment, ordinal int) string { + if !group.Spec.Networking.TCPEnabled() { + return "" + } + return r.p2pEndpointAddress(group, ordinal) +} + // scaleDown deletes SeiNodes with ordinals >= the desired replica count. func (r *SeiNodeDeploymentReconciler) scaleDown(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) error { if group.Spec.Replicas <= 0 { @@ -289,6 +310,12 @@ func (r *SeiNodeDeploymentReconciler) scaleDown(ctx context.Context, group *seiv continue } if ord >= int(group.Spec.Replicas) { + // Delete the per-ordinal P2P endpoint Service before the + // child so the NLB is not stranded between scale-down and + // SND delete. Idempotent. + if err := r.deleteP2PEndpointForOrdinal(ctx, group, ord); err != nil { + return fmt.Errorf("deleting P2P endpoint Service for scaled-down ordinal %d: %w", ord, err) + } if err := r.Delete(ctx, node); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("deleting excess SeiNode %s: %w", node.Name, err) } diff --git a/internal/controller/nodedeployment/p2p_endpoint.go b/internal/controller/nodedeployment/p2p_endpoint.go new file mode 100644 index 0000000..dd9b0a3 --- /dev/null +++ b/internal/controller/nodedeployment/p2p_endpoint.go @@ -0,0 +1,225 @@ +// p2p_endpoint.go provisions per-pod P2P endpoint Services (LoadBalancer) +// and the deterministic vanity hostname for each. external-dns CNAMEs the +// hostname to the NLB; the child SeiNode reads Spec.ExternalAddress. +package nodedeployment + +import ( + "context" + "fmt" + "strconv" + + seiconfig "github.com/sei-protocol/sei-config" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" + "github.com/sei-protocol/sei-k8s-controller/internal/noderesource" +) + +const ( + p2pEndpointFieldOwner = client.FieldOwner("nodedeployment-controller-p2p-endpoint") + + p2pEndpointComponentLabel = "sei.io/component" + p2pEndpointComponentValue = "p2p-endpoint" + + externalDNSHostnameAnnotation = "external-dns.alpha.kubernetes.io/hostname" + + awsLBTypeAnnotation = "service.beta.kubernetes.io/aws-load-balancer-type" + awsLBSchemeAnnotation = "service.beta.kubernetes.io/aws-load-balancer-scheme" + awsLBTargetTypeAnnotation = "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" + awsLBCrossZoneAnnotation = "service.beta.kubernetes.io/aws-load-balancer-attributes" + awsLBTypeValue = "external" + awsLBSchemeValue = "internet-facing" + awsLBTargetTypeValue = "ip" + awsLBCrossZoneAttributeStr = "load_balancing.cross_zone.enabled=true" + + p2pEndpointServiceSuffix = "-p2p" +) + +// effectiveChainID returns the chain ID for child SeiNodes, mirroring +// generateSeiNode: template wins, Genesis is the validator fallback. +func effectiveChainID(group *seiv1alpha1.SeiNodeDeployment) string { + if id := group.Spec.Template.Spec.ChainID; id != "" { + return id + } + if group.Spec.Genesis != nil { + return group.Spec.Genesis.ChainID + } + return "" +} + +// p2pEndpointHostname returns "-p2p.." or "" when +// the P2P endpoint path is not configured. +func (r *SeiNodeDeploymentReconciler) p2pEndpointHostname(group *seiv1alpha1.SeiNodeDeployment, ordinal int) string { + if r.P2PEndpointDomain == "" { + return "" + } + chainID := effectiveChainID(group) + if chainID == "" { + return "" + } + return fmt.Sprintf("%s-p2p.%s.%s", seiNodeName(group, ordinal), chainID, r.P2PEndpointDomain) +} + +// p2pEndpointAddress returns ":" or "" when the +// hostname is unavailable. +func (r *SeiNodeDeploymentReconciler) p2pEndpointAddress(group *seiv1alpha1.SeiNodeDeployment, ordinal int) string { + host := r.p2pEndpointHostname(group, ordinal) + if host == "" { + return "" + } + return fmt.Sprintf("%s:%d", host, seiconfig.PortP2P) +} + +func p2pEndpointServiceName(group *seiv1alpha1.SeiNodeDeployment, ordinal int) string { + return seiNodeName(group, ordinal) + p2pEndpointServiceSuffix +} + +// reconcileP2PEndpoints server-side-applies one LoadBalancer Service +// per desired ordinal and trims orphans. +func (r *SeiNodeDeploymentReconciler) reconcileP2PEndpoints(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) error { + desiredNames := make(map[string]struct{}, group.Spec.Replicas) + for i := range int(group.Spec.Replicas) { + desired := generateP2PEndpointService(group, i, r.p2pEndpointHostname(group, i)) + desired.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) + if err := ctrl.SetControllerReference(group, desired, r.Scheme); err != nil { + return fmt.Errorf("setting owner reference on P2P endpoint Service %s: %w", desired.Name, err) + } + //nolint:staticcheck // SSA apply via untyped object mirrors the rest of this controller + if err := r.Patch(ctx, desired, client.Apply, p2pEndpointFieldOwner, client.ForceOwnership); err != nil { + return fmt.Errorf("applying P2P endpoint Service %s: %w", desired.Name, err) + } + desiredNames[desired.Name] = struct{}{} + } + + if err := r.deleteOrphanP2PEndpoints(ctx, group, desiredNames); err != nil { + return fmt.Errorf("trimming orphan P2P endpoint Services: %w", err) + } + + setCondition(group, seiv1alpha1.ConditionNetworkingReady, metav1.ConditionTrue, + "P2PEndpointsApplied", + fmt.Sprintf("%d P2P endpoint Service(s) reconciled", len(desiredNames))) + return nil +} + +// deleteOrphanP2PEndpoints removes P2P endpoint Services owned by the +// SND whose names are not in desiredNames. Pass nil to delete all. +func (r *SeiNodeDeploymentReconciler) deleteOrphanP2PEndpoints(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment, desiredNames map[string]struct{}) error { + list := &corev1.ServiceList{} + if err := r.List(ctx, list, + client.InNamespace(group.Namespace), + client.MatchingLabels{ + groupLabel: group.Name, + p2pEndpointComponentLabel: p2pEndpointComponentValue, + }, + ); err != nil { + return fmt.Errorf("listing P2P endpoint Services: %w", err) + } + for i := range list.Items { + svc := &list.Items[i] + if _, ok := desiredNames[svc.Name]; ok { + continue + } + if !metav1.IsControlledBy(svc, group) { + continue + } + if err := r.Delete(ctx, svc); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("deleting orphan P2P endpoint Service %s: %w", svc.Name, err) + } + } + return nil +} + +func (r *SeiNodeDeploymentReconciler) deleteP2PEndpoints(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) error { + return r.deleteOrphanP2PEndpoints(ctx, group, nil) +} + +// orphanP2PEndpoints drops the SND owner ref so the Services survive +// SND deletion under DeletionPolicyRetain. +func (r *SeiNodeDeploymentReconciler) orphanP2PEndpoints(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment) error { + list := &corev1.ServiceList{} + if err := r.List(ctx, list, + client.InNamespace(group.Namespace), + client.MatchingLabels{ + groupLabel: group.Name, + p2pEndpointComponentLabel: p2pEndpointComponentValue, + }, + ); err != nil { + return fmt.Errorf("listing P2P endpoint Services for orphan: %w", err) + } + for i := range list.Items { + if err := r.removeOwnerRef(ctx, &list.Items[i], group); err != nil { + return fmt.Errorf("orphaning P2P endpoint Service %s: %w", list.Items[i].Name, err) + } + } + return nil +} + +// deleteP2PEndpointForOrdinal removes the per-ordinal Service before +// scaleDown deletes the child. Idempotent. +func (r *SeiNodeDeploymentReconciler) deleteP2PEndpointForOrdinal(ctx context.Context, group *seiv1alpha1.SeiNodeDeployment, ordinal int) error { + name := p2pEndpointServiceName(group, ordinal) + svc := &corev1.Service{} + err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: group.Namespace}, svc) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("fetching P2P endpoint Service %s for delete: %w", name, err) + } + if !metav1.IsControlledBy(svc, group) { + return nil + } + if err := r.Delete(ctx, svc); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("deleting P2P endpoint Service %s: %w", name, err) + } + return nil +} + +func generateP2PEndpointService(group *seiv1alpha1.SeiNodeDeployment, ordinal int, hostname string) *corev1.Service { + name := p2pEndpointServiceName(group, ordinal) + childName := seiNodeName(group, ordinal) + + labels := map[string]string{ + groupLabel: group.Name, + groupOrdinalLabel: strconv.Itoa(ordinal), + p2pEndpointComponentLabel: p2pEndpointComponentValue, + } + + annotations := map[string]string{ + externalDNSHostnameAnnotation: hostname, + awsLBTypeAnnotation: awsLBTypeValue, + awsLBSchemeAnnotation: awsLBSchemeValue, + awsLBTargetTypeAnnotation: awsLBTargetTypeValue, + awsLBCrossZoneAnnotation: awsLBCrossZoneAttributeStr, + managedByAnnotation: controllerName, + } + + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: group.Namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + AllocateLoadBalancerNodePorts: new(bool), + Selector: map[string]string{ + noderesource.NodeLabel: childName, + }, + Ports: []corev1.ServicePort{{ + Name: "p2p", + Port: seiconfig.PortP2P, + TargetPort: intstr.FromInt32(seiconfig.PortP2P), + Protocol: corev1.ProtocolTCP, + }}, + }, + } +} diff --git a/internal/controller/nodedeployment/p2p_endpoint_test.go b/internal/controller/nodedeployment/p2p_endpoint_test.go new file mode 100644 index 0000000..46fce55 --- /dev/null +++ b/internal/controller/nodedeployment/p2p_endpoint_test.go @@ -0,0 +1,150 @@ +package nodedeployment + +import ( + "testing" + + . "github.com/onsi/gomega" + seiconfig "github.com/sei-protocol/sei-config" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + seiv1alpha1 "github.com/sei-protocol/sei-k8s-controller/api/v1alpha1" + "github.com/sei-protocol/sei-k8s-controller/internal/noderesource" +) + +const ( + testChainAtlantic = "atlantic-2" + testP2PEndpointProd = "prod.platform.sei.io" +) + +func TestP2PEndpointHostname_Table(t *testing.T) { + t.Parallel() + cases := []struct { + name string + sndName string + templateChainID string + genesisChainID string + ordinal int + p2pEndpointDomain string + want string + }{ + { + name: "atlantic-2 ordinal 0", + sndName: testChainAtlantic, + templateChainID: testChainAtlantic, + ordinal: 0, + p2pEndpointDomain: testP2PEndpointProd, + want: "atlantic-2-0-p2p.atlantic-2.prod.platform.sei.io", + }, + { + name: "ordinal 5 of multi-replica fleet", + sndName: "validators", + templateChainID: testNamespace, + ordinal: 5, + p2pEndpointDomain: testP2PEndpointProd, + want: "validators-5-p2p.pacific-1.prod.platform.sei.io", + }, + { + name: "genesis-only chainID (validator SND)", + sndName: "newchain-validators", + genesisChainID: "newchain-1", + ordinal: 1, + p2pEndpointDomain: "test.platform.sei.io", + want: "newchain-validators-1-p2p.newchain-1.test.platform.sei.io", + }, + { + name: "empty publishable domain returns empty", + sndName: testChainAtlantic, + templateChainID: testChainAtlantic, + ordinal: 0, + p2pEndpointDomain: "", + want: "", + }, + { + name: "empty chain ID returns empty", + sndName: testChainAtlantic, + ordinal: 0, + p2pEndpointDomain: testP2PEndpointProd, + want: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + snd := &seiv1alpha1.SeiNodeDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: tc.sndName}, + Spec: seiv1alpha1.SeiNodeDeploymentSpec{ + Template: seiv1alpha1.SeiNodeTemplate{ + Spec: seiv1alpha1.SeiNodeSpec{ChainID: tc.templateChainID}, + }, + }, + } + if tc.genesisChainID != "" { + snd.Spec.Genesis = &seiv1alpha1.GenesisCeremonyConfig{ChainID: tc.genesisChainID} + } + r := &SeiNodeDeploymentReconciler{P2PEndpointDomain: tc.p2pEndpointDomain} + g.Expect(r.p2pEndpointHostname(snd, tc.ordinal)).To(Equal(tc.want)) + }) + } +} + +func TestP2PEndpointExternalAddress_AppendsP2PPort(t *testing.T) { + g := NewWithT(t) + snd := &seiv1alpha1.SeiNodeDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: testChainAtlantic}, + Spec: seiv1alpha1.SeiNodeDeploymentSpec{ + Template: seiv1alpha1.SeiNodeTemplate{ + Spec: seiv1alpha1.SeiNodeSpec{ChainID: testChainAtlantic}, + }, + }, + } + r := &SeiNodeDeploymentReconciler{P2PEndpointDomain: testP2PEndpointProd} + g.Expect(r.p2pEndpointAddress(snd, 0)).To(Equal("atlantic-2-0-p2p.atlantic-2.prod.platform.sei.io:26656")) +} + +func TestP2PEndpointExternalAddress_EmptyWhenHostnameRejected(t *testing.T) { + g := NewWithT(t) + snd := &seiv1alpha1.SeiNodeDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: testChainAtlantic}, + } + r := &SeiNodeDeploymentReconciler{P2PEndpointDomain: testP2PEndpointProd} + g.Expect(r.p2pEndpointAddress(snd, 0)).To(BeEmpty()) +} + +func TestP2PEndpointServiceName(t *testing.T) { + g := NewWithT(t) + snd := &seiv1alpha1.SeiNodeDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: testChainAtlantic}, + } + g.Expect(p2pEndpointServiceName(snd, 0)).To(Equal("atlantic-2-0-p2p")) + g.Expect(p2pEndpointServiceName(snd, 7)).To(Equal("atlantic-2-7-p2p")) +} + +func TestGeneratePublishableService_Annotations(t *testing.T) { + g := NewWithT(t) + snd := &seiv1alpha1.SeiNodeDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: testChainAtlantic, Namespace: "sei-test-1"}, + } + svc := generateP2PEndpointService(snd, 0, "atlantic-2-0-p2p.atlantic-2.prod.platform.sei.io") + + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "external-dns.alpha.kubernetes.io/hostname", + "atlantic-2-0-p2p.atlantic-2.prod.platform.sei.io", + )) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-type", "external")) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-scheme", "internet-facing")) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type", "ip")) + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-attributes", + "load_balancing.cross_zone.enabled=true")) + + g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeLoadBalancer)) + g.Expect(svc.Spec.ExternalTrafficPolicy).To(Equal(corev1.ServiceExternalTrafficPolicyTypeLocal)) + g.Expect(svc.Spec.Ports).To(HaveLen(1)) + g.Expect(svc.Spec.Ports[0].Port).To(Equal(seiconfig.PortP2P)) + g.Expect(svc.Spec.Selector).To(HaveKeyWithValue(noderesource.NodeLabel, "atlantic-2-0")) +} diff --git a/internal/controller/nodedeployment/status.go b/internal/controller/nodedeployment/status.go index 1eef56b..54d2b0b 100644 --- a/internal/controller/nodedeployment/status.go +++ b/internal/controller/nodedeployment/status.go @@ -103,20 +103,39 @@ func (r *SeiNodeDeploymentReconciler) buildNetworkingStatus(group *seiv1alpha1.S if group.Spec.Networking == nil { return nil } - routes := resolveEffectiveRoutes(group, r.GatewayDomain, r.GatewayPublicDomain) - if len(routes) == 0 { - return nil + + out := &seiv1alpha1.NetworkingStatus{} + + if group.Spec.Networking.HTTPEnabled() { + routes := resolveEffectiveRoutes(group, r.GatewayDomain, r.GatewayPublicDomain) + for _, er := range routes { + for _, h := range er.Hostnames { + out.Routes = append(out.Routes, seiv1alpha1.RouteStatus{ + Hostname: h, + Protocol: er.Protocol, + }) + } + } } - var rs []seiv1alpha1.RouteStatus - for _, er := range routes { - for _, h := range er.Hostnames { - rs = append(rs, seiv1alpha1.RouteStatus{ - Hostname: h, - Protocol: er.Protocol, + + if group.Spec.Networking.TCPEnabled() && r.P2PEndpointDomain != "" { + for i := range int(group.Spec.Replicas) { + host := r.p2pEndpointAddress(group, i) + if host == "" { + continue + } + out.P2PEndpoints = append(out.P2PEndpoints, seiv1alpha1.P2PEndpoint{ + Ordinal: int32(i), + SeiNodeName: seiNodeName(group, i), + Hostname: host, }) } } - return &seiv1alpha1.NetworkingStatus{Routes: rs} + + if len(out.Routes) == 0 && len(out.P2PEndpoints) == 0 { + return nil + } + return out } func setNodesReadyCondition(group *seiv1alpha1.SeiNodeDeployment, ready, desired int32, nodes []seiv1alpha1.SeiNode) { diff --git a/internal/planner/common_overrides_test.go b/internal/planner/common_overrides_test.go index c3f7d08..42ab6a1 100644 --- a/internal/planner/common_overrides_test.go +++ b/internal/planner/common_overrides_test.go @@ -12,7 +12,7 @@ import ( func TestCommonOverrides_WithExternalAddress(t *testing.T) { node := &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, - Status: seiv1alpha1.SeiNodeStatus{ + Spec: seiv1alpha1.SeiNodeSpec{ ExternalAddress: "p2p.atlantic-2.seinetwork.io:26656", }, } @@ -45,16 +45,14 @@ func TestCommonOverrides_UserOverrideTakesPrecedence(t *testing.T) { node := &seiv1alpha1.SeiNode{ ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, Spec: seiv1alpha1.SeiNodeSpec{ - ChainID: "test-1", - Image: "seid:v1", - FullNode: &seiv1alpha1.FullNodeSpec{}, + ChainID: "test-1", + Image: "seid:v1", + FullNode: &seiv1alpha1.FullNodeSpec{}, + ExternalAddress: "lb.address:26656", Overrides: map[string]string{ seiconfig.KeyP2PExternalAddress: "custom.address:26656", }, }, - Status: seiv1alpha1.SeiNodeStatus{ - ExternalAddress: "lb.address:26656", - }, } common := commonOverrides(node) diff --git a/internal/planner/planner.go b/internal/planner/planner.go index c2c33bd..0b31441 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -694,8 +694,8 @@ func commonOverrides(node *seiv1alpha1.SeiNode) map[string]string { out := map[string]string{ "logging.level": "error", } - if node.Status.ExternalAddress != "" { - out[seiconfig.KeyP2PExternalAddress] = node.Status.ExternalAddress + if node.Spec.ExternalAddress != "" { + out[seiconfig.KeyP2PExternalAddress] = node.Spec.ExternalAddress } return out } diff --git a/manifests/sei.io_seinodedeployments.yaml b/manifests/sei.io_seinodedeployments.yaml index b16679c..56b6d45 100644 --- a/manifests/sei.io_seinodedeployments.yaml +++ b/manifests/sei.io_seinodedeployments.yaml @@ -106,10 +106,15 @@ spec: type: object type: array chainId: - description: ChainID for the new network. + description: |- + ChainID for the new network. + Constrained to DNS-1123 label characters because child SeiNodes + compose it into P2P endpoint hostnames when the SND has + `spec.networking.tcp` set; the address is a one-way door once peers + cache it. maxLength: 64 minLength: 1 - pattern: ^[a-z0-9][a-z0-9-]*[a-z0-9]$ + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string maxCeremonyDuration: description: |- @@ -222,8 +227,14 @@ spec: type: object type: object chainId: - description: ChainID of the chain this node belongs to. + description: |- + ChainID of the chain this node belongs to. + Constrained to DNS-1123 label characters because the controller composes + it into P2P endpoint hostnames (e.g. `-p2p..`) when + the parent SND opts into TCP networking; the address is a one-way door + once peers cache it. minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string dataVolume: description: |- @@ -261,6 +272,13 @@ spec: x-kubernetes-validations: - message: import cannot be unset once configured rule: (!has(oldSelf.import) || has(self.import)) + externalAddress: + description: |- + ExternalAddress is the routable P2P host:port written into seid's + `p2p.external_address`. SND-managed nodes get this stamped by the + SND reconciler when TCP networking is enabled. Standalone SeiNodes + can set it directly. + type: string fullNode: description: FullNode configures a chain-following full node (absorbs the "rpc" role). @@ -1072,6 +1090,45 @@ spec: description: NetworkingStatus reports the observed state of networking resources. properties: + p2pEndpoints: + description: |- + P2PEndpoints lists the per-ordinal P2P endpoint + hostnames stamped by the SND when `spec.networking.tcp` is set. + Each entry mirrors the value injected into the child SeiNode's + `spec.externalAddress` (hostname:port). Hostnames are deterministic + from the SND identity, so this slice is populated without reading + Service status back. + items: + description: |- + P2PEndpoint is the observed state of one child's publishable + P2P endpoint. Stamped by the SND networking reconciler. + properties: + hostname: + description: |- + Hostname is the vanity host:port advertised to peers via + `p2p.external_address`. Equals the value injected into the + child SeiNode's `spec.externalAddress`. + type: string + ordinal: + description: |- + Ordinal is the child's replica index within the SND + (matches `sei.io/nodedeployment-ordinal`). + format: int32 + type: integer + seiNodeName: + description: |- + SeiNodeName is the child SeiNode resource name (also the + per-pod headless Service name). + type: string + required: + - hostname + - ordinal + - seiNodeName + type: object + type: array + x-kubernetes-list-map-keys: + - ordinal + x-kubernetes-list-type: map routes: description: Routes lists the HTTPRoute hostnames managed by this deployment. diff --git a/manifests/sei.io_seinodes.yaml b/manifests/sei.io_seinodes.yaml index 8110658..8b2f86e 100644 --- a/manifests/sei.io_seinodes.yaml +++ b/manifests/sei.io_seinodes.yaml @@ -88,8 +88,14 @@ spec: type: object type: object chainId: - description: ChainID of the chain this node belongs to. + description: |- + ChainID of the chain this node belongs to. + Constrained to DNS-1123 label characters because the controller composes + it into P2P endpoint hostnames (e.g. `-p2p..`) when + the parent SND opts into TCP networking; the address is a one-way door + once peers cache it. minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string dataVolume: description: |- @@ -127,6 +133,13 @@ spec: x-kubernetes-validations: - message: import cannot be unset once configured rule: (!has(oldSelf.import) || has(self.import)) + externalAddress: + description: |- + ExternalAddress is the routable P2P host:port written into seid's + `p2p.external_address`. SND-managed nodes get this stamped by the + SND reconciler when TCP networking is enabled. Standalone SeiNodes + can set it directly. + type: string fullNode: description: FullNode configures a chain-following full node (absorbs the "rpc" role). @@ -800,14 +813,6 @@ spec: Parent controllers compare this against spec.image to determine whether a spec change has been fully actuated. type: string - externalAddress: - description: |- - ExternalAddress is the routable P2P address for this node — bare - host:port, no nodeId@ prefix. Populated by the SeiNode controller - from the per-pod LoadBalancer Service when the parent SND has - Spec.Networking.TCP set; consumed by the planner to set - p2p.external_address in CometBFT config. - type: string phase: description: Phase is the high-level lifecycle state. enum: