topology-updater: wait for NodeResourceTopology CRD at startup

This fixes an issue where nfd-topology-updater pods would crash repeatedly
(CrashLoopBackOff) when the NodeResourceTopology CRD was not installed,
providing an unhelpful error message.

Changes:
- Add waitForNodeResourceTopologyCRD() that waits with exponential backoff
  (5s to 60s max) for the CRD to become available
- Move HTTP server startup earlier so health probes pass during CRD wait
- Add RBAC permission for topology-updater to read CRDs
- Handle Forbidden errors gracefully (skip wait if no permission)
- Update documentation with CRD requirements section

This handles race conditions during deployment and scenarios where the
CRD is installed after the topology-updater pods start.

Author: ArangoGutierrez

deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml

@@ -36,3 +36,11 @@ rules:
   - create
   - get
   - update
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch

deployment/helm/node-feature-discovery/templates/clusterrole.yaml

@@ -99,6 +99,14 @@ rules:
   - create
   - get
   - update
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
 {{- end }}
 
 {{- if and .Values.gc.enable .Values.gc.rbac.create }}

deployment/helm/node-feature-discovery/values.yaml

@@ -754,7 +754,10 @@ topologyUpdater:
   # -- Specifies whether nfd-topology-updater should be deployed.
   # @section -- NFD-Topology-Updater
   enable: false
-  # -- Create CustomResourceDefinitions for the TopologyUpdater.
+  # -- Create the NodeResourceTopology CRD. This MUST be set to true when
+  # 'enable' is true, unless the CRD is installed separately (e.g., by another
+  # Helm release or external tool). If the CRD is missing, the topology-updater
+  # pods will fail with "NodeResourceTopology CRD is not installed" error.
   # @section -- NFD-Topology-Updater
   createCRDs: false
   # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/args

docs/usage/nfd-topology-updater.md

@@ -56,6 +56,29 @@ Preceding Kubernetes v1.23, the `kubelet` must be started with
 Starting from Kubernetes v1.23, the `KubeletPodResourcesGetAllocatable`
 [feature gate][feature-gate].  is enabled by default
 
+### NodeResourceTopology CRD
+
+The NFD-Topology-Updater requires the `NodeResourceTopology` Custom Resource
+Definition (CRD) to be installed in the cluster. Without this CRD, the
+topology-updater pods will fail with an error:
+
+```
+NodeResourceTopology CRD "noderesourcetopologies.topology.node.k8s.io" is not installed
+```
+
+When deploying with Helm, you **must** set `topologyUpdater.createCRDs=true`
+along with `topologyUpdater.enable=true`:
+
+```bash
+helm install nfd node-feature-discovery \
+  --set topologyUpdater.enable=true \
+  --set topologyUpdater.createCRDs=true
+```
+
+If you manage CRDs separately (e.g., through a dedicated CRD management tool or
+another Helm release), ensure the `NodeResourceTopology` CRD is installed before
+deploying the topology-updater
+
 ## Topology-Updater Configuration
 
 NFD-Topology-Updater supports configuration through a configuration file. The

pkg/nfd-topology-updater/nfd-topology-updater.go

@@ -24,11 +24,14 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	k8sclient "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 
@@ -48,10 +51,14 @@ import (
 )
 
 const (
-	// TopologyManagerPolicyAttributeName represents an attribute which defines Topology Manager Policy
+	// TopologyManagerPolicyAttributeName represents an attribute which defines
+	// Topology Manager Policy
 	TopologyManagerPolicyAttributeName = "topologyManagerPolicy"
-	// TopologyManagerScopeAttributeName represents an attribute which defines Topology Manager Policy Scope
+	// TopologyManagerScopeAttributeName represents an attribute which defines
+	// Topology Manager Policy Scope
 	TopologyManagerScopeAttributeName = "topologyManagerScope"
+	// NodeResourceTopologyCRDName is the name of the NodeResourceTopology CRD
+	NodeResourceTopologyCRDName = "noderesourcetopologies.topology.node.k8s.io"
 )
 
 // Args are the command line arguments
@@ -141,6 +148,26 @@ func (w *nfdTopologyUpdater) Healthz(writer http.ResponseWriter, _ *http.Request
 func (w *nfdTopologyUpdater) Run() error {
 	klog.InfoS("Node Feature Discovery Topology Updater", "version", version.Get(), "nodeName", w.nodeName)
 
+	// Start HTTP server early so health probes work during initialization.
+	// This is important because we may wait for the CRD to be available.
+	httpMux := http.NewServeMux()
+	httpMux.HandleFunc("/healthz", w.Healthz)
+
+	// Register to metrics server
+	promRegistry := prometheus.NewRegistry()
+	promRegistry.MustRegister(
+		buildInfo,
+		scanErrors)
+	httpMux.Handle("/metrics", promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{}))
+	registerVersion(version.Get())
+
+	httpServer := http.Server{Addr: fmt.Sprintf(":%d", w.args.Port), Handler: httpMux}
+	go func() {
+		klog.InfoS("http server starting", "port", httpServer.Addr)
+		klog.InfoS("http server stopped", "exitCode", httpServer.ListenAndServe())
+	}()
+	defer httpServer.Close() // nolint: errcheck
+
 	podResClient, err := podres.GetPodResClient(w.resourcemonitorArgs.PodResourceSocketPath)
 	if err != nil {
 		return fmt.Errorf("failed to get PodResource Client: %w", err)
@@ -152,7 +179,7 @@ func (w *nfdTopologyUpdater) Run() error {
 	}
 	topoClient, err := topologyclientset.NewForConfig(kubeconfig)
 	if err != nil {
-		return nil
+		return fmt.Errorf("failed to create topology client: %w", err)
 	}
 	w.topoClient = topoClient
 
@@ -162,20 +189,17 @@ func (w *nfdTopologyUpdater) Run() error {
 	}
 	w.k8sClient = k8sClient
 
+	// Wait for the NodeResourceTopology CRD to be available before proceeding.
+	// This handles race conditions during deployment and scenarios where the
+	// CRD is installed after the topology-updater.
+	if err := waitForNodeResourceTopologyCRD(kubeconfig, w.stop); err != nil {
+		return err
+	}
+
 	if err := w.configure(); err != nil {
 		return fmt.Errorf("faild to configure Node Feature Discovery Topology Updater: %w", err)
 	}
 
-	httpMux := http.NewServeMux()
-
-	// Register to metrics server
-	promRegistry := prometheus.NewRegistry()
-	promRegistry.MustRegister(
-		buildInfo,
-		scanErrors)
-	httpMux.Handle("/metrics", promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{}))
-	registerVersion(version.Get())
-
 	var resScan resourcemonitor.ResourcesScanner
 
 	resScan, err = resourcemonitor.NewPodResourcesScanner(w.resourcemonitorArgs.Namespace, podResClient, k8sClient, w.resourcemonitorArgs.PodSetFingerprint)
@@ -194,17 +218,6 @@ func (w *nfdTopologyUpdater) Run() error {
 		return fmt.Errorf("failed to obtain node resource information: %w", err)
 	}
 
-	// Register health probe (at this point we're "ready and live")
-	httpMux.HandleFunc("/healthz", w.Healthz)
-
-	// Start HTTP server
-	httpServer := http.Server{Addr: fmt.Sprintf(":%d", w.args.Port), Handler: httpMux}
-	go func() {
-		klog.InfoS("http server starting", "port", httpServer.Addr)
-		klog.InfoS("http server stopped", "exitCode", httpServer.ListenAndServe())
-	}()
-	defer httpServer.Close() // nolint: errcheck
-
 	for {
 		select {
 		case info := <-w.eventSource:
@@ -282,6 +295,12 @@ func (w *nfdTopologyUpdater) updateNodeResourceTopology(zoneInfo v1alpha2.ZoneLi
 		updateAttributes(&nrtNew.Attributes, scanResponse.Attributes)
 
 		if _, err := w.topoClient.TopologyV1alpha2().NodeResourceTopologies().Create(context.TODO(), &nrtNew, metav1.CreateOptions{}); err != nil {
+			if errors.IsNotFound(err) {
+				return fmt.Errorf("failed to create NodeResourceTopology: %w. "+
+					"The NodeResourceTopology CRD may not be installed. "+
+					"If using Helm, ensure 'topologyUpdater.createCRDs=true' is set",
+					err)
+			}
 			return fmt.Errorf("failed to create NodeResourceTopology: %w", err)
 		}
 		return nil
@@ -380,6 +399,66 @@ func (w *nfdTopologyUpdater) configure() error {
 	return nil
 }
 
+// waitForNodeResourceTopologyCRD waits for the NodeResourceTopology CRD to be
+// available in the cluster. This handles race conditions during deployment and
+// scenarios where the CRD is installed after the topology-updater pods start.
+// The function will retry indefinitely until the CRD is found or the stop
+// channel is closed.
+func waitForNodeResourceTopologyCRD(config *restclient.Config, stop <-chan struct{}) error {
+	const (
+		initialBackoff = 5 * time.Second
+		maxBackoff     = 60 * time.Second
+	)
+
+	apiextClient, err := apiextensionsclient.NewForConfig(config)
+	if err != nil {
+		klog.V(2).InfoS("unable to create apiextensions client for CRD check, skipping wait",
+			"error", err)
+		// Don't block startup, the error will be caught later when creating NRT
+		return nil
+	}
+
+	backoff := initialBackoff
+	for {
+		_, err = apiextClient.ApiextensionsV1().CustomResourceDefinitions().Get(
+			context.TODO(), NodeResourceTopologyCRDName, metav1.GetOptions{})
+		if err == nil {
+			klog.InfoS("NodeResourceTopology CRD is available",
+				"crd", NodeResourceTopologyCRDName)
+			return nil
+		}
+
+		// If we don't have permission to check CRDs, skip waiting and let the
+		// actual NRT creation fail with a more descriptive error
+		if errors.IsForbidden(err) {
+			klog.V(2).InfoS("no permission to check CRD existence, skipping wait",
+				"crd", NodeResourceTopologyCRDName, "error", err)
+			return nil
+		}
+
+		if errors.IsNotFound(err) {
+			klog.InfoS("waiting for NodeResourceTopology CRD to be created. "+
+				"If using Helm, ensure 'topologyUpdater.createCRDs=true' is set",
+				"crd", NodeResourceTopologyCRDName, "retryIn", backoff)
+		} else {
+			klog.V(2).InfoS("error checking for CRD, will retry",
+				"crd", NodeResourceTopologyCRDName, "error", err, "retryIn", backoff)
+		}
+
+		select {
+		case <-stop:
+			return fmt.Errorf("stopped while waiting for CRD %q",
+				NodeResourceTopologyCRDName)
+		case <-time.After(backoff):
+			// Exponential backoff with max cap
+			backoff *= 2
+			if backoff > maxBackoff {
+				backoff = maxBackoff
+			}
+		}
+	}
+}
+
 func createTopologyAttributes(policy string, scope string) v1alpha2.AttributeList {
 	return v1alpha2.AttributeList{
 		{