44 "context"
55 goerrors "errors"
66 "fmt"
7+ "io/ioutil"
78 "math"
89 "strings"
910 "time"
@@ -17,8 +18,10 @@ import (
1718 "k8s.io/apimachinery/pkg/api/errors"
1819 "k8s.io/apimachinery/pkg/api/resource"
1920 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21+ "k8s.io/apimachinery/pkg/labels"
2022 "k8s.io/apimachinery/pkg/runtime"
2123 "k8s.io/apimachinery/pkg/types"
24+ "k8s.io/client-go/kubernetes"
2225 "k8s.io/client-go/tools/record"
2326 ctrl "sigs.k8s.io/controller-runtime"
2427 "sigs.k8s.io/controller-runtime/pkg/client"
@@ -59,14 +62,15 @@ func (r *ReconcileComplianceScan) SetupWithManager(mgr ctrl.Manager) error {
5962
6063// Add creates a new ComplianceScan Controller and adds it to the Manager. The Manager will set fields on the Controller
6164// and Start it when the Manager is Started.
62- func Add (mgr manager.Manager , met * metrics.Metrics , si utils.CtlplaneSchedulingInfo ) error {
63- return add (mgr , newReconciler (mgr , met , si ))
65+ func Add (mgr manager.Manager , met * metrics.Metrics , si utils.CtlplaneSchedulingInfo , kubeClient * kubernetes. Clientset ) error {
66+ return add (mgr , newReconciler (mgr , met , si , kubeClient ))
6467}
6568
6669// newReconciler returns a new reconcile.Reconciler
67- func newReconciler (mgr manager.Manager , met * metrics.Metrics , si utils.CtlplaneSchedulingInfo ) reconcile.Reconciler {
70+ func newReconciler (mgr manager.Manager , met * metrics.Metrics , si utils.CtlplaneSchedulingInfo , kubeClient * kubernetes. Clientset ) reconcile.Reconciler {
6871 return & ReconcileComplianceScan {
6972 Client : mgr .GetClient (),
73+ ClientSet : kubeClient ,
7074 Scheme : mgr .GetScheme (),
7175 Recorder : mgr .GetEventRecorderFor ("scanctrl" ),
7276 Metrics : met ,
@@ -89,10 +93,11 @@ var _ reconcile.Reconciler = &ReconcileComplianceScan{}
8993type ReconcileComplianceScan struct {
9094 // This Client, initialized using mgr.Client() above, is a split Client
9195 // that reads objects from the cache and writes to the apiserver
92- Client client.Client
93- Scheme * runtime.Scheme
94- Recorder record.EventRecorder
95- Metrics * metrics.Metrics
96+ Client client.Client
97+ ClientSet * kubernetes.Clientset
98+ Scheme * runtime.Scheme
99+ Recorder record.EventRecorder
100+ Metrics * metrics.Metrics
96101 // helps us schedule platform scans on the nodes labeled for the
97102 // compliance operator's control plane
98103 schedulingInfo utils.CtlplaneSchedulingInfo
@@ -104,6 +109,7 @@ type ReconcileComplianceScan struct {
104109//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,persistentvolumes,verbs=watch,create,get,list,delete
105110//+kubebuilder:rbac:groups="",resources=pods,configmaps,events,verbs=create,get,list,watch,patch,update,delete,deletecollection
106111//+kubebuilder:rbac:groups="",resources=secrets,verbs=create,get,list,update,watch,delete
112+ //+kubebuilder:rbac:groups="",resources=nodes,nodes/proxy,verbs=get,list,watch
107113//+kubebuilder:rbac:groups=apps,resources=replicasets,deployments,verbs=get,list,watch,create,update,delete
108114//+kubebuilder:rbac:groups=compliance.openshift.io,resources=compliancescans,verbs=create,watch,patch,get,list
109115//+kubebuilder:rbac:groups=compliance.openshift.io,resources=*,verbs=*
@@ -275,7 +281,6 @@ func (r *ReconcileComplianceScan) validate(instance *compv1alpha1.ComplianceScan
275281
276282func (r * ReconcileComplianceScan ) phasePendingHandler (instance * compv1alpha1.ComplianceScan , logger logr.Logger ) (reconcile.Result , error ) {
277283 logger .Info ("Phase: Pending" )
278-
279284 // Remove annotation if needed
280285 if instance .NeedsRescan () {
281286 instanceCopy := instance .DeepCopy ()
@@ -350,6 +355,11 @@ func (r *ReconcileComplianceScan) phaseLaunchingHandler(h scanTypeHandler, logge
350355 return reconcile.Result {}, err
351356 }
352357
358+ if err = r .handleRuntimeKubeletConfig (scan , logger ); err != nil {
359+ logger .Error (err , "Cannot handle runtime kubelet config" )
360+ return reconcile.Result {}, err
361+ }
362+
353363 if err = h .createScanWorkload (); err != nil {
354364 if ! common .IsRetriable (err ) {
355365 // Surface non-retriable errors to the CR
@@ -380,6 +390,101 @@ func (r *ReconcileComplianceScan) phaseLaunchingHandler(h scanTypeHandler, logge
380390 return reconcile.Result {}, nil
381391}
382392
393+ // getRuntimeKubeletConfig gets the runtime kubeletconfig for a node
394+ // by fetching /api/v1/nodes/$nodeName/proxy/configz endpoint use the kubeClientset
395+ // and store it in a configmap for each node
396+ func (r * ReconcileComplianceScan ) getRuntimeKubeletConfig (nodeName string ) (string , error ) {
397+ // get the runtime kubeletconfig
398+ kubeletConfigIO , err := r .ClientSet .CoreV1 ().RESTClient ().Get ().RequestURI ("/api/v1/nodes/" + nodeName + "/proxy/configz" ).Stream (context .TODO ())
399+ if err != nil {
400+ return "" , err
401+ }
402+ defer kubeletConfigIO .Close ()
403+ kubeletConfig , err := ioutil .ReadAll (kubeletConfigIO )
404+ if err != nil {
405+ return "" , err
406+ }
407+ // kubeletConfig is a byte array, we need to convert it to string
408+ kubeletConfigStr := string (kubeletConfig )
409+ return kubeletConfigStr , nil
410+ }
411+
412+ func (r * ReconcileComplianceScan ) createKubeletConfigCM (instance * compv1alpha1.ComplianceScan , node * corev1.Node ) error {
413+ kubeletConfig , err := r .getRuntimeKubeletConfig (node .Name )
414+ if err != nil {
415+ return fmt .Errorf ("cannot get the runtime kubelet config for node %s: %v" , node .Name , err )
416+ }
417+ trueP := true
418+ // create a configmap for the node
419+ kubeletConfigCM := & corev1.ConfigMap {
420+ ObjectMeta : metav1.ObjectMeta {
421+ Name : getKubeletCMNameForScan (instance , node ),
422+ Namespace : instance .Namespace ,
423+ CreationTimestamp : metav1.Time {
424+ Time : time .Now (),
425+ },
426+ Labels : map [string ]string {
427+ compv1alpha1 .ComplianceScanLabel : instance .Name ,
428+ compv1alpha1 .KubeletConfigLabel : "" ,
429+ },
430+ },
431+ Data : map [string ]string {
432+ KubeletConfigMapName : kubeletConfig ,
433+ },
434+ Immutable : & trueP ,
435+ }
436+ err = r .Client .Create (context .TODO (), kubeletConfigCM )
437+ if err != nil {
438+ return err
439+ }
440+ return nil
441+ }
442+
443+ func (r * ReconcileComplianceScan ) getNodesForScan (instance * compv1alpha1.ComplianceScan ) (corev1.NodeList , error ) {
444+ nodes := corev1.NodeList {}
445+ nodeScanSelector := map [string ]string {"kubernetes.io/os" : "linux" }
446+ listOpts := client.ListOptions {
447+ LabelSelector : labels .SelectorFromSet (labels .Merge (instance .Spec .NodeSelector , nodeScanSelector )),
448+ }
449+
450+ if err := r .Client .List (context .TODO (), & nodes , & listOpts ); err != nil {
451+ return nodes , err
452+ }
453+ return nodes , nil
454+ }
455+
456+ func (r * ReconcileComplianceScan ) handleRuntimeKubeletConfig (instance * compv1alpha1.ComplianceScan , logger logr.Logger ) error {
457+ // only handle node scans
458+ if instance .Spec .ScanType != compv1alpha1 .ScanTypeNode {
459+ return nil
460+ }
461+ nodes , err := r .getNodesForScan (instance )
462+ if err != nil {
463+ logger .Error (err , "Cannot get the nodes for the scan" )
464+ return err
465+ }
466+ for _ , node := range nodes .Items {
467+ // check if the configmap already exists for the node
468+ kubeletConfigCM := & corev1.ConfigMap {}
469+ err := r .Client .Get (context .TODO (), types.NamespacedName {Name : getKubeletCMNameForScan (instance , & node ), Namespace : instance .Namespace }, kubeletConfigCM )
470+ if err != nil {
471+ if ! errors .IsNotFound (err ) {
472+ logger .Error (err , "Error getting the configmap for the runtime kubeletconfig" , "node" , node .Name )
473+ return err
474+ }
475+ // create a configmap for the node
476+ err = r .createKubeletConfigCM (instance , & node )
477+ logger .Info ("Created a new configmap for the node" , "configmap" , kubeletConfigCM .Name , "node" , node .Name )
478+ if err != nil {
479+ logger .Error (err , "Error creating the configmap for the runtime kubeletconfig" , "node" , node .Name )
480+ return err
481+ }
482+ continue
483+ }
484+ }
485+ return nil
486+ }
487+
383488func (r * ReconcileComplianceScan ) phaseRunningHandler (h scanTypeHandler , logger logr.Logger ) (reconcile.Result , error ) {
384489 logger .Info ("Phase: Running" )
385490
@@ -575,6 +680,7 @@ func (r *ReconcileComplianceScan) phaseDoneHandler(h scanTypeHandler, instance *
575680 logger .Error (err , "Cannot delete aggregator" )
576681 return reconcile.Result {}, err
577682 }
683+
578684 }
579685
580686 // We need to remove resources before doing a re-scan
@@ -605,6 +711,11 @@ func (r *ReconcileComplianceScan) phaseDoneHandler(h scanTypeHandler, instance *
605711 return reconcile.Result {}, err
606712 }
607713
714+ if err := r .deleteKubeletConfigConfigMaps (instance , logger ); err != nil {
715+ logger .Error (err , "Cannot delete KubeletConfig ConfigMaps" )
716+ return reconcile.Result {}, err
717+ }
718+
608719 if instance .NeedsRescan () {
609720 if err = r .deleteResultConfigMaps (instance , logger ); err != nil {
610721 logger .Error (err , "Cannot delete result ConfigMaps" )
@@ -752,6 +863,19 @@ func (r *ReconcileComplianceScan) deleteResultConfigMaps(instance *compv1alpha1.
752863 return nil
753864}
754865
866+ func (r * ReconcileComplianceScan ) deleteKubeletConfigConfigMaps (instance * compv1alpha1.ComplianceScan , logger logr.Logger ) error {
867+ inNs := client .InNamespace (common .GetComplianceOperatorNamespace ())
868+ withLabel := client.MatchingLabels {
869+ compv1alpha1 .ComplianceScanLabel : instance .Name ,
870+ compv1alpha1 .KubeletConfigLabel : "" ,
871+ }
872+ err := r .Client .DeleteAllOf (context .Background (), & corev1.ConfigMap {}, inNs , withLabel )
873+ if err != nil {
874+ return err
875+ }
876+ return nil
877+ }
878+
755879// returns true if the pod is still running, false otherwise
756880func isPodRunningInNode (r * ReconcileComplianceScan , scanInstance * compv1alpha1.ComplianceScan , node * corev1.Node , timeout time.Duration , logger logr.Logger ) (bool , error ) {
757881 podName := getPodForNodeName (scanInstance .Name , node .Name )
0 commit comments