Add a retry around updating a secrets-encrypt node annotations (#9039)

* Add a retry around updating a se node annotations

Signed-off-by: Derek Nola <derek.nola@suse.com>

pkg/server/secrets-encrypt.go

@@ -23,6 +23,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/utils/pointer"
 )
 
@@ -222,13 +223,16 @@ func encryptionPrepare(ctx context.Context, server *config.Control, force bool)
 		return err
 	}
 	nodeName := os.Getenv("NODE_NAME")
-	node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare)
+	})
 	if err != nil {
 		return err
 	}
-	if err = secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare); err != nil {
-		return err
-	}
 	return cluster.Save(ctx, server, true)
 }
 
@@ -250,13 +254,16 @@ func encryptionRotate(ctx context.Context, server *config.Control, force bool) e
 	}
 	logrus.Infoln("Encryption keys right rotated")
 	nodeName := os.Getenv("NODE_NAME")
-	node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate)
+	})
 	if err != nil {
 		return err
 	}
-	if err := secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate); err != nil {
-		return err
-	}
 	return cluster.Save(ctx, server, true)
 }