From aac491c8bd5bd1c3668e666e683be80c8e6f70f3 Mon Sep 17 00:00:00 2001
From: Derek Nola <derek.nola@suse.com>
Date: Fri, 22 Apr 2022 15:02:07 -0700
Subject: [PATCH] [Release-1.22] Secrets Encryption: Add RetryOnConflict around
 updating nodes (#5498)

* Add RetryOnConflict around updating nodes

Signed-off-by: Derek Nola <derek.nola@suse.com>
---
 pkg/secretsencrypt/controller.go | 39 ++++++++++++++++++++++++--------
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go
index 4c2372041a..e3519fa36e 100644
--- a/pkg/secretsencrypt/controller.go
+++ b/pkg/secretsencrypt/controller.go
@@ -18,6 +18,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/pager"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/retry"
 )
 
 const (
@@ -57,7 +58,7 @@ func Register(
 }
 
 // onChangeNode handles changes to Nodes. We are looking for a specific annotation change
-func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, error) {
+func (h *handler) onChangeNode(nodeName string, node *corev1.Node) (*corev1.Node, error) {
 	if node == nil {
 		return nil, nil
 	}
@@ -80,8 +81,16 @@ func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, err
 		return node, err
 	}
 	ann = EncryptionReencryptActive + "-" + reencryptHash
-	node.Annotations[EncryptionHashAnnotation] = ann
-	node, err = h.nodes.Update(node)
+
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		node, err = h.nodes.Get(nodeName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		node.Annotations[EncryptionHashAnnotation] = ann
+		_, err = h.nodes.Update(node)
+		return err
+	})
 	if err != nil {
 		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
 		return node, err
@@ -94,11 +103,16 @@ func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, err
 
 	// If skipping, revert back to the previous stage
 	if h.controlConfig.EncryptSkip {
-		BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime)
-		if node, err := h.nodes.Update(node); err != nil {
-			return node, err
-		}
-		return node, nil
+		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			node, err = h.nodes.Get(nodeName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime)
+			_, err = h.nodes.Update(node)
+			return err
+		})
+		return node, err
 	}
 
 	// Remove last key
@@ -118,7 +132,14 @@ func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, err
 		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
 		return node, err
 	}
-	if err := WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished); err != nil {
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		node, err = h.nodes.Get(nodeName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		return WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished)
+	})
+	if err != nil {
 		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
 		return node, err
 	}