Non leaders should overwrite any local copies of keys they have with what the leader has.

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -247,7 +247,9 @@ def setup_non_leader_authentication():
     known_tokens = '/root/cdk/known_tokens.csv'
 
     keys = [service_key, basic_auth, known_tokens]
-    if not get_keys_from_leader(keys):
+    # The source of truth for non-leaders is the leader.
+    # Therefore we overwrite_local with whatever the leader has.
+    if not get_keys_from_leader(keys, overwrite_local=True):
         # the keys were not retrieved. Non-leaders have to retry.
         return
 
@@ -268,7 +270,7 @@ def setup_non_leader_authentication():
     set_state('authentication.setup')
 
 
-def get_keys_from_leader(keys):
+def get_keys_from_leader(keys, overwrite_local=False):
     """
     Gets the broadcasted keys from the leader and stores them in
     the corresponding files.
@@ -285,7 +287,7 @@ def get_keys_from_leader(keys):
 
     for k in keys:
         # If the path does not exist, assume we need it
-        if not os.path.exists(k):
+        if not os.path.exists(k) or overwrite_local:
             # Fetch data from leadership broadcast
             contents = charms.leadership.leader_get(k)
             # Default to logging the warning and wait for leader data to be set