From 533d4bfd546a7a11fb6b358265e0c6facc4e24ae Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Fri, 24 Mar 2017 12:46:55 +0200 Subject: [PATCH 1/2] Fixes: 1. Get certs for a dead leader. 2. Append tokens. --- .../reactive/kubernetes_master.py | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index cfa718780c..2b98aff059 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -158,18 +158,28 @@ def setup_leader_authentication(): api_opts.add('--token-auth-file', known_tokens) api_opts.add('--service-cluster-ip-range', service_cidr()) hookenv.status_set('maintenance', 'Rendering authentication templates.') - if not os.path.isfile(basic_auth): - setup_basic_auth('admin', 'admin', 'admin') - if not os.path.isfile(known_tokens): - setup_tokens(None, 'admin', 'admin') - setup_tokens(None, 'kubelet', 'kubelet') - setup_tokens(None, 'kube_proxy', 'kube_proxy') - # Generate the default service account token key - os.makedirs('/etc/kubernetes', exist_ok=True) - cmd = ['openssl', 'genrsa', '-out', service_key, - '2048'] - check_call(cmd) + # Try to fetch data from leadership broadcast. + contents = charms.leadership.leader_get(service_key) + if contents is not None: + # Since there was a leader in the past and we are bootstrapping + # the leader here, all masters were killed and a new one is + # added again. + keys = [service_key, basic_auth, known_tokens] + get_keys_from_leader(keys) + else: + if not os.path.isfile(basic_auth): + setup_basic_auth('admin', 'admin', 'admin') + if not os.path.isfile(known_tokens): + setup_tokens(None, 'admin', 'admin') + setup_tokens(None, 'kubelet', 'kubelet') + setup_tokens(None, 'kube_proxy', 'kube_proxy') + # Generate the default service account token key + os.makedirs('/etc/kubernetes', exist_ok=True) + cmd = ['openssl', 'genrsa', '-out', service_key, + '2048'] + check_call(cmd) + api_opts.add('--service-account-key-file', service_key) controller_opts.add('--service-account-private-key-file', service_key) @@ -199,15 +209,27 @@ def setup_non_leader_authentication(): basic_auth = '/srv/kubernetes/basic_auth.csv' known_tokens = '/srv/kubernetes/known_tokens.csv' + hookenv.status_set('maintenance', 'Rendering authentication templates.') + + # Set an array for looping logic + keys = [service_key, basic_auth, known_tokens] + get_keys_from_leader(keys) + + api_opts.add('--basic-auth-file', basic_auth) + api_opts.add('--token-auth-file', known_tokens) + api_opts.add('--service-cluster-ip-range', service_cidr()) + api_opts.add('--service-account-key-file', service_key) + controller_opts.add('--service-account-private-key-file', service_key) + + set_state('authentication.setup') + + +def get_keys_from_leader(keys): # This races with other codepaths, and seems to require being created first # This block may be extracted later, but for now seems to work as intended os.makedirs('/etc/kubernetes', exist_ok=True) os.makedirs('/srv/kubernetes', exist_ok=True) - hookenv.status_set('maintenance', 'Rendering authentication templates.') - - # Set an array for looping logic - keys = [service_key, basic_auth, known_tokens] for k in keys: # If the path does not exist, assume we need it if not os.path.exists(k): @@ -223,14 +245,6 @@ def setup_non_leader_authentication(): with open(k, 'w+') as fp: fp.write(contents) - api_opts.add('--basic-auth-file', basic_auth) - api_opts.add('--token-auth-file', known_tokens) - api_opts.add('--service-cluster-ip-range', service_cidr()) - api_opts.add('--service-account-key-file', service_key) - controller_opts.add('--service-account-private-key-file', service_key) - - set_state('authentication.setup') - @when('kubernetes-master.components.installed') def set_app_version(): @@ -760,8 +774,8 @@ def setup_tokens(token, username, user): if not token: alpha = string.ascii_letters + string.digits token = ''.join(random.SystemRandom().choice(alpha) for _ in range(32)) - with open(known_tokens, 'w') as stream: - stream.write('{0},{1},{2}'.format(token, username, user)) + with open(known_tokens, 'a') as stream: + stream.write('{0},{1},{2}\n'.format(token, username, user)) def all_kube_system_pods_running(): From 12a5c3a2f31a758d85c1cb9bce2d999d73b00f1d Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Mon, 27 Mar 2017 16:59:43 +0300 Subject: [PATCH 2/2] Fixing bug: non-leaders should retry fetching auth keys --- .../reactive/kubernetes_master.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 2b98aff059..2487483f0e 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -159,15 +159,9 @@ def setup_leader_authentication(): api_opts.add('--service-cluster-ip-range', service_cidr()) hookenv.status_set('maintenance', 'Rendering authentication templates.') - # Try to fetch data from leadership broadcast. - contents = charms.leadership.leader_get(service_key) - if contents is not None: - # Since there was a leader in the past and we are bootstrapping - # the leader here, all masters were killed and a new one is - # added again. - keys = [service_key, basic_auth, known_tokens] - get_keys_from_leader(keys) - else: + # Try first to fetch data from an old leadership broadcast. + keys = [service_key, basic_auth, known_tokens] + if not get_keys_from_leader(keys): if not os.path.isfile(basic_auth): setup_basic_auth('admin', 'admin', 'admin') if not os.path.isfile(known_tokens): @@ -211,9 +205,10 @@ def setup_non_leader_authentication(): hookenv.status_set('maintenance', 'Rendering authentication templates.') - # Set an array for looping logic keys = [service_key, basic_auth, known_tokens] - get_keys_from_leader(keys) + if not get_keys_from_leader(keys): + # the keys were not retrieved. Non-leaders have to retry. + return api_opts.add('--basic-auth-file', basic_auth) api_opts.add('--token-auth-file', known_tokens) @@ -225,6 +220,16 @@ def setup_non_leader_authentication(): def get_keys_from_leader(keys): + """ + Gets the broadcasted keys from the leader and stores them in + the corresponding files. + + Args: + keys: list of keys. Keys are actually files on the FS. + + Returns: True if all key were fetched, False if not. + + """ # This races with other codepaths, and seems to require being created first # This block may be extracted later, but for now seems to work as intended os.makedirs('/etc/kubernetes', exist_ok=True) @@ -240,10 +245,11 @@ def get_keys_from_leader(keys): msg = "Waiting on leaders crypto keys." hookenv.status_set('waiting', msg) hookenv.log('Missing content for file {}'.format(k)) - return + return False # Write out the file and move on to the next item with open(k, 'w+') as fp: fp.write(contents) + return True @when('kubernetes-master.components.installed')