#!/usr/bin/env python # Copyright 2015 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from shlex import split from subprocess import call from subprocess import check_call from subprocess import check_output from charms.docker.compose import Compose from charms.reactive import hook from charms.reactive import remove_state from charms.reactive import set_state from charms.reactive import when from charms.reactive import when_any from charms.reactive import when_not from charmhelpers.core import hookenv from charmhelpers.core.hookenv import is_leader from charmhelpers.core.hookenv import leader_set from charmhelpers.core.hookenv import leader_get from charmhelpers.core.templating import render from charmhelpers.core import unitdata from charmhelpers.core.host import chdir import tlslib @when('leadership.is_leader') def i_am_leader(): '''The leader is the Kubernetes master node. ''' leader_set({'master-address': hookenv.unit_private_ip()}) @when_not('tls.client.authorization.required') def configure_easrsa(): '''Require the tls layer to generate certificates with "clientAuth". ''' # By default easyrsa generates the server certificates without clientAuth # Setting this state before easyrsa is configured ensures the tls layer is # configured to generate certificates with client authentication. set_state('tls.client.authorization.required') domain = hookenv.config().get('dns_domain') cidr = hookenv.config().get('cidr') sdn_ip = get_sdn_ip(cidr) # Create extra sans that the tls layer will add to the server cert. extra_sans = [ sdn_ip, 'kubernetes', 'kubernetes.{0}'.format(domain), 'kubernetes.default', 'kubernetes.default.svc', 'kubernetes.default.svc.{0}'.format(domain) ] unitdata.kv().set('extra_sans', extra_sans) @hook('config-changed') def config_changed(): '''If the configuration values change, remove the available states.''' config = hookenv.config() if any(config.changed(key) for key in config.keys()): hookenv.log('The configuration options have changed.') # Use the Compose class that encapsulates the docker-compose commands. compose = Compose('files/kubernetes') if is_leader(): hookenv.log('Removing master container and kubelet.available state.') # noqa # Stop and remove the Kubernetes kubelet container. compose.kill('master') compose.rm('master') compose.kill('proxy') compose.rm('proxy') # Remove the state so the code can react to restarting kubelet. remove_state('kubelet.available') else: hookenv.log('Removing kubelet container and kubelet.available state.') # noqa # Stop and remove the Kubernetes kubelet container. compose.kill('kubelet') compose.rm('kubelet') # Remove the state so the code can react to restarting kubelet. remove_state('kubelet.available') hookenv.log('Removing proxy container and proxy.available state.') # Stop and remove the Kubernetes proxy container. compose.kill('proxy') compose.rm('proxy') # Remove the state so the code can react to restarting proxy. remove_state('proxy.available') if config.changed('version'): hookenv.log('The version changed removing the states so the new ' 'version of kubectl will be downloaded.') remove_state('kubectl.downloaded') remove_state('kubeconfig.created') @when('tls.server.certificate available') @when_not('k8s.server.certificate available') def server_cert(): '''When the server certificate is available, get the server certificate from the charm unitdata and write it to the kubernetes directory. ''' server_cert = '/srv/kubernetes/server.crt' server_key = '/srv/kubernetes/server.key' # Save the server certificate from unit data to the destination. tlslib.server_cert(None, server_cert, user='ubuntu', group='ubuntu') # Copy the server key from the default location to the destination. tlslib.server_key(None, server_key, user='ubuntu', group='ubuntu') set_state('k8s.server.certificate available') @when('tls.client.certificate available') @when_not('k8s.client.certficate available') def client_cert(): '''When the client certificate is available, get the client certificate from the charm unitdata and write it to the kubernetes directory. ''' client_cert = '/srv/kubernetes/client.crt' client_key = '/srv/kubernetes/client.key' # Save the client certificate from the default location to the destination. tlslib.client_cert(None, client_cert, user='ubuntu', group='ubuntu') # Copy the client key from the default location to the destination. tlslib.client_key(None, client_key, user='ubuntu', group='ubuntu') set_state('k8s.client.certficate available') @when('tls.certificate.authority available') @when_not('k8s.certificate.authority available') def ca(): '''When the Certificate Authority is available, copy the CA from the default location to the /srv/kubernetes directory. ''' ca_crt = '/srv/kubernetes/ca.crt' # Copy the Certificate Authority to the destination directory. tlslib.ca(None, ca_crt, user='ubuntu', group='ubuntu') set_state('k8s.certificate.authority available') @when('kubelet.available', 'leadership.is_leader') @when_not('kubedns.available', 'skydns.available') def launch_dns(): '''Create the "kube-system" namespace, the kubedns resource controller, and the kubedns service. ''' hookenv.log('Creating kubernetes kubedns on the master node.') # Only launch and track this state on the leader. # Launching duplicate kubeDNS rc will raise an error # Run a command to check if the apiserver is responding. return_code = call(split('kubectl cluster-info')) if return_code != 0: hookenv.log('kubectl command failed, waiting for apiserver to start.') remove_state('kubedns.available') # Return without setting kubedns.available so this method will retry. return # Check for the "kube-system" namespace. return_code = call(split('kubectl get namespace kube-system')) if return_code != 0: # Create the kube-system namespace that is used by the kubedns files. check_call(split('kubectl create namespace kube-system')) # Check for the kubedns replication controller. return_code = call(split('kubectl get -f files/manifests/kubedns-controller.yaml')) if return_code != 0: # Create the kubedns replication controller from the rendered file. check_call(split('kubectl create -f files/manifests/kubedns-controller.yaml')) # Check for the kubedns service. return_code = call(split('kubectl get -f files/manifests/kubedns-svc.yaml')) if return_code != 0: # Create the kubedns service from the rendered file. check_call(split('kubectl create -f files/manifests/kubedns-svc.yaml')) set_state('kubedns.available') @when('skydns.available', 'leadership.is_leader') def convert_to_kubedns(): '''Delete the skydns containers to make way for the kubedns containers.''' hookenv.log('Deleteing the old skydns deployment.') # Delete the skydns replication controller. return_code = call(split('kubectl delete rc kube-dns-v11')) # Delete the skydns service. return_code = call(split('kubectl delete svc kube-dns')) remove_state('skydns.available') @when('docker.available') @when_not('etcd.available') def relation_message(): '''Take over messaging to let the user know they are pending a relationship to the ETCD cluster before going any further. ''' status_set('waiting', 'Waiting for relation to ETCD') @when('kubeconfig.created') @when('etcd.available') @when_not('kubelet.available', 'proxy.available') def start_kubelet(etcd): '''Run the hyperkube container that starts the kubernetes services. When the leader, run the master services (apiserver, controller, scheduler, proxy) using the master.json from the rendered manifest directory. When a follower, start the node services (kubelet, and proxy). ''' render_files(etcd) # Use the Compose class that encapsulates the docker-compose commands. compose = Compose('files/kubernetes') status_set('maintenance', 'Starting the Kubernetes services.') if is_leader(): compose.up('master') compose.up('proxy') set_state('kubelet.available') # Open the secure port for api-server. hookenv.open_port(6443) else: # Start the Kubernetes kubelet container using docker-compose. compose.up('kubelet') set_state('kubelet.available') # Start the Kubernetes proxy container using docker-compose. compose.up('proxy') set_state('proxy.available') status_set('active', 'Kubernetes services started') @when('docker.available') @when_not('kubectl.downloaded') def download_kubectl(): '''Download the kubectl binary to test and interact with the cluster.''' status_set('maintenance', 'Downloading the kubectl binary') version = hookenv.config()['version'] cmd = 'wget -nv -O /usr/local/bin/kubectl https://storage.googleapis.com' \ '/kubernetes-release/release/{0}/bin/linux/{1}/kubectl' cmd = cmd.format(version, arch()) hookenv.log('Downloading kubelet: {0}'.format(cmd)) check_call(split(cmd)) cmd = 'chmod +x /usr/local/bin/kubectl' check_call(split(cmd)) set_state('kubectl.downloaded') @when('kubectl.downloaded', 'leadership.is_leader', 'k8s.certificate.authority available', 'k8s.client.certficate available') # noqa @when_not('kubeconfig.created') def master_kubeconfig(): '''Create the kubernetes configuration for the master unit. The master should create a package with the client credentials so the user can interact securely with the apiserver.''' hookenv.log('Creating Kubernetes configuration for master node.') directory = '/srv/kubernetes' ca = '/srv/kubernetes/ca.crt' key = '/srv/kubernetes/client.key' cert = '/srv/kubernetes/client.crt' # Get the public address of the apiserver so users can access the master. server = 'https://{0}:{1}'.format(hookenv.unit_public_ip(), '6443') # Create the client kubeconfig so users can access the master node. create_kubeconfig(directory, server, ca, key, cert) # Copy the kubectl binary to this directory. cmd = 'cp -v /usr/local/bin/kubectl {0}'.format(directory) check_call(split(cmd)) # Use a context manager to run the tar command in a specific directory. with chdir(directory): # Create a package with kubectl and the files to use it externally. cmd = 'tar -cvzf /home/ubuntu/kubectl_package.tar.gz ca.crt ' \ 'client.key client.crt kubectl kubeconfig' check_call(split(cmd)) # This sets up the client workspace consistently on the leader and nodes. node_kubeconfig() set_state('kubeconfig.created') @when('kubectl.downloaded', 'k8s.certificate.authority available', 'k8s.server.certificate available') # noqa @when_not('kubeconfig.created', 'leadership.is_leader') def node_kubeconfig(): '''Create the kubernetes configuration (kubeconfig) for this unit. The the nodes will create a kubeconfig with the server credentials so the services can interact securely with the apiserver.''' hookenv.log('Creating Kubernetes configuration for worker node.') directory = '/var/lib/kubelet' ca = '/srv/kubernetes/ca.crt' cert = '/srv/kubernetes/server.crt' key = '/srv/kubernetes/server.key' # Get the private address of the apiserver for communication between units. server = 'https://{0}:{1}'.format(leader_get('master-address'), '6443') # Create the kubeconfig for the other services. kubeconfig = create_kubeconfig(directory, server, ca, key, cert) # Install the kubeconfig in the root user's home directory. install_kubeconfig(kubeconfig, '/root/.kube', 'root') # Install the kubeconfig in the ubunut user's home directory. install_kubeconfig(kubeconfig, '/home/ubuntu/.kube', 'ubuntu') set_state('kubeconfig.created') @when('proxy.available') @when_not('cadvisor.available') def start_cadvisor(): '''Start the cAdvisor container that gives metrics about the other application containers on this system. ''' compose = Compose('files/kubernetes') compose.up('cadvisor') hookenv.open_port(8088) status_set('active', 'cadvisor running on port 8088') set_state('cadvisor.available') @when('kubelet.available', 'kubeconfig.created') @when_any('proxy.available', 'cadvisor.available', 'kubedns.available') def final_message(): '''Issue some final messages when the services are started. ''' # TODO: Run a simple/quick health checks before issuing this message. status_set('active', 'Kubernetes running.') def gather_sdn_data(): '''Get the Software Defined Network (SDN) information and return it as a dictionary. ''' sdn_data = {} # The dictionary named 'pillar' is a construct of the k8s template files. pillar = {} # SDN Providers pass data via the unitdata.kv module db = unitdata.kv() # Ideally the DNS address should come from the sdn cidr. subnet = db.get('sdn_subnet') if subnet: # Generate the DNS ip address on the SDN cidr (this is desired). pillar['dns_server'] = get_dns_ip(subnet) else: # There is no SDN cider fall back to the kubernetes config cidr option. pillar['dns_server'] = get_dns_ip(hookenv.config().get('cidr')) # The pillar['dns_domain'] value is used in the kubedns-controller.yaml pillar['dns_domain'] = hookenv.config().get('dns_domain') # Use a 'pillar' dictionary so we can reuse the upstream kubedns templates. sdn_data['pillar'] = pillar return sdn_data def install_kubeconfig(kubeconfig, directory, user): '''Copy the a file from the target to a new directory creating directories if necessary. ''' # The file and directory must be owned by the correct user. chown = 'chown {0}:{0} {1}' if not os.path.isdir(directory): os.makedirs(directory) # Change the ownership of the config file to the right user. check_call(split(chown.format(user, directory))) # kubectl looks for a file named "config" in the ~/.kube directory. config = os.path.join(directory, 'config') # Copy the kubeconfig file to the directory renaming it to "config". cmd = 'cp -v {0} {1}'.format(kubeconfig, config) check_call(split(cmd)) # Change the ownership of the config file to the right user. check_call(split(chown.format(user, config))) def create_kubeconfig(directory, server, ca, key, cert, user='ubuntu'): '''Create a configuration for kubernetes in a specific directory using the supplied arguments, return the path to the file.''' context = 'default-context' cluster_name = 'kubernetes' # Ensure the destination directory exists. if not os.path.isdir(directory): os.makedirs(directory) # The configuration file should be in this directory named kubeconfig. kubeconfig = os.path.join(directory, 'kubeconfig') # Create the config file with the address of the master server. cmd = 'kubectl config set-cluster --kubeconfig={0} {1} ' \ '--server={2} --certificate-authority={3}' check_call(split(cmd.format(kubeconfig, cluster_name, server, ca))) # Create the credentials using the client flags. cmd = 'kubectl config set-credentials --kubeconfig={0} {1} ' \ '--client-key={2} --client-certificate={3}' check_call(split(cmd.format(kubeconfig, user, key, cert))) # Create a default context with the cluster. cmd = 'kubectl config set-context --kubeconfig={0} {1} ' \ '--cluster={2} --user={3}' check_call(split(cmd.format(kubeconfig, context, cluster_name, user))) # Make the config use this new context. cmd = 'kubectl config use-context --kubeconfig={0} {1}' check_call(split(cmd.format(kubeconfig, context))) hookenv.log('kubectl configuration created at {0}.'.format(kubeconfig)) return kubeconfig def get_dns_ip(cidr): '''Get an IP address for the DNS server on the provided cidr.''' # Remove the range from the cidr. ip = cidr.split('/')[0] # Take the last octet off the IP address and replace it with 10. return '.'.join(ip.split('.')[0:-1]) + '.10' def get_sdn_ip(cidr): '''Get the IP address for the SDN gateway based on the provided cidr.''' # Remove the range from the cidr. ip = cidr.split('/')[0] # Remove the last octet and replace it with 1. return '.'.join(ip.split('.')[0:-1]) + '.1' def render_files(reldata=None): '''Use jinja templating to render the docker-compose.yml and master.json file to contain the dynamic data for the configuration files.''' context = {} # Load the context data with SDN data. context.update(gather_sdn_data()) # Add the charm configuration data to the context. context.update(hookenv.config()) if reldata: connection_string = reldata.get_connection_string() # Define where the etcd tls files will be kept. etcd_dir = '/etc/ssl/etcd' # Create paths to the etcd client ca, key, and cert file locations. ca = os.path.join(etcd_dir, 'client-ca.pem') key = os.path.join(etcd_dir, 'client-key.pem') cert = os.path.join(etcd_dir, 'client-cert.pem') # Save the client credentials (in relation data) to the paths provided. reldata.save_client_credentials(key, cert, ca) # Update the context so the template has the etcd information. context.update({'etcd_dir': etcd_dir, 'connection_string': connection_string, 'etcd_ca': ca, 'etcd_key': key, 'etcd_cert': cert}) charm_dir = hookenv.charm_dir() rendered_kube_dir = os.path.join(charm_dir, 'files/kubernetes') if not os.path.exists(rendered_kube_dir): os.makedirs(rendered_kube_dir) rendered_manifest_dir = os.path.join(charm_dir, 'files/manifests') if not os.path.exists(rendered_manifest_dir): os.makedirs(rendered_manifest_dir) # Update the context with extra values, arch, manifest dir, and private IP. context.update({'arch': arch(), 'master_address': leader_get('master-address'), 'manifest_directory': rendered_manifest_dir, 'public_address': hookenv.unit_get('public-address'), 'private_address': hookenv.unit_get('private-address')}) # Adapted from: http://kubernetes.io/docs/getting-started-guides/docker/ target = os.path.join(rendered_kube_dir, 'docker-compose.yml') # Render the files/kubernetes/docker-compose.yml file that contains the # definition for kubelet and proxy. render('docker-compose.yml', target, context) if is_leader(): # Source: https://github.com/kubernetes/...master/cluster/images/hyperkube # noqa target = os.path.join(rendered_manifest_dir, 'master.json') # Render the files/manifests/master.json that contains parameters for # the apiserver, controller, and controller-manager render('master.json', target, context) # Source: ...cluster/addons/dns/kubedns-svc.yaml.in target = os.path.join(rendered_manifest_dir, 'kubedns-svc.yaml') # Render files/kubernetes/kubedns-svc.yaml for the DNS service. render('kubedns-svc.yaml', target, context) # Source: ...cluster/addons/dns/kubedns-controller.yaml.in target = os.path.join(rendered_manifest_dir, 'kubedns-controller.yaml') # Render files/kubernetes/kubedns-controller.yaml for the DNS pod. render('kubedns-controller.yaml', target, context) def status_set(level, message): '''Output status message with leadership information.''' if is_leader(): message = '{0} (master) '.format(message) hookenv.status_set(level, message) def arch(): '''Return the package architecture as a string. Raise an exception if the architecture is not supported by kubernetes.''' # Get the package architecture for this system. architecture = check_output(['dpkg', '--print-architecture']).rstrip() # Convert the binary result into a string. architecture = architecture.decode('utf-8') # Validate the architecture is supported by kubernetes. if architecture not in ['amd64', 'arm', 'arm64', 'ppc64le', 's390x']: message = 'Unsupported machine architecture: {0}'.format(architecture) status_set('blocked', message) raise Exception(message) return architecture