From 422a7ec49fcf72793f0ca20ff40ed05d59097662 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Mon, 15 Apr 2019 16:19:09 -0500 Subject: [PATCH] re-home juju related source --- cluster/juju/README.md | 7 + cluster/juju/bundles/OWNERS | 4 - cluster/juju/bundles/README.md | 208 -- cluster/juju/bundles/local.yaml.base | 18 - cluster/juju/config-default.sh | 16 - cluster/juju/config-test.sh | 17 - cluster/juju/identify-leaders.py | 31 - cluster/juju/kube-system-ns.yaml | 4 - .../layers/kubeapi-load-balancer/README.md | 5 - .../layers/kubeapi-load-balancer/config.yaml | 15 - .../layers/kubeapi-load-balancer/copyright | 13 - .../layers/kubeapi-load-balancer/icon.svg | 412 ---- .../layers/kubeapi-load-balancer/layer.yaml | 14 - .../kubeapi-load-balancer/metadata.yaml | 23 - .../layers/kubeapi-load-balancer/metrics.yaml | 2 - .../reactive/load_balancer.py | 249 --- .../templates/apilb.conf | 41 - cluster/juju/layers/kubernetes-e2e/README.md | 135 -- .../juju/layers/kubernetes-e2e/actions.yaml | 19 - .../juju/layers/kubernetes-e2e/actions/test | 53 - .../juju/layers/kubernetes-e2e/config.yaml | 6 - cluster/juju/layers/kubernetes-e2e/icon.svg | 362 ---- cluster/juju/layers/kubernetes-e2e/layer.yaml | 12 - .../juju/layers/kubernetes-e2e/metadata.yaml | 32 - .../kubernetes-e2e/reactive/kubernetes_e2e.py | 220 -- cluster/juju/layers/kubernetes-e2e/tox.ini | 12 - .../juju/layers/kubernetes-master/README.md | 132 -- .../layers/kubernetes-master/actions.yaml | 50 - .../kubernetes-master/actions/create-rbd-pv | 325 --- .../actions/namespace-create | 59 - .../actions/namespace-delete | 1 - .../kubernetes-master/actions/namespace-list | 1 - .../layers/kubernetes-master/actions/restart | 14 - .../layers/kubernetes-master/actions/upgrade | 5 - .../juju/layers/kubernetes-master/config.yaml | 164 -- .../juju/layers/kubernetes-master/copyright | 13 - .../kubernetes-master/debug-scripts/kubectl | 15 - .../debug-scripts/kubernetes-master-services | 9 - .../exec.d/vmware-patch/charm-pre-install | 17 - .../juju/layers/kubernetes-master/icon.svg | 362 ---- .../juju/layers/kubernetes-master/layer.yaml | 37 - .../lib/charms/kubernetes/common.py | 71 - .../layers/kubernetes-master/metadata.yaml | 74 - .../layers/kubernetes-master/metrics.yaml | 34 - .../reactive/kubernetes_master.py | 1903 ----------------- .../templates/ceph-secret.yaml | 7 - .../kubernetes-master/templates/ceph.conf | 18 - .../templates/create-namespace.yaml.j2 | 6 - .../templates/rbd-persistent-volume.yaml | 25 - cluster/juju/layers/kubernetes-master/tox.ini | 12 - .../juju/layers/kubernetes-worker/HACKING.md | 25 - .../juju/layers/kubernetes-worker/README.md | 131 -- .../layers/kubernetes-worker/actions.yaml | 56 - .../layers/kubernetes-worker/actions/microbot | 76 - .../layers/kubernetes-worker/actions/pause | 28 - .../layers/kubernetes-worker/actions/registry | 139 -- .../layers/kubernetes-worker/actions/resume | 8 - .../layers/kubernetes-worker/actions/upgrade | 5 - .../juju/layers/kubernetes-worker/config.yaml | 108 - .../juju/layers/kubernetes-worker/copyright | 13 - .../kubernetes-worker/debug-scripts/inotify | 8 - .../kubernetes-worker/debug-scripts/kubectl | 15 - .../debug-scripts/kubernetes-worker-services | 9 - .../exec.d/docker-compose/charm-pre-install | 2 - .../exec.d/vmware-patch/charm-pre-install | 17 - .../juju/layers/kubernetes-worker/icon.svg | 362 ---- .../juju/layers/kubernetes-worker/layer.yaml | 42 - .../lib/charms/kubernetes/common.py | 35 - .../layers/kubernetes-worker/metadata.yaml | 73 - .../layers/kubernetes-worker/metrics.yaml | 2 - .../reactive/kubernetes_worker.py | 1500 ------------- .../kubernetes-worker/registry-configmap.yaml | 6 - .../templates/default-http-backend.yaml | 44 - .../templates/ingress-daemon-set.yaml | 179 -- .../templates/microbot-example.yaml | 63 - .../templates/nfs-provisioner.yaml | 44 - .../kubernetes-worker/templates/registry.yaml | 118 - .../layers/kubernetes-worker/wheelhouse.txt | 1 - cluster/juju/prereqs/OWNERS | 5 - cluster/juju/prereqs/ubuntu-juju.sh | 48 - cluster/juju/return-node-ips.py | 29 - cluster/juju/util.sh | 153 -- 82 files changed, 7 insertions(+), 8621 deletions(-) create mode 100644 cluster/juju/README.md delete mode 100644 cluster/juju/bundles/OWNERS delete mode 100644 cluster/juju/bundles/README.md delete mode 100644 cluster/juju/bundles/local.yaml.base delete mode 100644 cluster/juju/config-default.sh delete mode 100644 cluster/juju/config-test.sh delete mode 100755 cluster/juju/identify-leaders.py delete mode 100644 cluster/juju/kube-system-ns.yaml delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/README.md delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/config.yaml delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/copyright delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/icon.svg delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/layer.yaml delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/metadata.yaml delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/metrics.yaml delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/reactive/load_balancer.py delete mode 100644 cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf delete mode 100644 cluster/juju/layers/kubernetes-e2e/README.md delete mode 100644 cluster/juju/layers/kubernetes-e2e/actions.yaml delete mode 100755 cluster/juju/layers/kubernetes-e2e/actions/test delete mode 100644 cluster/juju/layers/kubernetes-e2e/config.yaml delete mode 100644 cluster/juju/layers/kubernetes-e2e/icon.svg delete mode 100644 cluster/juju/layers/kubernetes-e2e/layer.yaml delete mode 100644 cluster/juju/layers/kubernetes-e2e/metadata.yaml delete mode 100644 cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py delete mode 100644 cluster/juju/layers/kubernetes-e2e/tox.ini delete mode 100644 cluster/juju/layers/kubernetes-master/README.md delete mode 100644 cluster/juju/layers/kubernetes-master/actions.yaml delete mode 100755 cluster/juju/layers/kubernetes-master/actions/create-rbd-pv delete mode 100755 cluster/juju/layers/kubernetes-master/actions/namespace-create delete mode 120000 cluster/juju/layers/kubernetes-master/actions/namespace-delete delete mode 120000 cluster/juju/layers/kubernetes-master/actions/namespace-list delete mode 100755 cluster/juju/layers/kubernetes-master/actions/restart delete mode 100755 cluster/juju/layers/kubernetes-master/actions/upgrade delete mode 100644 cluster/juju/layers/kubernetes-master/config.yaml delete mode 100644 cluster/juju/layers/kubernetes-master/copyright delete mode 100755 cluster/juju/layers/kubernetes-master/debug-scripts/kubectl delete mode 100755 cluster/juju/layers/kubernetes-master/debug-scripts/kubernetes-master-services delete mode 100755 cluster/juju/layers/kubernetes-master/exec.d/vmware-patch/charm-pre-install delete mode 100644 cluster/juju/layers/kubernetes-master/icon.svg delete mode 100644 cluster/juju/layers/kubernetes-master/layer.yaml delete mode 100644 cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py delete mode 100644 cluster/juju/layers/kubernetes-master/metadata.yaml delete mode 100644 cluster/juju/layers/kubernetes-master/metrics.yaml delete mode 100644 cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py delete mode 100644 cluster/juju/layers/kubernetes-master/templates/ceph-secret.yaml delete mode 100644 cluster/juju/layers/kubernetes-master/templates/ceph.conf delete mode 100644 cluster/juju/layers/kubernetes-master/templates/create-namespace.yaml.j2 delete mode 100644 cluster/juju/layers/kubernetes-master/templates/rbd-persistent-volume.yaml delete mode 100644 cluster/juju/layers/kubernetes-master/tox.ini delete mode 100644 cluster/juju/layers/kubernetes-worker/HACKING.md delete mode 100644 cluster/juju/layers/kubernetes-worker/README.md delete mode 100644 cluster/juju/layers/kubernetes-worker/actions.yaml delete mode 100755 cluster/juju/layers/kubernetes-worker/actions/microbot delete mode 100755 cluster/juju/layers/kubernetes-worker/actions/pause delete mode 100755 cluster/juju/layers/kubernetes-worker/actions/registry delete mode 100755 cluster/juju/layers/kubernetes-worker/actions/resume delete mode 100755 cluster/juju/layers/kubernetes-worker/actions/upgrade delete mode 100644 cluster/juju/layers/kubernetes-worker/config.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/copyright delete mode 100755 cluster/juju/layers/kubernetes-worker/debug-scripts/inotify delete mode 100755 cluster/juju/layers/kubernetes-worker/debug-scripts/kubectl delete mode 100755 cluster/juju/layers/kubernetes-worker/debug-scripts/kubernetes-worker-services delete mode 100644 cluster/juju/layers/kubernetes-worker/exec.d/docker-compose/charm-pre-install delete mode 100755 cluster/juju/layers/kubernetes-worker/exec.d/vmware-patch/charm-pre-install delete mode 100644 cluster/juju/layers/kubernetes-worker/icon.svg delete mode 100644 cluster/juju/layers/kubernetes-worker/layer.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/lib/charms/kubernetes/common.py delete mode 100644 cluster/juju/layers/kubernetes-worker/metadata.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/metrics.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py delete mode 100644 cluster/juju/layers/kubernetes-worker/registry-configmap.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/templates/default-http-backend.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/templates/ingress-daemon-set.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/templates/microbot-example.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/templates/nfs-provisioner.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/templates/registry.yaml delete mode 100644 cluster/juju/layers/kubernetes-worker/wheelhouse.txt delete mode 100644 cluster/juju/prereqs/OWNERS delete mode 100644 cluster/juju/prereqs/ubuntu-juju.sh delete mode 100755 cluster/juju/return-node-ips.py delete mode 100755 cluster/juju/util.sh diff --git a/cluster/juju/README.md b/cluster/juju/README.md new file mode 100644 index 0000000000..17486a8770 --- /dev/null +++ b/cluster/juju/README.md @@ -0,0 +1,7 @@ +# Juju Provider + +Source for charms and bundles related to Juju have moved to the +[Charmed Kubernetes][cdk-repo] GitHub organization. This directory will be +removed in a future release. + +[cdk-repo]: https://github.com/charmed-kubernetes diff --git a/cluster/juju/bundles/OWNERS b/cluster/juju/bundles/OWNERS deleted file mode 100644 index 1b8211edec..0000000000 --- a/cluster/juju/bundles/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: -- jlowdermilk diff --git a/cluster/juju/bundles/README.md b/cluster/juju/bundles/README.md deleted file mode 100644 index 0833fd1dc5..0000000000 --- a/cluster/juju/bundles/README.md +++ /dev/null @@ -1,208 +0,0 @@ -# kubernetes-bundle - -The kubernetes-bundle allows you to deploy the many services of -Kubernetes to a cloud environment and get started using the Kubernetes -technology quickly. - -## Kubernetes - -Kubernetes is an open source system for managing containerized -applications. Kubernetes uses [Docker](http://docker.com) to run -containerized applications. - -## Juju TL;DR - -The [Juju](https://jujucharms.com) system provides provisioning and -orchestration across a variety of clouds and bare metal. A juju bundle -describes collection of services and how they interrelate. `juju -quickstart` allows you to bootstrap a deployment environment and -deploy a bundle. - -## Dive in! - -#### Install Juju Quickstart - -You will need to -[install the Juju client](https://jujucharms.com/get-started) and -`juju-quickstart` as prerequisites. To deploy the bundle use -`juju-quickstart` which runs on Mac OS (`brew install -juju-quickstart`) or Ubuntu (`apt-get install juju-quickstart`). - -### Deploy a Kubernetes Bundle - -Use the 'juju quickstart' command to deploy a Kubernetes cluster to any cloud -supported by Juju. - -The charm store version of the Kubernetes bundle can be deployed as follows: - - juju quickstart u/kubernetes/kubernetes-cluster - -> Note: The charm store bundle may be locked to a specific Kubernetes release. - -Alternately you could deploy a Kubernetes bundle straight from github or a file: - - juju quickstart -i https://raw.githubusercontent.com/whitmo/bundle-kubernetes/master/bundles.yaml - -The command above does few things for you: - -- Starts a curses based gui for managing your cloud or MAAS credentials -- Looks for a bootstrapped deployment environment, and bootstraps if - required. This will launch a bootstrap node in your chosen - deployment environment (machine 0). -- Deploys the Juju GUI to your environment onto the bootstrap node. -- Provisions 4 machines, and deploys the Kubernetes services on top of - them (Kubernetes-master, two Kubernetes nodes using flannel, and etcd). -- Orchestrates the relations among the services, and exits. - -Now you should have a running Kubernetes. Run `juju status ---format=oneline` to see the address of your kubernetes-master unit. - -For further reading on [Juju Quickstart](https://pypi.python.org/pypi/juju-quickstart) - -Go to the [Getting started with Juju guide](https://kubernetes.io/docs/getting-started-guides/ubuntu/installation/#setting-up-kubernetes-with-juju) -for more information about deploying a development Kubernetes cluster. - -### Using the Kubernetes Client - -You'll need the Kubernetes command line client, -[kubectl](https://github.com/kubernetes/kubernetes/blob/master/docs/user-guide/kubectl/kubectl.md) -to interact with the created cluster. The kubectl command is -installed on the kubernetes-master charm. If you want to work with -the cluster from your computer you will need to install the binary -locally. - -You can access kubectl by a number ways using juju. - -via juju run: - - juju run --service kubernetes-master/0 "sudo kubectl get nodes" - -via juju ssh: - - juju ssh kubernetes-master/0 -t "sudo kubectl get nodes" - -You may also SSH to the kubernetes-master unit (`juju ssh kubernetes-master/0`) -and call kubectl from the command prompt. - -See the -[kubectl documentation](https://github.com/kubernetes/kubernetes/blob/master/docs/user-guide/kubectl/kubectl.md) -for more details of what can be done with the command line tool. - -### Scaling up the cluster - -You can add capacity by adding more Docker units: - - juju add-unit docker - -### Known Limitations - -Kubernetes currently has several platform specific functionality. For -example load balancers and persistence volumes only work with the -Google Compute provider at this time. - -The Juju integration uses the Kubernetes null provider. This means -external load balancers and storage can't be directly driven through -Kubernetes config files at this time. We look forward to adding these -capabilities to the charms. - - -## More about the components the bundle deploys - -### Kubernetes master - -The master controls the Kubernetes cluster. It manages for the worker -nodes and provides the primary interface for control by the user. - -### Kubernetes node - -The nodes are the servers that perform the work. Nodes must -communicate with the master and run the workloads that are assigned to -them. - -### Flannel-docker - -Flannel provides individual subnets for each machine in the cluster by -creating a -[software defined networking](http://en.wikipedia.org/wiki/Software-defined_networking). - -### Docker - -An open platform for distributed applications for developers and sysadmins. - -### Etcd - -Etcd persists state for Flannel and Kubernetes. It is a distributed -key-value store with an http interface. - - -## For further information on getting started with Juju - -Juju has complete documentation with regard to setup, and cloud -configuration on it's own -[documentation site](https://jujucharms.com/docs/). - -- [Getting Started](https://jujucharms.com/docs/stable/getting-started) -- [Using Juju](https://jujucharms.com/docs/stable/charms) - - -## Installing the kubectl outside of kubernetes-master unit - -Download the Kubernetes release from: -https://github.com/kubernetes/kubernetes/releases and extract -the release, you can then just directly use the cli binary at -./kubernetes/platforms/linux/amd64/kubectl - -You'll need the address of the kubernetes-master as environment variable : - - juju status kubernetes-master/0 - -Grab the public-address there and export it as KUBERNETES_MASTER -environment variable : - - export KUBERNETES_MASTER=$(juju status --format=oneline kubernetes-master | grep kubernetes-master | cut -d' ' -f3):8080 - -And now you can run kubectl on the command line : - - kubectl get no - -See the -[kubectl documentation](https://github.com/kubernetes/kubernetes/blob/master/docs/user-guide/kubectl/kubectl.md) -for more details of what can be done with the command line tool. - - -## Hacking on the kubernetes-bundle and associated charms - -The kubernetes-bundle is open source and available on github.com. If -you want to get started developing on the bundle you can clone it from -github. - - git clone https://github.com/kubernetes/kubernetes.git - -Go to the [Getting started with Juju guide](https://kubernetes.io/docs/getting-started-guides/ubuntu/installation/#setting-up-kubernetes-with-juju) -for more information about the bundle or charms. - -## How to contribute - -Send us pull requests! We'll send you a cookie if they include tests and docs. - - -## Current and Most Complete Information - -The charms and bundles are in the [kubernetes](https://github.com/kubernetes/kubernetes) -repository in github. - - - [kubernetes-master charm on GitHub](https://github.com/kubernetes/kubernetes/tree/master/cluster/juju/charms/trusty/kubernetes-master) - - [kubernetes charm on GitHub](https://github.com/kubernetes/kubernetes/tree/master/cluster/juju/charms/trusty/kubernetes) - - -More information about the -[Kubernetes project](https://github.com/kubernetes/kubernetes) -or check out the -[Kubernetes Documentation](https://github.com/kubernetes/kubernetes/tree/master/docs) -for more details about the Kubernetes concepts and terminology. - -Having a problem? Check the [Kubernetes issues database](https://github.com/kubernetes/kubernetes/issues) -for related issues. - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/juju/bundles/README.md?pixel)]() diff --git a/cluster/juju/bundles/local.yaml.base b/cluster/juju/bundles/local.yaml.base deleted file mode 100644 index dc01cf16b9..0000000000 --- a/cluster/juju/bundles/local.yaml.base +++ /dev/null @@ -1,18 +0,0 @@ -services: - kubernetes: - charm: __CHARM_DIR__/builds/kubernetes - annotations: - "gui-x": "600" - "gui-y": "0" - expose: true - num_units: 2 - etcd: - charm: cs:~containers/etcd - annotations: - "gui-x": "300" - "gui-y": "0" - num_units: 1 -relations: - - - "kubernetes:etcd" - - "etcd:db" -series: xenial diff --git a/cluster/juju/config-default.sh b/cluster/juju/config-default.sh deleted file mode 100644 index 05e5d63ea6..0000000000 --- a/cluster/juju/config-default.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/cluster/juju/config-test.sh b/cluster/juju/config-test.sh deleted file mode 100644 index eb58d012af..0000000000 --- a/cluster/juju/config-test.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -NUM_NODES=${NUM_NODES:-2} diff --git a/cluster/juju/identify-leaders.py b/cluster/juju/identify-leaders.py deleted file mode 100755 index dd0e80e339..0000000000 --- a/cluster/juju/identify-leaders.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from subprocess import check_output -import yaml - - -cmd = ['juju', 'run', '--application', 'kubernetes', '--format=yaml', 'is-leader'] -out = check_output(cmd) -try: - parsed_output = yaml.safe_load(out) - for unit in parsed_output: - standard_out = unit['Stdout'].rstrip() - unit_id = unit['UnitId'] - if 'True' in standard_out: - print(unit_id) -except: - pass diff --git a/cluster/juju/kube-system-ns.yaml b/cluster/juju/kube-system-ns.yaml deleted file mode 100644 index 986f4b4822..0000000000 --- a/cluster/juju/kube-system-ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kube-system diff --git a/cluster/juju/layers/kubeapi-load-balancer/README.md b/cluster/juju/layers/kubeapi-load-balancer/README.md deleted file mode 100644 index 5f5c1bb337..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# kubeapi-load-balancer - -Simple NGINX reverse proxy to lend a hand in HA kubernetes-master deployments. - - diff --git a/cluster/juju/layers/kubeapi-load-balancer/config.yaml b/cluster/juju/layers/kubeapi-load-balancer/config.yaml deleted file mode 100644 index 245d789063..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -options: - port: - type: int - default: 443 - description: The port to run the loadbalancer - extra_sans: - type: string - default: "" - description: | - Space-separated list of extra SAN entries to add to the x509 certificate - created for the load balancers. - proxy_read_timeout: - type: int - default: 90 - description: Timeout in seconds for reading a response from proxy server. diff --git a/cluster/juju/layers/kubeapi-load-balancer/copyright b/cluster/juju/layers/kubeapi-load-balancer/copyright deleted file mode 100644 index ac5e525c8e..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/copyright +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2016 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/cluster/juju/layers/kubeapi-load-balancer/icon.svg b/cluster/juju/layers/kubeapi-load-balancer/icon.svg deleted file mode 100644 index 7f2998ed40..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/icon.svg +++ /dev/null @@ -1,412 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - diff --git a/cluster/juju/layers/kubeapi-load-balancer/layer.yaml b/cluster/juju/layers/kubeapi-load-balancer/layer.yaml deleted file mode 100644 index 9cb22a3d4a..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/layer.yaml +++ /dev/null @@ -1,14 +0,0 @@ -repo: https://github.com/kubernetes/kubernetes.git -includes: - - 'layer:metrics' - - 'layer:nagios' - - 'layer:nginx' - - 'layer:tls-client' - - 'interface:public-address' -options: - tls-client: - ca_certificate_path: '/srv/kubernetes/ca.crt' - server_certificate_path: '/srv/kubernetes/server.crt' - server_key_path: '/srv/kubernetes/server.key' - client_certificate_path: '/srv/kubernetes/client.crt' - client_key_path: '/srv/kubernetes/client.key' diff --git a/cluster/juju/layers/kubeapi-load-balancer/metadata.yaml b/cluster/juju/layers/kubeapi-load-balancer/metadata.yaml deleted file mode 100644 index 0cabd1bf62..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/metadata.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: kubeapi-load-balancer -summary: Nginx Load Balancer -maintainers: - - Tim Van Steenburgh - - George Kraft - - Rye Terrell - - Konstantinos Tsakalozos - - Charles Butler - - Matthew Bruzek -description: | - A round robin Nginx load balancer to distribute traffic for kubernetes apiservers. -tags: - - misc -subordinate: false -series: - - xenial - - bionic -requires: - apiserver: - interface: http -provides: - loadbalancer: - interface: public-address diff --git a/cluster/juju/layers/kubeapi-load-balancer/metrics.yaml b/cluster/juju/layers/kubeapi-load-balancer/metrics.yaml deleted file mode 100644 index 0fcb3c1c48..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/metrics.yaml +++ /dev/null @@ -1,2 +0,0 @@ -metrics: - juju-units: {} diff --git a/cluster/juju/layers/kubeapi-load-balancer/reactive/load_balancer.py b/cluster/juju/layers/kubeapi-load-balancer/reactive/load_balancer.py deleted file mode 100644 index a1576bb4ab..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/reactive/load_balancer.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import socket -import subprocess - -from charms import layer -from charms.reactive import when, when_any, when_not -from charms.reactive import set_state, remove_state -from charms.reactive import hook -from charmhelpers.core import hookenv -from charmhelpers.core import host -from charmhelpers.contrib.charmsupport import nrpe -from charms.reactive.helpers import data_changed - -from charms.layer import nginx -from charms.layer import tls_client - -from subprocess import Popen -from subprocess import PIPE -from subprocess import STDOUT -from subprocess import CalledProcessError - - -apilb_nginx = """/var/log/nginx.*.log { - daily - missingok - rotate 14 - compress - delaycompress - notifempty - create 0640 www-data adm - sharedscripts - prerotate - if [ -d /etc/logrotate.d/httpd-prerotate ]; then \\ - run-parts /etc/logrotate.d/httpd-prerotate; \\ - fi \\ - endscript - postrotate - invoke-rc.d nginx rotate >/dev/null 2>&1 - endscript -}""" - - -def get_ingress_address(relation): - try: - network_info = hookenv.network_get(relation.relation_name) - except NotImplementedError: - network_info = [] - - if network_info and 'ingress-addresses' in network_info: - # just grab the first one for now, maybe be more robust here? - return network_info['ingress-addresses'][0] - else: - # if they don't have ingress-addresses they are running a juju that - # doesn't support spaces, so just return the private address - return hookenv.unit_get('private-address') - - -@when('certificates.available', 'website.available') -def request_server_certificates(tls, website): - '''Send the data that is required to create a server certificate for - this server.''' - # Use the public ip of this unit as the Common Name for the certificate. - common_name = hookenv.unit_public_ip() - # Create SANs that the tls layer will add to the server cert. - sans = [ - hookenv.unit_public_ip(), - get_ingress_address(website), - socket.gethostname(), - ] - # maybe they have extra names they want as SANs - extra_sans = hookenv.config('extra_sans') - if extra_sans and not extra_sans == "": - sans.extend(extra_sans.split()) - # Create a path safe name by removing path characters from the unit name. - certificate_name = hookenv.local_unit().replace('/', '_') - # Request a server cert with this information. - tls.request_server_cert(common_name, sans, certificate_name) - - -@when('config.changed.extra_sans', 'certificates.available', - 'website.available') -def update_certificate(tls, website): - # Using the config.changed.extra_sans flag to catch changes. - # IP changes will take ~5 minutes or so to propagate, but - # it will update. - request_server_certificates(tls, website) - - -@when('certificates.server.cert.available', - 'nginx.available', 'tls_client.server.certificate.written') -def kick_nginx(tls): - # we are just going to sighup it, but still want to avoid kicking it - # without need - if data_changed('cert', tls.get_server_cert()): - # certificate changed, so sighup nginx - hookenv.log("Certificate information changed, sending SIGHUP to nginx") - host.service_restart('nginx') - tls_client.reset_certificate_write_flag('server') - - -@when('config.changed.port') -def close_old_port(): - config = hookenv.config() - old_port = config.previous('port') - if not old_port: - return - try: - hookenv.close_port(old_port) - except CalledProcessError: - hookenv.log('Port %d already closed, skipping.' % old_port) - - -def maybe_write_apilb_logrotate_config(): - filename = '/etc/logrotate.d/apilb_nginx' - if not os.path.exists(filename): - # Set log rotation for apilb log file - with open(filename, 'w+') as fp: - fp.write(apilb_nginx) - - -@when('nginx.available', 'apiserver.available', - 'certificates.server.cert.available') -def install_load_balancer(apiserver, tls): - ''' Create the default vhost template for load balancing ''' - # Get the tls paths from the layer data. - layer_options = layer.options('tls-client') - server_cert_path = layer_options.get('server_certificate_path') - cert_exists = server_cert_path and os.path.isfile(server_cert_path) - server_key_path = layer_options.get('server_key_path') - key_exists = server_key_path and os.path.isfile(server_key_path) - # Do both the key and certificate exist? - if cert_exists and key_exists: - # At this point the cert and key exist, and they are owned by root. - chown = ['chown', 'www-data:www-data', server_cert_path] - - # Change the owner to www-data so the nginx process can read the cert. - subprocess.call(chown) - chown = ['chown', 'www-data:www-data', server_key_path] - - # Change the owner to www-data so the nginx process can read the key. - subprocess.call(chown) - - port = hookenv.config('port') - hookenv.open_port(port) - services = apiserver.services() - nginx.configure_site( - 'apilb', - 'apilb.conf', - server_name='_', - services=services, - port=port, - server_certificate=server_cert_path, - server_key=server_key_path, - proxy_read_timeout=hookenv.config('proxy_read_timeout') - ) - - maybe_write_apilb_logrotate_config() - hookenv.status_set('active', 'Loadbalancer ready.') - - -@hook('upgrade-charm') -def upgrade_charm(): - maybe_write_apilb_logrotate_config() - - -@when('nginx.available') -def set_nginx_version(): - ''' Surface the currently deployed version of nginx to Juju ''' - cmd = 'nginx -v' - p = Popen(cmd, shell=True, - stdin=PIPE, - stdout=PIPE, - stderr=STDOUT, - close_fds=True) - raw = p.stdout.read() - # The version comes back as: - # nginx version: nginx/1.10.0 (Ubuntu) - version = raw.split(b'/')[-1].split(b' ')[0] - hookenv.application_version_set(version.rstrip()) - - -@when('website.available') -def provide_application_details(website): - ''' re-use the nginx layer website relation to relay the hostname/port - to any consuming kubernetes-workers, or other units that require the - kubernetes API ''' - website.configure(port=hookenv.config('port')) - - -@when('loadbalancer.available') -def provide_loadbalancing(loadbalancer): - '''Send the public address and port to the public-address interface, so - the subordinates can get the public address of this loadbalancer.''' - loadbalancer.set_address_port(hookenv.unit_get('public-address'), - hookenv.config('port')) - - -@when('nrpe-external-master.available') -@when_not('nrpe-external-master.initial-config') -def initial_nrpe_config(nagios=None): - set_state('nrpe-external-master.initial-config') - update_nrpe_config(nagios) - - -@when('nginx.available') -@when('nrpe-external-master.available') -@when_any('config.changed.nagios_context', - 'config.changed.nagios_servicegroups') -def update_nrpe_config(unused=None): - services = ('nginx',) - - hostname = nrpe.get_nagios_hostname() - current_unit = nrpe.get_nagios_unit_name() - nrpe_setup = nrpe.NRPE(hostname=hostname) - nrpe.add_init_service_checks(nrpe_setup, services, current_unit) - nrpe_setup.write() - - -@when_not('nrpe-external-master.available') -@when('nrpe-external-master.initial-config') -def remove_nrpe_config(nagios=None): - remove_state('nrpe-external-master.initial-config') - - # List of systemd services for which the checks will be removed - services = ('nginx',) - - # The current nrpe-external-master interface doesn't handle a lot of logic, - # use the charm-helpers code for now. - hostname = nrpe.get_nagios_hostname() - nrpe_setup = nrpe.NRPE(hostname=hostname) - - for service in services: - nrpe_setup.remove_check(shortname=service) diff --git a/cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf b/cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf deleted file mode 100644 index f771b6d974..0000000000 --- a/cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf +++ /dev/null @@ -1,41 +0,0 @@ -{% for app in services -%} -upstream target_service { - {% for host in app['hosts'] -%} - server {{ host['hostname'] }}:{{ host['port'] }}; - {% endfor %} -} -{% endfor %} - - -server { - listen {{ port }} ssl http2; - server_name {{ server_name }}; - - access_log /var/log/nginx.access.log; - error_log /var/log/nginx.error.log; - - ssl on; - ssl_session_cache builtin:1000 shared:SSL:10m; - ssl_certificate {{ server_certificate }}; - ssl_certificate_key {{ server_key }}; - ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4; - ssl_prefer_server_ciphers on; - - - location / { - proxy_buffering off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Proto-Version $http2; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $http_connection; - proxy_set_header X-Stream-Protocol-Version $http_x_stream_protocol_version; - - add_header X-Stream-Protocol-Version $upstream_http_x_stream_protocol_version; - - proxy_pass https://target_service; - proxy_read_timeout {{ proxy_read_timeout }}; - } -} diff --git a/cluster/juju/layers/kubernetes-e2e/README.md b/cluster/juju/layers/kubernetes-e2e/README.md deleted file mode 100644 index 906f3ece66..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/README.md +++ /dev/null @@ -1,135 +0,0 @@ -# Kubernetes end to end - -End-to-end (e2e) tests for Kubernetes provide a mechanism to test end-to-end -behavior of the system, and is the last signal to ensure end user operations -match developer specifications. Although unit and integration tests provide a -good signal, in a distributed system like Kubernetes it is not uncommon that a -minor change may pass all unit and integration tests, but cause unforeseen -changes at the system level. - -The primary objectives of the e2e tests are to ensure a consistent and reliable -behavior of the kubernetes code base, and to catch hard-to-test bugs before -users do, when unit and integration tests are insufficient. - - -## Usage - -To deploy the end-to-end test suite, it is best to deploy the -[kubernetes-core bundle](https://github.com/juju-solutions/bundle-kubernetes-core) -and then relate the `kubernetes-e2e` charm. - -```shell -juju deploy kubernetes-core -juju deploy cs:~containers/kubernetes-e2e -juju add-relation kubernetes-e2e:kube-control kubernetes-master:kube-control -juju add-relation kubernetes-e2e:kubernetes-master kubernetes-master:kube-api-endpoint -juju add-relation kubernetes-e2e easyrsa -``` - - -Once the relations have settled, and the `kubernetes-e2e` charm reports - `Ready to test.` - you may kick off an end to end validation test. - -### Running the e2e test - -The e2e test is encapsulated as an action to ensure consistent runs of the -end to end test. The defaults are sensible for most deployments. - -```shell -juju run-action kubernetes-e2e/0 test -``` - -### Tuning the e2e test - -The e2e test is configurable. By default it will focus on or skip the declared -conformance tests in a cloud agnostic way. Default behaviors are configurable. -This allows the operator to test only a subset of the conformance tests, or to -test more behaviors not enabled by default. You can see all tunable options on -the charm by inspecting the schema output of the actions: - -```shell -$ juju actions kubernetes-e2e --format=yaml --schema -test: - description: Run end-to-end validation test suite - properties: - focus: - default: \[Conformance\] - description: Regex focus for executing the test - type: string - skip: - default: \[Flaky\] - description: Regex of tests to skip - type: string - timeout: - default: 30000 - description: Timeout in nanoseconds - type: integer - title: test - type: object -``` - - -As an example, you can run a more limited set of tests for rapid validation of -a deployed cluster. The following example will skip the `Flaky`, `Slow`, and -`Feature` labeled tests: - -```shell -juju run-action kubernetes-e2e/0 test skip='\[(Flaky|Slow|Feature:.*)\]' -``` - -> Note: the escaping of the regex due to how bash handles brackets. - -To see the different types of tests the Kubernetes end-to-end charm has access -to, we encourage you to see the upstream documentation on the different types -of tests, and to strongly understand what subsets of the tests you are running. - -[Kinds of tests](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md#kinds-of-tests) - -### More information on end-to-end testing - -Along with the above descriptions, end-to-end testing is a much larger subject -than this readme can encapsulate. There is far more information in the -[end-to-end testing guide](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md). - -### Evaluating end-to-end results - -It is not enough to just simply run the test. Result output is stored in two -places. The raw output of the e2e run is available in the `juju show-action-output` -command, as well as a flat file on disk on the `kubernetes-e2e` unit that -executed the test. - -> Note: The results will only be available once the action has -completed the test run. End-to-end testing can be quite time intensive. Often -times taking **greater than 1 hour**, depending on configuration. - -##### Flat file - -```shell -$ juju run-action kubernetes-e2e/0 test -Action queued with id: 4ceed33a-d96d-465a-8f31-20d63442e51b - -$ juju scp kubernetes-e2e/0:4ceed33a-d96d-465a-8f31-20d63442e51b.log . -``` - -##### Action result output - -```shell -$ juju run-action kubernetes-e2e/0 test -Action queued with id: 4ceed33a-d96d-465a-8f31-20d63442e51b - -$ juju show-action-output 4ceed33a-d96d-465a-8f31-20d63442e51b -``` - -## Known issues - -The e2e test suite assumes egress network access. It will pull container -images from `gcr.io`. You will need to have this registry unblocked in your -firewall to successfully run e2e test results. Or you may use the exposed -proxy settings [properly configured](https://github.com/juju-solutions/bundle-canonical-kubernetes#proxy-configuration) -on the kubernetes-worker units. - -## Help resources: - -- [Bug Tracker](https://github.com/juju-solutions/bundle-canonical-kubernetes/issues) -- [Github Repository](https://github.com/kubernetes/kubernetes/) -- [Mailing List](mailto:juju@lists.ubuntu.com) diff --git a/cluster/juju/layers/kubernetes-e2e/actions.yaml b/cluster/juju/layers/kubernetes-e2e/actions.yaml deleted file mode 100644 index 1a92754fb8..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/actions.yaml +++ /dev/null @@ -1,19 +0,0 @@ -test: - description: "Execute an end to end test." - params: - focus: - default: "\\[Conformance\\]" - description: Run tests matching the focus regex pattern. - type: string - parallelism: - default: 25 - description: The number of test nodes to run in parallel. - type: integer - skip: - default: "\\[Flaky\\]|\\[Serial\\]" - description: Skip tests matching the skip regex pattern. - type: string - timeout: - default: 30000 - description: Timeout in nanoseconds - type: integer diff --git a/cluster/juju/layers/kubernetes-e2e/actions/test b/cluster/juju/layers/kubernetes-e2e/actions/test deleted file mode 100755 index 56b930deec..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/actions/test +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -export PATH="$PATH:/snap/bin" - -# Grab the action parameter values -FOCUS=$(action-get focus) -SKIP=$(action-get skip) -PARALLELISM=$(action-get parallelism) - -if [ ! -f /home/ubuntu/.kube/config ] -then - action-fail "Missing Kubernetes configuration." - action-set suggestion="Relate to the certificate authority, and kubernetes-master" - exit 0 -fi - -# get the host from the config file -SERVER=$(cat /home/ubuntu/.kube/config | grep server | sed 's/ server: //') - -ACTION_HOME=/home/ubuntu -ACTION_LOG=$ACTION_HOME/${JUJU_ACTION_UUID}.log -ACTION_LOG_TGZ=$ACTION_LOG.tar.gz -ACTION_JUNIT=$ACTION_HOME/${JUJU_ACTION_UUID}-junit -ACTION_JUNIT_TGZ=$ACTION_JUNIT.tar.gz - -# This initializes an e2e build log with the START TIMESTAMP. -echo "JUJU_E2E_START=$(date -u +%s)" | tee $ACTION_LOG -echo "JUJU_E2E_VERSION=$(kubectl version | grep Server | cut -d " " -f 5 | cut -d ":" -f 2 | sed s/\"// | sed s/\",//)" | tee -a $ACTION_LOG -GINKGO_ARGS="-nodes=$PARALLELISM" kubernetes-test.e2e \ - -kubeconfig /home/ubuntu/.kube/config \ - -host $SERVER \ - -ginkgo.focus $FOCUS \ - -ginkgo.skip "$SKIP" \ - -report-dir $ACTION_JUNIT 2>&1 | tee -a $ACTION_LOG - -# This appends the END TIMESTAMP to the e2e build log -echo "JUJU_E2E_END=$(date -u +%s)" | tee -a $ACTION_LOG - -# set cwd to /home/ubuntu and tar the artifacts using a minimal directory -# path. Extracing "home/ubuntu/1412341234/foobar.log is cumbersome in ci -cd $ACTION_HOME/${JUJU_ACTION_UUID}-junit -tar -czf $ACTION_JUNIT_TGZ * -cd .. -tar -czf $ACTION_LOG_TGZ ${JUJU_ACTION_UUID}.log - -action-set log="$ACTION_LOG_TGZ" -action-set junit="$ACTION_JUNIT_TGZ" - -if tail ${JUJU_ACTION_UUID}.log | grep -q "Test Suite Failed"; then - action-fail "Failure detected in the logs" -fi diff --git a/cluster/juju/layers/kubernetes-e2e/config.yaml b/cluster/juju/layers/kubernetes-e2e/config.yaml deleted file mode 100644 index ee92840b76..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -options: - channel: - type: string - default: "1.11/stable" - description: | - Snap channel to install Kubernetes snaps from diff --git a/cluster/juju/layers/kubernetes-e2e/icon.svg b/cluster/juju/layers/kubernetes-e2e/icon.svg deleted file mode 100644 index 0ab453f84e..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/icon.svg +++ /dev/null @@ -1,362 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - diff --git a/cluster/juju/layers/kubernetes-e2e/layer.yaml b/cluster/juju/layers/kubernetes-e2e/layer.yaml deleted file mode 100644 index b913883bb0..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/layer.yaml +++ /dev/null @@ -1,12 +0,0 @@ -repo: https://github.com/juju-solutions/layer-kubernetes-e2e -includes: - - layer:basic - - layer:tls-client - - layer:snap - - interface:http - - interface:kube-control -options: - tls-client: - ca_certificate_path: '/srv/kubernetes/ca.crt' - client_certificate_path: '/srv/kubernetes/client.crt' - client_key_path: '/srv/kubernetes/client.key' diff --git a/cluster/juju/layers/kubernetes-e2e/metadata.yaml b/cluster/juju/layers/kubernetes-e2e/metadata.yaml deleted file mode 100644 index d93f649b0a..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/metadata.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: kubernetes-e2e -summary: Run end-2-end validation of a clusters conformance -maintainers: - - Tim Van Steenburgh - - George Kraft - - Rye Terrell - - Konstantinos Tsakalozos - - Charles Butler - - Matthew Bruzek -description: | - Deploy the Kubernetes e2e framework and validate the conformance of a - deployed kubernetes cluster -tags: - - validation - - conformance -series: - - xenial - - bionic -requires: - kubernetes-master: - interface: http - kube-control: - interface: kube-control -resources: - kubectl: - type: file - filename: kubectl.snap - description: kubectl snap - kubernetes-test: - type: file - filename: kubernetes-test.snap - description: kubernetes-test snap diff --git a/cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py b/cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py deleted file mode 100644 index c573ef78eb..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charms import layer -from charms.layer import snap - -from charms.reactive import hook -from charms.reactive import is_state -from charms.reactive import set_state -from charms.reactive import when -from charms.reactive import when_not -from charms.reactive.helpers import data_changed - -from charmhelpers.core import hookenv, unitdata - -from shlex import split - -from subprocess import check_call -from subprocess import check_output - -db = unitdata.kv() -USER = 'system:e2e' - - -@hook('upgrade-charm') -def reset_delivery_states(): - ''' Remove the state set when resources are unpacked. ''' - install_snaps() - - -@when('kubernetes-e2e.installed') -def report_status(): - ''' Report the status of the charm. ''' - messaging() - - -def messaging(): - ''' Probe our relations to determine the proper messaging to the - end user ''' - - missing_services = [] - if not is_state('kubernetes-master.available'): - missing_services.append('kubernetes-master:http') - if not is_state('certificates.available'): - missing_services.append('certificates') - if not is_state('kubeconfig.ready'): - missing_services.append('kubernetes-master:kube-control') - - if missing_services: - if len(missing_services) > 1: - subject = 'relations' - else: - subject = 'relation' - - services = ','.join(missing_services) - message = 'Missing {0}: {1}'.format(subject, services) - hookenv.status_set('blocked', message) - return - - hookenv.status_set('active', 'Ready to test.') - - -@when('config.changed.channel') -def channel_changed(): - install_snaps() - - -def install_snaps(): - ''' Deliver the e2e and kubectl components from the binary resource stream - packages declared in the charm ''' - channel = hookenv.config('channel') - hookenv.status_set('maintenance', 'Installing kubectl snap') - snap.install('kubectl', channel=channel, classic=True) - hookenv.status_set('maintenance', 'Installing kubernetes-test snap') - snap.install('kubernetes-test', channel=channel, classic=True) - set_state('kubernetes-e2e.installed') - - -@when('tls_client.ca.saved', 'tls_client.client.certificate.saved', - 'tls_client.client.key.saved', 'kubernetes-master.available', - 'kubernetes-e2e.installed', 'e2e.auth.bootstrapped') -@when_not('kubeconfig.ready') -def prepare_kubeconfig_certificates(master): - ''' Prepare the data to feed to create the kubeconfig file. ''' - - layer_options = layer.options('tls-client') - # Get all the paths to the tls information required for kubeconfig. - ca = layer_options.get('ca_certificate_path') - creds = db.get('credentials') - data_changed('kube-control.creds', creds) - - servers = get_kube_api_servers(master) - - # pedantry - kubeconfig_path = '/home/ubuntu/.kube/config' - - # Create kubernetes configuration in the default location for ubuntu. - create_kubeconfig('/root/.kube/config', servers[0], ca, - token=creds['client_token'], user='root') - create_kubeconfig(kubeconfig_path, servers[0], ca, - token=creds['client_token'], user='ubuntu') - # Set permissions on the ubuntu users kubeconfig to ensure a consistent UX - cmd = ['chown', 'ubuntu:ubuntu', kubeconfig_path] - check_call(cmd) - messaging() - set_state('kubeconfig.ready') - - -@when('kube-control.connected') -def request_credentials(kube_control): - """ Request authorization creds.""" - - # Ask for a user, although we will be using the 'client_token' - kube_control.set_auth_request(USER) - - -@when('kube-control.auth.available') -def catch_change_in_creds(kube_control): - """Request a service restart in case credential updates were detected.""" - creds = kube_control.get_auth_credentials(USER) - if creds \ - and data_changed('kube-control.creds', creds) \ - and creds['user'] == USER: - # We need to cache the credentials here because if the - # master changes (master leader dies and replaced by a new one) - # the new master will have no recollection of our certs. - db.set('credentials', creds) - set_state('e2e.auth.bootstrapped') - - -@when('kubernetes-e2e.installed', 'kubeconfig.ready') -def set_app_version(): - ''' Declare the application version to juju ''' - cmd = ['kubectl', 'version', '--client'] - from subprocess import CalledProcessError - try: - version = check_output(cmd).decode('utf-8') - except CalledProcessError: - message = "Missing kubeconfig causes errors. Skipping version set." - hookenv.log(message) - return - git_version = version.split('GitVersion:"v')[-1] - version_from = git_version.split('",')[0] - hookenv.application_version_set(version_from.rstrip()) - - -def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None, - user='ubuntu', context='juju-context', - cluster='juju-cluster', password=None, token=None): - '''Create a configuration for Kubernetes based on path using the supplied - arguments for values of the Kubernetes server, CA, key, certificate, user - context and cluster.''' - if not key and not certificate and not password and not token: - raise ValueError('Missing authentication mechanism.') - - # token and password are mutually exclusive. Error early if both are - # present. The developer has requested an impossible situation. - # see: kubectl config set-credentials --help - if token and password: - raise ValueError('Token and Password are mutually exclusive.') - # Create the config file with the address of the master server. - cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \ - '--server={2} --certificate-authority={3} --embed-certs=true' - check_call(split(cmd.format(kubeconfig, cluster, server, ca))) - # Delete old users - cmd = 'kubectl config --kubeconfig={0} unset users' - check_call(split(cmd.format(kubeconfig))) - # Create the credentials using the client flags. - cmd = 'kubectl config --kubeconfig={0} ' \ - 'set-credentials {1} '.format(kubeconfig, user) - - if key and certificate: - cmd = '{0} --client-key={1} --client-certificate={2} '\ - '--embed-certs=true'.format(cmd, key, certificate) - if password: - cmd = "{0} --username={1} --password={2}".format(cmd, user, password) - # This is mutually exclusive from password. They will not work together. - if token: - cmd = "{0} --token={1}".format(cmd, token) - check_call(split(cmd)) - # Create a default context with the cluster. - cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \ - '--cluster={2} --user={3}' - check_call(split(cmd.format(kubeconfig, context, cluster, user))) - # Make the config use this new context. - cmd = 'kubectl config --kubeconfig={0} use-context {1}' - check_call(split(cmd.format(kubeconfig, context))) - - -def get_kube_api_servers(master): - '''Return the kubernetes api server address and port for this - relationship.''' - hosts = [] - # Iterate over every service from the relation object. - for service in master.services(): - for unit in service['hosts']: - hosts.append('https://{0}:{1}'.format(unit['hostname'], - unit['port'])) - return hosts - - -def determine_arch(): - ''' dpkg wrapper to surface the architecture we are tied to''' - cmd = ['dpkg', '--print-architecture'] - output = check_output(cmd).decode('utf-8') - - return output.rstrip() diff --git a/cluster/juju/layers/kubernetes-e2e/tox.ini b/cluster/juju/layers/kubernetes-e2e/tox.ini deleted file mode 100644 index 0b8b27a9d5..0000000000 --- a/cluster/juju/layers/kubernetes-e2e/tox.ini +++ /dev/null @@ -1,12 +0,0 @@ -[tox] -skipsdist=True -envlist = py34, py35 -skip_missing_interpreters = True - -[testenv] -commands = py.test -v -deps = - -r{toxinidir}/requirements.txt - -[flake8] -exclude=docs diff --git a/cluster/juju/layers/kubernetes-master/README.md b/cluster/juju/layers/kubernetes-master/README.md deleted file mode 100644 index 6279a54cfc..0000000000 --- a/cluster/juju/layers/kubernetes-master/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# Kubernetes-master - -[Kubernetes](http://kubernetes.io/) is an open source system for managing -application containers across a cluster of hosts. The Kubernetes project was -started by Google in 2014, combining the experience of running production -workloads combined with best practices from the community. - -The Kubernetes project defines some new terms that may be unfamiliar to users -or operators. For more information please refer to the concept guide in the -[getting started guide](https://kubernetes.io/docs/home/). - -This charm is an encapsulation of the Kubernetes master processes and the -operations to run on any cloud for the entire lifecycle of the cluster. - -This charm is built from other charm layers using the Juju reactive framework. -The other layers focus on specific subset of operations making this layer -specific to operations of Kubernetes master processes. - -# Deployment - -This charm is not fully functional when deployed by itself. It requires other -charms to model a complete Kubernetes cluster. A Kubernetes cluster needs a -distributed key value store such as [Etcd](https://coreos.com/etcd/) and the -kubernetes-worker charm which delivers the Kubernetes node services. A cluster -requires a Software Defined Network (SDN) and Transport Layer Security (TLS) so -the components in a cluster communicate securely. - -Please take a look at the [Canonical Distribution of Kubernetes](https://jujucharms.com/canonical-kubernetes/) -or the [Kubernetes core](https://jujucharms.com/kubernetes-core/) bundles for -examples of complete models of Kubernetes clusters. - -# Resources - -The kubernetes-master charm takes advantage of the [Juju Resources](https://jujucharms.com/docs/2.0/developer-resources) -feature to deliver the Kubernetes software. - -In deployments on public clouds the Charm Store provides the resource to the -charm automatically with no user intervention. Some environments with strict -firewall rules may not be able to contact the Charm Store. In these network -restricted environments the resource can be uploaded to the model by the Juju -operator. - -#### Snap Refresh - -The kubernetes resources used by this charm are snap packages. When not -specified during deployment, these resources come from the public store. By -default, the `snapd` daemon will refresh all snaps installed from the store -four (4) times per day. A charm configuration option is provided for operators -to control this refresh frequency. - ->NOTE: this is a global configuration option and will affect the refresh -time for all snaps installed on a system. - -Examples: - -```sh -## refresh kubernetes-master snaps every tuesday -juju config kubernetes-master snapd_refresh="tue" - -## refresh snaps at 11pm on the last (5th) friday of the month -juju config kubernetes-master snapd_refresh="fri5,23:00" - -## delay the refresh as long as possible -juju config kubernetes-master snapd_refresh="max" - -## use the system default refresh timer -juju config kubernetes-master snapd_refresh="" -``` - -For more information on the possible values for `snapd_refresh`, see the -*refresh.timer* section in the [system options][] documentation. - -[system options]: https://forum.snapcraft.io/t/system-options/87 - -# Configuration - -This charm supports some configuration options to set up a Kubernetes cluster -that works in your environment: - -#### dns_domain - -The domain name to use for the Kubernetes cluster for DNS. - -#### enable-dashboard-addons - -Enables the installation of Kubernetes dashboard, Heapster, Grafana, and -InfluxDB. - -#### enable-rbac - -Enable RBAC and Node authorisation. - -# DNS for the cluster - -The DNS add-on allows the pods to have a DNS names in addition to IP addresses. -The Kubernetes cluster DNS server (based off the SkyDNS library) supports -forward lookups (A records), service lookups (SRV records) and reverse IP -address lookups (PTR records). More information about the DNS can be obtained -from the [Kubernetes DNS admin guide](http://kubernetes.io/docs/admin/dns/). - -# Actions - -The kubernetes-master charm models a few one time operations called -[Juju actions](https://jujucharms.com/docs/stable/actions) that can be run by -Juju users. - -#### create-rbd-pv - -This action creates RADOS Block Device (RBD) in Ceph and defines a Persistent -Volume in Kubernetes so the containers can use durable storage. This action -requires a relation to the ceph-mon charm before it can create the volume. - -#### restart - -This action restarts the master processes `kube-apiserver`, -`kube-controller-manager`, and `kube-scheduler` when the user needs a restart. - -# More information - - - [Kubernetes github project](https://github.com/kubernetes/kubernetes) - - [Kubernetes issue tracker](https://github.com/kubernetes/kubernetes/issues) - - [Kubernetes documentation](http://kubernetes.io/docs/) - - [Kubernetes releases](https://github.com/kubernetes/kubernetes/releases) - -# Contact - -The kubernetes-master charm is free and open source operations created -by the containers team at Canonical. - -Canonical also offers enterprise support and customization services. Please -refer to the [Kubernetes product page](https://www.ubuntu.com/cloud/kubernetes) -for more details. diff --git a/cluster/juju/layers/kubernetes-master/actions.yaml b/cluster/juju/layers/kubernetes-master/actions.yaml deleted file mode 100644 index 89cd502b04..0000000000 --- a/cluster/juju/layers/kubernetes-master/actions.yaml +++ /dev/null @@ -1,50 +0,0 @@ -restart: - description: Restart the Kubernetes master services on demand. -create-rbd-pv: - description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume. Note this is deprecated on Kubernetes >= 1.10 in favor of CSI, where PersistentVolumes are created dynamically to back PersistentVolumeClaims. - params: - name: - type: string - description: Name the persistent volume. - minLength: 1 - size: - type: integer - description: Size in MB of the RBD volume. - minimum: 1 - mode: - type: string - default: ReadWriteOnce - description: Access mode for the persistent volume. - filesystem: - type: string - default: xfs - description: File system type to format the volume. - skip-size-check: - type: boolean - default: false - description: Allow creation of overprovisioned RBD. - required: - - name - - size -namespace-list: - description: List existing k8s namespaces -namespace-create: - description: Create new namespace - params: - name: - type: string - description: Namespace name eg. staging - minLength: 2 - required: - - name -namespace-delete: - description: Delete namespace - params: - name: - type: string - description: Namespace name eg. staging - minLength: 2 - required: - - name -upgrade: - description: Upgrade the kubernetes snaps \ No newline at end of file diff --git a/cluster/juju/layers/kubernetes-master/actions/create-rbd-pv b/cluster/juju/layers/kubernetes-master/actions/create-rbd-pv deleted file mode 100755 index b6b229d63a..0000000000 --- a/cluster/juju/layers/kubernetes-master/actions/create-rbd-pv +++ /dev/null @@ -1,325 +0,0 @@ -#!/usr/local/sbin/charm-env python3 - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.core.templating import render -from charms.reactive import is_state -from charmhelpers.core.hookenv import action_get -from charmhelpers.core.hookenv import action_set -from charmhelpers.core.hookenv import action_fail -from subprocess import check_call -from subprocess import check_output -from subprocess import CalledProcessError -from tempfile import TemporaryDirectory -import json -import re -import os -import sys - - -os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') - - -def main(): - ''' Control logic to enlist Ceph RBD volumes as PersistentVolumes in - Kubernetes. This will invoke the validation steps, and only execute if - this script thinks the environment is 'sane' enough to provision volumes. - ''' - - # k8s >= 1.10 uses CSI and doesn't directly create persistent volumes - if get_version('kube-apiserver') >= (1, 10): - print('This action is deprecated in favor of CSI creation of persistent volumes') - print('in Kubernetes >= 1.10. Just create the PVC and a PV will be created') - print('for you.') - action_fail('Deprecated, just create PVC.') - return - - # validate relationship pre-reqs before additional steps can be taken - if not validate_relation(): - print('Failed ceph relationship check') - action_fail('Failed ceph relationship check') - return - - if not is_ceph_healthy(): - print('Ceph was not healthy.') - action_fail('Ceph was not healthy.') - return - - context = {} - - context['RBD_NAME'] = action_get_or_default('name').strip() - context['RBD_SIZE'] = action_get_or_default('size') - context['RBD_FS'] = action_get_or_default('filesystem').strip() - context['PV_MODE'] = action_get_or_default('mode').strip() - - # Ensure we're not exceeding available space in the pool - if not validate_space(context['RBD_SIZE']): - return - - # Ensure our parameters match - param_validation = validate_parameters(context['RBD_NAME'], - context['RBD_FS'], - context['PV_MODE']) - if not param_validation == 0: - return - - if not validate_unique_volume_name(context['RBD_NAME']): - action_fail('Volume name collision detected. Volume creation aborted.') - return - - context['monitors'] = get_monitors() - - # Invoke creation and format the mount device - create_rbd_volume(context['RBD_NAME'], - context['RBD_SIZE'], - context['RBD_FS']) - - # Create a temporary workspace to render our persistentVolume template, and - # enlist the RDB based PV we've just created - with TemporaryDirectory() as active_working_path: - temp_template = '{}/pv.yaml'.format(active_working_path) - render('rbd-persistent-volume.yaml', temp_template, context) - - cmd = ['kubectl', 'create', '-f', temp_template] - debug_command(cmd) - check_call(cmd) - - -def get_version(bin_name): - """Get the version of an installed Kubernetes binary. - - :param str bin_name: Name of binary - :return: 3-tuple version (maj, min, patch) - - Example:: - - >>> `get_version('kubelet') - (1, 6, 0) - - """ - cmd = '{} --version'.format(bin_name).split() - version_string = check_output(cmd).decode('utf-8') - return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3]) - - -def action_get_or_default(key): - ''' Convenience method to manage defaults since actions dont appear to - properly support defaults ''' - - value = action_get(key) - if value: - return value - elif key == 'filesystem': - return 'xfs' - elif key == 'size': - return 0 - elif key == 'mode': - return "ReadWriteOnce" - elif key == 'skip-size-check': - return False - else: - return '' - - -def create_rbd_volume(name, size, filesystem): - ''' Create the RBD volume in Ceph. Then mount it locally to format it for - the requested filesystem. - - :param name - The name of the RBD volume - :param size - The size in MB of the volume - :param filesystem - The type of filesystem to format the block device - ''' - - # Create the rbd volume - # $ rbd create foo --size 50 --image-feature layering - command = ['rbd', 'create', '--size', '{}'.format(size), '--image-feature', - 'layering', name] - debug_command(command) - check_call(command) - - # Lift the validation sequence to determine if we actually created the - # rbd volume - if validate_unique_volume_name(name): - # we failed to create the RBD volume. whoops - action_fail('RBD Volume not listed after creation.') - print('Ceph RBD volume {} not found in rbd list'.format(name)) - # hack, needs love if we're killing the process thread this deep in - # the call stack. - sys.exit(0) - - mount = ['rbd', 'map', name] - debug_command(mount) - device_path = check_output(mount).strip() - - try: - format_command = ['mkfs.{}'.format(filesystem), device_path] - debug_command(format_command) - check_call(format_command) - unmount = ['rbd', 'unmap', name] - debug_command(unmount) - check_call(unmount) - except CalledProcessError: - print('Failed to format filesystem and unmount. RBD created but not' - ' enlisted.') - action_fail('Failed to format filesystem and unmount.' - ' RDB created but not enlisted.') - - -def is_ceph_healthy(): - ''' Probe the remote ceph cluster for health status ''' - command = ['ceph', 'health'] - debug_command(command) - health_output = check_output(command) - if b'HEALTH_OK' in health_output: - return True - else: - return False - - -def get_monitors(): - ''' Parse the monitors out of /etc/ceph/ceph.conf ''' - found_hosts = [] - # This is kind of hacky. We should be piping this in from juju relations - with open('/etc/ceph/ceph.conf', 'r') as ceph_conf: - for line in ceph_conf.readlines(): - if 'mon host' in line: - # strip out the key definition - hosts = line.lstrip('mon host = ').split(' ') - for host in hosts: - found_hosts.append(host) - return found_hosts - - -def get_available_space(): - ''' Determine the space available in the RBD pool. Throw an exception if - the RBD pool ('rbd') isn't found. ''' - command = 'ceph df -f json'.split() - debug_command(command) - out = check_output(command).decode('utf-8') - data = json.loads(out) - for pool in data['pools']: - if pool['name'] == 'rbd': - return int(pool['stats']['max_avail'] / (1024 * 1024)) - raise UnknownAvailableSpaceException('Unable to determine available space.') # noqa - - -def validate_unique_volume_name(name): - ''' Poll the CEPH-MON services to determine if we have a unique rbd volume - name to use. If there is naming collisions, block the request for volume - provisioning. - - :param name - The name of the RBD volume - ''' - - command = ['rbd', 'list'] - debug_command(command) - raw_out = check_output(command) - - # Split the output on newlines - # output spec: - # $ rbd list - # foo - # foobar - volume_list = raw_out.decode('utf-8').splitlines() - - for volume in volume_list: - if volume.strip() == name: - return False - - return True - - -def validate_relation(): - ''' Determine if we are related to ceph. If we are not, we should - note this in the action output and fail this action run. We are relying - on specific files in specific paths to be placed in order for this function - to work. This method verifies those files are placed. ''' - - # TODO: Validate that the ceph-common package is installed - if not is_state('ceph-storage.available'): - message = 'Failed to detect connected ceph-mon' - print(message) - action_set({'pre-req.ceph-relation': message}) - return False - - if not os.path.isfile('/etc/ceph/ceph.conf'): - message = 'No Ceph configuration found in /etc/ceph/ceph.conf' - print(message) - action_set({'pre-req.ceph-configuration': message}) - return False - - # TODO: Validate ceph key - - return True - - -def validate_space(size): - if action_get_or_default('skip-size-check'): - return True - available_space = get_available_space() - if available_space < size: - msg = 'Unable to allocate RBD of size {}MB, only {}MB are available.' - action_fail(msg.format(size, available_space)) - return False - return True - - -def validate_parameters(name, fs, mode): - ''' Validate the user inputs to ensure they conform to what the - action expects. This method will check the naming characters used - for the rbd volume, ensure they have selected a fstype we are expecting - and the mode against our whitelist ''' - name_regex = '^[a-zA-z0-9][a-zA-Z0-9|-]' - - fs_whitelist = ['xfs', 'ext4'] - - # see http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes - # for supported operations on RBD volumes. - mode_whitelist = ['ReadWriteOnce', 'ReadOnlyMany'] - - fails = 0 - - if not re.match(name_regex, name): - message = 'Validation failed for RBD volume-name' - action_fail(message) - fails = fails + 1 - action_set({'validation.name': message}) - - if fs not in fs_whitelist: - message = 'Validation failed for file system' - action_fail(message) - fails = fails + 1 - action_set({'validation.filesystem': message}) - - if mode not in mode_whitelist: - message = "Validation failed for mode" - action_fail(message) - fails = fails + 1 - action_set({'validation.mode': message}) - - return fails - - -def debug_command(cmd): - ''' Print a debug statement of the command invoked ''' - print("Invoking {}".format(cmd)) - - -class UnknownAvailableSpaceException(Exception): - pass - - -if __name__ == '__main__': - main() diff --git a/cluster/juju/layers/kubernetes-master/actions/namespace-create b/cluster/juju/layers/kubernetes-master/actions/namespace-create deleted file mode 100755 index c1c2c64519..0000000000 --- a/cluster/juju/layers/kubernetes-master/actions/namespace-create +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/local/sbin/charm-env python3 -import os -from yaml import safe_load as load -from charmhelpers.core.hookenv import ( - action_get, - action_set, - action_fail, - action_name -) -from charmhelpers.core.templating import render -from subprocess import check_output - - -os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') - - -def kubectl(args): - cmd = ['kubectl'] + args - return check_output(cmd) - - -def namespace_list(): - y = load(kubectl(['get', 'namespaces', '-o', 'yaml'])) - ns = [i['metadata']['name'] for i in y['items']] - action_set({'namespaces': ', '.join(ns)+'.'}) - return ns - - -def namespace_create(): - name = action_get('name') - if name in namespace_list(): - action_fail('Namespace "{}" already exists.'.format(name)) - return - - render('create-namespace.yaml.j2', '/etc/kubernetes/addons/create-namespace.yaml', - context={'name': name}) - kubectl(['create', '-f', '/etc/kubernetes/addons/create-namespace.yaml']) - action_set({'msg': 'Namespace "{}" created.'.format(name)}) - - -def namespace_delete(): - name = action_get('name') - if name in ['default', 'kube-system']: - action_fail('Not allowed to delete "{}".'.format(name)) - return - if name not in namespace_list(): - action_fail('Namespace "{}" does not exist.'.format(name)) - return - kubectl(['delete', 'ns/'+name]) - action_set({'msg': 'Namespace "{}" deleted.'.format(name)}) - - -action = action_name().replace('namespace-', '') -if action == 'create': - namespace_create() -elif action == 'list': - namespace_list() -elif action == 'delete': - namespace_delete() diff --git a/cluster/juju/layers/kubernetes-master/actions/namespace-delete b/cluster/juju/layers/kubernetes-master/actions/namespace-delete deleted file mode 120000 index 358cddb12b..0000000000 --- a/cluster/juju/layers/kubernetes-master/actions/namespace-delete +++ /dev/null @@ -1 +0,0 @@ -namespace-create \ No newline at end of file diff --git a/cluster/juju/layers/kubernetes-master/actions/namespace-list b/cluster/juju/layers/kubernetes-master/actions/namespace-list deleted file mode 120000 index 358cddb12b..0000000000 --- a/cluster/juju/layers/kubernetes-master/actions/namespace-list +++ /dev/null @@ -1 +0,0 @@ -namespace-create \ No newline at end of file diff --git a/cluster/juju/layers/kubernetes-master/actions/restart b/cluster/juju/layers/kubernetes-master/actions/restart deleted file mode 100755 index d130733213..0000000000 --- a/cluster/juju/layers/kubernetes-master/actions/restart +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set +ex - -# Restart the apiserver, controller-manager, and scheduler - -systemctl restart snap.kube-apiserver.daemon -action-set apiserver.status='restarted' - -systemctl restart snap.kube-controller-manager.daemon -action-set controller-manager.status='restarted' - -systemctl restart snap.kube-scheduler.daemon -action-set kube-scheduler.status='restarted' diff --git a/cluster/juju/layers/kubernetes-master/actions/upgrade b/cluster/juju/layers/kubernetes-master/actions/upgrade deleted file mode 100755 index 7a115293b5..0000000000 --- a/cluster/juju/layers/kubernetes-master/actions/upgrade +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -eux - -charms.reactive set_state kubernetes-master.upgrade-specified -exec hooks/config-changed diff --git a/cluster/juju/layers/kubernetes-master/config.yaml b/cluster/juju/layers/kubernetes-master/config.yaml deleted file mode 100644 index e6acd54252..0000000000 --- a/cluster/juju/layers/kubernetes-master/config.yaml +++ /dev/null @@ -1,164 +0,0 @@ -options: - audit-policy: - type: string - default: | - apiVersion: audit.k8s.io/v1beta1 - kind: Policy - rules: - # Don't log read-only requests from the apiserver - - level: None - users: ["system:apiserver"] - verbs: ["get", "list", "watch"] - # Don't log kube-proxy watches - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - resources: ["endpoints", "services"] - # Don't log nodes getting their own status - - level: None - userGroups: ["system:nodes"] - verbs: ["get"] - resources: - - resources: ["nodes"] - # Don't log kube-controller-manager and kube-scheduler getting endpoints - - level: None - users: ["system:unsecured"] - namespaces: ["kube-system"] - verbs: ["get"] - resources: - - resources: ["endpoints"] - # Log everything else at the Request level. - - level: Request - omitStages: - - RequestReceived - description: | - Audit policy passed to kube-apiserver via --audit-policy-file. - For more info, please refer to the upstream documentation at - https://kubernetes.io/docs/tasks/debug-application-cluster/audit/ - audit-webhook-config: - type: string - default: "" - description: | - Audit webhook config passed to kube-apiserver via --audit-webhook-config-file. - For more info, please refer to the upstream documentation at - https://kubernetes.io/docs/tasks/debug-application-cluster/audit/ - addons-registry: - type: string - default: "" - description: Specify the docker registry to use when applying addons - enable-dashboard-addons: - type: boolean - default: True - description: Deploy the Kubernetes Dashboard and Heapster addons - enable-kube-dns: - type: boolean - default: True - description: Deploy kube-dns addon - dns_domain: - type: string - default: cluster.local - description: The local domain for cluster dns - extra_sans: - type: string - default: "" - description: | - Space-separated list of extra SAN entries to add to the x509 certificate - created for the master nodes. - service-cidr: - type: string - default: 10.152.183.0/24 - description: CIDR to user for Kubernetes services. Cannot be changed after deployment. - allow-privileged: - type: string - default: "auto" - description: | - Allow kube-apiserver to run in privileged mode. Supported values are - "true", "false", and "auto". If "true", kube-apiserver will run in - privileged mode by default. If "false", kube-apiserver will never run in - privileged mode. If "auto", kube-apiserver will not run in privileged - mode by default, but will switch to privileged mode if gpu hardware is - detected on a worker node. - enable-nvidia-plugin: - type: string - default: "auto" - description: | - Load the nvidia device plugin daemonset. Supported values are - "auto" and "false". When "auto", the daemonset will be loaded - only if GPUs are detected. When "false" the nvidia device plugin - will not be loaded. - channel: - type: string - default: "1.11/stable" - description: | - Snap channel to install Kubernetes master services from - client_password: - type: string - default: "" - description: | - Password to be used for admin user (leave empty for random password). - api-extra-args: - type: string - default: "" - description: | - Space separated list of flags and key=value pairs that will be passed as arguments to - kube-apiserver. For example a value like this: - runtime-config=batch/v2alpha1=true profiling=true - will result in kube-apiserver being run with the following options: - --runtime-config=batch/v2alpha1=true --profiling=true - controller-manager-extra-args: - type: string - default: "" - description: | - Space separated list of flags and key=value pairs that will be passed as arguments to - kube-controller-manager. For example a value like this: - runtime-config=batch/v2alpha1=true profiling=true - will result in kube-controller-manager being run with the following options: - --runtime-config=batch/v2alpha1=true --profiling=true - scheduler-extra-args: - type: string - default: "" - description: | - Space separated list of flags and key=value pairs that will be passed as arguments to - kube-scheduler. For example a value like this: - runtime-config=batch/v2alpha1=true profiling=true - will result in kube-scheduler being run with the following options: - --runtime-config=batch/v2alpha1=true --profiling=true - authorization-mode: - type: string - default: "AlwaysAllow" - description: | - Comma separated authorization modes. Allowed values are - "RBAC", "Node", "Webhook", "ABAC", "AlwaysDeny" and "AlwaysAllow". - require-manual-upgrade: - type: boolean - default: true - description: | - When true, master nodes will not be upgraded until the user triggers - it manually by running the upgrade action. - storage-backend: - type: string - default: "auto" - description: | - The storage backend for kube-apiserver persistence. Can be "etcd2", "etcd3", or - "auto". Auto mode will select etcd3 on new installations, or etcd2 on upgrades. - enable-metrics: - type: boolean - default: true - description: | - If true the metrics server for Kubernetes will be deployed onto the cluster. - snapd_refresh: - default: "max" - type: string - description: | - How often snapd handles updates for installed snaps. Setting an empty - string will check 4x per day. Set to "max" to delay the refresh as long - as possible. You may also set a custom string as described in the - 'refresh.timer' section here: - https://forum.snapcraft.io/t/system-options/87 - default-storage: - type: string - default: "auto" - description: | - The storage class to make the default storage class. Allowed values are "auto", - "none", "ceph-xfs", "ceph-ext4". Note: Only works in Kubernetes >= 1.10 diff --git a/cluster/juju/layers/kubernetes-master/copyright b/cluster/juju/layers/kubernetes-master/copyright deleted file mode 100644 index 8aec8ece45..0000000000 --- a/cluster/juju/layers/kubernetes-master/copyright +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/cluster/juju/layers/kubernetes-master/debug-scripts/kubectl b/cluster/juju/layers/kubernetes-master/debug-scripts/kubectl deleted file mode 100755 index 0d62a0bbce..0000000000 --- a/cluster/juju/layers/kubernetes-master/debug-scripts/kubectl +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ux - -export PATH=$PATH:/snap/bin - -alias kubectl="kubectl --kubeconfig=/home/ubuntu/config" - -kubectl cluster-info > $DEBUG_SCRIPT_DIR/cluster-info -kubectl cluster-info dump > $DEBUG_SCRIPT_DIR/cluster-info-dump -for obj in pods svc ingress secrets pv pvc rc; do - kubectl describe $obj --all-namespaces > $DEBUG_SCRIPT_DIR/describe-$obj -done -for obj in nodes; do - kubectl describe $obj > $DEBUG_SCRIPT_DIR/describe-$obj -done diff --git a/cluster/juju/layers/kubernetes-master/debug-scripts/kubernetes-master-services b/cluster/juju/layers/kubernetes-master/debug-scripts/kubernetes-master-services deleted file mode 100755 index a1eef81d84..0000000000 --- a/cluster/juju/layers/kubernetes-master/debug-scripts/kubernetes-master-services +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -ux - -for service in kube-apiserver kube-controller-manager kube-scheduler; do - systemctl status snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-systemctl-status - journalctl -u snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-journal -done - -# FIXME: grab snap config or something diff --git a/cluster/juju/layers/kubernetes-master/exec.d/vmware-patch/charm-pre-install b/cluster/juju/layers/kubernetes-master/exec.d/vmware-patch/charm-pre-install deleted file mode 100755 index b5e6d97b59..0000000000 --- a/cluster/juju/layers/kubernetes-master/exec.d/vmware-patch/charm-pre-install +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -MY_HOSTNAME=$(hostname) - -: ${JUJU_UNIT_NAME:=`uuidgen`} - - -if [ "${MY_HOSTNAME}" == "ubuntuguest" ]; then - juju-log "Detected broken vsphere integration. Applying hostname override" - - FRIENDLY_HOSTNAME=$(echo $JUJU_UNIT_NAME | tr / -) - juju-log "Setting hostname to $FRIENDLY_HOSTNAME" - if [ ! -f /etc/hostname.orig ]; then - mv /etc/hostname /etc/hostname.orig - fi - echo "${FRIENDLY_HOSTNAME}" > /etc/hostname - hostname $FRIENDLY_HOSTNAME -fi diff --git a/cluster/juju/layers/kubernetes-master/icon.svg b/cluster/juju/layers/kubernetes-master/icon.svg deleted file mode 100644 index 0ab453f84e..0000000000 --- a/cluster/juju/layers/kubernetes-master/icon.svg +++ /dev/null @@ -1,362 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - diff --git a/cluster/juju/layers/kubernetes-master/layer.yaml b/cluster/juju/layers/kubernetes-master/layer.yaml deleted file mode 100644 index b1a1ed22c1..0000000000 --- a/cluster/juju/layers/kubernetes-master/layer.yaml +++ /dev/null @@ -1,37 +0,0 @@ -repo: https://github.com/kubernetes/kubernetes.git -includes: - - 'layer:basic' - - 'layer:snap' - - 'layer:tls-client' - - 'layer:leadership' - - 'layer:debug' - - 'layer:metrics' - - 'layer:nagios' - - 'layer:cdk-service-kicker' - - 'interface:ceph-admin' - - 'interface:etcd' - - 'interface:http' - - 'interface:kubernetes-cni' - - 'interface:kube-dns' - - 'interface:kube-control' - - 'interface:public-address' - - 'interface:aws-integration' - - 'interface:gcp-integration' - - 'interface:openstack-integration' - - 'interface:vsphere-integration' - - 'interface:azure-integration' -options: - basic: - packages: - - socat - tls-client: - ca_certificate_path: '/root/cdk/ca.crt' - server_certificate_path: '/root/cdk/server.crt' - server_key_path: '/root/cdk/server.key' - client_certificate_path: '/root/cdk/client.crt' - client_key_path: '/root/cdk/client.key' - cdk-service-kicker: - services: - - snap.kube-apiserver.daemon - - snap.kube-controller-manager.daemon - - snap.kube-scheduler.daemon diff --git a/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py b/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py deleted file mode 100644 index 0f9caa523f..0000000000 --- a/cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/common.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - -from time import sleep - - -def get_version(bin_name): - """Get the version of an installed Kubernetes binary. - - :param str bin_name: Name of binary - :return: 3-tuple version (maj, min, patch) - - Example:: - - >>> `get_version('kubelet') - (1, 6, 0) - - """ - cmd = '{} --version'.format(bin_name).split() - version_string = subprocess.check_output(cmd).decode('utf-8') - return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3]) - - -def retry(times, delay_secs): - """ Decorator for retrying a method call. - - Args: - times: How many times should we retry before giving up - delay_secs: Delay in secs - - Returns: A callable that would return the last call outcome - """ - - def retry_decorator(func): - """ Decorator to wrap the function provided. - - Args: - func: Provided function should return either True od False - - Returns: A callable that would return the last call outcome - - """ - def _wrapped(*args, **kwargs): - res = func(*args, **kwargs) - attempt = 0 - while not res and attempt < times: - sleep(delay_secs) - res = func(*args, **kwargs) - if res: - break - attempt += 1 - return res - return _wrapped - - return retry_decorator diff --git a/cluster/juju/layers/kubernetes-master/metadata.yaml b/cluster/juju/layers/kubernetes-master/metadata.yaml deleted file mode 100644 index 03746ad95a..0000000000 --- a/cluster/juju/layers/kubernetes-master/metadata.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: kubernetes-master -summary: The Kubernetes control plane. -maintainers: - - Tim Van Steenburgh - - George Kraft - - Rye Terrell - - Konstantinos Tsakalozos - - Charles Butler - - Matthew Bruzek -description: | - Kubernetes is an open-source platform for deploying, scaling, and operations - of application containers across a cluster of hosts. Kubernetes is portable - in that it works with public, private, and hybrid clouds. Extensible through - a pluggable infrastructure. Self healing in that it will automatically - restart and place containers on healthy nodes if a node ever goes away. -tags: - - infrastructure - - kubernetes - - master -subordinate: false -series: - - xenial - - bionic -provides: - kube-api-endpoint: - interface: http - cluster-dns: - # kube-dns is deprecated. Its functionality has been rolled into the - # kube-control interface. The cluster-dns relation will be removed in - # a future release. - interface: kube-dns - kube-control: - interface: kube-control - cni: - interface: kubernetes-cni - scope: container -requires: - etcd: - interface: etcd - loadbalancer: - interface: public-address - ceph-storage: - interface: ceph-admin - aws: - interface: aws-integration - gcp: - interface: gcp-integration - openstack: - interface: openstack-integration - vsphere: - interface: vsphere-integration - azure: - interface: azure-integration -resources: - kubectl: - type: file - filename: kubectl.snap - description: kubectl snap - kube-apiserver: - type: file - filename: kube-apiserver.snap - description: kube-apiserver snap - kube-controller-manager: - type: file - filename: kube-controller-manager.snap - description: kube-controller-manager snap - kube-scheduler: - type: file - filename: kube-scheduler.snap - description: kube-scheduler snap - cdk-addons: - type: file - filename: cdk-addons.snap - description: CDK addons snap diff --git a/cluster/juju/layers/kubernetes-master/metrics.yaml b/cluster/juju/layers/kubernetes-master/metrics.yaml deleted file mode 100644 index 40df025651..0000000000 --- a/cluster/juju/layers/kubernetes-master/metrics.yaml +++ /dev/null @@ -1,34 +0,0 @@ -metrics: - juju-units: {} - pods: - type: gauge - description: number of pods - command: /snap/bin/kubectl get po --all-namespaces | tail -n+2 | wc -l - services: - type: gauge - description: number of services - command: /snap/bin/kubectl get svc --all-namespaces | tail -n+2 | wc -l - replicasets: - type: gauge - description: number of replicasets - command: /snap/bin/kubectl get rs --all-namespaces | tail -n+2 | wc -l - replicationcontrollers: - type: gauge - description: number of replicationcontrollers - command: /snap/bin/kubectl get rc --all-namespaces | tail -n+2 | wc -l - nodes: - type: gauge - description: number of kubernetes nodes - command: /snap/bin/kubectl get nodes | tail -n+2 | wc -l - persistentvolume: - type: gauge - description: number of pv - command: /snap/bin/kubectl get pv | tail -n+2 | wc -l - persistentvolumeclaims: - type: gauge - description: number of claims - command: /snap/bin/kubectl get pvc --all-namespaces | tail -n+2 | wc -l - serviceaccounts: - type: gauge - description: number of sa - command: /snap/bin/kubectl get sa --all-namespaces | tail -n+2 | wc -l diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py deleted file mode 100644 index 6652aeafb1..0000000000 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ /dev/null @@ -1,1903 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import hashlib -import os -import re -import random -import shutil -import socket -import string -import json -import ipaddress - -from charms.leadership import leader_get, leader_set - -from shutil import move -from tempfile import TemporaryDirectory - -from pathlib import Path -from shlex import split -from subprocess import check_call -from subprocess import check_output -from subprocess import CalledProcessError -from urllib.request import Request, urlopen - -from charms import layer -from charms.layer import snap -from charms.reactive import hook -from charms.reactive import remove_state -from charms.reactive import set_state -from charms.reactive import is_state -from charms.reactive import endpoint_from_flag -from charms.reactive import when, when_any, when_not, when_none -from charms.reactive.helpers import data_changed, any_file_changed -from charms.kubernetes.common import get_version -from charms.kubernetes.common import retry - -from charms.layer import tls_client - -from charmhelpers.core import hookenv -from charmhelpers.core import host -from charmhelpers.core import unitdata -from charmhelpers.core.host import service_stop -from charmhelpers.core.templating import render -from charmhelpers.fetch import apt_install -from charmhelpers.contrib.charmsupport import nrpe - - -# Override the default nagios shortname regex to allow periods, which we -# need because our bin names contain them (e.g. 'snap.foo.daemon'). The -# default regex in charmhelpers doesn't allow periods, but nagios itself does. -nrpe.Check.shortname_re = '[\.A-Za-z0-9-_]+$' - -gcp_creds_env_key = 'GOOGLE_APPLICATION_CREDENTIALS' -snap_resources = ['kubectl', 'kube-apiserver', 'kube-controller-manager', - 'kube-scheduler', 'cdk-addons'] - -os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') -db = unitdata.kv() - - -def set_upgrade_needed(forced=False): - set_state('kubernetes-master.upgrade-needed') - config = hookenv.config() - previous_channel = config.previous('channel') - require_manual = config.get('require-manual-upgrade') - hookenv.log('set upgrade needed') - if previous_channel is None or not require_manual or forced: - hookenv.log('forcing upgrade') - set_state('kubernetes-master.upgrade-specified') - - -@when('config.changed.channel') -def channel_changed(): - set_upgrade_needed() - - -def service_cidr(): - ''' Return the charm's service-cidr config ''' - frozen_cidr = db.get('kubernetes-master.service-cidr') - return frozen_cidr or hookenv.config('service-cidr') - - -def freeze_service_cidr(): - ''' Freeze the service CIDR. Once the apiserver has started, we can no - longer safely change this value. ''' - db.set('kubernetes-master.service-cidr', service_cidr()) - - -@hook('upgrade-charm') -def check_for_upgrade_needed(): - '''An upgrade charm event was triggered by Juju, react to that here.''' - hookenv.status_set('maintenance', 'Checking resources') - - # migrate to new flags - if is_state('kubernetes-master.restarted-for-cloud'): - remove_state('kubernetes-master.restarted-for-cloud') - set_state('kubernetes-master.cloud.ready') - if is_state('kubernetes-master.cloud-request-sent'): - # minor change, just for consistency - remove_state('kubernetes-master.cloud-request-sent') - set_state('kubernetes-master.cloud.request-sent') - - migrate_from_pre_snaps() - add_rbac_roles() - set_state('reconfigure.authentication.setup') - remove_state('authentication.setup') - - if not db.get('snap.resources.fingerprint.initialised'): - # We are here on an upgrade from non-rolling master - # Since this upgrade might also include resource updates eg - # juju upgrade-charm kubernetes-master --resource kube-any=my.snap - # we take no risk and forcibly upgrade the snaps. - # Forcibly means we do not prompt the user to call the upgrade action. - set_upgrade_needed(forced=True) - - migrate_resource_checksums() - check_resources_for_upgrade_needed() - - # Set the auto storage backend to etcd2. - auto_storage_backend = leader_get('auto_storage_backend') - is_leader = is_state('leadership.is_leader') - if not auto_storage_backend and is_leader: - leader_set(auto_storage_backend='etcd2') - - -def get_resource_checksum_db_key(resource): - ''' Convert a resource name to a resource checksum database key. ''' - return 'kubernetes-master.resource-checksums.' + resource - - -def calculate_resource_checksum(resource): - ''' Calculate a checksum for a resource ''' - md5 = hashlib.md5() - path = hookenv.resource_get(resource) - if path: - with open(path, 'rb') as f: - data = f.read() - md5.update(data) - return md5.hexdigest() - - -def migrate_resource_checksums(): - ''' Migrate resource checksums from the old schema to the new one ''' - for resource in snap_resources: - new_key = get_resource_checksum_db_key(resource) - if not db.get(new_key): - path = hookenv.resource_get(resource) - if path: - # old key from charms.reactive.helpers.any_file_changed - old_key = 'reactive.files_changed.' + path - old_checksum = db.get(old_key) - db.set(new_key, old_checksum) - else: - # No resource is attached. Previously, this meant no checksum - # would be calculated and stored. But now we calculate it as if - # it is a 0-byte resource, so let's go ahead and do that. - zero_checksum = hashlib.md5().hexdigest() - db.set(new_key, zero_checksum) - - -def check_resources_for_upgrade_needed(): - hookenv.status_set('maintenance', 'Checking resources') - for resource in snap_resources: - key = get_resource_checksum_db_key(resource) - old_checksum = db.get(key) - new_checksum = calculate_resource_checksum(resource) - if new_checksum != old_checksum: - set_upgrade_needed() - - -def calculate_and_store_resource_checksums(): - for resource in snap_resources: - key = get_resource_checksum_db_key(resource) - checksum = calculate_resource_checksum(resource) - db.set(key, checksum) - - -def add_rbac_roles(): - '''Update the known_tokens file with proper groups.''' - - tokens_fname = '/root/cdk/known_tokens.csv' - tokens_backup_fname = '/root/cdk/known_tokens.csv.backup' - move(tokens_fname, tokens_backup_fname) - with open(tokens_fname, 'w') as ftokens: - with open(tokens_backup_fname, 'r') as stream: - for line in stream: - record = line.strip().split(',') - # token, username, user, groups - if record[2] == 'admin' and len(record) == 3: - towrite = '{0},{1},{2},"{3}"\n'.format(record[0], - record[1], - record[2], - 'system:masters') - ftokens.write(towrite) - continue - if record[2] == 'kube_proxy': - towrite = '{0},{1},{2}\n'.format(record[0], - 'system:kube-proxy', - 'kube-proxy') - ftokens.write(towrite) - continue - if record[2] == 'kubelet' and record[1] == 'kubelet': - continue - - ftokens.write('{}'.format(line)) - - -def rename_file_idempotent(source, destination): - if os.path.isfile(source): - os.rename(source, destination) - - -def migrate_from_pre_snaps(): - # remove old states - remove_state('kubernetes.components.installed') - remove_state('kubernetes.dashboard.available') - remove_state('kube-dns.available') - remove_state('kubernetes-master.app_version.set') - - # disable old services - services = ['kube-apiserver', - 'kube-controller-manager', - 'kube-scheduler'] - for service in services: - hookenv.log('Stopping {0} service.'.format(service)) - host.service_stop(service) - - # rename auth files - os.makedirs('/root/cdk', exist_ok=True) - rename_file_idempotent('/etc/kubernetes/serviceaccount.key', - '/root/cdk/serviceaccount.key') - rename_file_idempotent('/srv/kubernetes/basic_auth.csv', - '/root/cdk/basic_auth.csv') - rename_file_idempotent('/srv/kubernetes/known_tokens.csv', - '/root/cdk/known_tokens.csv') - - # cleanup old files - files = [ - "/lib/systemd/system/kube-apiserver.service", - "/lib/systemd/system/kube-controller-manager.service", - "/lib/systemd/system/kube-scheduler.service", - "/etc/default/kube-defaults", - "/etc/default/kube-apiserver.defaults", - "/etc/default/kube-controller-manager.defaults", - "/etc/default/kube-scheduler.defaults", - "/srv/kubernetes", - "/home/ubuntu/kubectl", - "/usr/local/bin/kubectl", - "/usr/local/bin/kube-apiserver", - "/usr/local/bin/kube-controller-manager", - "/usr/local/bin/kube-scheduler", - "/etc/kubernetes" - ] - for file in files: - if os.path.isdir(file): - hookenv.log("Removing directory: " + file) - shutil.rmtree(file) - elif os.path.isfile(file): - hookenv.log("Removing file: " + file) - os.remove(file) - - -@when('kubernetes-master.upgrade-specified') -def do_upgrade(): - install_snaps() - remove_state('kubernetes-master.upgrade-needed') - remove_state('kubernetes-master.upgrade-specified') - - -def install_snaps(): - channel = hookenv.config('channel') - hookenv.status_set('maintenance', 'Installing kubectl snap') - snap.install('kubectl', channel=channel, classic=True) - hookenv.status_set('maintenance', 'Installing kube-apiserver snap') - snap.install('kube-apiserver', channel=channel) - hookenv.status_set('maintenance', - 'Installing kube-controller-manager snap') - snap.install('kube-controller-manager', channel=channel) - hookenv.status_set('maintenance', 'Installing kube-scheduler snap') - snap.install('kube-scheduler', channel=channel) - hookenv.status_set('maintenance', 'Installing cdk-addons snap') - snap.install('cdk-addons', channel=channel) - calculate_and_store_resource_checksums() - db.set('snap.resources.fingerprint.initialised', True) - set_state('kubernetes-master.snaps.installed') - remove_state('kubernetes-master.components.started') - - -@when('config.changed.client_password', 'leadership.is_leader') -def password_changed(): - """Handle password change via the charms config.""" - password = hookenv.config('client_password') - if password == "" and is_state('client.password.initialised'): - # password_changed is called during an upgrade. Nothing to do. - return - elif password == "": - # Password not initialised - password = token_generator() - setup_basic_auth(password, "admin", "admin", "system:masters") - set_state('reconfigure.authentication.setup') - remove_state('authentication.setup') - set_state('client.password.initialised') - - -@when('config.changed.storage-backend') -def storage_backend_changed(): - remove_state('kubernetes-master.components.started') - - -@when('cni.connected') -@when_not('cni.configured') -def configure_cni(cni): - ''' Set master configuration on the CNI relation. This lets the CNI - subordinate know that we're the master so it can respond accordingly. ''' - cni.set_config(is_master=True, kubeconfig_path='') - - -@when('leadership.is_leader') -@when_not('authentication.setup') -def setup_leader_authentication(): - '''Setup basic authentication and token access for the cluster.''' - service_key = '/root/cdk/serviceaccount.key' - basic_auth = '/root/cdk/basic_auth.csv' - known_tokens = '/root/cdk/known_tokens.csv' - - hookenv.status_set('maintenance', 'Rendering authentication templates.') - - keys = [service_key, basic_auth, known_tokens] - # Try first to fetch data from an old leadership broadcast. - if not get_keys_from_leader(keys) \ - or is_state('reconfigure.authentication.setup'): - last_pass = get_password('basic_auth.csv', 'admin') - setup_basic_auth(last_pass, 'admin', 'admin', 'system:masters') - - if not os.path.isfile(known_tokens): - touch(known_tokens) - - # Generate the default service account token key - os.makedirs('/root/cdk', exist_ok=True) - if not os.path.isfile(service_key): - cmd = ['openssl', 'genrsa', '-out', service_key, - '2048'] - check_call(cmd) - remove_state('reconfigure.authentication.setup') - - # read service account key for syndication - leader_data = {} - for f in [known_tokens, basic_auth, service_key]: - with open(f, 'r') as fp: - leader_data[f] = fp.read() - - # this is slightly opaque, but we are sending file contents under its file - # path as a key. - # eg: - # {'/root/cdk/serviceaccount.key': 'RSA:2471731...'} - leader_set(leader_data) - remove_state('kubernetes-master.components.started') - set_state('authentication.setup') - - -@when_not('leadership.is_leader') -def setup_non_leader_authentication(): - - service_key = '/root/cdk/serviceaccount.key' - basic_auth = '/root/cdk/basic_auth.csv' - known_tokens = '/root/cdk/known_tokens.csv' - - keys = [service_key, basic_auth, known_tokens] - # The source of truth for non-leaders is the leader. - # Therefore we overwrite_local with whatever the leader has. - if not get_keys_from_leader(keys, overwrite_local=True): - # the keys were not retrieved. Non-leaders have to retry. - return - - if not any_file_changed(keys) and is_state('authentication.setup'): - # No change detected and we have already setup the authentication - return - - hookenv.status_set('maintenance', 'Rendering authentication templates.') - - remove_state('kubernetes-master.components.started') - set_state('authentication.setup') - - -def get_keys_from_leader(keys, overwrite_local=False): - """ - Gets the broadcasted keys from the leader and stores them in - the corresponding files. - - Args: - keys: list of keys. Keys are actually files on the FS. - - Returns: True if all key were fetched, False if not. - - """ - # This races with other codepaths, and seems to require being created first - # This block may be extracted later, but for now seems to work as intended - os.makedirs('/root/cdk', exist_ok=True) - - for k in keys: - # If the path does not exist, assume we need it - if not os.path.exists(k) or overwrite_local: - # Fetch data from leadership broadcast - contents = leader_get(k) - # Default to logging the warning and wait for leader data to be set - if contents is None: - hookenv.log('Missing content for file {}'.format(k)) - return False - # Write out the file and move on to the next item - with open(k, 'w+') as fp: - fp.write(contents) - fp.write('\n') - - return True - - -@when('kubernetes-master.snaps.installed') -def set_app_version(): - ''' Declare the application version to juju ''' - version = check_output(['kube-apiserver', '--version']) - hookenv.application_version_set(version.split(b' v')[-1].rstrip()) - - -@when('kubernetes-master.snaps.installed') -@when('snap.refresh.set') -@when('leadership.is_leader') -def process_snapd_timer(): - ''' Set the snapd refresh timer on the leader so all cluster members - (present and future) will refresh near the same time. ''' - # Get the current snapd refresh timer; we know layer-snap has set this - # when the 'snap.refresh.set' flag is present. - timer = snap.get(snapname='core', key='refresh.timer').decode('utf-8') - - # The first time through, data_changed will be true. Subsequent calls - # should only update leader data if something changed. - if data_changed('master_snapd_refresh', timer): - hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) - leader_set({'snapd_refresh': timer}) - - -@when('kubernetes-master.snaps.installed') -@when('snap.refresh.set') -@when('leadership.changed.snapd_refresh') -@when_not('leadership.is_leader') -def set_snapd_timer(): - ''' Set the snapd refresh.timer on non-leader cluster members. ''' - # NB: This method should only be run when 'snap.refresh.set' is present. - # Layer-snap will always set a core refresh.timer, which may not be the - # same as our leader. Gating with 'snap.refresh.set' ensures layer-snap - # has finished and we are free to set our config to the leader's timer. - timer = leader_get('snapd_refresh') - hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) - snap.set_refresh_timer(timer) - - -@hookenv.atexit -def set_final_status(): - ''' Set the final status of the charm as we leave hook execution ''' - try: - goal_state = hookenv.goal_state() - except NotImplementedError: - goal_state = {} - - vsphere_joined = is_state('endpoint.vsphere.joined') - azure_joined = is_state('endpoint.azure.joined') - cloud_blocked = is_state('kubernetes-master.cloud.blocked') - if vsphere_joined and cloud_blocked: - hookenv.status_set('blocked', - 'vSphere integration requires K8s 1.12 or greater') - return - if azure_joined and cloud_blocked: - hookenv.status_set('blocked', - 'Azure integration requires K8s 1.11 or greater') - return - - if is_state('kubernetes-master.cloud.pending'): - hookenv.status_set('waiting', 'Waiting for cloud integration') - return - - if not is_state('kube-api-endpoint.available'): - if 'kube-api-endpoint' in goal_state.get('relations', {}): - status = 'waiting' - else: - status = 'blocked' - hookenv.status_set(status, 'Waiting for kube-api-endpoint relation') - return - - if not is_state('kube-control.connected'): - if 'kube-control' in goal_state.get('relations', {}): - status = 'waiting' - else: - status = 'blocked' - hookenv.status_set(status, 'Waiting for workers.') - return - - upgrade_needed = is_state('kubernetes-master.upgrade-needed') - upgrade_specified = is_state('kubernetes-master.upgrade-specified') - if upgrade_needed and not upgrade_specified: - msg = 'Needs manual upgrade, run the upgrade action' - hookenv.status_set('blocked', msg) - return - - if is_state('kubernetes-master.components.started'): - # All services should be up and running at this point. Double-check... - failing_services = master_services_down() - if len(failing_services) != 0: - msg = 'Stopped services: {}'.format(','.join(failing_services)) - hookenv.status_set('blocked', msg) - return - - is_leader = is_state('leadership.is_leader') - authentication_setup = is_state('authentication.setup') - if not is_leader and not authentication_setup: - hookenv.status_set('waiting', 'Waiting on leaders crypto keys.') - return - - components_started = is_state('kubernetes-master.components.started') - addons_configured = is_state('cdk-addons.configured') - if components_started and not addons_configured: - hookenv.status_set('waiting', 'Waiting to retry addon deployment') - return - - if addons_configured and not all_kube_system_pods_running(): - hookenv.status_set('waiting', 'Waiting for kube-system pods to start') - return - - if hookenv.config('service-cidr') != service_cidr(): - msg = 'WARN: cannot change service-cidr, still using ' + service_cidr() - hookenv.status_set('active', msg) - return - - gpu_available = is_state('kube-control.gpu.available') - gpu_enabled = is_state('kubernetes-master.gpu.enabled') - if gpu_available and not gpu_enabled: - msg = 'GPUs available. Set allow-privileged="auto" to enable.' - hookenv.status_set('active', msg) - return - - hookenv.status_set('active', 'Kubernetes master running.') - - -def master_services_down(): - """Ensure master services are up and running. - - Return: list of failing services""" - services = ['kube-apiserver', - 'kube-controller-manager', - 'kube-scheduler'] - failing_services = [] - for service in services: - daemon = 'snap.{}.daemon'.format(service) - if not host.service_running(daemon): - failing_services.append(service) - return failing_services - - -@when('etcd.available', 'tls_client.server.certificate.saved', - 'authentication.setup') -@when('leadership.set.auto_storage_backend') -@when_not('kubernetes-master.components.started', - 'kubernetes-master.cloud.pending', - 'kubernetes-master.cloud.blocked') -def start_master(etcd): - '''Run the Kubernetes master components.''' - hookenv.status_set('maintenance', - 'Configuring the Kubernetes master services.') - freeze_service_cidr() - if not etcd.get_connection_string(): - # etcd is not returning a connection string. This happens when - # the master unit disconnects from etcd and is ready to terminate. - # No point in trying to start master services and fail. Just return. - return - - # TODO: Make sure below relation is handled on change - # https://github.com/kubernetes/kubernetes/issues/43461 - handle_etcd_relation(etcd) - - # Add CLI options to all components - configure_apiserver(etcd.get_connection_string()) - configure_controller_manager() - configure_scheduler() - set_state('kubernetes-master.components.started') - hookenv.open_port(6443) - - -@when('etcd.available') -def etcd_data_change(etcd): - ''' Etcd scale events block master reconfiguration due to the - kubernetes-master.components.started state. We need a way to - handle these events consistently only when the number of etcd - units has actually changed ''' - - # key off of the connection string - connection_string = etcd.get_connection_string() - - # If the connection string changes, remove the started state to trigger - # handling of the master components - if data_changed('etcd-connect', connection_string): - remove_state('kubernetes-master.components.started') - - # We are the leader and the auto_storage_backend is not set meaning - # this is the first time we connect to etcd. - auto_storage_backend = leader_get('auto_storage_backend') - is_leader = is_state('leadership.is_leader') - if is_leader and not auto_storage_backend: - if etcd.get_version().startswith('3.'): - leader_set(auto_storage_backend='etcd3') - else: - leader_set(auto_storage_backend='etcd2') - - -@when('kube-control.connected') -@when('cdk-addons.configured') -def send_cluster_dns_detail(kube_control): - ''' Send cluster DNS info ''' - enableKubeDNS = hookenv.config('enable-kube-dns') - dnsDomain = hookenv.config('dns_domain') - dns_ip = None - if enableKubeDNS: - try: - dns_ip = get_dns_ip() - except CalledProcessError: - hookenv.log("kubedns not ready yet") - return - kube_control.set_dns(53, dnsDomain, dns_ip, enableKubeDNS) - - -@when('kube-control.connected') -@when('snap.installed.kubectl') -@when('leadership.is_leader') -def create_service_configs(kube_control): - """Create the users for kubelet""" - should_restart = False - # generate the username/pass for the requesting unit - proxy_token = get_token('system:kube-proxy') - if not proxy_token: - setup_tokens(None, 'system:kube-proxy', 'kube-proxy') - proxy_token = get_token('system:kube-proxy') - should_restart = True - - client_token = get_token('admin') - if not client_token: - setup_tokens(None, 'admin', 'admin', "system:masters") - client_token = get_token('admin') - should_restart = True - - requests = kube_control.auth_user() - for request in requests: - username = request[1]['user'] - group = request[1]['group'] - kubelet_token = get_token(username) - if not kubelet_token and username and group: - # Usernames have to be in the form of system:node: - userid = "kubelet-{}".format(request[0].split('/')[1]) - setup_tokens(None, username, userid, group) - kubelet_token = get_token(username) - kube_control.sign_auth_request(request[0], username, - kubelet_token, proxy_token, - client_token) - should_restart = True - - if should_restart: - host.service_restart('snap.kube-apiserver.daemon') - remove_state('authentication.setup') - - -@when('kube-api-endpoint.available') -def push_service_data(kube_api): - ''' Send configuration to the load balancer, and close access to the - public interface ''' - kube_api.configure(port=6443) - - -def get_ingress_address(relation_name): - try: - network_info = hookenv.network_get(relation_name) - except NotImplementedError: - network_info = [] - - if network_info and 'ingress-addresses' in network_info: - # just grab the first one for now, maybe be more robust here? - return network_info['ingress-addresses'][0] - else: - # if they don't have ingress-addresses they are running a juju that - # doesn't support spaces, so just return the private address - return hookenv.unit_get('private-address') - - -@when('certificates.available', 'kube-api-endpoint.available') -def send_data(tls, kube_api_endpoint): - '''Send the data that is required to create a server certificate for - this server.''' - # Use the public ip of this unit as the Common Name for the certificate. - common_name = hookenv.unit_public_ip() - - # Get the SDN gateway based on the cidr address. - kubernetes_service_ip = get_kubernetes_service_ip() - - # Get ingress address - ingress_ip = get_ingress_address(kube_api_endpoint.relation_name) - - domain = hookenv.config('dns_domain') - # Create SANs that the tls layer will add to the server cert. - sans = [ - hookenv.unit_public_ip(), - ingress_ip, - socket.gethostname(), - kubernetes_service_ip, - 'kubernetes', - 'kubernetes.{0}'.format(domain), - 'kubernetes.default', - 'kubernetes.default.svc', - 'kubernetes.default.svc.{0}'.format(domain) - ] - - # maybe they have extra names they want as SANs - extra_sans = hookenv.config('extra_sans') - if extra_sans and not extra_sans == "": - sans.extend(extra_sans.split()) - - # Create a path safe name by removing path characters from the unit name. - certificate_name = hookenv.local_unit().replace('/', '_') - # Request a server cert with this information. - tls.request_server_cert(common_name, sans, certificate_name) - - -@when('config.changed.extra_sans', 'certificates.available', - 'kube-api-endpoint.available') -def update_certificate(tls, kube_api_endpoint): - # Using the config.changed.extra_sans flag to catch changes. - # IP changes will take ~5 minutes or so to propagate, but - # it will update. - send_data(tls, kube_api_endpoint) - - -@when('certificates.server.cert.available', - 'kubernetes-master.components.started', - 'tls_client.server.certificate.written') -def kick_api_server(tls): - # need to be idempotent and don't want to kick the api server - # without need - if data_changed('cert', tls.get_server_cert()): - # certificate changed, so restart the api server - hookenv.log("Certificate information changed, restarting api server") - restart_apiserver() - tls_client.reset_certificate_write_flag('server') - - -@when_any('kubernetes-master.components.started', 'ceph-storage.configured') -@when('leadership.is_leader') -def configure_cdk_addons(): - ''' Configure CDK addons ''' - remove_state('cdk-addons.configured') - load_gpu_plugin = hookenv.config('enable-nvidia-plugin').lower() - gpuEnable = (get_version('kube-apiserver') >= (1, 9) and - load_gpu_plugin == "auto" and - is_state('kubernetes-master.gpu.enabled')) - registry = hookenv.config('addons-registry') - dbEnabled = str(hookenv.config('enable-dashboard-addons')).lower() - dnsEnabled = str(hookenv.config('enable-kube-dns')).lower() - metricsEnabled = str(hookenv.config('enable-metrics')).lower() - if (is_state('ceph-storage.configured') and - get_version('kube-apiserver') >= (1, 10)): - cephEnabled = "true" - else: - cephEnabled = "false" - ceph_ep = endpoint_from_flag('ceph-storage.available') - ceph = {} - default_storage = '' - if ceph_ep: - b64_ceph_key = base64.b64encode(ceph_ep.key().encode('utf-8')) - ceph['admin_key'] = b64_ceph_key.decode('ascii') - ceph['kubernetes_key'] = b64_ceph_key.decode('ascii') - ceph['mon_hosts'] = ceph_ep.mon_hosts() - default_storage = hookenv.config('default-storage') - - args = [ - 'arch=' + arch(), - 'dns-ip=' + get_deprecated_dns_ip(), - 'dns-domain=' + hookenv.config('dns_domain'), - 'registry=' + registry, - 'enable-dashboard=' + dbEnabled, - 'enable-kube-dns=' + dnsEnabled, - 'enable-metrics=' + metricsEnabled, - 'enable-gpu=' + str(gpuEnable).lower(), - 'enable-ceph=' + cephEnabled, - 'ceph-admin-key=' + (ceph.get('admin_key', '')), - 'ceph-kubernetes-key=' + (ceph.get('admin_key', '')), - 'ceph-mon-hosts="' + (ceph.get('mon_hosts', '')) + '"', - 'default-storage=' + default_storage, - ] - check_call(['snap', 'set', 'cdk-addons'] + args) - if not addons_ready(): - remove_state('cdk-addons.configured') - return - - set_state('cdk-addons.configured') - - -@retry(times=3, delay_secs=20) -def addons_ready(): - """ - Test if the add ons got installed - - Returns: True is the addons got applied - - """ - try: - check_call(['cdk-addons.apply']) - return True - except CalledProcessError: - hookenv.log("Addons are not ready yet.") - return False - - -@when('loadbalancer.available', 'certificates.ca.available', - 'certificates.client.cert.available', 'authentication.setup') -def loadbalancer_kubeconfig(loadbalancer, ca, client): - # Get the potential list of loadbalancers from the relation object. - hosts = loadbalancer.get_addresses_ports() - # Get the public address of loadbalancers so users can access the cluster. - address = hosts[0].get('public-address') - # Get the port of the loadbalancer so users can access the cluster. - port = hosts[0].get('port') - server = 'https://{0}:{1}'.format(address, port) - build_kubeconfig(server) - - -@when('certificates.ca.available', 'certificates.client.cert.available', - 'authentication.setup') -@when_not('loadbalancer.available') -def create_self_config(ca, client): - '''Create a kubernetes configuration for the master unit.''' - server = 'https://{0}:{1}'.format(hookenv.unit_get('public-address'), 6443) - build_kubeconfig(server) - - -@when('ceph-storage.available') -def ceph_state_control(ceph_admin): - ''' Determine if we should remove the state that controls the re-render - and execution of the ceph-relation-changed event because there - are changes in the relationship data, and we should re-render any - configs, keys, and/or service pre-reqs ''' - - ceph_relation_data = { - 'mon_hosts': ceph_admin.mon_hosts(), - 'fsid': ceph_admin.fsid(), - 'auth_supported': ceph_admin.auth(), - 'hostname': socket.gethostname(), - 'key': ceph_admin.key() - } - - # Re-execute the rendering if the data has changed. - if data_changed('ceph-config', ceph_relation_data): - remove_state('ceph-storage.configured') - - -@when('ceph-storage.available') -@when_not('ceph-storage.configured') -def ceph_storage(ceph_admin): - '''Ceph on kubernetes will require a few things - namely a ceph - configuration, and the ceph secret key file used for authentication. - This method will install the client package, and render the requisit files - in order to consume the ceph-storage relation.''' - - # deprecated in 1.10 in favor of using CSI - if get_version('kube-apiserver') >= (1, 10): - # this is actually false, but by setting this flag we won't keep - # running this function for no reason. Also note that we watch this - # flag to run cdk-addons.apply. - set_state('ceph-storage.configured') - return - - ceph_context = { - 'mon_hosts': ceph_admin.mon_hosts(), - 'fsid': ceph_admin.fsid(), - 'auth_supported': ceph_admin.auth(), - 'use_syslog': "true", - 'ceph_public_network': '', - 'ceph_cluster_network': '', - 'loglevel': 1, - 'hostname': socket.gethostname(), - } - # Install the ceph common utilities. - apt_install(['ceph-common'], fatal=True) - - etc_ceph_directory = '/etc/ceph' - if not os.path.isdir(etc_ceph_directory): - os.makedirs(etc_ceph_directory) - charm_ceph_conf = os.path.join(etc_ceph_directory, 'ceph.conf') - # Render the ceph configuration from the ceph conf template - render('ceph.conf', charm_ceph_conf, ceph_context) - - # The key can rotate independently of other ceph config, so validate it - admin_key = os.path.join(etc_ceph_directory, - 'ceph.client.admin.keyring') - try: - with open(admin_key, 'w') as key_file: - key_file.write("[client.admin]\n\tkey = {}\n".format( - ceph_admin.key())) - except IOError as err: - hookenv.log("IOError writing admin.keyring: {}".format(err)) - - # Enlist the ceph-admin key as a kubernetes secret - if ceph_admin.key(): - encoded_key = base64.b64encode(ceph_admin.key().encode('utf-8')) - else: - # We didn't have a key, and cannot proceed. Do not set state and - # allow this method to re-execute - return - - context = {'secret': encoded_key.decode('ascii')} - render('ceph-secret.yaml', '/tmp/ceph-secret.yaml', context) - try: - # At first glance this is deceptive. The apply stanza will create if - # it doesn't exist, otherwise it will update the entry, ensuring our - # ceph-secret is always reflective of what we have in /etc/ceph - # assuming we have invoked this anytime that file would change. - cmd = ['kubectl', 'apply', '-f', '/tmp/ceph-secret.yaml'] - check_call(cmd) - os.remove('/tmp/ceph-secret.yaml') - except: # NOQA - # the enlistment in kubernetes failed, return and prepare for re-exec - return - - # when complete, set a state relating to configuration of the storage - # backend that will allow other modules to hook into this and verify we - # have performed the necessary pre-req steps to interface with a ceph - # deployment. - set_state('ceph-storage.configured') - - -@when('nrpe-external-master.available') -@when_not('nrpe-external-master.initial-config') -def initial_nrpe_config(nagios=None): - set_state('nrpe-external-master.initial-config') - update_nrpe_config(nagios) - - -@when('config.changed.authorization-mode', - 'kubernetes-master.components.started') -def switch_auth_mode(): - config = hookenv.config() - mode = config.get('authorization-mode') - if data_changed('auth-mode', mode): - remove_state('kubernetes-master.components.started') - - -@when('kubernetes-master.components.started') -@when('nrpe-external-master.available') -@when_any('config.changed.nagios_context', - 'config.changed.nagios_servicegroups') -def update_nrpe_config(unused=None): - services = ( - 'snap.kube-apiserver.daemon', - 'snap.kube-controller-manager.daemon', - 'snap.kube-scheduler.daemon' - ) - hostname = nrpe.get_nagios_hostname() - current_unit = nrpe.get_nagios_unit_name() - nrpe_setup = nrpe.NRPE(hostname=hostname) - nrpe.add_init_service_checks(nrpe_setup, services, current_unit) - nrpe_setup.write() - - -@when_not('nrpe-external-master.available') -@when('nrpe-external-master.initial-config') -def remove_nrpe_config(nagios=None): - remove_state('nrpe-external-master.initial-config') - - # List of systemd services for which the checks will be removed - services = ( - 'snap.kube-apiserver.daemon', - 'snap.kube-controller-manager.daemon', - 'snap.kube-scheduler.daemon' - ) - - # The current nrpe-external-master interface doesn't handle a lot of logic, - # use the charm-helpers code for now. - hostname = nrpe.get_nagios_hostname() - nrpe_setup = nrpe.NRPE(hostname=hostname) - - for service in services: - nrpe_setup.remove_check(shortname=service) - - -def is_privileged(): - """Return boolean indicating whether or not to set allow-privileged=true. - - """ - privileged = hookenv.config('allow-privileged').lower() - if privileged == 'auto': - return is_state('kubernetes-master.gpu.enabled') - else: - return privileged == 'true' - - -@when('config.changed.allow-privileged') -@when('kubernetes-master.components.started') -def on_config_allow_privileged_change(): - """React to changed 'allow-privileged' config value. - - """ - remove_state('kubernetes-master.components.started') - remove_state('config.changed.allow-privileged') - - -@when_any('config.changed.api-extra-args', - 'config.changed.audit-policy', - 'config.changed.audit-webhook-config') -@when('kubernetes-master.components.started') -@when('leadership.set.auto_storage_backend') -@when('etcd.available') -def reconfigure_apiserver(etcd): - configure_apiserver(etcd.get_connection_string()) - - -@when('config.changed.controller-manager-extra-args') -@when('kubernetes-master.components.started') -def on_config_controller_manager_extra_args_change(): - configure_controller_manager() - - -@when('config.changed.scheduler-extra-args') -@when('kubernetes-master.components.started') -def on_config_scheduler_extra_args_change(): - configure_scheduler() - - -@when('kube-control.gpu.available') -@when('kubernetes-master.components.started') -@when_not('kubernetes-master.gpu.enabled') -def on_gpu_available(kube_control): - """The remote side (kubernetes-worker) is gpu-enabled. - - We need to run in privileged mode. - - """ - kube_version = get_version('kube-apiserver') - config = hookenv.config() - if (config['allow-privileged'].lower() == "false" and - kube_version < (1, 9)): - return - - remove_state('kubernetes-master.components.started') - set_state('kubernetes-master.gpu.enabled') - - -@when('kubernetes-master.gpu.enabled') -@when('kubernetes-master.components.started') -@when_not('kubernetes-master.privileged') -def gpu_with_no_privileged(): - """We were in gpu mode, but the operator has set allow-privileged="false", - so we can't run in gpu mode anymore. - - """ - if get_version('kube-apiserver') < (1, 9): - remove_state('kubernetes-master.gpu.enabled') - - -@when('kube-control.connected') -@when_not('kube-control.gpu.available') -@when('kubernetes-master.gpu.enabled') -@when('kubernetes-master.components.started') -def gpu_departed(kube_control): - """We were in gpu mode, but the workers informed us there is - no gpu support anymore. - - """ - remove_state('kubernetes-master.gpu.enabled') - - -@hook('stop') -def shutdown(): - """ Stop the kubernetes master services - - """ - service_stop('snap.kube-apiserver.daemon') - service_stop('snap.kube-controller-manager.daemon') - service_stop('snap.kube-scheduler.daemon') - - -def restart_apiserver(): - hookenv.status_set('maintenance', 'Restarting kube-apiserver') - host.service_restart('snap.kube-apiserver.daemon') - - -def restart_controller_manager(): - hookenv.status_set('maintenance', 'Restarting kube-controller-manager') - host.service_restart('snap.kube-controller-manager.daemon') - - -def restart_scheduler(): - hookenv.status_set('maintenance', 'Restarting kube-scheduler') - host.service_restart('snap.kube-scheduler.daemon') - - -def arch(): - '''Return the package architecture as a string. Raise an exception if the - architecture is not supported by kubernetes.''' - # Get the package architecture for this system. - architecture = check_output(['dpkg', '--print-architecture']).rstrip() - # Convert the binary result into a string. - architecture = architecture.decode('utf-8') - return architecture - - -def build_kubeconfig(server): - '''Gather the relevant data for Kubernetes configuration objects and create - a config object with that information.''' - # Get the options from the tls-client layer. - layer_options = layer.options('tls-client') - # Get all the paths to the tls information required for kubeconfig. - ca = layer_options.get('ca_certificate_path') - ca_exists = ca and os.path.isfile(ca) - client_pass = get_password('basic_auth.csv', 'admin') - # Do we have everything we need? - if ca_exists and client_pass: - # Create an absolute path for the kubeconfig file. - kubeconfig_path = os.path.join(os.sep, 'home', 'ubuntu', 'config') - # Create the kubeconfig on this system so users can access the cluster. - - create_kubeconfig(kubeconfig_path, server, ca, - user='admin', password=client_pass) - # Make the config file readable by the ubuntu users so juju scp works. - cmd = ['chown', 'ubuntu:ubuntu', kubeconfig_path] - check_call(cmd) - - -def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None, - user='ubuntu', context='juju-context', - cluster='juju-cluster', password=None, token=None): - '''Create a configuration for Kubernetes based on path using the supplied - arguments for values of the Kubernetes server, CA, key, certificate, user - context and cluster.''' - if not key and not certificate and not password and not token: - raise ValueError('Missing authentication mechanism.') - - # token and password are mutually exclusive. Error early if both are - # present. The developer has requested an impossible situation. - # see: kubectl config set-credentials --help - if token and password: - raise ValueError('Token and Password are mutually exclusive.') - # Create the config file with the address of the master server. - cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \ - '--server={2} --certificate-authority={3} --embed-certs=true' - check_call(split(cmd.format(kubeconfig, cluster, server, ca))) - # Delete old users - cmd = 'kubectl config --kubeconfig={0} unset users' - check_call(split(cmd.format(kubeconfig))) - # Create the credentials using the client flags. - cmd = 'kubectl config --kubeconfig={0} ' \ - 'set-credentials {1} '.format(kubeconfig, user) - - if key and certificate: - cmd = '{0} --client-key={1} --client-certificate={2} '\ - '--embed-certs=true'.format(cmd, key, certificate) - if password: - cmd = "{0} --username={1} --password={2}".format(cmd, user, password) - # This is mutually exclusive from password. They will not work together. - if token: - cmd = "{0} --token={1}".format(cmd, token) - check_call(split(cmd)) - # Create a default context with the cluster. - cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \ - '--cluster={2} --user={3}' - check_call(split(cmd.format(kubeconfig, context, cluster, user))) - # Make the config use this new context. - cmd = 'kubectl config --kubeconfig={0} use-context {1}' - check_call(split(cmd.format(kubeconfig, context))) - - -def get_dns_ip(): - cmd = "kubectl get service --namespace kube-system kube-dns --output json" - output = check_output(cmd, shell=True).decode() - svc = json.loads(output) - return svc['spec']['clusterIP'] - - -def get_deprecated_dns_ip(): - '''We previously hardcoded the dns ip. This function returns the old - hardcoded value for use with older versions of cdk_addons.''' - interface = ipaddress.IPv4Interface(service_cidr()) - ip = interface.network.network_address + 10 - return ip.exploded - - -def get_kubernetes_service_ip(): - '''Get the IP address for the kubernetes service based on the cidr.''' - interface = ipaddress.IPv4Interface(service_cidr()) - # Add .1 at the end of the network - ip = interface.network.network_address + 1 - return ip.exploded - - -def handle_etcd_relation(reldata): - ''' Save the client credentials and set appropriate daemon flags when - etcd declares itself as available''' - # Define where the etcd tls files will be kept. - etcd_dir = '/root/cdk/etcd' - - # Create paths to the etcd client ca, key, and cert file locations. - ca = os.path.join(etcd_dir, 'client-ca.pem') - key = os.path.join(etcd_dir, 'client-key.pem') - cert = os.path.join(etcd_dir, 'client-cert.pem') - - # Save the client credentials (in relation data) to the paths provided. - reldata.save_client_credentials(key, cert, ca) - - -def parse_extra_args(config_key): - elements = hookenv.config().get(config_key, '').split() - args = {} - - for element in elements: - if '=' in element: - key, _, value = element.partition('=') - args[key] = value - else: - args[element] = 'true' - - return args - - -def configure_kubernetes_service(service, base_args, extra_args_key): - prev_args_key = 'kubernetes-master.prev_args.' + service - prev_args = db.get(prev_args_key) or {} - - extra_args = parse_extra_args(extra_args_key) - - args = {} - for arg in prev_args: - # remove previous args by setting to null - # note this is so we remove them from the snap's config - args[arg] = 'null' - for k, v in base_args.items(): - args[k] = v - for k, v in extra_args.items(): - args[k] = v - - cmd = ['snap', 'set', service] + ['%s=%s' % item for item in args.items()] - check_call(cmd) - - db.set(prev_args_key, args) - - -def remove_if_exists(path): - try: - os.remove(path) - except FileNotFoundError: - pass - - -def write_audit_config_file(path, contents): - with open(path, 'w') as f: - header = '# Autogenerated by kubernetes-master charm' - f.write(header + '\n' + contents) - - -def configure_apiserver(etcd_connection_string): - api_opts = {} - - # Get the tls paths from the layer data. - layer_options = layer.options('tls-client') - ca_cert_path = layer_options.get('ca_certificate_path') - client_cert_path = layer_options.get('client_certificate_path') - client_key_path = layer_options.get('client_key_path') - server_cert_path = layer_options.get('server_certificate_path') - server_key_path = layer_options.get('server_key_path') - - # at one point in time, this code would set ca-client-cert, - # but this was removed. This was before configure_kubernetes_service - # kept track of old arguments and removed them, so client-ca-cert - # was able to hang around forever stored in the snap configuration. - # This removes that stale configuration from the snap if it still - # exists. - api_opts['client-ca-file'] = 'null' - - if is_privileged(): - api_opts['allow-privileged'] = 'true' - set_state('kubernetes-master.privileged') - else: - api_opts['allow-privileged'] = 'false' - remove_state('kubernetes-master.privileged') - - # Handle static options for now - api_opts['service-cluster-ip-range'] = service_cidr() - api_opts['min-request-timeout'] = '300' - api_opts['v'] = '4' - api_opts['tls-cert-file'] = server_cert_path - api_opts['tls-private-key-file'] = server_key_path - api_opts['kubelet-certificate-authority'] = ca_cert_path - api_opts['kubelet-client-certificate'] = client_cert_path - api_opts['kubelet-client-key'] = client_key_path - api_opts['logtostderr'] = 'true' - api_opts['insecure-bind-address'] = '127.0.0.1' - api_opts['insecure-port'] = '8080' - api_opts['storage-backend'] = getStorageBackend() - api_opts['basic-auth-file'] = '/root/cdk/basic_auth.csv' - - api_opts['token-auth-file'] = '/root/cdk/known_tokens.csv' - api_opts['service-account-key-file'] = '/root/cdk/serviceaccount.key' - api_opts['kubelet-preferred-address-types'] = \ - '[InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP]' - api_opts['advertise-address'] = get_ingress_address('kube-control') - - etcd_dir = '/root/cdk/etcd' - etcd_ca = os.path.join(etcd_dir, 'client-ca.pem') - etcd_key = os.path.join(etcd_dir, 'client-key.pem') - etcd_cert = os.path.join(etcd_dir, 'client-cert.pem') - - api_opts['etcd-cafile'] = etcd_ca - api_opts['etcd-keyfile'] = etcd_key - api_opts['etcd-certfile'] = etcd_cert - api_opts['etcd-servers'] = etcd_connection_string - - admission_control_pre_1_9 = [ - 'NamespaceLifecycle', - 'LimitRanger', - 'ServiceAccount', - 'ResourceQuota', - 'DefaultTolerationSeconds' - ] - - admission_control = [ - 'NamespaceLifecycle', - 'LimitRanger', - 'ServiceAccount', - 'PersistentVolumeLabel', - 'DefaultStorageClass', - 'DefaultTolerationSeconds', - 'MutatingAdmissionWebhook', - 'ValidatingAdmissionWebhook', - 'ResourceQuota' - ] - - auth_mode = hookenv.config('authorization-mode') - if 'Node' in auth_mode: - admission_control.append('NodeRestriction') - - api_opts['authorization-mode'] = auth_mode - - kube_version = get_version('kube-apiserver') - if kube_version < (1, 6): - hookenv.log('Removing DefaultTolerationSeconds from admission-control') - admission_control_pre_1_9.remove('DefaultTolerationSeconds') - if kube_version < (1, 9): - api_opts['admission-control'] = ','.join(admission_control_pre_1_9) - else: - api_opts['admission-control'] = ','.join(admission_control) - - if kube_version > (1, 6) and \ - hookenv.config('enable-metrics'): - api_opts['requestheader-client-ca-file'] = ca_cert_path - api_opts['requestheader-allowed-names'] = 'client' - api_opts['requestheader-extra-headers-prefix'] = 'X-Remote-Extra-' - api_opts['requestheader-group-headers'] = 'X-Remote-Group' - api_opts['requestheader-username-headers'] = 'X-Remote-User' - api_opts['proxy-client-cert-file'] = client_cert_path - api_opts['proxy-client-key-file'] = client_key_path - api_opts['enable-aggregator-routing'] = 'true' - api_opts['client-ca-file'] = ca_cert_path - - if is_state('endpoint.aws.ready'): - api_opts['cloud-provider'] = 'aws' - elif is_state('endpoint.gcp.ready'): - cloud_config_path = _cloud_config_path('kube-apiserver') - api_opts['cloud-provider'] = 'gce' - api_opts['cloud-config'] = str(cloud_config_path) - elif is_state('endpoint.openstack.ready'): - cloud_config_path = _cloud_config_path('kube-apiserver') - api_opts['cloud-provider'] = 'openstack' - api_opts['cloud-config'] = str(cloud_config_path) - elif (is_state('endpoint.vsphere.ready') and - get_version('kube-apiserver') >= (1, 12)): - cloud_config_path = _cloud_config_path('kube-apiserver') - api_opts['cloud-provider'] = 'vsphere' - api_opts['cloud-config'] = str(cloud_config_path) - elif is_state('endpoint.azure.ready'): - cloud_config_path = _cloud_config_path('kube-apiserver') - api_opts['cloud-provider'] = 'azure' - api_opts['cloud-config'] = str(cloud_config_path) - - audit_root = '/root/cdk/audit' - os.makedirs(audit_root, exist_ok=True) - - audit_log_path = audit_root + '/audit.log' - api_opts['audit-log-path'] = audit_log_path - api_opts['audit-log-maxsize'] = '100' - api_opts['audit-log-maxbackup'] = '9' - - audit_policy_path = audit_root + '/audit-policy.yaml' - audit_policy = hookenv.config('audit-policy') - if audit_policy: - write_audit_config_file(audit_policy_path, audit_policy) - api_opts['audit-policy-file'] = audit_policy_path - else: - remove_if_exists(audit_policy_path) - - audit_webhook_config_path = audit_root + '/audit-webhook-config.yaml' - audit_webhook_config = hookenv.config('audit-webhook-config') - if audit_webhook_config: - write_audit_config_file(audit_webhook_config_path, - audit_webhook_config) - api_opts['audit-webhook-config-file'] = audit_webhook_config_path - else: - remove_if_exists(audit_webhook_config_path) - - configure_kubernetes_service('kube-apiserver', api_opts, 'api-extra-args') - restart_apiserver() - - -def configure_controller_manager(): - controller_opts = {} - - # Get the tls paths from the layer data. - layer_options = layer.options('tls-client') - ca_cert_path = layer_options.get('ca_certificate_path') - - # Default to 3 minute resync. TODO: Make this configurable? - controller_opts['min-resync-period'] = '3m' - controller_opts['v'] = '2' - controller_opts['root-ca-file'] = ca_cert_path - controller_opts['logtostderr'] = 'true' - controller_opts['master'] = 'http://127.0.0.1:8080' - - controller_opts['service-account-private-key-file'] = \ - '/root/cdk/serviceaccount.key' - - if is_state('endpoint.aws.ready'): - controller_opts['cloud-provider'] = 'aws' - elif is_state('endpoint.gcp.ready'): - cloud_config_path = _cloud_config_path('kube-controller-manager') - controller_opts['cloud-provider'] = 'gce' - controller_opts['cloud-config'] = str(cloud_config_path) - elif is_state('endpoint.openstack.ready'): - cloud_config_path = _cloud_config_path('kube-controller-manager') - controller_opts['cloud-provider'] = 'openstack' - controller_opts['cloud-config'] = str(cloud_config_path) - elif (is_state('endpoint.vsphere.ready') and - get_version('kube-apiserver') >= (1, 12)): - cloud_config_path = _cloud_config_path('kube-controller-manager') - controller_opts['cloud-provider'] = 'vsphere' - controller_opts['cloud-config'] = str(cloud_config_path) - elif is_state('endpoint.azure.ready'): - cloud_config_path = _cloud_config_path('kube-controller-manager') - controller_opts['cloud-provider'] = 'azure' - controller_opts['cloud-config'] = str(cloud_config_path) - - configure_kubernetes_service('kube-controller-manager', controller_opts, - 'controller-manager-extra-args') - restart_controller_manager() - - -def configure_scheduler(): - scheduler_opts = {} - - scheduler_opts['v'] = '2' - scheduler_opts['logtostderr'] = 'true' - scheduler_opts['master'] = 'http://127.0.0.1:8080' - - configure_kubernetes_service('kube-scheduler', scheduler_opts, - 'scheduler-extra-args') - - restart_scheduler() - - -def setup_basic_auth(password=None, username='admin', uid='admin', - groups=None): - '''Create the htacces file and the tokens.''' - root_cdk = '/root/cdk' - if not os.path.isdir(root_cdk): - os.makedirs(root_cdk) - htaccess = os.path.join(root_cdk, 'basic_auth.csv') - if not password: - password = token_generator() - with open(htaccess, 'w') as stream: - if groups: - stream.write('{0},{1},{2},"{3}"'.format(password, - username, uid, groups)) - else: - stream.write('{0},{1},{2}'.format(password, username, uid)) - - -def setup_tokens(token, username, user, groups=None): - '''Create a token file for kubernetes authentication.''' - root_cdk = '/root/cdk' - if not os.path.isdir(root_cdk): - os.makedirs(root_cdk) - known_tokens = os.path.join(root_cdk, 'known_tokens.csv') - if not token: - token = token_generator() - with open(known_tokens, 'a') as stream: - if groups: - stream.write('{0},{1},{2},"{3}"\n'.format(token, - username, - user, - groups)) - else: - stream.write('{0},{1},{2}\n'.format(token, username, user)) - - -def get_password(csv_fname, user): - '''Get the password of user within the csv file provided.''' - root_cdk = '/root/cdk' - tokens_fname = os.path.join(root_cdk, csv_fname) - if not os.path.isfile(tokens_fname): - return None - with open(tokens_fname, 'r') as stream: - for line in stream: - record = line.split(',') - if record[1] == user: - return record[0] - return None - - -def get_token(username): - """Grab a token from the static file if present. """ - return get_password('known_tokens.csv', username) - - -def set_token(password, save_salt): - ''' Store a token so it can be recalled later by token_generator. - - param: password - the password to be stored - param: save_salt - the key to store the value of the token.''' - db.set(save_salt, password) - return db.get(save_salt) - - -def token_generator(length=32): - ''' Generate a random token for use in passwords and account tokens. - - param: length - the length of the token to generate''' - alpha = string.ascii_letters + string.digits - token = ''.join(random.SystemRandom().choice(alpha) for _ in range(length)) - return token - - -@retry(times=3, delay_secs=10) -def all_kube_system_pods_running(): - ''' Check pod status in the kube-system namespace. Returns True if all - pods are running, False otherwise. ''' - cmd = ['kubectl', 'get', 'po', '-n', 'kube-system', '-o', 'json'] - - try: - output = check_output(cmd).decode('utf-8') - result = json.loads(output) - except CalledProcessError: - hookenv.log('failed to get kube-system pod status') - return False - hookenv.log('Checking system pods status: {}'.format(', '.join( - '='.join([pod['metadata']['name'], pod['status']['phase']]) - for pod in result['items']))) - - all_pending = all(pod['status']['phase'] == 'Pending' - for pod in result['items']) - if is_state('endpoint.gcp.ready') and all_pending: - poke_network_unavailable() - return False - - # All pods must be Running or Evicted (which should re-spawn) - all_running = all(pod['status']['phase'] == 'Running' or - pod['status'].get('reason', '') == 'Evicted' - for pod in result['items']) - return all_running - - -def poke_network_unavailable(): - """ - Work around https://github.com/kubernetes/kubernetes/issues/44254 by - manually poking the status into the API server to tell the nodes they have - a network route. - - This is needed because kubelet sets the NetworkUnavailable flag and expects - the network plugin to clear it, which only kubenet does. There is some - discussion about refactoring the affected code but nothing has happened - in a while. - """ - cmd = ['kubectl', 'get', 'nodes', '-o', 'json'] - - try: - output = check_output(cmd).decode('utf-8') - nodes = json.loads(output)['items'] - except CalledProcessError: - hookenv.log('failed to get kube-system nodes') - return - except (KeyError, json.JSONDecodeError) as e: - hookenv.log('failed to parse kube-system node status ' - '({}): {}'.format(e, output), hookenv.ERROR) - return - - for node in nodes: - node_name = node['metadata']['name'] - url = 'http://localhost:8080/api/v1/nodes/{}/status'.format(node_name) - with urlopen(url) as response: - code = response.getcode() - body = response.read().decode('utf8') - if code != 200: - hookenv.log('failed to get node status from {} [{}]: {}'.format( - url, code, body), hookenv.ERROR) - return - try: - node_info = json.loads(body) - conditions = node_info['status']['conditions'] - i = [c['type'] for c in conditions].index('NetworkUnavailable') - if conditions[i]['status'] == 'True': - hookenv.log('Clearing NetworkUnavailable from {}'.format( - node_name)) - conditions[i] = { - "type": "NetworkUnavailable", - "status": "False", - "reason": "RouteCreated", - "message": "Manually set through k8s api", - } - req = Request(url, method='PUT', - data=json.dumps(node_info).encode('utf8'), - headers={'Content-Type': 'application/json'}) - with urlopen(req) as response: - code = response.getcode() - body = response.read().decode('utf8') - if code not in (200, 201, 202): - hookenv.log('failed to update node status [{}]: {}'.format( - code, body), hookenv.ERROR) - return - except (json.JSONDecodeError, KeyError): - hookenv.log('failed to parse node status: {}'.format(body), - hookenv.ERROR) - return - - -def apiserverVersion(): - cmd = 'kube-apiserver --version'.split() - version_string = check_output(cmd).decode('utf-8') - return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3]) - - -def touch(fname): - try: - os.utime(fname, None) - except OSError: - open(fname, 'a').close() - - -def getStorageBackend(): - storage_backend = hookenv.config('storage-backend') - if storage_backend == 'auto': - storage_backend = leader_get('auto_storage_backend') - return storage_backend - - -@when('leadership.is_leader') -@when_not('leadership.set.cluster_tag') -def create_cluster_tag(): - cluster_tag = 'kubernetes-{}'.format(token_generator().lower()) - leader_set(cluster_tag=cluster_tag) - - -@when('leadership.set.cluster_tag', - 'kube-control.connected') -@when_not('kubernetes-master.cluster-tag-sent') -def send_cluster_tag(): - cluster_tag = leader_get('cluster_tag') - kube_control = endpoint_from_flag('kube-control.connected') - kube_control.set_cluster_tag(cluster_tag) - set_state('kubernetes-master.cluster-tag-sent') - - -@when_not('kube-control.connected') -def clear_cluster_tag_sent(): - remove_state('kubernetes-master.cluster-tag-sent') - - -@when_any('endpoint.aws.joined', - 'endpoint.gcp.joined', - 'endpoint.openstack.joined', - 'endpoint.vsphere.joined', - 'endpoint.azure.joined') -@when_not('kubernetes-master.cloud.ready') -def set_cloud_pending(): - k8s_version = get_version('kube-apiserver') - k8s_1_11 = k8s_version >= (1, 11) - k8s_1_12 = k8s_version >= (1, 12) - vsphere_joined = is_state('endpoint.vsphere.joined') - azure_joined = is_state('endpoint.azure.joined') - if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): - set_state('kubernetes-master.cloud.blocked') - else: - remove_state('kubernetes-master.cloud.blocked') - set_state('kubernetes-master.cloud.pending') - - -@when_any('endpoint.aws.joined', - 'endpoint.gcp.joined', - 'endpoint.azure.joined') -@when('leadership.set.cluster_tag') -@when_not('kubernetes-master.cloud.request-sent') -def request_integration(): - hookenv.status_set('maintenance', 'requesting cloud integration') - cluster_tag = leader_get('cluster_tag') - if is_state('endpoint.aws.joined'): - cloud = endpoint_from_flag('endpoint.aws.joined') - cloud.tag_instance({ - 'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned', - 'k8s.io/role/master': 'true', - }) - cloud.tag_instance_security_group({ - 'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned', - }) - cloud.tag_instance_subnet({ - 'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned', - }) - cloud.enable_object_storage_management(['kubernetes-*']) - cloud.enable_load_balancer_management() - elif is_state('endpoint.gcp.joined'): - cloud = endpoint_from_flag('endpoint.gcp.joined') - cloud.label_instance({ - 'k8s-io-cluster-name': cluster_tag, - 'k8s-io-role-master': 'master', - }) - cloud.enable_object_storage_management() - cloud.enable_security_management() - elif is_state('endpoint.azure.joined'): - cloud = endpoint_from_flag('endpoint.azure.joined') - cloud.tag_instance({ - 'k8s-io-cluster-name': cluster_tag, - 'k8s-io-role-master': 'master', - }) - cloud.enable_object_storage_management() - cloud.enable_security_management() - cloud.enable_instance_inspection() - cloud.enable_network_management() - cloud.enable_dns_management() - cloud.enable_block_storage_management() - set_state('kubernetes-master.cloud.request-sent') - - -@when_none('endpoint.aws.joined', - 'endpoint.gcp.joined', - 'endpoint.openstack.joined', - 'endpoint.vsphere.joined', - 'endpoint.azure.joined') -def clear_cloud_flags(): - remove_state('kubernetes-master.cloud.pending') - remove_state('kubernetes-master.cloud.request-sent') - remove_state('kubernetes-master.cloud.blocked') - remove_state('kubernetes-master.cloud.ready') - - -@when_any('endpoint.aws.ready', - 'endpoint.gcp.ready', - 'endpoint.openstack.ready', - 'endpoint.vsphere.ready', - 'endpoint.azure.ready') -@when_not('kubernetes-master.cloud.blocked', - 'kubernetes-master.cloud.ready') -def cloud_ready(): - if is_state('endpoint.gcp.ready'): - _write_gcp_snap_config('kube-apiserver') - _write_gcp_snap_config('kube-controller-manager') - elif is_state('endpoint.openstack.ready'): - _write_openstack_snap_config('kube-apiserver') - _write_openstack_snap_config('kube-controller-manager') - elif is_state('endpoint.vsphere.ready'): - _write_vsphere_snap_config('kube-apiserver') - _write_vsphere_snap_config('kube-controller-manager') - elif is_state('endpoint.azure.ready'): - _write_azure_snap_config('kube-apiserver') - _write_azure_snap_config('kube-controller-manager') - remove_state('kubernetes-master.cloud.pending') - set_state('kubernetes-master.cloud.ready') - remove_state('kubernetes-master.components.started') # force restart - - -def _snap_common_path(component): - return Path('/var/snap/{}/common'.format(component)) - - -def _cloud_config_path(component): - return _snap_common_path(component) / 'cloud-config.conf' - - -def _gcp_creds_path(component): - return _snap_common_path(component) / 'gcp-creds.json' - - -def _daemon_env_path(component): - return _snap_common_path(component) / 'environment' - - -def _cdk_addons_template_path(): - return Path('/snap/cdk-addons/current/templates') - - -def _write_gcp_snap_config(component): - # gcp requires additional credentials setup - gcp = endpoint_from_flag('endpoint.gcp.ready') - creds_path = _gcp_creds_path(component) - with creds_path.open('w') as fp: - os.fchmod(fp.fileno(), 0o600) - fp.write(gcp.credentials) - - # create a cloud-config file that sets token-url to nil to make the - # services use the creds env var instead of the metadata server, as - # well as making the cluster multizone - cloud_config_path = _cloud_config_path(component) - cloud_config_path.write_text('[Global]\n' - 'token-url = nil\n' - 'multizone = true\n') - - daemon_env_path = _daemon_env_path(component) - if daemon_env_path.exists(): - daemon_env = daemon_env_path.read_text() - if not daemon_env.endswith('\n'): - daemon_env += '\n' - else: - daemon_env = '' - if gcp_creds_env_key not in daemon_env: - daemon_env += '{}={}\n'.format(gcp_creds_env_key, creds_path) - daemon_env_path.parent.mkdir(parents=True, exist_ok=True) - daemon_env_path.write_text(daemon_env) - - -def _write_openstack_snap_config(component): - # openstack requires additional credentials setup - openstack = endpoint_from_flag('endpoint.openstack.ready') - - cloud_config_path = _cloud_config_path(component) - cloud_config_path.write_text('\n'.join([ - '[Global]', - 'auth-url = {}'.format(openstack.auth_url), - 'username = {}'.format(openstack.username), - 'password = {}'.format(openstack.password), - 'tenant-name = {}'.format(openstack.project_name), - 'domain-name = {}'.format(openstack.user_domain_name), - ])) - - -def _write_vsphere_snap_config(component): - # vsphere requires additional cloud config - vsphere = endpoint_from_flag('endpoint.vsphere.ready') - - # NB: vsphere provider will ask kube-apiserver and -controller-manager to - # find a uuid from sysfs unless a global config value is set. Our strict - # snaps cannot read sysfs, so let's do it in the charm. An invalid uuid is - # not fatal for storage, but it will muddy the logs; try to get it right. - uuid_file = '/sys/class/dmi/id/product_uuid' - try: - with open(uuid_file, 'r') as f: - uuid = f.read().strip() - except IOError as err: - hookenv.log("Unable to read UUID from sysfs: {}".format(err)) - uuid = 'UNKNOWN' - - cloud_config_path = _cloud_config_path(component) - cloud_config_path.write_text('\n'.join([ - '[Global]', - 'insecure-flag = true', - 'datacenters = "{}"'.format(vsphere.datacenter), - 'vm-uuid = "VMware-{}"'.format(uuid), - '[VirtualCenter "{}"]'.format(vsphere.vsphere_ip), - 'user = {}'.format(vsphere.user), - 'password = {}'.format(vsphere.password), - '[Workspace]', - 'server = {}'.format(vsphere.vsphere_ip), - 'datacenter = "{}"'.format(vsphere.datacenter), - 'default-datastore = "{}"'.format(vsphere.datastore), - 'folder = "kubernetes"', - 'resourcepool-path = ""', - '[Disk]', - 'scsicontrollertype = "pvscsi"', - ])) - - -def _write_azure_snap_config(component): - azure = endpoint_from_flag('endpoint.azure.ready') - cloud_config_path = _cloud_config_path(component) - cloud_config_path.write_text(json.dumps({ - 'useInstanceMetadata': True, - 'useManagedIdentityExtension': True, - 'subscriptionId': azure.subscription_id, - 'resourceGroup': azure.resource_group, - 'location': azure.resource_group_location, - 'vnetName': azure.vnet_name, - 'vnetResourceGroup': azure.vnet_resource_group, - 'subnetName': azure.subnet_name, - 'securityGroupName': azure.security_group_name, - })) diff --git a/cluster/juju/layers/kubernetes-master/templates/ceph-secret.yaml b/cluster/juju/layers/kubernetes-master/templates/ceph-secret.yaml deleted file mode 100644 index fc6b1fb914..0000000000 --- a/cluster/juju/layers/kubernetes-master/templates/ceph-secret.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: ceph-secret -type: kubernetes.io/rbd -data: - key: {{ secret }} diff --git a/cluster/juju/layers/kubernetes-master/templates/ceph.conf b/cluster/juju/layers/kubernetes-master/templates/ceph.conf deleted file mode 100644 index d27c522d8f..0000000000 --- a/cluster/juju/layers/kubernetes-master/templates/ceph.conf +++ /dev/null @@ -1,18 +0,0 @@ -[global] -auth cluster required = {{ auth_supported }} -auth service required = {{ auth_supported }} -auth client required = {{ auth_supported }} -keyring = /etc/ceph/$cluster.$name.keyring -mon host = {{ mon_hosts }} -fsid = {{ fsid }} - -log to syslog = {{ use_syslog }} -err to syslog = {{ use_syslog }} -clog to syslog = {{ use_syslog }} -mon cluster log to syslog = {{ use_syslog }} -debug mon = {{ loglevel }}/5 -debug osd = {{ loglevel }}/5 - -[client] -log file = /var/log/ceph.log - diff --git a/cluster/juju/layers/kubernetes-master/templates/create-namespace.yaml.j2 b/cluster/juju/layers/kubernetes-master/templates/create-namespace.yaml.j2 deleted file mode 100644 index a121ecc677..0000000000 --- a/cluster/juju/layers/kubernetes-master/templates/create-namespace.yaml.j2 +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ name }} - labels: - name: {{ name }} diff --git a/cluster/juju/layers/kubernetes-master/templates/rbd-persistent-volume.yaml b/cluster/juju/layers/kubernetes-master/templates/rbd-persistent-volume.yaml deleted file mode 100644 index 84248e5432..0000000000 --- a/cluster/juju/layers/kubernetes-master/templates/rbd-persistent-volume.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# JUJU Internal Template used to enlist RBD volumes from the -# `create-rbd-pv` action. This is a temporary file on disk to enlist resources. -apiVersion: v1 -kind: PersistentVolume -metadata: - name: {{ RBD_NAME }} -spec: - capacity: - storage: {{ RBD_SIZE }}M - accessModes: - - {{ PV_MODE }} - storageClassName: "rbd" - rbd: - monitors: - {% for host in monitors %} - - {{ host }} - {% endfor %} - pool: rbd - image: {{ RBD_NAME }} - user: admin - secretRef: - name: ceph-secret - fsType: {{ RBD_FS }} - readOnly: false - # persistentVolumeReclaimPolicy: Recycle diff --git a/cluster/juju/layers/kubernetes-master/tox.ini b/cluster/juju/layers/kubernetes-master/tox.ini deleted file mode 100644 index 0b8b27a9d5..0000000000 --- a/cluster/juju/layers/kubernetes-master/tox.ini +++ /dev/null @@ -1,12 +0,0 @@ -[tox] -skipsdist=True -envlist = py34, py35 -skip_missing_interpreters = True - -[testenv] -commands = py.test -v -deps = - -r{toxinidir}/requirements.txt - -[flake8] -exclude=docs diff --git a/cluster/juju/layers/kubernetes-worker/HACKING.md b/cluster/juju/layers/kubernetes-worker/HACKING.md deleted file mode 100644 index 28e380cc9e..0000000000 --- a/cluster/juju/layers/kubernetes-worker/HACKING.md +++ /dev/null @@ -1,25 +0,0 @@ - # Kubernetes Worker - -### Building from the layer - -You can clone the kubernetes-worker layer with git and build locally if you -have the charm package/snap installed. - -```shell -# Instal the snap -sudo snap install charm --channel=edge - -# Set the build environment -export JUJU_REPOSITORY=$HOME - -# Clone the layer and build it to our JUJU_REPOSITORY -git clone https://github.com/juju-solutions/kubernetes -cd kubernetes/cluster/juju/layers/kubernetes-worker -charm build -r -``` - -### Contributing - -TBD - - diff --git a/cluster/juju/layers/kubernetes-worker/README.md b/cluster/juju/layers/kubernetes-worker/README.md deleted file mode 100644 index a661696f65..0000000000 --- a/cluster/juju/layers/kubernetes-worker/README.md +++ /dev/null @@ -1,131 +0,0 @@ -# Kubernetes Worker - -## Usage - -This charm deploys a container runtime, and additionally stands up the Kubernetes -worker applications: kubelet, and kube-proxy. - -In order for this charm to be useful, it should be deployed with its companion -charm [kubernetes-master](https://jujucharms.com/u/containers/kubernetes-master) -and linked with an SDN-Plugin. - -This charm has also been bundled up for your convenience so you can skip the -above steps, and deploy it with a single command: - -```shell -juju deploy canonical-kubernetes -``` - -For more information about [Canonical Kubernetes](https://jujucharms.com/canonical-kubernetes) -consult the bundle `README.md` file. - - -## Scale out - -To add additional compute capacity to your Kubernetes workers, you may -`juju add-unit` scale the cluster of applications. They will automatically -join any related kubernetes-master, and enlist themselves as ready once the -deployment is complete. - -## Snap Configuration - -The kubernetes resources used by this charm are snap packages. When not -specified during deployment, these resources come from the public store. By -default, the `snapd` daemon will refresh all snaps installed from the store -four (4) times per day. A charm configuration option is provided for operators -to control this refresh frequency. - ->NOTE: this is a global configuration option and will affect the refresh -time for all snaps installed on a system. - -Examples: - -```sh -## refresh kubernetes-worker snaps every tuesday -juju config kubernetes-worker snapd_refresh="tue" - -## refresh snaps at 11pm on the last (5th) friday of the month -juju config kubernetes-worker snapd_refresh="fri5,23:00" - -## delay the refresh as long as possible -juju config kubernetes-worker snapd_refresh="max" - -## use the system default refresh timer -juju config kubernetes-worker snapd_refresh="" -``` - -For more information on the possible values for `snapd_refresh`, see the -*refresh.timer* section in the [system options][] documentation. - -[system options]: https://forum.snapcraft.io/t/system-options/87 - -## Operational actions - -The kubernetes-worker charm supports the following Operational Actions: - -#### Pause - -Pausing the workload enables administrators to both [drain](http://kubernetes.io/docs/user-guide/kubectl/kubectl_drain/) and [cordon](http://kubernetes.io/docs/user-guide/kubectl/kubectl_cordon/) -a unit for maintenance. - - -#### Resume - -Resuming the workload will [uncordon](http://kubernetes.io/docs/user-guide/kubectl/kubectl_uncordon/) a paused unit. Workloads will automatically migrate unless otherwise directed via their application declaration. - -## Private registry - -With the "registry" action that is part for the kubernetes-worker charm, you can very easily create a private docker registry, with authentication, and available over TLS. Please note that the registry deployed with the action is not HA, and uses storage tied to the kubernetes node where the pod is running. So if the registry pod changes is migrated from one node to another for whatever reason, you will need to re-publish the images. - -### Example usage - -Create the relevant authentication files. Let's say you want user `userA` to authenticate with the password `passwordA`. Then you'll do : - - echo -n "userA:passwordA" > htpasswd-plain - htpasswd -c -b -B htpasswd userA passwordA - -(the `htpasswd` program comes with the `apache2-utils` package) - -Supposing your registry will be reachable at `myregistry.company.com`, and that you already have your TLS key in the `registry.key` file, and your TLS certificate (with `myregistry.company.com` as Common Name) in the `registry.crt` file, you would then run : - - juju run-action kubernetes-worker/0 registry domain=myregistry.company.com htpasswd="$(base64 -w0 htpasswd)" htpasswd-plain="$(base64 -w0 htpasswd-plain)" tlscert="$(base64 -w0 registry.crt)" tlskey="$(base64 -w0 registry.key)" ingress=true - -If you then decide that you want do delete the registry, just run : - - juju run-action kubernetes-worker/0 registry delete=true ingress=true - -## Known Limitations - -Kubernetes workers currently only support 'phaux' HA scenarios. Even when configured with an HA cluster string, they will only ever contact the first unit in the cluster map. To enable a proper HA story, kubernetes-worker units are encouraged to proxy through a [kubeapi-load-balancer](https://jujucharms.com/kubeapi-load-balancer) -application. This enables a HA deployment without the need to -re-render configuration and disrupt the worker services. - -External access to pods must be performed through a [Kubernetes -Ingress Resource](http://kubernetes.io/docs/user-guide/ingress/). - -When using NodePort type networking, there is no automation in exposing the -ports selected by kubernetes or chosen by the user. They will need to be -opened manually and can be performed across an entire worker pool. - -If your NodePort service port selected is `30510` you can open this across all -members of a worker pool named `kubernetes-worker` like so: - -``` -juju run --application kubernetes-worker open-port 30510/tcp -``` - -Don't forget to expose the kubernetes-worker application if its not already -exposed, as this can cause confusion once the port has been opened and the -service is not reachable. - -Note: When debugging connection issues with NodePort services, its important -to first check the kube-proxy service on the worker units. If kube-proxy is not -running, the associated port-mapping will not be configured in the iptables -rulechains. - -If you need to close the NodePort once a workload has been terminated, you can -follow the same steps inversely. - -``` -juju run --application kubernetes-worker close-port 30510 -``` diff --git a/cluster/juju/layers/kubernetes-worker/actions.yaml b/cluster/juju/layers/kubernetes-worker/actions.yaml deleted file mode 100644 index 94363e3a24..0000000000 --- a/cluster/juju/layers/kubernetes-worker/actions.yaml +++ /dev/null @@ -1,56 +0,0 @@ -pause: - description: | - Cordon the unit, draining all active workloads. - params: - delete-local-data: - type: boolean - description: Force deletion of local storage to enable a drain - default: False - force: - type: boolean - description: | - Continue even if there are pods not managed by a RC, RS, Job, DS or SS - default: False - -resume: - description: | - UnCordon the unit, enabling workload scheduling. -microbot: - description: Launch microbot containers - params: - replicas: - type: integer - default: 3 - description: Number of microbots to launch in Kubernetes. - delete: - type: boolean - default: False - description: Remove a microbots deployment, service, and ingress if True. -upgrade: - description: Upgrade the kubernetes snaps -registry: - description: Create a private Docker registry - params: - htpasswd: - type: string - description: base64 encoded htpasswd file used for authentication. - htpasswd-plain: - type: string - description: base64 encoded plaintext version of the htpasswd file, needed by docker daemons to authenticate to the registry. - tlscert: - type: string - description: base64 encoded TLS certificate for the registry. Common Name must match the domain name of the registry. - tlskey: - type: string - description: base64 encoded TLS key for the registry. - domain: - type: string - description: The domain name for the registry. Must match the Common Name of the certificate. - ingress: - type: boolean - default: false - description: Create an Ingress resource for the registry (or delete resource object if "delete" is True) - delete: - type: boolean - default: false - description: Remove a registry replication controller, service, and ingress if True. diff --git a/cluster/juju/layers/kubernetes-worker/actions/microbot b/cluster/juju/layers/kubernetes-worker/actions/microbot deleted file mode 100755 index 85d7bf13b9..0000000000 --- a/cluster/juju/layers/kubernetes-worker/actions/microbot +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/local/sbin/charm-env python3 - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -from charmhelpers.core.hookenv import action_get -from charmhelpers.core.hookenv import action_set -from charmhelpers.core.hookenv import unit_public_ip -from charms.templating.jinja2 import render -from subprocess import call, check_output - -os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') - -context = {} -context['replicas'] = action_get('replicas') -context['delete'] = action_get('delete') -context['public_address'] = unit_public_ip() - -arch = check_output(['dpkg', '--print-architecture']).rstrip() -context['arch'] = arch.decode('utf-8') - -if not context['replicas']: - context['replicas'] = 3 - -# Declare a kubectl template when invoking kubectl -kubectl = ['kubectl', '--kubeconfig=/root/.kube/config'] - -# Remove deployment if requested -if context['delete']: - service_del = kubectl + ['delete', 'svc', 'microbot'] - service_response = call(service_del) - deploy_del = kubectl + ['delete', 'deployment', 'microbot'] - deploy_response = call(deploy_del) - ingress_del = kubectl + ['delete', 'ing', 'microbot-ingress'] - ingress_response = call(ingress_del) - - if ingress_response != 0: - action_set({'microbot-ing': - 'Failed removal of microbot ingress resource.'}) - if deploy_response != 0: - action_set({'microbot-deployment': - 'Failed removal of microbot deployment resource.'}) - if service_response != 0: - action_set({'microbot-service': - 'Failed removal of microbot service resource.'}) - sys.exit(0) - -# Creation request - -render('microbot-example.yaml', '/root/cdk/addons/microbot.yaml', - context) - -create_command = kubectl + ['create', '-f', - '/root/cdk/addons/microbot.yaml'] - -create_response = call(create_command) - -if create_response == 0: - action_set({'address': - 'microbot.{}.xip.io'.format(context['public_address'])}) -else: - action_set({'microbot-create': 'Failed microbot creation.'}) diff --git a/cluster/juju/layers/kubernetes-worker/actions/pause b/cluster/juju/layers/kubernetes-worker/actions/pause deleted file mode 100755 index 184b10a67a..0000000000 --- a/cluster/juju/layers/kubernetes-worker/actions/pause +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -export PATH=$PATH:/snap/bin - -DELETE_LOCAL_DATA=$(action-get delete-local-data) -FORCE=$(action-get force) - -# placeholder for additional flags to the command -export EXTRA_FLAGS="" - -# Determine if we have extra flags -if [[ "${DELETE_LOCAL_DATA}" == "True" || "${DELETE_LOCAL_DATA}" == "true" ]]; then - EXTRA_FLAGS="${EXTRA_FLAGS} --delete-local-data=true" -fi - -if [[ "${FORCE}" == "True" || "${FORCE}" == "true" ]]; then - EXTRA_FLAGS="${EXTRA_FLAGS} --force" -fi - - -# Cordon and drain the unit -kubectl --kubeconfig=/root/.kube/config cordon $(hostname) -kubectl --kubeconfig=/root/.kube/config drain $(hostname) ${EXTRA_FLAGS} - -# Set status to indicate the unit is paused and under maintenance. -status-set 'waiting' 'Kubernetes unit paused' diff --git a/cluster/juju/layers/kubernetes-worker/actions/registry b/cluster/juju/layers/kubernetes-worker/actions/registry deleted file mode 100755 index 81c8bc0380..0000000000 --- a/cluster/juju/layers/kubernetes-worker/actions/registry +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/local/sbin/charm-env python3 -# -# For a usage examples, see README.md -# -# TODO -# -# - make the action idempotent (i.e. if you run it multiple times, the first -# run will create/delete the registry, and the reset will be a no-op and won't -# error out) -# -# - take only a plain authentication file, and create the encrypted version in -# the action -# -# - validate the parameters (make sure tlscert is a certificate, that tlskey is a -# proper key, etc) -# -# - when https://bugs.launchpad.net/juju/+bug/1661015 is fixed, handle the -# base64 encoding the parameters in the action itself - -import os -import sys - -from base64 import b64encode - -from charmhelpers.core.hookenv import action_get -from charmhelpers.core.hookenv import action_set -from charms.templating.jinja2 import render -from subprocess import call, check_output - -os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') - -deletion = action_get('delete') - -context = {} - -arch = check_output(['dpkg', '--print-architecture']).rstrip() -context['arch'] = arch.decode('utf-8') - -# These config options must be defined in the case of a creation -param_error = False -for param in ('tlscert', 'tlskey', 'domain', 'htpasswd', 'htpasswd-plain'): - value = action_get(param) - if not value and not deletion: - key = "registry-create-parameter-{}".format(param) - error = "failure, parameter {} is required".format(param) - action_set({key: error}) - param_error = True - - context[param] = value - -# Create the dockercfg template variable -dockercfg = '{"%s": {"auth": "%s", "email": "root@localhost"}}' % \ - (context['domain'], context['htpasswd-plain']) -context['dockercfg'] = b64encode(dockercfg.encode()).decode('ASCII') - -if param_error: - sys.exit(0) - -# This one is either true or false, no need to check if it has a "good" value. -context['ingress'] = action_get('ingress') - -# Declare a kubectl template when invoking kubectl -kubectl = ['kubectl', '--kubeconfig=/root/.kube/config'] - -# Remove deployment if requested -if deletion: - resources = ['svc/kube-registry', 'rc/kube-registry-v0', 'secrets/registry-tls-data', - 'secrets/registry-auth-data', 'secrets/registry-access'] - - if action_get('ingress'): - resources.append('ing/registry-ing') - - delete_command = kubectl + ['delete', '--ignore-not-found=true'] + resources - delete_response = call(delete_command) - if delete_response == 0: - action_set({'registry-delete': 'success'}) - else: - action_set({'registry-delete': 'failure'}) - - sys.exit(0) - -# Creation request -render('registry.yaml', '/root/cdk/addons/registry.yaml', - context) - -create_command = kubectl + ['create', '-f', - '/root/cdk/addons/registry.yaml'] - -create_response = call(create_command) - -if create_response == 0: - action_set({'registry-create': 'success'}) - - # Create a ConfigMap if it doesn't exist yet, else patch it. - # A ConfigMap is needed to change the default value for nginx' client_max_body_size. - # The default is 1MB, and this is the maximum size of images that can be - # pushed on the registry. 1MB images aren't useful, so we bump this value to 1024MB. - cm_name = 'nginx-load-balancer-conf' - check_cm_command = kubectl + ['get', 'cm', cm_name] - check_cm_response = call(check_cm_command) - - if check_cm_response == 0: - # There is an existing ConfigMap, patch it - patch = '{"data":{"body-size":"1024m"}}' - patch_cm_command = kubectl + ['patch', 'cm', cm_name, '-p', patch] - patch_cm_response = call(patch_cm_command) - - if patch_cm_response == 0: - action_set({'configmap-patch': 'success'}) - else: - action_set({'configmap-patch': 'failure'}) - - else: - # No existing ConfigMap, create it - render('registry-configmap.yaml', '/root/cdk/addons/registry-configmap.yaml', - context) - create_cm_command = kubectl + ['create', '-f', '/root/cdk/addons/registry-configmap.yaml'] - create_cm_response = call(create_cm_command) - - if create_cm_response == 0: - action_set({'configmap-create': 'success'}) - else: - action_set({'configmap-create': 'failure'}) - - # Patch the "default" serviceaccount with an imagePullSecret. - # This will allow the docker daemons to authenticate to our private - # registry automatically - patch = '{"imagePullSecrets":[{"name":"registry-access"}]}' - patch_sa_command = kubectl + ['patch', 'sa', 'default', '-p', patch] - patch_sa_response = call(patch_sa_command) - - if patch_sa_response == 0: - action_set({'serviceaccount-patch': 'success'}) - else: - action_set({'serviceaccount-patch': 'failure'}) - - -else: - action_set({'registry-create': 'failure'}) diff --git a/cluster/juju/layers/kubernetes-worker/actions/resume b/cluster/juju/layers/kubernetes-worker/actions/resume deleted file mode 100755 index 2092185ae8..0000000000 --- a/cluster/juju/layers/kubernetes-worker/actions/resume +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -export PATH=$PATH:/snap/bin - -kubectl --kubeconfig=/root/.kube/config uncordon $(hostname) -status-set 'active' 'Kubernetes unit resumed' diff --git a/cluster/juju/layers/kubernetes-worker/actions/upgrade b/cluster/juju/layers/kubernetes-worker/actions/upgrade deleted file mode 100755 index a97c19b3db..0000000000 --- a/cluster/juju/layers/kubernetes-worker/actions/upgrade +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -eux - -charms.reactive set_state kubernetes-worker.snaps.upgrade-specified -exec hooks/config-changed diff --git a/cluster/juju/layers/kubernetes-worker/config.yaml b/cluster/juju/layers/kubernetes-worker/config.yaml deleted file mode 100644 index 2f15134f25..0000000000 --- a/cluster/juju/layers/kubernetes-worker/config.yaml +++ /dev/null @@ -1,108 +0,0 @@ -options: - ingress: - type: boolean - default: true - description: | - Deploy the default http backend and ingress controller to handle - ingress requests. - labels: - type: string - default: "" - description: | - Labels can be used to organize and to select subsets of nodes in the - cluster. Declare node labels in key=value format, separated by spaces. - allow-privileged: - type: string - default: "true" - description: | - Allow privileged containers to run on worker nodes. Supported values are - "true", "false", and "auto". If "true", kubelet will run in privileged - mode by default. If "false", kubelet will never run in privileged mode. - If "auto", kubelet will not run in privileged mode by default, but will - switch to privileged mode if gpu hardware is detected. Pod security - policies (PSP) should be used to restrict container privileges. - channel: - type: string - default: "1.11/stable" - description: | - Snap channel to install Kubernetes worker services from - require-manual-upgrade: - type: boolean - default: true - description: | - When true, worker services will not be upgraded until the user triggers - it manually by running the upgrade action. - kubelet-extra-args: - type: string - default: "" - description: | - Space separated list of flags and key=value pairs that will be passed as arguments to - kubelet. For example a value like this: - runtime-config=batch/v2alpha1=true profiling=true - will result in kube-apiserver being run with the following options: - --runtime-config=batch/v2alpha1=true --profiling=true - proxy-extra-args: - type: string - default: "" - description: | - Space separated list of flags and key=value pairs that will be passed as arguments to - kube-proxy. For example a value like this: - runtime-config=batch/v2alpha1=true profiling=true - will result in kube-apiserver being run with the following options: - --runtime-config=batch/v2alpha1=true --profiling=true - docker-logins: - type: string - default: "[]" - description: | - Docker login credentials. Setting this config allows Kubelet to pull images from - registries where auth is required. - - The value for this config must be a JSON array of credential objects, like this: - [{"server": "my.registry", "username": "myUser", "password": "myPass"}] - ingress-ssl-chain-completion: - type: boolean - default: false - description: | - Enable chain completion for TLS certificates used by the nginx ingress - controller. Set this to true if you would like the ingress controller - to attempt auto-retrieval of intermediate certificates. The default - (false) is recommended for all production kubernetes installations, and - any environment which does not have outbound Internet access. - nginx-image: - type: string - default: "auto" - description: | - Docker image to use for the nginx ingress controller. Auto will select an image - based on architecture. - default-backend-image: - type: string - default: "auto" - description: | - Docker image to use for the default backend. Auto will select an image - based on architecture. - snapd_refresh: - default: "max" - type: string - description: | - How often snapd handles updates for installed snaps. Setting an empty - string will check 4x per day. Set to "max" to delay the refresh as long - as possible. You may also set a custom string as described in the - 'refresh.timer' section here: - https://forum.snapcraft.io/t/system-options/87 - kubelet-extra-config: - default: "{}" - type: string - description: | - Extra configuration to be passed to kubelet. Any values specified in this - config will be merged into a KubeletConfiguration file that is passed to - the kubelet service via the --config flag. This can be used to override - values provided by the charm. - - Requires Kubernetes 1.10+. - - The value for this config must be a YAML mapping that can be safely - merged with a KubeletConfiguration file. For example: - {evictionHard: {memory.available: 200Mi}} - - For more information about KubeletConfiguration, see upstream docs: - https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ diff --git a/cluster/juju/layers/kubernetes-worker/copyright b/cluster/juju/layers/kubernetes-worker/copyright deleted file mode 100644 index ac5e525c8e..0000000000 --- a/cluster/juju/layers/kubernetes-worker/copyright +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2016 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/cluster/juju/layers/kubernetes-worker/debug-scripts/inotify b/cluster/juju/layers/kubernetes-worker/debug-scripts/inotify deleted file mode 100755 index 350e20ff87..0000000000 --- a/cluster/juju/layers/kubernetes-worker/debug-scripts/inotify +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -ux - -# We had to bump inotify limits once in the past, hence why this oddly specific -# script lives here in kubernetes-worker. - -sysctl fs.inotify > $DEBUG_SCRIPT_DIR/sysctl-limits -ls -l /proc/*/fd/* | grep inotify > $DEBUG_SCRIPT_DIR/inotify-instances diff --git a/cluster/juju/layers/kubernetes-worker/debug-scripts/kubectl b/cluster/juju/layers/kubernetes-worker/debug-scripts/kubectl deleted file mode 100755 index 1192c3c9a1..0000000000 --- a/cluster/juju/layers/kubernetes-worker/debug-scripts/kubectl +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ux - -export PATH=$PATH:/snap/bin - -alias kubectl="kubectl --kubeconfig=/root/cdk/kubeconfig" - -kubectl cluster-info > $DEBUG_SCRIPT_DIR/cluster-info -kubectl cluster-info dump > $DEBUG_SCRIPT_DIR/cluster-info-dump -for obj in pods svc ingress secrets pv pvc rc; do - kubectl describe $obj --all-namespaces > $DEBUG_SCRIPT_DIR/describe-$obj -done -for obj in nodes; do - kubectl describe $obj > $DEBUG_SCRIPT_DIR/describe-$obj -done diff --git a/cluster/juju/layers/kubernetes-worker/debug-scripts/kubernetes-worker-services b/cluster/juju/layers/kubernetes-worker/debug-scripts/kubernetes-worker-services deleted file mode 100755 index 4f9dfa23b3..0000000000 --- a/cluster/juju/layers/kubernetes-worker/debug-scripts/kubernetes-worker-services +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -ux - -for service in kubelet kube-proxy; do - systemctl status snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-systemctl-status - journalctl -u snap.$service.daemon > $DEBUG_SCRIPT_DIR/$service-journal -done - -# FIXME: get the snap config or something diff --git a/cluster/juju/layers/kubernetes-worker/exec.d/docker-compose/charm-pre-install b/cluster/juju/layers/kubernetes-worker/exec.d/docker-compose/charm-pre-install deleted file mode 100644 index 4940430c95..0000000000 --- a/cluster/juju/layers/kubernetes-worker/exec.d/docker-compose/charm-pre-install +++ /dev/null @@ -1,2 +0,0 @@ -# This stubs out charm-pre-install coming from layer-docker as a workaround for -# offline installs until https://github.com/juju/charm-tools/issues/301 is fixed. diff --git a/cluster/juju/layers/kubernetes-worker/exec.d/vmware-patch/charm-pre-install b/cluster/juju/layers/kubernetes-worker/exec.d/vmware-patch/charm-pre-install deleted file mode 100755 index b5e6d97b59..0000000000 --- a/cluster/juju/layers/kubernetes-worker/exec.d/vmware-patch/charm-pre-install +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -MY_HOSTNAME=$(hostname) - -: ${JUJU_UNIT_NAME:=`uuidgen`} - - -if [ "${MY_HOSTNAME}" == "ubuntuguest" ]; then - juju-log "Detected broken vsphere integration. Applying hostname override" - - FRIENDLY_HOSTNAME=$(echo $JUJU_UNIT_NAME | tr / -) - juju-log "Setting hostname to $FRIENDLY_HOSTNAME" - if [ ! -f /etc/hostname.orig ]; then - mv /etc/hostname /etc/hostname.orig - fi - echo "${FRIENDLY_HOSTNAME}" > /etc/hostname - hostname $FRIENDLY_HOSTNAME -fi diff --git a/cluster/juju/layers/kubernetes-worker/icon.svg b/cluster/juju/layers/kubernetes-worker/icon.svg deleted file mode 100644 index dca16eecaf..0000000000 --- a/cluster/juju/layers/kubernetes-worker/icon.svg +++ /dev/null @@ -1,362 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - diff --git a/cluster/juju/layers/kubernetes-worker/layer.yaml b/cluster/juju/layers/kubernetes-worker/layer.yaml deleted file mode 100644 index e7958853ae..0000000000 --- a/cluster/juju/layers/kubernetes-worker/layer.yaml +++ /dev/null @@ -1,42 +0,0 @@ -repo: https://github.com/kubernetes/kubernetes.git -includes: - - 'layer:basic' - - 'layer:debug' - - 'layer:snap' - - 'layer:leadership' - - 'layer:docker' - - 'layer:metrics' - - 'layer:nagios' - - 'layer:tls-client' - - 'layer:cdk-service-kicker' - - 'interface:http' - - 'interface:kubernetes-cni' - - 'interface:kube-dns' - - 'interface:kube-control' - - 'interface:aws-integration' - - 'interface:gcp-integration' - - 'interface:openstack-integration' - - 'interface:vsphere-integration' - - 'interface:azure-integration' - - 'interface:mount' -config: - deletes: - - install_from_upstream -options: - basic: - packages: - - 'cifs-utils' - - 'ceph-common' - - 'nfs-common' - - 'socat' - - 'virt-what' - tls-client: - ca_certificate_path: '/root/cdk/ca.crt' - server_certificate_path: '/root/cdk/server.crt' - server_key_path: '/root/cdk/server.key' - client_certificate_path: '/root/cdk/client.crt' - client_key_path: '/root/cdk/client.key' - cdk-service-kicker: - services: - - 'snap.kubelet.daemon' - - 'snap.kube-proxy.daemon' diff --git a/cluster/juju/layers/kubernetes-worker/lib/charms/kubernetes/common.py b/cluster/juju/layers/kubernetes-worker/lib/charms/kubernetes/common.py deleted file mode 100644 index 054399aeef..0000000000 --- a/cluster/juju/layers/kubernetes-worker/lib/charms/kubernetes/common.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - - -def get_version(bin_name): - """Get the version of an installed Kubernetes binary. - - :param str bin_name: Name of binary - :return: 3-tuple version (maj, min, patch) - - Example:: - - >>> `get_version('kubelet') - (1, 6, 0) - - """ - cmd = '{} --version'.format(bin_name).split() - version_string = subprocess.check_output(cmd).decode('utf-8') - return tuple(int(q) for q in re.findall("[0-9]+", version_string)[:3]) diff --git a/cluster/juju/layers/kubernetes-worker/metadata.yaml b/cluster/juju/layers/kubernetes-worker/metadata.yaml deleted file mode 100644 index d4040fb449..0000000000 --- a/cluster/juju/layers/kubernetes-worker/metadata.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: kubernetes-worker -summary: The workload bearing units of a kubernetes cluster -maintainers: - - Tim Van Steenburgh - - George Kraft - - Rye Terrell - - Konstantinos Tsakalozos - - Charles Butler - - Matthew Bruzek - - Mike Wilson -description: | - Kubernetes is an open-source platform for deploying, scaling, and operations - of application containers across a cluster of hosts. Kubernetes is portable - in that it works with public, private, and hybrid clouds. Extensible through - a pluggable infrastructure. Self healing in that it will automatically - restart and place containers on healthy nodes if a node ever goes away. -tags: - - misc -series: - - xenial - - bionic -subordinate: false -requires: - kube-api-endpoint: - interface: http - kube-dns: - # kube-dns is deprecated. Its functionality has been rolled into the - # kube-control interface. The kube-dns relation will be removed in - # a future release. - interface: kube-dns - kube-control: - interface: kube-control - aws: - interface: aws-integration - gcp: - interface: gcp-integration - openstack: - interface: openstack-integration - vsphere: - interface: vsphere-integration - azure: - interface: azure-integration - nfs: - interface: mount -provides: - cni: - interface: kubernetes-cni - scope: container -resources: - cni-amd64: - type: file - filename: cni.tgz - description: CNI plugins for amd64 - cni-arm64: - type: file - filename: cni.tgz - description: CNI plugins for arm64 - cni-s390x: - type: file - filename: cni.tgz - description: CNI plugins for s390x - kubectl: - type: file - filename: kubectl.snap - description: kubectl snap - kubelet: - type: file - filename: kubelet.snap - description: kubelet snap - kube-proxy: - type: file - filename: kube-proxy.snap - description: kube-proxy snap diff --git a/cluster/juju/layers/kubernetes-worker/metrics.yaml b/cluster/juju/layers/kubernetes-worker/metrics.yaml deleted file mode 100644 index 0fcb3c1c48..0000000000 --- a/cluster/juju/layers/kubernetes-worker/metrics.yaml +++ /dev/null @@ -1,2 +0,0 @@ -metrics: - juju-units: {} diff --git a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py b/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py deleted file mode 100644 index 3b6270e5ed..0000000000 --- a/cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py +++ /dev/null @@ -1,1500 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import json -import os -import random -import shutil -import subprocess -import time -import yaml - -from charms.leadership import leader_get, leader_set - -from pathlib import Path -from shlex import split -from subprocess import check_call, check_output -from subprocess import CalledProcessError -from socket import gethostname, getfqdn - -from charms import layer -from charms.layer import snap -from charms.reactive import hook -from charms.reactive import endpoint_from_flag -from charms.reactive import set_state, remove_state, is_state -from charms.reactive import when, when_any, when_not, when_none - -from charms.kubernetes.common import get_version - -from charms.reactive.helpers import data_changed -from charms.templating.jinja2 import render - -from charmhelpers.core import hookenv, unitdata -from charmhelpers.core.host import service_stop, service_restart -from charmhelpers.contrib.charmsupport import nrpe - -# Override the default nagios shortname regex to allow periods, which we -# need because our bin names contain them (e.g. 'snap.foo.daemon'). The -# default regex in charmhelpers doesn't allow periods, but nagios itself does. -nrpe.Check.shortname_re = '[\.A-Za-z0-9-_]+$' - -kubeconfig_path = '/root/cdk/kubeconfig' -kubeproxyconfig_path = '/root/cdk/kubeproxyconfig' -kubeclientconfig_path = '/root/.kube/config' -gcp_creds_env_key = 'GOOGLE_APPLICATION_CREDENTIALS' -snap_resources = ['kubectl', 'kubelet', 'kube-proxy'] - -os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin') -db = unitdata.kv() - - -@hook('upgrade-charm') -def upgrade_charm(): - # migrate to new flags - if is_state('kubernetes-worker.restarted-for-cloud'): - remove_state('kubernetes-worker.restarted-for-cloud') - set_state('kubernetes-worker.cloud.ready') - if is_state('kubernetes-worker.cloud-request-sent'): - # minor change, just for consistency - remove_state('kubernetes-worker.cloud-request-sent') - set_state('kubernetes-worker.cloud.request-sent') - - # Trigger removal of PPA docker installation if it was previously set. - set_state('config.changed.install_from_upstream') - hookenv.atexit(remove_state, 'config.changed.install_from_upstream') - - cleanup_pre_snap_services() - migrate_resource_checksums() - check_resources_for_upgrade_needed() - - # Remove the RC for nginx ingress if it exists - if hookenv.config().get('ingress'): - kubectl_success('delete', 'rc', 'nginx-ingress-controller') - - # Remove gpu.enabled state so we can reconfigure gpu-related kubelet flags, - # since they can differ between k8s versions - if is_state('kubernetes-worker.gpu.enabled'): - remove_state('kubernetes-worker.gpu.enabled') - try: - disable_gpu() - except ApplyNodeLabelFailed: - # Removing node label failed. Probably the master is unavailable. - # Proceed with the upgrade in hope GPUs will still be there. - hookenv.log('Failed to remove GPU labels. Proceed with upgrade.') - - remove_state('kubernetes-worker.cni-plugins.installed') - remove_state('kubernetes-worker.config.created') - remove_state('kubernetes-worker.ingress.available') - remove_state('worker.auth.bootstrapped') - set_state('kubernetes-worker.restart-needed') - - -def get_resource_checksum_db_key(resource): - ''' Convert a resource name to a resource checksum database key. ''' - return 'kubernetes-worker.resource-checksums.' + resource - - -def calculate_resource_checksum(resource): - ''' Calculate a checksum for a resource ''' - md5 = hashlib.md5() - path = hookenv.resource_get(resource) - if path: - with open(path, 'rb') as f: - data = f.read() - md5.update(data) - return md5.hexdigest() - - -def migrate_resource_checksums(): - ''' Migrate resource checksums from the old schema to the new one ''' - for resource in snap_resources: - new_key = get_resource_checksum_db_key(resource) - if not db.get(new_key): - path = hookenv.resource_get(resource) - if path: - # old key from charms.reactive.helpers.any_file_changed - old_key = 'reactive.files_changed.' + path - old_checksum = db.get(old_key) - db.set(new_key, old_checksum) - else: - # No resource is attached. Previously, this meant no checksum - # would be calculated and stored. But now we calculate it as if - # it is a 0-byte resource, so let's go ahead and do that. - zero_checksum = hashlib.md5().hexdigest() - db.set(new_key, zero_checksum) - - -def check_resources_for_upgrade_needed(): - hookenv.status_set('maintenance', 'Checking resources') - for resource in snap_resources: - key = get_resource_checksum_db_key(resource) - old_checksum = db.get(key) - new_checksum = calculate_resource_checksum(resource) - if new_checksum != old_checksum: - set_upgrade_needed() - - -def calculate_and_store_resource_checksums(): - for resource in snap_resources: - key = get_resource_checksum_db_key(resource) - checksum = calculate_resource_checksum(resource) - db.set(key, checksum) - - -def set_upgrade_needed(): - set_state('kubernetes-worker.snaps.upgrade-needed') - config = hookenv.config() - previous_channel = config.previous('channel') - require_manual = config.get('require-manual-upgrade') - if previous_channel is None or not require_manual: - set_state('kubernetes-worker.snaps.upgrade-specified') - - -def cleanup_pre_snap_services(): - # remove old states - remove_state('kubernetes-worker.components.installed') - - # disable old services - services = ['kubelet', 'kube-proxy'] - for service in services: - hookenv.log('Stopping {0} service.'.format(service)) - service_stop(service) - - # cleanup old files - files = [ - "/lib/systemd/system/kubelet.service", - "/lib/systemd/system/kube-proxy.service", - "/etc/default/kube-default", - "/etc/default/kubelet", - "/etc/default/kube-proxy", - "/srv/kubernetes", - "/usr/local/bin/kubectl", - "/usr/local/bin/kubelet", - "/usr/local/bin/kube-proxy", - "/etc/kubernetes" - ] - for file in files: - if os.path.isdir(file): - hookenv.log("Removing directory: " + file) - shutil.rmtree(file) - elif os.path.isfile(file): - hookenv.log("Removing file: " + file) - os.remove(file) - - -@when('config.changed.channel') -def channel_changed(): - set_upgrade_needed() - - -@when('kubernetes-worker.snaps.upgrade-specified') -def install_snaps(): - channel = hookenv.config('channel') - hookenv.status_set('maintenance', 'Installing kubectl snap') - snap.install('kubectl', channel=channel, classic=True) - hookenv.status_set('maintenance', 'Installing kubelet snap') - snap.install('kubelet', channel=channel, classic=True) - hookenv.status_set('maintenance', 'Installing kube-proxy snap') - snap.install('kube-proxy', channel=channel, classic=True) - calculate_and_store_resource_checksums() - set_state('kubernetes-worker.snaps.installed') - set_state('kubernetes-worker.restart-needed') - remove_state('kubernetes-worker.snaps.upgrade-needed') - remove_state('kubernetes-worker.snaps.upgrade-specified') - - -@hook('stop') -def shutdown(): - ''' When this unit is destroyed: - - delete the current node - - stop the worker services - ''' - try: - if os.path.isfile(kubeconfig_path): - kubectl('delete', 'node', get_node_name()) - except CalledProcessError: - hookenv.log('Failed to unregister node.') - service_stop('snap.kubelet.daemon') - service_stop('snap.kube-proxy.daemon') - - -@when('docker.available') -@when_not('kubernetes-worker.cni-plugins.installed') -def install_cni_plugins(): - ''' Unpack the cni-plugins resource ''' - charm_dir = os.getenv('CHARM_DIR') - - # Get the resource via resource_get - try: - resource_name = 'cni-{}'.format(arch()) - archive = hookenv.resource_get(resource_name) - except Exception: - message = 'Error fetching the cni resource.' - hookenv.log(message) - hookenv.status_set('blocked', message) - return - - if not archive: - hookenv.log('Missing cni resource.') - hookenv.status_set('blocked', 'Missing cni resource.') - return - - # Handle null resource publication, we check if filesize < 1mb - filesize = os.stat(archive).st_size - if filesize < 1000000: - hookenv.status_set('blocked', 'Incomplete cni resource.') - return - - hookenv.status_set('maintenance', 'Unpacking cni resource.') - - unpack_path = '{}/files/cni'.format(charm_dir) - os.makedirs(unpack_path, exist_ok=True) - cmd = ['tar', 'xfvz', archive, '-C', unpack_path] - hookenv.log(cmd) - check_call(cmd) - - apps = [ - {'name': 'loopback', 'path': '/opt/cni/bin'} - ] - - for app in apps: - unpacked = '{}/{}'.format(unpack_path, app['name']) - app_path = os.path.join(app['path'], app['name']) - install = ['install', '-v', '-D', unpacked, app_path] - hookenv.log(install) - check_call(install) - - # Used by the "registry" action. The action is run on a single worker, but - # the registry pod can end up on any worker, so we need this directory on - # all the workers. - os.makedirs('/srv/registry', exist_ok=True) - - set_state('kubernetes-worker.cni-plugins.installed') - - -@when('kubernetes-worker.snaps.installed') -def set_app_version(): - ''' Declare the application version to juju ''' - cmd = ['kubelet', '--version'] - version = check_output(cmd) - hookenv.application_version_set(version.split(b' v')[-1].rstrip()) - - -@when('kubernetes-worker.snaps.installed') -@when('snap.refresh.set') -@when('leadership.is_leader') -def process_snapd_timer(): - ''' Set the snapd refresh timer on the leader so all cluster members - (present and future) will refresh near the same time. ''' - # Get the current snapd refresh timer; we know layer-snap has set this - # when the 'snap.refresh.set' flag is present. - timer = snap.get(snapname='core', key='refresh.timer').decode('utf-8') - - # The first time through, data_changed will be true. Subsequent calls - # should only update leader data if something changed. - if data_changed('worker_snapd_refresh', timer): - hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) - leader_set({'snapd_refresh': timer}) - - -@when('kubernetes-worker.snaps.installed') -@when('snap.refresh.set') -@when('leadership.changed.snapd_refresh') -@when_not('leadership.is_leader') -def set_snapd_timer(): - ''' Set the snapd refresh.timer on non-leader cluster members. ''' - # NB: This method should only be run when 'snap.refresh.set' is present. - # Layer-snap will always set a core refresh.timer, which may not be the - # same as our leader. Gating with 'snap.refresh.set' ensures layer-snap - # has finished and we are free to set our config to the leader's timer. - timer = leader_get('snapd_refresh') - hookenv.log('setting snapd_refresh timer to: {}'.format(timer)) - snap.set_refresh_timer(timer) - - -@hookenv.atexit -def charm_status(): - '''Update the status message with the current status of kubelet.''' - vsphere_joined = is_state('endpoint.vsphere.joined') - azure_joined = is_state('endpoint.azure.joined') - cloud_blocked = is_state('kubernetes-worker.cloud.blocked') - if vsphere_joined and cloud_blocked: - hookenv.status_set('blocked', - 'vSphere integration requires K8s 1.12 or greater') - return - if azure_joined and cloud_blocked: - hookenv.status_set('blocked', - 'Azure integration requires K8s 1.11 or greater') - return - if is_state('kubernetes-worker.cloud.pending'): - hookenv.status_set('waiting', 'Waiting for cloud integration') - return - if not is_state('kube-control.dns.available'): - # During deployment the worker has to start kubelet without cluster dns - # configured. If this is the first unit online in a service pool - # waiting to self host the dns pod, and configure itself to query the - # dns service declared in the kube-system namespace - hookenv.status_set('waiting', 'Waiting for cluster DNS.') - return - if is_state('kubernetes-worker.snaps.upgrade-specified'): - hookenv.status_set('waiting', 'Upgrade pending') - return - if is_state('kubernetes-worker.snaps.upgrade-needed'): - hookenv.status_set('blocked', - 'Needs manual upgrade, run the upgrade action') - return - if is_state('kubernetes-worker.snaps.installed'): - update_kubelet_status() - return - else: - pass # will have been set by snap layer or other handler - - -def update_kubelet_status(): - ''' There are different states that the kubelet can be in, where we are - waiting for dns, waiting for cluster turnup, or ready to serve - applications.''' - services = [ - 'kubelet', - 'kube-proxy' - ] - failing_services = [] - for service in services: - daemon = 'snap.{}.daemon'.format(service) - if not _systemctl_is_active(daemon): - failing_services.append(service) - - if len(failing_services) == 0: - hookenv.status_set('active', 'Kubernetes worker running.') - else: - msg = 'Waiting for {} to start.'.format(','.join(failing_services)) - hookenv.status_set('waiting', msg) - - -def get_ingress_address(relation): - try: - network_info = hookenv.network_get(relation.relation_name) - except NotImplementedError: - network_info = [] - - if network_info and 'ingress-addresses' in network_info: - # just grab the first one for now, maybe be more robust here? - return network_info['ingress-addresses'][0] - else: - # if they don't have ingress-addresses they are running a juju that - # doesn't support spaces, so just return the private address - return hookenv.unit_get('private-address') - - -@when('certificates.available', 'kube-control.connected') -def send_data(tls, kube_control): - '''Send the data that is required to create a server certificate for - this server.''' - # Use the public ip of this unit as the Common Name for the certificate. - common_name = hookenv.unit_public_ip() - - ingress_ip = get_ingress_address(kube_control) - - # Create SANs that the tls layer will add to the server cert. - sans = [ - hookenv.unit_public_ip(), - ingress_ip, - gethostname() - ] - - # Create a path safe name by removing path characters from the unit name. - certificate_name = hookenv.local_unit().replace('/', '_') - - # Request a server cert with this information. - tls.request_server_cert(common_name, sans, certificate_name) - - -@when('kube-api-endpoint.available', 'kube-control.dns.available', - 'cni.available') -def watch_for_changes(kube_api, kube_control, cni): - ''' Watch for configuration changes and signal if we need to restart the - worker services ''' - servers = get_kube_api_servers(kube_api) - dns = kube_control.get_dns() - cluster_cidr = cni.get_config()['cidr'] - - if (data_changed('kube-api-servers', servers) or - data_changed('kube-dns', dns) or - data_changed('cluster-cidr', cluster_cidr)): - - set_state('kubernetes-worker.restart-needed') - - -@when('kubernetes-worker.snaps.installed', 'kube-api-endpoint.available', - 'tls_client.ca.saved', 'tls_client.client.certificate.saved', - 'tls_client.client.key.saved', 'tls_client.server.certificate.saved', - 'tls_client.server.key.saved', - 'kube-control.dns.available', 'kube-control.auth.available', - 'cni.available', 'kubernetes-worker.restart-needed', - 'worker.auth.bootstrapped') -@when_not('kubernetes-worker.cloud.pending', - 'kubernetes-worker.cloud.blocked') -def start_worker(kube_api, kube_control, auth_control, cni): - ''' Start kubelet using the provided API and DNS info.''' - servers = get_kube_api_servers(kube_api) - # Note that the DNS server doesn't necessarily exist at this point. We know - # what its IP will eventually be, though, so we can go ahead and configure - # kubelet with that info. This ensures that early pods are configured with - # the correct DNS even though the server isn't ready yet. - - dns = kube_control.get_dns() - ingress_ip = get_ingress_address(kube_control) - cluster_cidr = cni.get_config()['cidr'] - - if cluster_cidr is None: - hookenv.log('Waiting for cluster cidr.') - return - - creds = db.get('credentials') - data_changed('kube-control.creds', creds) - - create_config(random.choice(servers), creds) - configure_kubelet(dns, ingress_ip) - configure_kube_proxy(servers, cluster_cidr) - set_state('kubernetes-worker.config.created') - restart_unit_services() - update_kubelet_status() - set_state('kubernetes-worker.label-config-required') - remove_state('kubernetes-worker.restart-needed') - - -@when('cni.connected') -@when_not('cni.configured') -def configure_cni(cni): - ''' Set worker configuration on the CNI relation. This lets the CNI - subordinate know that we're the worker so it can respond accordingly. ''' - cni.set_config(is_master=False, kubeconfig_path=kubeconfig_path) - - -@when('config.changed.ingress') -def toggle_ingress_state(): - ''' Ingress is a toggled state. Remove ingress.available if set when - toggled ''' - remove_state('kubernetes-worker.ingress.available') - - -@when('docker.sdn.configured') -def sdn_changed(): - '''The Software Defined Network changed on the container so restart the - kubernetes services.''' - restart_unit_services() - update_kubelet_status() - remove_state('docker.sdn.configured') - - -@when('kubernetes-worker.config.created') -@when_not('kubernetes-worker.ingress.available') -def render_and_launch_ingress(): - ''' If configuration has ingress daemon set enabled, launch the ingress - load balancer and default http backend. Otherwise attempt deletion. ''' - config = hookenv.config() - # If ingress is enabled, launch the ingress controller - if config.get('ingress'): - launch_default_ingress_controller() - else: - hookenv.log('Deleting the http backend and ingress.') - kubectl_manifest('delete', - '/root/cdk/addons/default-http-backend.yaml') - kubectl_manifest('delete', - '/root/cdk/addons/ingress-daemon-set.yaml') # noqa - hookenv.close_port(80) - hookenv.close_port(443) - - -@when('config.changed.labels') -def handle_labels_changed(): - set_state('kubernetes-worker.label-config-required') - - -@when('kubernetes-worker.label-config-required', - 'kubernetes-worker.config.created') -def apply_node_labels(): - ''' Parse the labels configuration option and apply the labels to the - node. ''' - # Get the user's configured labels. - config = hookenv.config() - user_labels = {} - for item in config.get('labels').split(' '): - if '=' in item: - key, val = item.split('=') - user_labels[key] = val - else: - hookenv.log('Skipping malformed option: {}.'.format(item)) - # Collect the current label state. - current_labels = db.get('current_labels') or {} - # Remove any labels that the user has removed from the config. - for key in list(current_labels.keys()): - if key not in user_labels: - try: - remove_label(key) - del current_labels[key] - db.set('current_labels', current_labels) - except ApplyNodeLabelFailed as e: - hookenv.log(str(e)) - return - # Add any new labels. - for key, val in user_labels.items(): - try: - set_label(key, val) - current_labels[key] = val - db.set('current_labels', current_labels) - except ApplyNodeLabelFailed as e: - hookenv.log(str(e)) - return - # Set the juju-application label. - try: - set_label('juju-application', hookenv.service_name()) - except ApplyNodeLabelFailed as e: - hookenv.log(str(e)) - return - # Label configuration complete. - remove_state('kubernetes-worker.label-config-required') - - -@when_any('config.changed.kubelet-extra-args', - 'config.changed.proxy-extra-args', - 'config.changed.kubelet-extra-config') -def config_changed_requires_restart(): - set_state('kubernetes-worker.restart-needed') - - -@when('config.changed.docker-logins') -def docker_logins_changed(): - """Set a flag to handle new docker login options. - - If docker daemon options have also changed, set a flag to ensure the - daemon is restarted prior to running docker login. - """ - config = hookenv.config() - - if data_changed('docker-opts', config['docker-opts']): - hookenv.log('Found new docker daemon options. Requesting a restart.') - # State will be removed by layer-docker after restart - set_state('docker.restart') - - set_state('kubernetes-worker.docker-login') - - -@when('kubernetes-worker.docker-login') -@when_not('docker.restart') -def run_docker_login(): - """Login to a docker registry with configured credentials.""" - config = hookenv.config() - - previous_logins = config.previous('docker-logins') - logins = config['docker-logins'] - logins = json.loads(logins) - - if previous_logins: - previous_logins = json.loads(previous_logins) - next_servers = {login['server'] for login in logins} - previous_servers = {login['server'] for login in previous_logins} - servers_to_logout = previous_servers - next_servers - for server in servers_to_logout: - cmd = ['docker', 'logout', server] - subprocess.check_call(cmd) - - for login in logins: - server = login['server'] - username = login['username'] - password = login['password'] - cmd = ['docker', 'login', server, '-u', username, '-p', password] - subprocess.check_call(cmd) - - remove_state('kubernetes-worker.docker-login') - set_state('kubernetes-worker.restart-needed') - - -def arch(): - '''Return the package architecture as a string. Raise an exception if the - architecture is not supported by kubernetes.''' - # Get the package architecture for this system. - architecture = check_output(['dpkg', '--print-architecture']).rstrip() - # Convert the binary result into a string. - architecture = architecture.decode('utf-8') - return architecture - - -def create_config(server, creds): - '''Create a kubernetes configuration for the worker unit.''' - # Get the options from the tls-client layer. - layer_options = layer.options('tls-client') - # Get all the paths to the tls information required for kubeconfig. - ca = layer_options.get('ca_certificate_path') - - # Create kubernetes configuration in the default location for ubuntu. - create_kubeconfig('/home/ubuntu/.kube/config', server, ca, - token=creds['client_token'], user='ubuntu') - # Make the config dir readable by the ubuntu users so juju scp works. - cmd = ['chown', '-R', 'ubuntu:ubuntu', '/home/ubuntu/.kube'] - check_call(cmd) - # Create kubernetes configuration in the default location for root. - create_kubeconfig(kubeclientconfig_path, server, ca, - token=creds['client_token'], user='root') - # Create kubernetes configuration for kubelet, and kube-proxy services. - create_kubeconfig(kubeconfig_path, server, ca, - token=creds['kubelet_token'], user='kubelet') - create_kubeconfig(kubeproxyconfig_path, server, ca, - token=creds['proxy_token'], user='kube-proxy') - - -def parse_extra_args(config_key): - elements = hookenv.config().get(config_key, '').split() - args = {} - - for element in elements: - if '=' in element: - key, _, value = element.partition('=') - args[key] = value - else: - args[element] = 'true' - - return args - - -def configure_kubernetes_service(service, base_args, extra_args_key): - db = unitdata.kv() - - prev_args_key = 'kubernetes-worker.prev_args.' + service - prev_args = db.get(prev_args_key) or {} - - extra_args = parse_extra_args(extra_args_key) - - args = {} - for arg in prev_args: - # remove previous args by setting to null - args[arg] = 'null' - for k, v in base_args.items(): - args[k] = v - for k, v in extra_args.items(): - args[k] = v - - cmd = ['snap', 'set', service] + ['%s=%s' % item for item in args.items()] - check_call(cmd) - - db.set(prev_args_key, args) - - -def merge_kubelet_extra_config(config, extra_config): - ''' Updates config to include the contents of extra_config. This is done - recursively to allow deeply nested dictionaries to be merged. - - This is destructive: it modifies the config dict that is passed in. - ''' - for k, extra_config_value in extra_config.items(): - if isinstance(extra_config_value, dict): - config_value = config.setdefault(k, {}) - merge_kubelet_extra_config(config_value, extra_config_value) - else: - config[k] = extra_config_value - - -def configure_kubelet(dns, ingress_ip): - layer_options = layer.options('tls-client') - ca_cert_path = layer_options.get('ca_certificate_path') - server_cert_path = layer_options.get('server_certificate_path') - server_key_path = layer_options.get('server_key_path') - - kubelet_opts = {} - kubelet_opts['require-kubeconfig'] = 'true' - kubelet_opts['kubeconfig'] = kubeconfig_path - kubelet_opts['network-plugin'] = 'cni' - kubelet_opts['v'] = '0' - kubelet_opts['logtostderr'] = 'true' - kubelet_opts['node-ip'] = ingress_ip - kubelet_opts['allow-privileged'] = set_privileged() - - if is_state('endpoint.aws.ready'): - kubelet_opts['cloud-provider'] = 'aws' - elif is_state('endpoint.gcp.ready'): - cloud_config_path = _cloud_config_path('kubelet') - kubelet_opts['cloud-provider'] = 'gce' - kubelet_opts['cloud-config'] = str(cloud_config_path) - elif is_state('endpoint.openstack.ready'): - cloud_config_path = _cloud_config_path('kubelet') - kubelet_opts['cloud-provider'] = 'openstack' - kubelet_opts['cloud-config'] = str(cloud_config_path) - elif is_state('endpoint.vsphere.joined'): - # vsphere just needs to be joined on the worker (vs 'ready') - cloud_config_path = _cloud_config_path('kubelet') - kubelet_opts['cloud-provider'] = 'vsphere' - # NB: vsphere maps node product-id to its uuid (no config file needed). - uuid_file = '/sys/class/dmi/id/product_uuid' - with open(uuid_file, 'r') as f: - uuid = f.read().strip() - kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid) - elif is_state('endpoint.azure.ready'): - azure = endpoint_from_flag('endpoint.azure.ready') - cloud_config_path = _cloud_config_path('kubelet') - kubelet_opts['cloud-provider'] = 'azure' - kubelet_opts['cloud-config'] = str(cloud_config_path) - kubelet_opts['provider-id'] = azure.vm_id - - if get_version('kubelet') >= (1, 10): - # Put together the KubeletConfiguration data - kubelet_config = { - 'apiVersion': 'kubelet.config.k8s.io/v1beta1', - 'kind': 'KubeletConfiguration', - 'address': '0.0.0.0', - 'authentication': { - 'anonymous': { - 'enabled': False - }, - 'x509': { - 'clientCAFile': ca_cert_path - } - }, - 'clusterDomain': dns['domain'], - 'failSwapOn': False, - 'port': 10250, - 'tlsCertFile': server_cert_path, - 'tlsPrivateKeyFile': server_key_path - } - if dns['enable-kube-dns']: - kubelet_config['clusterDNS'] = [dns['sdn-ip']] - if is_state('kubernetes-worker.gpu.enabled'): - kubelet_config['featureGates'] = { - 'DevicePlugins': True - } - - # Add kubelet-extra-config. This needs to happen last so that it - # overrides any config provided by the charm. - kubelet_extra_config = hookenv.config('kubelet-extra-config') - kubelet_extra_config = yaml.load(kubelet_extra_config) - merge_kubelet_extra_config(kubelet_config, kubelet_extra_config) - - # Render the file and configure Kubelet to use it - os.makedirs('/root/cdk/kubelet', exist_ok=True) - with open('/root/cdk/kubelet/config.yaml', 'w') as f: - f.write('# Generated by kubernetes-worker charm, do not edit\n') - yaml.dump(kubelet_config, f) - kubelet_opts['config'] = '/root/cdk/kubelet/config.yaml' - else: - # NOTE: This is for 1.9. Once we've dropped 1.9 support, we can remove - # this whole block and the parent if statement. - kubelet_opts['address'] = '0.0.0.0' - kubelet_opts['anonymous-auth'] = 'false' - kubelet_opts['client-ca-file'] = ca_cert_path - kubelet_opts['cluster-domain'] = dns['domain'] - kubelet_opts['fail-swap-on'] = 'false' - kubelet_opts['port'] = '10250' - kubelet_opts['tls-cert-file'] = server_cert_path - kubelet_opts['tls-private-key-file'] = server_key_path - if dns['enable-kube-dns']: - kubelet_opts['cluster-dns'] = dns['sdn-ip'] - if is_state('kubernetes-worker.gpu.enabled'): - kubelet_opts['feature-gates'] = 'DevicePlugins=true' - - if get_version('kubelet') >= (1, 11): - kubelet_opts['dynamic-config-dir'] = '/root/cdk/kubelet/dynamic-config' - - configure_kubernetes_service('kubelet', kubelet_opts, 'kubelet-extra-args') - - -def configure_kube_proxy(api_servers, cluster_cidr): - kube_proxy_opts = {} - kube_proxy_opts['cluster-cidr'] = cluster_cidr - kube_proxy_opts['kubeconfig'] = kubeproxyconfig_path - kube_proxy_opts['logtostderr'] = 'true' - kube_proxy_opts['v'] = '0' - kube_proxy_opts['master'] = random.choice(api_servers) - kube_proxy_opts['hostname-override'] = get_node_name() - - if b'lxc' in check_output('virt-what', shell=True): - kube_proxy_opts['conntrack-max-per-core'] = '0' - - configure_kubernetes_service('kube-proxy', kube_proxy_opts, - 'proxy-extra-args') - - -def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None, - user='ubuntu', context='juju-context', - cluster='juju-cluster', password=None, token=None): - '''Create a configuration for Kubernetes based on path using the supplied - arguments for values of the Kubernetes server, CA, key, certificate, user - context and cluster.''' - if not key and not certificate and not password and not token: - raise ValueError('Missing authentication mechanism.') - - # token and password are mutually exclusive. Error early if both are - # present. The developer has requested an impossible situation. - # see: kubectl config set-credentials --help - if token and password: - raise ValueError('Token and Password are mutually exclusive.') - # Create the config file with the address of the master server. - cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \ - '--server={2} --certificate-authority={3} --embed-certs=true' - check_call(split(cmd.format(kubeconfig, cluster, server, ca))) - # Delete old users - cmd = 'kubectl config --kubeconfig={0} unset users' - check_call(split(cmd.format(kubeconfig))) - # Create the credentials using the client flags. - cmd = 'kubectl config --kubeconfig={0} ' \ - 'set-credentials {1} '.format(kubeconfig, user) - - if key and certificate: - cmd = '{0} --client-key={1} --client-certificate={2} '\ - '--embed-certs=true'.format(cmd, key, certificate) - if password: - cmd = "{0} --username={1} --password={2}".format(cmd, user, password) - # This is mutually exclusive from password. They will not work together. - if token: - cmd = "{0} --token={1}".format(cmd, token) - check_call(split(cmd)) - # Create a default context with the cluster. - cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \ - '--cluster={2} --user={3}' - check_call(split(cmd.format(kubeconfig, context, cluster, user))) - # Make the config use this new context. - cmd = 'kubectl config --kubeconfig={0} use-context {1}' - check_call(split(cmd.format(kubeconfig, context))) - - -@when_any('config.changed.default-backend-image', - 'config.changed.ingress-ssl-chain-completion', - 'config.changed.nginx-image') -@when('kubernetes-worker.config.created') -def launch_default_ingress_controller(): - ''' Launch the Kubernetes ingress controller & default backend (404) ''' - config = hookenv.config() - - # need to test this in case we get in - # here from a config change to the image - if not config.get('ingress'): - return - - context = {} - context['arch'] = arch() - addon_path = '/root/cdk/addons/{}' - - context['defaultbackend_image'] = config.get('default-backend-image') - if (context['defaultbackend_image'] == "" or - context['defaultbackend_image'] == "auto"): - if context['arch'] == 's390x': - context['defaultbackend_image'] = \ - "k8s.gcr.io/defaultbackend-s390x:1.5" - elif context['arch'] == 'arm64': - context['defaultbackend_image'] = \ - "k8s.gcr.io/defaultbackend-arm64:1.5" - else: - context['defaultbackend_image'] = \ - "k8s.gcr.io/defaultbackend-amd64:1.5" - - # Render the default http backend (404) replicationcontroller manifest - manifest = addon_path.format('default-http-backend.yaml') - render('default-http-backend.yaml', manifest, context) - hookenv.log('Creating the default http backend.') - try: - kubectl('apply', '-f', manifest) - except CalledProcessError as e: - hookenv.log(e) - hookenv.log('Failed to create default-http-backend. Will attempt again next update.') # noqa - hookenv.close_port(80) - hookenv.close_port(443) - return - - # Render the ingress daemon set controller manifest - context['ssl_chain_completion'] = config.get( - 'ingress-ssl-chain-completion') - context['ingress_image'] = config.get('nginx-image') - if context['ingress_image'] == "" or context['ingress_image'] == "auto": - images = {'amd64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1', # noqa - 'arm64': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.16.1', # noqa - 's390x': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-s390x:0.16.1', # noqa - 'ppc64el': 'quay.io/kubernetes-ingress-controller/nginx-ingress-controller-ppc64le:0.16.1', # noqa - } - context['ingress_image'] = images.get(context['arch'], images['amd64']) - if get_version('kubelet') < (1, 9): - context['daemonset_api_version'] = 'extensions/v1beta1' - else: - context['daemonset_api_version'] = 'apps/v1' - context['juju_application'] = hookenv.service_name() - manifest = addon_path.format('ingress-daemon-set.yaml') - render('ingress-daemon-set.yaml', manifest, context) - hookenv.log('Creating the ingress daemon set.') - try: - kubectl('apply', '-f', manifest) - except CalledProcessError as e: - hookenv.log(e) - hookenv.log('Failed to create ingress controller. Will attempt again next update.') # noqa - hookenv.close_port(80) - hookenv.close_port(443) - return - - set_state('kubernetes-worker.ingress.available') - hookenv.open_port(80) - hookenv.open_port(443) - - -def restart_unit_services(): - '''Restart worker services.''' - hookenv.log('Restarting kubelet and kube-proxy.') - services = ['kube-proxy', 'kubelet'] - for service in services: - service_restart('snap.%s.daemon' % service) - - -def get_kube_api_servers(kube_api): - '''Return the kubernetes api server address and port for this - relationship.''' - hosts = [] - # Iterate over every service from the relation object. - for service in kube_api.services(): - for unit in service['hosts']: - hosts.append('https://{0}:{1}'.format(unit['hostname'], - unit['port'])) - return hosts - - -def kubectl(*args): - ''' Run a kubectl cli command with a config file. Returns stdout and throws - an error if the command fails. ''' - command = ['kubectl', '--kubeconfig=' + kubeclientconfig_path] + list(args) - hookenv.log('Executing {}'.format(command)) - return check_output(command) - - -def kubectl_success(*args): - ''' Runs kubectl with the given args. Returns True if successful, False if - not. ''' - try: - kubectl(*args) - return True - except CalledProcessError: - return False - - -def kubectl_manifest(operation, manifest): - ''' Wrap the kubectl creation command when using filepath resources - :param operation - one of get, create, delete, replace - :param manifest - filepath to the manifest - ''' - # Deletions are a special case - if operation == 'delete': - # Ensure we immediately remove requested resources with --now - return kubectl_success(operation, '-f', manifest, '--now') - else: - # Guard against an error re-creating the same manifest multiple times - if operation == 'create': - # If we already have the definition, its probably safe to assume - # creation was true. - if kubectl_success('get', '-f', manifest): - hookenv.log('Skipping definition for {}'.format(manifest)) - return True - # Execute the requested command that did not match any of the special - # cases above - return kubectl_success(operation, '-f', manifest) - - -@when('nrpe-external-master.available') -@when_not('nrpe-external-master.initial-config') -def initial_nrpe_config(nagios=None): - set_state('nrpe-external-master.initial-config') - update_nrpe_config(nagios) - - -@when('kubernetes-worker.config.created') -@when('nrpe-external-master.available') -@when_any('config.changed.nagios_context', - 'config.changed.nagios_servicegroups') -def update_nrpe_config(unused=None): - services = ('snap.kubelet.daemon', 'snap.kube-proxy.daemon') - hostname = nrpe.get_nagios_hostname() - current_unit = nrpe.get_nagios_unit_name() - nrpe_setup = nrpe.NRPE(hostname=hostname) - nrpe.add_init_service_checks(nrpe_setup, services, current_unit) - nrpe_setup.write() - - -@when_not('nrpe-external-master.available') -@when('nrpe-external-master.initial-config') -def remove_nrpe_config(nagios=None): - remove_state('nrpe-external-master.initial-config') - - # List of systemd services for which the checks will be removed - services = ('snap.kubelet.daemon', 'snap.kube-proxy.daemon') - - # The current nrpe-external-master interface doesn't handle a lot of logic, - # use the charm-helpers code for now. - hostname = nrpe.get_nagios_hostname() - nrpe_setup = nrpe.NRPE(hostname=hostname) - - for service in services: - nrpe_setup.remove_check(shortname=service) - - -def set_privileged(): - """Return 'true' if privileged containers are needed. - This is when a) the user requested them - b) user does not care (auto) and GPUs are available in a pre - 1.9 era - """ - privileged = hookenv.config('allow-privileged').lower() - gpu_needs_privileged = (is_state('kubernetes-worker.gpu.enabled') and - get_version('kubelet') < (1, 9)) - - if privileged == 'auto': - privileged = 'true' if gpu_needs_privileged else 'false' - - if privileged == 'false' and gpu_needs_privileged: - disable_gpu() - remove_state('kubernetes-worker.gpu.enabled') - # No need to restart kubernetes (set the restart-needed state) - # because set-privileged is already in the restart path - - return privileged - - -@when('config.changed.allow-privileged') -@when('kubernetes-worker.config.created') -def on_config_allow_privileged_change(): - """React to changed 'allow-privileged' config value. - - """ - set_state('kubernetes-worker.restart-needed') - remove_state('config.changed.allow-privileged') - - -@when('nvidia-docker.installed') -@when('kubernetes-worker.config.created') -@when_not('kubernetes-worker.gpu.enabled') -def enable_gpu(): - """Enable GPU usage on this node. - - """ - if get_version('kubelet') < (1, 9): - hookenv.status_set( - 'active', - 'Upgrade to snap channel >= 1.9/stable to enable GPU support.' - ) - return - - hookenv.log('Enabling gpu mode') - try: - # Not sure why this is necessary, but if you don't run this, k8s will - # think that the node has 0 gpus (as shown by the output of - # `kubectl get nodes -o yaml` - check_call(['nvidia-smi']) - except CalledProcessError as cpe: - hookenv.log('Unable to communicate with the NVIDIA driver.') - hookenv.log(cpe) - return - - set_label('gpu', 'true') - set_label('cuda', 'true') - - set_state('kubernetes-worker.gpu.enabled') - set_state('kubernetes-worker.restart-needed') - - -@when('kubernetes-worker.gpu.enabled') -@when_not('nvidia-docker.installed') -@when_not('kubernetes-worker.restart-needed') -def nvidia_departed(): - """Cuda departed, probably due to the docker layer switching to a - non nvidia-docker.""" - disable_gpu() - remove_state('kubernetes-worker.gpu.enabled') - set_state('kubernetes-worker.restart-needed') - - -def disable_gpu(): - """Disable GPU usage on this node. - - """ - hookenv.log('Disabling gpu mode') - - # Remove node labels - remove_label('gpu') - remove_label('cuda') - - -@when('kubernetes-worker.gpu.enabled') -@when('kube-control.connected') -def notify_master_gpu_enabled(kube_control): - """Notify kubernetes-master that we're gpu-enabled. - - """ - kube_control.set_gpu(True) - - -@when_not('kubernetes-worker.gpu.enabled') -@when('kube-control.connected') -def notify_master_gpu_not_enabled(kube_control): - """Notify kubernetes-master that we're not gpu-enabled. - - """ - kube_control.set_gpu(False) - - -@when('kube-control.connected') -def request_kubelet_and_proxy_credentials(kube_control): - """ Request kubelet node authorization with a well formed kubelet user. - This also implies that we are requesting kube-proxy auth. """ - - # The kube-cotrol interface is created to support RBAC. - # At this point we might as well do the right thing and return the hostname - # even if it will only be used when we enable RBAC - nodeuser = 'system:node:{}'.format(get_node_name().lower()) - kube_control.set_auth_request(nodeuser) - - -@when('kube-control.connected') -def catch_change_in_creds(kube_control): - """Request a service restart in case credential updates were detected.""" - nodeuser = 'system:node:{}'.format(get_node_name().lower()) - creds = kube_control.get_auth_credentials(nodeuser) - if creds and creds['user'] == nodeuser: - # We need to cache the credentials here because if the - # master changes (master leader dies and replaced by a new one) - # the new master will have no recollection of our certs. - db.set('credentials', creds) - set_state('worker.auth.bootstrapped') - if data_changed('kube-control.creds', creds): - set_state('kubernetes-worker.restart-needed') - - -@when_not('kube-control.connected') -def missing_kube_control(): - """Inform the operator they need to add the kube-control relation. - - If deploying via bundle this won't happen, but if operator is upgrading a - a charm in a deployment that pre-dates the kube-control relation, it'll be - missing. - - """ - try: - goal_state = hookenv.goal_state() - except NotImplementedError: - goal_state = {} - - if 'kube-control' in goal_state.get('relations', {}): - hookenv.status_set( - 'waiting', - 'Waiting for kubernetes-master to become ready') - else: - hookenv.status_set( - 'blocked', - 'Relate {}:kube-control kubernetes-master:kube-control'.format( - hookenv.service_name())) - - -@when('docker.ready') -def fix_iptables_for_docker_1_13(): - """ Fix iptables FORWARD policy for Docker >=1.13 - https://github.com/kubernetes/kubernetes/issues/40182 - https://github.com/kubernetes/kubernetes/issues/39823 - """ - cmd = ['iptables', '-w', '300', '-P', 'FORWARD', 'ACCEPT'] - check_call(cmd) - - -def _systemctl_is_active(application): - ''' Poll systemctl to determine if the application is running ''' - cmd = ['systemctl', 'is-active', application] - try: - raw = check_output(cmd) - return b'active' in raw - except Exception: - return False - - -def get_node_name(): - kubelet_extra_args = parse_extra_args('kubelet-extra-args') - cloud_provider = kubelet_extra_args.get('cloud-provider', '') - if is_state('endpoint.aws.ready'): - cloud_provider = 'aws' - elif is_state('endpoint.gcp.ready'): - cloud_provider = 'gce' - elif is_state('endpoint.openstack.ready'): - cloud_provider = 'openstack' - elif is_state('endpoint.vsphere.ready'): - cloud_provider = 'vsphere' - elif is_state('endpoint.azure.ready'): - cloud_provider = 'azure' - if cloud_provider == 'aws': - return getfqdn().lower() - else: - return gethostname().lower() - - -class ApplyNodeLabelFailed(Exception): - pass - - -def persistent_call(cmd, retry_message): - deadline = time.time() + 180 - while time.time() < deadline: - code = subprocess.call(cmd) - if code == 0: - return True - hookenv.log(retry_message) - time.sleep(1) - else: - return False - - -def set_label(label, value): - nodename = get_node_name() - cmd = 'kubectl --kubeconfig={0} label node {1} {2}={3} --overwrite' - cmd = cmd.format(kubeconfig_path, nodename, label, value) - cmd = cmd.split() - retry = 'Failed to apply label %s=%s. Will retry.' % (label, value) - if not persistent_call(cmd, retry): - raise ApplyNodeLabelFailed(retry) - - -def remove_label(label): - nodename = get_node_name() - cmd = 'kubectl --kubeconfig={0} label node {1} {2}-' - cmd = cmd.format(kubeconfig_path, nodename, label) - cmd = cmd.split() - retry = 'Failed to remove label {0}. Will retry.'.format(label) - if not persistent_call(cmd, retry): - raise ApplyNodeLabelFailed(retry) - - -@when_any('endpoint.aws.joined', - 'endpoint.gcp.joined', - 'endpoint.openstack.joined', - 'endpoint.vsphere.joined', - 'endpoint.azure.joined') -@when_not('kubernetes-worker.cloud.ready') -def set_cloud_pending(): - k8s_version = get_version('kubelet') - k8s_1_11 = k8s_version >= (1, 11) - k8s_1_12 = k8s_version >= (1, 12) - vsphere_joined = is_state('endpoint.vsphere.joined') - azure_joined = is_state('endpoint.azure.joined') - if (vsphere_joined and not k8s_1_12) or (azure_joined and not k8s_1_11): - set_state('kubernetes-worker.cloud.blocked') - else: - remove_state('kubernetes-worker.cloud.blocked') - set_state('kubernetes-worker.cloud.pending') - - -@when_any('endpoint.aws.joined', - 'endpoint.gcp.joined', - 'endpoint.azure.joined') -@when('kube-control.cluster_tag.available') -@when_not('kubernetes-worker.cloud.request-sent') -def request_integration(): - hookenv.status_set('maintenance', 'requesting cloud integration') - kube_control = endpoint_from_flag('kube-control.cluster_tag.available') - cluster_tag = kube_control.get_cluster_tag() - if is_state('endpoint.aws.joined'): - cloud = endpoint_from_flag('endpoint.aws.joined') - cloud.tag_instance({ - 'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned', - }) - cloud.tag_instance_security_group({ - 'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned', - }) - cloud.tag_instance_subnet({ - 'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned', - }) - cloud.enable_object_storage_management(['kubernetes-*']) - elif is_state('endpoint.gcp.joined'): - cloud = endpoint_from_flag('endpoint.gcp.joined') - cloud.label_instance({ - 'k8s-io-cluster-name': cluster_tag, - }) - cloud.enable_object_storage_management() - elif is_state('endpoint.azure.joined'): - cloud = endpoint_from_flag('endpoint.azure.joined') - cloud.tag_instance({ - 'k8s-io-cluster-name': cluster_tag, - }) - cloud.enable_object_storage_management() - cloud.enable_instance_inspection() - cloud.enable_dns_management() - set_state('kubernetes-worker.cloud.request-sent') - hookenv.status_set('waiting', 'Waiting for cloud integration') - - -@when_none('endpoint.aws.joined', - 'endpoint.gcp.joined', - 'endpoint.openstack.joined', - 'endpoint.vsphere.joined', - 'endpoint.azure.joined') -def clear_cloud_flags(): - remove_state('kubernetes-worker.cloud.pending') - remove_state('kubernetes-worker.cloud.request-sent') - remove_state('kubernetes-worker.cloud.blocked') - remove_state('kubernetes-worker.cloud.ready') - - -@when_any('endpoint.aws.ready', - 'endpoint.gcp.ready', - 'endpoint.openstack.ready', - 'endpoint.vsphere.ready', - 'endpoint.azure.ready') -@when_not('kubernetes-worker.cloud.blocked', - 'kubernetes-worker.cloud.ready') -def cloud_ready(): - remove_state('kubernetes-worker.cloud.pending') - if is_state('endpoint.gcp.ready'): - _write_gcp_snap_config('kubelet') - elif is_state('endpoint.openstack.ready'): - _write_openstack_snap_config('kubelet') - elif is_state('endpoint.azure.ready'): - _write_azure_snap_config('kubelet') - set_state('kubernetes-worker.cloud.ready') - set_state('kubernetes-worker.restart-needed') # force restart - - -def _snap_common_path(component): - return Path('/var/snap/{}/common'.format(component)) - - -def _cloud_config_path(component): - return _snap_common_path(component) / 'cloud-config.conf' - - -def _gcp_creds_path(component): - return _snap_common_path(component) / 'gcp-creds.json' - - -def _daemon_env_path(component): - return _snap_common_path(component) / 'environment' - - -def _write_gcp_snap_config(component): - # gcp requires additional credentials setup - gcp = endpoint_from_flag('endpoint.gcp.ready') - creds_path = _gcp_creds_path(component) - with creds_path.open('w') as fp: - os.fchmod(fp.fileno(), 0o600) - fp.write(gcp.credentials) - - # create a cloud-config file that sets token-url to nil to make the - # services use the creds env var instead of the metadata server, as - # well as making the cluster multizone - cloud_config_path = _cloud_config_path(component) - cloud_config_path.write_text('[Global]\n' - 'token-url = nil\n' - 'multizone = true\n') - - daemon_env_path = _daemon_env_path(component) - if daemon_env_path.exists(): - daemon_env = daemon_env_path.read_text() - if not daemon_env.endswith('\n'): - daemon_env += '\n' - else: - daemon_env = '' - if gcp_creds_env_key not in daemon_env: - daemon_env += '{}={}\n'.format(gcp_creds_env_key, creds_path) - daemon_env_path.parent.mkdir(parents=True, exist_ok=True) - daemon_env_path.write_text(daemon_env) - - -def _write_openstack_snap_config(component): - # openstack requires additional credentials setup - openstack = endpoint_from_flag('endpoint.openstack.ready') - - cloud_config_path = _cloud_config_path(component) - cloud_config_path.write_text('\n'.join([ - '[Global]', - 'auth-url = {}'.format(openstack.auth_url), - 'username = {}'.format(openstack.username), - 'password = {}'.format(openstack.password), - 'tenant-name = {}'.format(openstack.project_name), - 'domain-name = {}'.format(openstack.user_domain_name), - ])) - - -def _write_azure_snap_config(component): - azure = endpoint_from_flag('endpoint.azure.ready') - cloud_config_path = _cloud_config_path(component) - cloud_config_path.write_text(json.dumps({ - 'useInstanceMetadata': True, - 'useManagedIdentityExtension': True, - 'subscriptionId': azure.subscription_id, - 'resourceGroup': azure.resource_group, - 'location': azure.resource_group_location, - 'vnetName': azure.vnet_name, - 'vnetResourceGroup': azure.vnet_resource_group, - 'subnetName': azure.subnet_name, - 'securityGroupName': azure.security_group_name, - })) - - -def get_first_mount(mount_relation): - mount_relation_list = mount_relation.mounts() - if mount_relation_list and len(mount_relation_list) > 0: - # mount relation list is a list of the mount layer relations - # for now we just use the first one that is nfs - for mount in mount_relation_list: - # for now we just check the first mount and use that. - # the nfs charm only supports one for now. - if ('mounts' in mount and - mount['mounts'][0]['fstype'] == 'nfs'): - return mount['mounts'][0] - return None - - -@when('nfs.available') -def nfs_state_control(mount): - ''' Determine if we should remove the state that controls the re-render - and execution of the nfs-relation-changed event because there - are changes in the relationship data, and we should re-render any - configs ''' - - mount_data = get_first_mount(mount) - if mount_data: - nfs_relation_data = { - 'options': mount_data['options'], - 'host': mount_data['hostname'], - 'mountpoint': mount_data['mountpoint'], - 'fstype': mount_data['fstype'] - } - - # Re-execute the rendering if the data has changed. - if data_changed('nfs-config', nfs_relation_data): - hookenv.log('reconfiguring nfs') - remove_state('nfs.configured') - - -@when('nfs.available') -@when_not('nfs.configured') -def nfs_storage(mount): - '''NFS on kubernetes requires nfs config rendered into a deployment of - the nfs client provisioner. That will handle the persistent volume claims - with no persistent volume to back them.''' - - mount_data = get_first_mount(mount) - if not mount_data: - return - - addon_path = '/root/cdk/addons/{}' - # Render the NFS deployment - manifest = addon_path.format('nfs-provisioner.yaml') - render('nfs-provisioner.yaml', manifest, mount_data) - hookenv.log('Creating the nfs provisioner.') - try: - kubectl('apply', '-f', manifest) - except CalledProcessError as e: - hookenv.log(e) - hookenv.log('Failed to create nfs provisioner. Will attempt again next update.') # noqa - return - - set_state('nfs.configured') diff --git a/cluster/juju/layers/kubernetes-worker/registry-configmap.yaml b/cluster/juju/layers/kubernetes-worker/registry-configmap.yaml deleted file mode 100644 index 4800ff3c04..0000000000 --- a/cluster/juju/layers/kubernetes-worker/registry-configmap.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -data: - body-size: 1024m -kind: ConfigMap -metadata: - name: nginx-load-balancer-conf diff --git a/cluster/juju/layers/kubernetes-worker/templates/default-http-backend.yaml b/cluster/juju/layers/kubernetes-worker/templates/default-http-backend.yaml deleted file mode 100644 index dbef52093c..0000000000 --- a/cluster/juju/layers/kubernetes-worker/templates/default-http-backend.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: default-http-backend -spec: - replicas: 1 - selector: - app: default-http-backend - template: - metadata: - labels: - app: default-http-backend - spec: - terminationGracePeriodSeconds: 60 - containers: - - name: default-http-backend - # Any image is permissible as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: {{ defaultbackend_image }} - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: default-http-backend -# namespace: kube-system - labels: - k8s-app: default-http-backend -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 80 - selector: - app: default-http-backend diff --git a/cluster/juju/layers/kubernetes-worker/templates/ingress-daemon-set.yaml b/cluster/juju/layers/kubernetes-worker/templates/ingress-daemon-set.yaml deleted file mode 100644 index b305d32e7f..0000000000 --- a/cluster/juju/layers/kubernetes-worker/templates/ingress-daemon-set.yaml +++ /dev/null @@ -1,179 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-{{ juju_application }}-serviceaccount ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: nginx-ingress-{{ juju_application }}-clusterrole -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-{{ juju_application }}-role -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - create - - update ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-nisa-{{ juju_application }}-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-{{ juju_application }}-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-{{ juju_application }}-serviceaccount ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-nisa-{{ juju_application }}-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-{{ juju_application }}-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-{{ juju_application }}-serviceaccount - namespace: default ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-load-balancer-{{ juju_application }}-conf ---- -apiVersion: {{ daemonset_api_version }} -kind: DaemonSet -metadata: - name: nginx-ingress-{{ juju_application }}-controller - labels: - juju-application: nginx-ingress-{{ juju_application }} -spec: - selector: - matchLabels: - name: nginx-ingress-{{ juju_application }} - template: - metadata: - labels: - name: nginx-ingress-{{ juju_application }} - spec: - nodeSelector: - juju-application: {{ juju_application }} - terminationGracePeriodSeconds: 60 - # hostPort doesn't work with CNI, so we have to use hostNetwork instead - # see https://github.com/kubernetes/kubernetes/issues/23920 - hostNetwork: true - serviceAccountName: nginx-ingress-{{ juju_application }}-serviceaccount - containers: - - image: {{ ingress_image }} - name: nginx-ingress-{{ juju_application }} - livenessProbe: - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - # use downward API - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - containerPort: 80 - - containerPort: 443 - args: - - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/default-http-backend - - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf - - --enable-ssl-chain-completion={{ ssl_chain_completion }} diff --git a/cluster/juju/layers/kubernetes-worker/templates/microbot-example.yaml b/cluster/juju/layers/kubernetes-worker/templates/microbot-example.yaml deleted file mode 100644 index 81bb731a6b..0000000000 --- a/cluster/juju/layers/kubernetes-worker/templates/microbot-example.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - creationTimestamp: null - labels: - app: microbot - name: microbot -spec: - replicas: {{ replicas }} - selector: - matchLabels: - app: microbot - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - app: microbot - spec: - containers: - - image: cdkbot/microbot-{{ arch }}:latest - imagePullPolicy: "" - name: microbot - ports: - - containerPort: 80 - livenessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 5 - timeoutSeconds: 30 - resources: {} - restartPolicy: Always - serviceAccountName: "" -status: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: microbot - labels: - app: microbot -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 80 - selector: - app: microbot ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: microbot-ingress -spec: - rules: - - host: microbot.{{ public_address }}.xip.io - http: - paths: - - path: / - backend: - serviceName: microbot - servicePort: 80 diff --git a/cluster/juju/layers/kubernetes-worker/templates/nfs-provisioner.yaml b/cluster/juju/layers/kubernetes-worker/templates/nfs-provisioner.yaml deleted file mode 100644 index 66aa9fd65b..0000000000 --- a/cluster/juju/layers/kubernetes-worker/templates/nfs-provisioner.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: default - annotations: - storageclass.kubernetes.io/is-default-class: "true" -provisioner: fuseim.pri/ifs ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: nfs-client-provisioner - labels: - app: nfs-client-provisioner -spec: - replicas: 1 - selector: - matchLabels: - app: nfs-client-provisioner - strategy: - type: Recreate - template: - metadata: - labels: - app: nfs-client-provisioner - spec: - containers: - - name: nfs-client-provisioner - image: quay.io/external_storage/nfs-client-provisioner:latest - volumeMounts: - - name: nfs-client-root - mountPath: /persistentvolumes - env: - - name: PROVISIONER_NAME - value: fuseim.pri/ifs - - name: NFS_SERVER - value: {{ hostname }} - - name: NFS_PATH - value: {{ mountpoint }} - volumes: - - name: nfs-client-root - nfs: - server: {{ hostname }} - path: {{ mountpoint }} diff --git a/cluster/juju/layers/kubernetes-worker/templates/registry.yaml b/cluster/juju/layers/kubernetes-worker/templates/registry.yaml deleted file mode 100644 index 2503f3abfb..0000000000 --- a/cluster/juju/layers/kubernetes-worker/templates/registry.yaml +++ /dev/null @@ -1,118 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: registry-tls-data -type: Opaque -data: - tls.crt: {{ tlscert }} - tls.key: {{ tlskey }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: registry-auth-data -type: Opaque -data: - htpasswd: {{ htpasswd }} ---- -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-registry-v0 - labels: - k8s-app: kube-registry - version: v0 - kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-registry - version: v0 - template: - metadata: - labels: - k8s-app: kube-registry - version: v0 - kubernetes.io/cluster-service: "true" - spec: - containers: - - name: registry - image: cdkbot/registry-{{ arch }}:2.6 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - env: - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - - name: REGISTRY_AUTH_HTPASSWD_REALM - value: basic_realm - - name: REGISTRY_AUTH_HTPASSWD_PATH - value: /auth/htpasswd - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - - name: auth-dir - mountPath: /auth - ports: - - containerPort: 5000 - name: registry - protocol: TCP - volumes: - - name: image-store - hostPath: - path: /srv/registry - - name: auth-dir - secret: - secretName: registry-auth-data ---- -apiVersion: v1 -kind: Service -metadata: - name: kube-registry - labels: - k8s-app: kube-registry - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeRegistry" -spec: - selector: - k8s-app: kube-registry - type: LoadBalancer - ports: - - name: registry - port: 5000 - protocol: TCP ---- -apiVersion: v1 -kind: Secret -metadata: - name: registry-access -data: - .dockercfg: {{ dockercfg }} -type: kubernetes.io/dockercfg -{%- if ingress %} ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: registry-ing -spec: - tls: - - hosts: - - {{ domain }} - secretName: registry-tls-data - rules: - - host: {{ domain }} - http: - paths: - - backend: - serviceName: kube-registry - servicePort: 5000 - path: / -{% endif %} diff --git a/cluster/juju/layers/kubernetes-worker/wheelhouse.txt b/cluster/juju/layers/kubernetes-worker/wheelhouse.txt deleted file mode 100644 index 0891dbc14b..0000000000 --- a/cluster/juju/layers/kubernetes-worker/wheelhouse.txt +++ /dev/null @@ -1 +0,0 @@ -charms.templating.jinja2>=0.0.1,<2.0.0 diff --git a/cluster/juju/prereqs/OWNERS b/cluster/juju/prereqs/OWNERS deleted file mode 100644 index a30810befa..0000000000 --- a/cluster/juju/prereqs/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: -- eparis -- david-mcmahon diff --git a/cluster/juju/prereqs/ubuntu-juju.sh b/cluster/juju/prereqs/ubuntu-juju.sh deleted file mode 100644 index ad774978a0..0000000000 --- a/cluster/juju/prereqs/ubuntu-juju.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -o errexit -set -o nounset -set -o pipefail - - -function check_for_ppa() { - local repo="$1" - grep -qsw "${repo}" /etc/apt/sources.list /etc/apt/sources.list.d/* -} - -function package_status() { - local pkgname=$1 - local pkgstatus - pkgstatus=$(dpkg-query -W --showformat='${Status}\n' "${pkgname}") - if [[ "${pkgstatus}" != "install ok installed" ]]; then - echo "Missing package ${pkgname}" - sudo apt-get --force-yes --yes install "${pkgname}" - fi -} - -function gather_installation_reqs() { - if ! check_for_ppa "juju"; then - echo "... Detected missing dependencies.. running" - echo "... add-apt-repository ppa:juju/stable" - sudo add-apt-repository -y ppa:juju/stable - sudo apt-get update - fi - - package_status 'juju' - package_status 'charm-tools' -} diff --git a/cluster/juju/return-node-ips.py b/cluster/juju/return-node-ips.py deleted file mode 100755 index 7d7125582d..0000000000 --- a/cluster/juju/return-node-ips.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import sys -# This script helps parse out the private IP addresses from the -# `juju run` command's JSON object, see cluster/juju/util.sh - -if len(sys.argv) > 1: - # It takes the JSON output as the first argument. - nodes = json.loads(sys.argv[1]) - # There can be multiple nodes to print the Stdout. - for num in nodes: - print num['Stdout'].rstrip() -else: - exit(1) diff --git a/cluster/juju/util.sh b/cluster/juju/util.sh deleted file mode 100755 index ede773a4a2..0000000000 --- a/cluster/juju/util.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -o errexit -set -o nounset -set -o pipefail -#set -o xtrace - -UTIL_SCRIPT=$(readlink -m "${BASH_SOURCE[0]}") -JUJU_PATH=$(dirname "${UTIL_SCRIPT}") -KUBE_ROOT=$(readlink -m "${JUJU_PATH}/../../") -# Use the config file specified in $KUBE_CONFIG_FILE, or config-default.sh. -source "${JUJU_PATH}/${KUBE_CONFIG_FILE-config-default.sh}" -# This attempts installation of Juju - This really needs to support multiple -# providers/distros - but I'm super familiar with ubuntu so assume that for now. -source "${JUJU_PATH}/prereqs/ubuntu-juju.sh" -export JUJU_REPOSITORY="${JUJU_PATH}/charms" -KUBE_BUNDLE_PATH="${JUJU_PATH}/bundles/local.yaml" -# The directory for the kubectl binary, this is one of the paths in kubectl.sh. -KUBECTL_DIR="${KUBE_ROOT}/platforms/linux/amd64" - - -function build-local() { - # This used to build the kubernetes project. Now it rebuilds the charm(s) - # living in `cluster/juju/layers` - - charm build "${JUJU_PATH}/layers/kubernetes" -o "$JUJU_REPOSITORY" -r --no-local-layers -} - -function detect-master() { - local kubestatus - - # Capturing a newline, and my awk-fu was weak - pipe through tr -d - kubestatus=$(juju status --format=oneline kubernetes | grep "${KUBE_MASTER_NAME}" | awk '{print $3}' | tr -d "\n") - export KUBE_MASTER_IP=${kubestatus} - export KUBE_SERVER=https://${KUBE_MASTER_IP}:6433 - -} - -function detect-nodes() { - # Run the Juju command that gets the minion private IP addresses. - local ipoutput - ipoutput=$(juju run --application kubernetes "unit-get private-address" --format=json) - # [ - # {"MachineId":"2","Stdout":"192.168.122.188\n","UnitId":"kubernetes/0"}, - # {"MachineId":"3","Stdout":"192.168.122.166\n","UnitId":"kubernetes/1"} - # ] - - # Strip out the IP addresses - while IFS=$'\n' read -r ip; - do KUBE_NODE_IP_ADDRESSES+=("$ip"); - done < <("${JUJU_PATH}/return-node-ips.py" "${ipoutput}") - export KUBE_NODE_IP_ADDRESSES - # echo "Kubernetes minions: " ${KUBE_NODE_IP_ADDRESSES[@]} 1>&2 - export NUM_NODES=${#KUBE_NODE_IP_ADDRESSES[@]} -} - -function kube-up() { - build-local - - # Replace the charm directory in the bundle. - sed "s|__CHARM_DIR__|${JUJU_REPOSITORY}|" < "${KUBE_BUNDLE_PATH}.base" > "${KUBE_BUNDLE_PATH}" - - # The juju-deployer command will deploy the bundle and can be run - # multiple times to continue deploying the parts that fail. - juju deploy "${KUBE_BUNDLE_PATH}" - - source "${KUBE_ROOT}/cluster/common.sh" - - # Sleep due to juju bug http://pad.lv/1432759 - sleep-status - detect-master - detect-nodes - - # Copy kubectl, the cert and key to this machine from master. - ( - umask 077 - mkdir -p "${KUBECTL_DIR}" - juju scp "${KUBE_MASTER_NAME}:kubectl_package.tar.gz" "${KUBECTL_DIR}" - tar xfz "${KUBECTL_DIR}/kubectl_package.tar.gz" -C "${KUBECTL_DIR}" - ) - # Export the location of the kubectl configuration file. - export KUBECONFIG="${KUBECTL_DIR}/kubeconfig" -} - -function kube-down() { - local force="${1-}" - local jujuenv - jujuenv=$(juju switch) - juju destroy-model "${jujuenv}" "${force}" || true - # Clean up the generated charm files. - rm -rf "${KUBE_ROOT}/cluster/juju/charms" - # Clean up the kubectl binary and config file. - rm -rf "${KUBECTL_DIR}" -} - -function prepare-e2e() { - echo "prepare-e2e() The Juju provider does not need any preparations for e2e." 1>&2 -} - -function sleep-status() { - local i - local maxtime - local jujustatus - i=0 - maxtime=900 - jujustatus='' - echo "Waiting up to 15 minutes to allow the cluster to come online... wait for it..." 1>&2 - - while [[ $i < $maxtime && -z $jujustatus ]]; do - sleep 15 - i=$((i + 15)) - jujustatus=$("${JUJU_PATH}/identify-leaders.py") - export KUBE_MASTER_NAME=${jujustatus} - done - -} - -# Execute prior to running tests to build a release if required for environment. -function test-build-release { - echo "test-build-release() " 1>&2 -} - -# Execute prior to running tests to initialize required structure. This is -# called from hack/e2e.go only when running -up. -function test-setup { - "${KUBE_ROOT}/cluster/kube-up.sh" -} - -# Execute after running tests to perform any required clean-up. This is called -# from hack/e2e.go -function test-teardown() { - kube-down "-y" -} - -# Verify the prerequisites are statisfied before running. -function verify-prereqs() { - gather_installation_reqs -}