k3s/pkg/volume/flocker/plugin.go

/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flocker

import (
	"fmt"
	"path"
	"time"

	flockerclient "github.com/ClusterHQ/flocker-go"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/env"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
)

const (
	flockerPluginName = "kubernetes.io/flocker"

	defaultHost           = "localhost"
	defaultPort           = 4523
	defaultCACertFile     = "/etc/flocker/cluster.crt"
	defaultClientKeyFile  = "/etc/flocker/apiuser.key"
	defaultClientCertFile = "/etc/flocker/apiuser.crt"

	timeoutWaitingForVolume = 2 * time.Minute
	tickerWaitingForVolume  = 5 * time.Second
)

func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{&flockerPlugin{}}
}

type flockerPlugin struct {
	host volume.VolumeHost
}

type flocker struct {
	datasetName string
	path        string
	pod         *api.Pod
	mounter     mount.Interface
	plugin      *flockerPlugin
}

func (p *flockerPlugin) Init(host volume.VolumeHost) error {
	p.host = host
	return nil
}

func (p flockerPlugin) Name() string {
	return flockerPluginName
}

func (p flockerPlugin) CanSupport(spec *volume.Spec) bool {
	return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Flocker != nil) ||
		(spec.Volume != nil && spec.Volume.Flocker != nil)
}

func (p *flockerPlugin) getFlockerVolumeSource(spec *volume.Spec) (*api.FlockerVolumeSource, bool) {
	// AFAIK this will always be r/w, but perhaps for the future it will be needed
	readOnly := false

	if spec.Volume != nil && spec.Volume.Flocker != nil {
		return spec.Volume.Flocker, readOnly
	}
	return spec.PersistentVolume.Spec.Flocker, readOnly
}

func (p *flockerPlugin) NewMounter(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	source, readOnly := p.getFlockerVolumeSource(spec)
	mounter := flockerMounter{
		flocker: &flocker{
			datasetName: source.DatasetName,
			pod:         pod,
			mounter:     p.host.GetMounter(),
			plugin:      p,
		},
		exe:      exec.New(),
		opts:     opts,
		readOnly: readOnly,
	}
	return &mounter, nil
}

func (p *flockerPlugin) NewUnmounter(datasetName string, podUID types.UID) (volume.Unmounter, error) {
	// Flocker agent will take care of this, there is nothing we can do here
	return nil, nil
}

type flockerMounter struct {
	*flocker
	client   flockerclient.Clientable
	exe      exec.Interface
	opts     volume.VolumeOptions
	readOnly bool
	volume.MetricsNil
}

func (b flockerMounter) GetAttributes() volume.Attributes {
	return volume.Attributes{
		ReadOnly:        b.readOnly,
		Managed:         false,
		SupportsSELinux: false,
	}
}
func (b flockerMounter) GetPath() string {
	return b.flocker.path
}

func (b flockerMounter) SetUp(fsGroup *int64) error {
	return b.SetUpAt(b.flocker.datasetName, fsGroup)
}

// newFlockerClient uses environment variables and pod attributes to return a
// flocker client capable of talking with the Flocker control service.
func (b flockerMounter) newFlockerClient() (*flockerclient.Client, error) {
	host := env.GetEnvAsStringOrFallback("FLOCKER_CONTROL_SERVICE_HOST", defaultHost)
	port, err := env.GetEnvAsIntOrFallback("FLOCKER_CONTROL_SERVICE_PORT", defaultPort)

	if err != nil {
		return nil, err
	}
	caCertPath := env.GetEnvAsStringOrFallback("FLOCKER_CONTROL_SERVICE_CA_FILE", defaultCACertFile)
	keyPath := env.GetEnvAsStringOrFallback("FLOCKER_CONTROL_SERVICE_CLIENT_KEY_FILE", defaultClientKeyFile)
	certPath := env.GetEnvAsStringOrFallback("FLOCKER_CONTROL_SERVICE_CLIENT_CERT_FILE", defaultClientCertFile)

	c, err := flockerclient.NewClient(host, port, b.flocker.pod.Status.HostIP, caCertPath, keyPath, certPath)
	return c, err
}

func (b *flockerMounter) getMetaDir() string {
	return path.Join(
		b.plugin.host.GetPodPluginDir(
			b.flocker.pod.UID, strings.EscapeQualifiedNameForDisk(flockerPluginName),
		),
		b.datasetName,
	)
}

/*
SetUpAt will setup a Flocker volume following this flow of calls to the Flocker
control service:

1. Get the dataset id for the given volume name/dir
2. It should already be there, if it's not the user needs to manually create it
3. Check the current Primary UUID
4. If it doesn't match with the Primary UUID that we got on 2, then we will
   need to update the Primary UUID for this volume.
5. Wait until the Primary UUID was updated or timeout.
*/
func (b flockerMounter) SetUpAt(dir string, fsGroup *int64) error {
	if volumeutil.IsReady(b.getMetaDir()) {
		return nil
	}

	if b.client == nil {
		c, err := b.newFlockerClient()
		if err != nil {
			return err
		}
		b.client = c
	}

	datasetID, err := b.client.GetDatasetID(dir)
	if err != nil {
		return err
	}

	s, err := b.client.GetDatasetState(datasetID)
	if err != nil {
		return fmt.Errorf("The volume '%s' is not available in Flocker. You need to create this manually with Flocker CLI before using it.", dir)
	}

	primaryUUID, err := b.client.GetPrimaryUUID()
	if err != nil {
		return err
	}

	if s.Primary != primaryUUID {
		if err := b.updateDatasetPrimary(datasetID, primaryUUID); err != nil {
			return err
		}
		newState, err := b.client.GetDatasetState(datasetID)
		if err != nil {
			return fmt.Errorf("The volume '%s' migrated unsuccessfully.", datasetID)
		}
		b.flocker.path = newState.Path
	} else {
		b.flocker.path = s.Path
	}

	volumeutil.SetReady(b.getMetaDir())
	return nil
}

// updateDatasetPrimary will update the primary in Flocker and wait for it to
// be ready. If it never gets to ready state it will timeout and error.
func (b flockerMounter) updateDatasetPrimary(datasetID, primaryUUID string) error {
	// We need to update the primary and wait for it to be ready
	_, err := b.client.UpdatePrimaryForDataset(primaryUUID, datasetID)
	if err != nil {
		return err
	}

	timeoutChan := time.NewTimer(timeoutWaitingForVolume).C
	tickChan := time.NewTicker(tickerWaitingForVolume).C

	for {
		if s, err := b.client.GetDatasetState(datasetID); err == nil && s.Primary == primaryUUID {
			return nil
		}

		select {
		case <-timeoutChan:
			return fmt.Errorf(
				"Timed out waiting for the dataset_id: '%s' to be moved to the primary: '%s'\n%v",
				datasetID, primaryUUID, err,
			)
		case <-tickChan:
			break
		}
	}

}