portainer/api/internal/upgrade/upgrade_kubernetes.go

package upgrade

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/pkg/errors"
	portainer "github.com/portainer/portainer/api"
	"github.com/rs/zerolog/log"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

func ptr[T any](i T) *T { return &i }

func (service *service) upgradeKubernetes(environment *portainer.Endpoint, licenseKey, version string) error {
	ctx := context.TODO()

	kubeCLI, err := service.kubernetesClientFactory.CreateClient(environment)
	if err != nil {
		return errors.WithMessage(err, "failed to get kubernetes client")
	}

	namespace := "portainer"
	taskName := fmt.Sprintf("portainer-upgrade-%d", time.Now().Unix())

	jobsCli := kubeCLI.BatchV1().Jobs(namespace)

	updaterImage := os.Getenv(updaterImageEnvVar)
	if updaterImage == "" {
		updaterImage = "portainer/portainer-updater:latest"
	}

	portainerImagePrefix := os.Getenv(portainerImagePrefixEnvVar)
	if portainerImagePrefix == "" {
		portainerImagePrefix = "portainer/portainer-ee"
	}

	image := fmt.Sprintf("%s:%s", portainerImagePrefix, version)

	if err := service.checkImageForKubernetes(ctx, kubeCLI, namespace, image); err != nil {
		return err
	}

	job, err := jobsCli.Create(ctx, &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Name:      taskName,
			Namespace: namespace,
		},

		Spec: batchv1.JobSpec{
			TTLSecondsAfterFinished: ptr[int32](5 * 60), // cleanup after 5 minutes
			BackoffLimit:            ptr[int32](0),
			Template: corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{

					RestartPolicy:      "Never",
					ServiceAccountName: "portainer-sa-clusteradmin",
					Containers: []corev1.Container{
						{
							Name:  taskName,
							Image: updaterImage,
							Args: []string{
								"--pretty-log",
								"--log-level", "DEBUG",
								"portainer",
								"--env-type", "kubernetes",
								"--image", image,
								"--license", licenseKey,
							},
						},
					},
				},
			},
		},
	}, metav1.CreateOptions{})

	if err != nil {
		return errors.WithMessage(err, "failed to create upgrade job")
	}

	watcher, err := jobsCli.Watch(ctx, metav1.ListOptions{
		FieldSelector:  "metadata.name=" + taskName,
		TimeoutSeconds: ptr[int64](60),
	})
	if err != nil {
		return errors.WithMessage(err, "failed to watch upgrade job")
	}

	for event := range watcher.ResultChan() {
		job, ok := event.Object.(*batchv1.Job)
		if !ok {
			continue
		}

		for _, c := range job.Status.Conditions {
			if c.Type == batchv1.JobComplete {
				log.Debug().
					Str("job", job.Name).
					Msg("Upgrade job completed")
				return nil
			}

			if c.Type == batchv1.JobFailed {
				return fmt.Errorf("upgrade failed: %s", c.Message)
			}
		}
	}

	log.Debug().
		Str("job", job.Name).
		Msg("Upgrade job created")

	return errors.New("upgrade failed: server should have been restarted by the updater")

}

func (service *service) checkImageForKubernetes(ctx context.Context, kubeCLI *kubernetes.Clientset, namespace, image string) error {
	podsCli := kubeCLI.CoreV1().Pods(namespace)

	log.Debug().
		Str("image", image).
		Msg("Checking image")

	podName := fmt.Sprintf("portainer-image-check-%d", time.Now().Unix())
	_, err := podsCli.Create(ctx, &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: corev1.PodSpec{
			RestartPolicy: "Never",

			Containers: []corev1.Container{
				{
					Name:  fmt.Sprint(podName, "-container"),
					Image: image,
				},
			},
		},
	}, metav1.CreateOptions{})

	if err != nil {
		log.Warn().Err(err).Msg("failed to create image check pod")
		return errors.WithMessage(err, "failed to create image check pod")
	}

	defer func() {
		log.Debug().
			Str("pod", podName).
			Msg("Deleting image check pod")

		if err := podsCli.Delete(ctx, podName, metav1.DeleteOptions{}); err != nil {
			log.Warn().Err(err).Msg("failed to delete image check pod")
		}
	}()

	i := 0
	for {
		time.Sleep(2 * time.Second)

		log.Debug().
			Str("image", image).
			Int("try", i).
			Msg("Checking image")

		i++

		pod, err := podsCli.Get(ctx, podName, metav1.GetOptions{})
		if err != nil {
			return errors.WithMessage(err, "failed to get image check pod")
		}

		for _, containerStatus := range pod.Status.ContainerStatuses {
			if containerStatus.Ready {
				log.Debug().
					Str("image", image).
					Str("pod", podName).
					Msg("Image check container ready, assuming image is available")

				return nil
			}

			if containerStatus.State.Waiting != nil {
				if containerStatus.State.Waiting.Reason == "ErrImagePull" || containerStatus.State.Waiting.Reason == "ImagePullBackOff" {
					log.Debug().
						Str("image", image).
						Str("pod", podName).
						Str("reason", containerStatus.State.Waiting.Reason).
						Str("message", containerStatus.State.Waiting.Message).
						Str("container", containerStatus.Name).
						Msg("Image check container failed because of missing image")
					return fmt.Errorf("image %s not found", image)
				}
			}
		}
	}
}