portainer/api/kubernetes/cli/applications.go

607 lines
22 KiB
Go

package cli
import (
"context"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
autoscalingv2 "k8s.io/api/autoscaling/v2"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
labels "k8s.io/apimachinery/pkg/labels"
)
// PortainerApplicationResources contains collections of various Kubernetes resources
// associated with a Portainer application.
type PortainerApplicationResources struct {
Pods []corev1.Pod
ReplicaSets []appsv1.ReplicaSet
Deployments []appsv1.Deployment
StatefulSets []appsv1.StatefulSet
DaemonSets []appsv1.DaemonSet
Services []corev1.Service
HorizontalPodAutoscalers []autoscalingv2.HorizontalPodAutoscaler
}
// GetAllKubernetesApplications gets a list of kubernetes workloads (or applications) across all namespaces in the cluster
// if the user is an admin, all namespaces in the current k8s environment(endpoint) are fetched using the fetchApplications function.
// otherwise, namespaces the non-admin user has access to will be used to filter the applications based on the allowed namespaces.
func (kcl *KubeClient) GetApplications(namespace, nodeName string) ([]models.K8sApplication, error) {
if kcl.IsKubeAdmin {
return kcl.fetchApplications(namespace, nodeName)
}
return kcl.fetchApplicationsForNonAdmin(namespace, nodeName)
}
// fetchApplications fetches the applications in the namespaces the user has access to.
// This function is called when the user is an admin.
func (kcl *KubeClient) fetchApplications(namespace, nodeName string) ([]models.K8sApplication, error) {
podListOptions := metav1.ListOptions{}
if nodeName != "" {
podListOptions.FieldSelector = "spec.nodeName=" + nodeName
}
portainerApplicationResources, err := kcl.fetchAllApplicationsListResources(namespace, podListOptions)
if err != nil {
return nil, err
}
applications, err := kcl.convertPodsToApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
unhealthyApplications, err := fetchUnhealthyApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
return append(applications, unhealthyApplications...), nil
}
// fetchApplicationsForNonAdmin fetches the applications in the namespaces the user has access to.
// This function is called when the user is not an admin.
func (kcl *KubeClient) fetchApplicationsForNonAdmin(namespace, nodeName string) ([]models.K8sApplication, error) {
log.Debug().Msgf("Fetching applications for non-admin user: %v", kcl.NonAdminNamespaces)
if len(kcl.NonAdminNamespaces) == 0 {
return nil, nil
}
podListOptions := metav1.ListOptions{}
if nodeName != "" {
podListOptions.FieldSelector = "spec.nodeName=" + nodeName
}
portainerApplicationResources, err := kcl.fetchAllApplicationsListResources(namespace, podListOptions)
if err != nil {
return nil, err
}
applications, err := kcl.convertPodsToApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
unhealthyApplications, err := fetchUnhealthyApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sApplication, 0)
for _, application := range append(applications, unhealthyApplications...) {
if _, ok := nonAdminNamespaceSet[application.ResourcePool]; ok {
results = append(results, application)
}
}
return results, nil
}
// convertPodsToApplications processes pods and converts them to applications, ensuring uniqueness by owner reference.
func (kcl *KubeClient) convertPodsToApplications(portainerApplicationResources PortainerApplicationResources) ([]models.K8sApplication, error) {
applications := []models.K8sApplication{}
processedOwners := make(map[string]struct{})
for _, pod := range portainerApplicationResources.Pods {
if len(pod.OwnerReferences) > 0 {
ownerUID := string(pod.OwnerReferences[0].UID)
if _, exists := processedOwners[ownerUID]; exists {
continue
}
processedOwners[ownerUID] = struct{}{}
}
application, err := kcl.ConvertPodToApplication(pod, portainerApplicationResources, true)
if err != nil {
return nil, err
}
if application != nil {
applications = append(applications, *application)
}
}
return applications, nil
}
// GetClusterApplicationsResource returns the total resource requests and limits for all applications in a namespace
// for a cluster level resource, set the namespace to ""
func (kcl *KubeClient) GetApplicationsResource(namespace, node string) (models.K8sApplicationResource, error) {
resource := models.K8sApplicationResource{}
podListOptions := metav1.ListOptions{}
if node != "" {
podListOptions.FieldSelector = "spec.nodeName=" + node
}
pods, err := kcl.cli.CoreV1().Pods(namespace).List(context.Background(), podListOptions)
if err != nil {
return resource, err
}
for _, pod := range pods.Items {
podResources := calculatePodResourceUsage(pod)
resource.CPURequest += podResources.CPURequest
resource.CPULimit += podResources.CPULimit
resource.MemoryRequest += podResources.MemoryRequest
resource.MemoryLimit += podResources.MemoryLimit
}
return resource, nil
}
// GetApplicationsFromConfigMap gets a list of applications that use a specific ConfigMap
// by checking all pods in the same namespace as the ConfigMap
func (kcl *KubeClient) GetApplicationNamesFromConfigMap(configMap models.K8sConfigMap, pods []corev1.Pod, replicaSets []appsv1.ReplicaSet) ([]string, error) {
applications := []string{}
for _, pod := range pods {
if pod.Namespace == configMap.Namespace {
if isPodUsingConfigMap(&pod, configMap.Name) {
application, err := kcl.ConvertPodToApplication(pod, PortainerApplicationResources{
ReplicaSets: replicaSets,
}, false)
if err != nil {
return nil, err
}
applications = append(applications, application.Name)
}
}
}
return applications, nil
}
func (kcl *KubeClient) GetApplicationNamesFromSecret(secret models.K8sSecret, pods []corev1.Pod, replicaSets []appsv1.ReplicaSet) ([]string, error) {
applications := []string{}
for _, pod := range pods {
if pod.Namespace == secret.Namespace {
if isPodUsingSecret(&pod, secret.Name) {
application, err := kcl.ConvertPodToApplication(pod, PortainerApplicationResources{
ReplicaSets: replicaSets,
}, false)
if err != nil {
return nil, err
}
applications = append(applications, application.Name)
}
}
}
return applications, nil
}
// ConvertPodToApplication converts a pod to an application, updating owner references if necessary
func (kcl *KubeClient) ConvertPodToApplication(pod corev1.Pod, portainerApplicationResources PortainerApplicationResources, withResource bool) (*models.K8sApplication, error) {
if isReplicaSetOwner(pod) {
updateOwnerReferenceToDeployment(&pod, portainerApplicationResources.ReplicaSets)
}
application := createApplicationFromPod(&pod, portainerApplicationResources)
if application.ID == "" && application.Name == "" {
return nil, nil
}
if withResource {
podResources := calculatePodResourceUsage(pod)
// multiply by the number of requested pods in the application (not the running count)
application.Resource.CPURequest = podResources.CPURequest * float64(application.TotalPodsCount)
application.Resource.CPULimit = podResources.CPULimit * float64(application.TotalPodsCount)
application.Resource.MemoryRequest = podResources.MemoryRequest * int64(application.TotalPodsCount)
application.Resource.MemoryLimit = podResources.MemoryLimit * int64(application.TotalPodsCount)
}
return &application, nil
}
// createApplicationFromPod creates a K8sApplication object from a pod
// it sets the application name, namespace, kind, image, stack id, stack name, and labels
func createApplicationFromPod(pod *corev1.Pod, portainerApplicationResources PortainerApplicationResources) models.K8sApplication {
kind := "Pod"
name := pod.Name
if len(pod.OwnerReferences) > 0 {
kind = pod.OwnerReferences[0].Kind
name = pod.OwnerReferences[0].Name
}
application := models.K8sApplication{
Services: []corev1.Service{},
Metadata: &models.Metadata{},
}
switch kind {
case "Deployment":
for _, deployment := range portainerApplicationResources.Deployments {
if deployment.Name == name && deployment.Namespace == pod.Namespace {
populateApplicationFromDeployment(&application, deployment)
break
}
}
case "StatefulSet":
for _, statefulSet := range portainerApplicationResources.StatefulSets {
if statefulSet.Name == name && statefulSet.Namespace == pod.Namespace {
populateApplicationFromStatefulSet(&application, statefulSet)
break
}
}
case "DaemonSet":
for _, daemonSet := range portainerApplicationResources.DaemonSets {
if daemonSet.Name == name && daemonSet.Namespace == pod.Namespace {
populateApplicationFromDaemonSet(&application, daemonSet)
break
}
}
case "Pod":
populateApplicationFromPod(&application, *pod)
}
if application.ID != "" && application.Name != "" && len(portainerApplicationResources.Services) > 0 {
updateApplicationWithService(&application, portainerApplicationResources.Services)
}
if application.ID != "" && application.Name != "" && len(portainerApplicationResources.HorizontalPodAutoscalers) > 0 {
updateApplicationWithHorizontalPodAutoscaler(&application, portainerApplicationResources.HorizontalPodAutoscalers)
}
return application
}
// createApplicationFromDeployment creates a K8sApplication from a Deployment
func createApplicationFromDeployment(deployment appsv1.Deployment) models.K8sApplication {
var app models.K8sApplication
populateApplicationFromDeployment(&app, deployment)
return app
}
// createApplicationFromStatefulSet creates a K8sApplication from a StatefulSet
func createApplicationFromStatefulSet(statefulSet appsv1.StatefulSet) models.K8sApplication {
var app models.K8sApplication
populateApplicationFromStatefulSet(&app, statefulSet)
return app
}
// createApplicationFromDaemonSet creates a K8sApplication from a DaemonSet
func createApplicationFromDaemonSet(daemonSet appsv1.DaemonSet) models.K8sApplication {
var app models.K8sApplication
populateApplicationFromDaemonSet(&app, daemonSet)
return app
}
func populateApplicationFromDeployment(application *models.K8sApplication, deployment appsv1.Deployment) {
application.ApplicationType = "Deployment"
application.Kind = "Deployment"
application.ID = string(deployment.UID)
application.ResourcePool = deployment.Namespace
application.Name = deployment.Name
application.ApplicationOwner = deployment.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = deployment.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = deployment.Labels["io.portainer.kubernetes.application.stack"]
application.Labels = deployment.Labels
application.MatchLabels = deployment.Spec.Selector.MatchLabels
application.CreationDate = deployment.CreationTimestamp.Time
application.TotalPodsCount = 0
if deployment.Spec.Replicas != nil {
application.TotalPodsCount = int(*deployment.Spec.Replicas)
}
application.RunningPodsCount = int(deployment.Status.ReadyReplicas)
application.DeploymentType = "Replicated"
application.Metadata = &models.Metadata{
Labels: deployment.Labels,
}
// If the deployment has containers, use the first container's image
if len(deployment.Spec.Template.Spec.Containers) > 0 {
application.Image = deployment.Spec.Template.Spec.Containers[0].Image
}
}
func populateApplicationFromStatefulSet(application *models.K8sApplication, statefulSet appsv1.StatefulSet) {
application.Kind = "StatefulSet"
application.ApplicationType = "StatefulSet"
application.ID = string(statefulSet.UID)
application.ResourcePool = statefulSet.Namespace
application.Name = statefulSet.Name
application.ApplicationOwner = statefulSet.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = statefulSet.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = statefulSet.Labels["io.portainer.kubernetes.application.stack"]
application.Labels = statefulSet.Labels
application.MatchLabels = statefulSet.Spec.Selector.MatchLabels
application.CreationDate = statefulSet.CreationTimestamp.Time
application.TotalPodsCount = 0
if statefulSet.Spec.Replicas != nil {
application.TotalPodsCount = int(*statefulSet.Spec.Replicas)
}
application.RunningPodsCount = int(statefulSet.Status.ReadyReplicas)
application.DeploymentType = "Replicated"
application.Metadata = &models.Metadata{
Labels: statefulSet.Labels,
}
// If the statefulSet has containers, use the first container's image
if len(statefulSet.Spec.Template.Spec.Containers) > 0 {
application.Image = statefulSet.Spec.Template.Spec.Containers[0].Image
}
}
func populateApplicationFromDaemonSet(application *models.K8sApplication, daemonSet appsv1.DaemonSet) {
application.Kind = "DaemonSet"
application.ApplicationType = "DaemonSet"
application.ID = string(daemonSet.UID)
application.ResourcePool = daemonSet.Namespace
application.Name = daemonSet.Name
application.ApplicationOwner = daemonSet.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = daemonSet.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = daemonSet.Labels["io.portainer.kubernetes.application.stack"]
application.Labels = daemonSet.Labels
application.MatchLabels = daemonSet.Spec.Selector.MatchLabels
application.CreationDate = daemonSet.CreationTimestamp.Time
application.TotalPodsCount = int(daemonSet.Status.DesiredNumberScheduled)
application.RunningPodsCount = int(daemonSet.Status.NumberReady)
application.DeploymentType = "Global"
application.Metadata = &models.Metadata{
Labels: daemonSet.Labels,
}
if len(daemonSet.Spec.Template.Spec.Containers) > 0 {
application.Image = daemonSet.Spec.Template.Spec.Containers[0].Image
}
}
func populateApplicationFromPod(application *models.K8sApplication, pod corev1.Pod) {
runningPodsCount := 1
if pod.Status.Phase != corev1.PodRunning {
runningPodsCount = 0
}
application.ApplicationType = "Pod"
application.Kind = "Pod"
application.ID = string(pod.UID)
application.ResourcePool = pod.Namespace
application.Name = pod.Name
application.ApplicationOwner = pod.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = pod.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = pod.Labels["io.portainer.kubernetes.application.stack"]
application.Labels = pod.Labels
application.MatchLabels = pod.Labels
application.CreationDate = pod.CreationTimestamp.Time
application.TotalPodsCount = 1
application.RunningPodsCount = runningPodsCount
application.DeploymentType = string(pod.Status.Phase)
application.Metadata = &models.Metadata{
Labels: pod.Labels,
}
// If the pod has containers, use the first container's image
if len(pod.Spec.Containers) > 0 {
application.Image = pod.Spec.Containers[0].Image
}
}
// updateApplicationWithService updates the application with the services that match the application's selector match labels
// and are in the same namespace as the application
func updateApplicationWithService(application *models.K8sApplication, services []corev1.Service) {
for _, service := range services {
serviceSelector := labels.SelectorFromSet(service.Spec.Selector)
if service.Namespace == application.ResourcePool && !serviceSelector.Empty() && serviceSelector.Matches(labels.Set(application.MatchLabels)) {
application.ServiceType = string(service.Spec.Type)
application.Services = append(application.Services, service)
}
}
}
func updateApplicationWithHorizontalPodAutoscaler(application *models.K8sApplication, hpas []autoscalingv2.HorizontalPodAutoscaler) {
for _, hpa := range hpas {
// Check if HPA is in the same namespace as the application
if hpa.Namespace != application.ResourcePool {
continue
}
// Check if the scale target ref matches the application
scaleTargetRef := hpa.Spec.ScaleTargetRef
if scaleTargetRef.Name == application.Name && scaleTargetRef.Kind == application.Kind {
hpaCopy := hpa // Create a local copy
application.HorizontalPodAutoscaler = &hpaCopy
break
}
}
}
// calculatePodResourceUsage calculates the resource usage for a pod in CPU cores and Bytes
func calculatePodResourceUsage(pod corev1.Pod) models.K8sApplicationResource {
resource := models.K8sApplicationResource{}
for _, container := range pod.Spec.Containers {
// CPU cores as a decimal
resource.CPURequest += float64(container.Resources.Requests.Cpu().MilliValue()) / 1000
resource.CPULimit += float64(container.Resources.Limits.Cpu().MilliValue()) / 1000
// Bytes
resource.MemoryRequest += container.Resources.Requests.Memory().Value()
resource.MemoryLimit += container.Resources.Limits.Memory().Value()
}
return resource
}
// GetApplicationFromServiceSelector gets applications based on service selectors
// it matches the service selector with the pod labels
func (kcl *KubeClient) GetApplicationFromServiceSelector(pods []corev1.Pod, service models.K8sServiceInfo, replicaSets []appsv1.ReplicaSet) (*models.K8sApplication, error) {
servicesSelector := labels.SelectorFromSet(service.Selector)
if servicesSelector.Empty() {
return nil, nil
}
for _, pod := range pods {
if servicesSelector.Matches(labels.Set(pod.Labels)) {
if isReplicaSetOwner(pod) {
updateOwnerReferenceToDeployment(&pod, replicaSets)
}
return &models.K8sApplication{
Name: pod.OwnerReferences[0].Name,
Kind: pod.OwnerReferences[0].Kind,
}, nil
}
}
return nil, nil
}
// GetApplicationConfigurationOwnersFromConfigMap gets a list of applications that use a specific ConfigMap
// by checking all pods in the same namespace as the ConfigMap
func (kcl *KubeClient) GetApplicationConfigurationOwnersFromConfigMap(configMap models.K8sConfigMap, pods []corev1.Pod, replicaSets []appsv1.ReplicaSet) ([]models.K8sConfigurationOwnerResource, error) {
configurationOwners := []models.K8sConfigurationOwnerResource{}
for _, pod := range pods {
if pod.Namespace == configMap.Namespace {
if isPodUsingConfigMap(&pod, configMap.Name) {
application, err := kcl.ConvertPodToApplication(pod, PortainerApplicationResources{
ReplicaSets: replicaSets,
}, false)
if err != nil {
return nil, err
}
if application != nil {
configurationOwners = append(configurationOwners, models.K8sConfigurationOwnerResource{
Name: application.Name,
ResourceKind: application.Kind,
Id: application.UID,
})
}
}
}
}
return configurationOwners, nil
}
// GetApplicationConfigurationOwnersFromSecret gets a list of applications that use a specific Secret
// by checking all pods in the same namespace as the Secret
func (kcl *KubeClient) GetApplicationConfigurationOwnersFromSecret(secret models.K8sSecret, pods []corev1.Pod, replicaSets []appsv1.ReplicaSet) ([]models.K8sConfigurationOwnerResource, error) {
configurationOwners := []models.K8sConfigurationOwnerResource{}
for _, pod := range pods {
if pod.Namespace == secret.Namespace {
if isPodUsingSecret(&pod, secret.Name) {
application, err := kcl.ConvertPodToApplication(pod, PortainerApplicationResources{
ReplicaSets: replicaSets,
}, false)
if err != nil {
return nil, err
}
if application != nil {
configurationOwners = append(configurationOwners, models.K8sConfigurationOwnerResource{
Name: application.Name,
ResourceKind: application.Kind,
Id: application.UID,
})
}
}
}
}
return configurationOwners, nil
}
// fetchUnhealthyApplications fetches applications that failed to schedule any pods
// due to issues like missing resource limits or other scheduling constraints
func fetchUnhealthyApplications(resources PortainerApplicationResources) ([]models.K8sApplication, error) {
var unhealthyApplications []models.K8sApplication
// Process Deployments
for _, deployment := range resources.Deployments {
if hasNoScheduledPods(deployment) {
app := createApplicationFromDeployment(deployment)
addRelatedResourcesToApplication(&app, resources)
unhealthyApplications = append(unhealthyApplications, app)
}
}
// Process StatefulSets
for _, statefulSet := range resources.StatefulSets {
if hasNoScheduledPods(statefulSet) {
app := createApplicationFromStatefulSet(statefulSet)
addRelatedResourcesToApplication(&app, resources)
unhealthyApplications = append(unhealthyApplications, app)
}
}
// Process DaemonSets
for _, daemonSet := range resources.DaemonSets {
if hasNoScheduledPods(daemonSet) {
app := createApplicationFromDaemonSet(daemonSet)
addRelatedResourcesToApplication(&app, resources)
unhealthyApplications = append(unhealthyApplications, app)
}
}
return unhealthyApplications, nil
}
// addRelatedResourcesToApplication adds Services and HPA information to the application
func addRelatedResourcesToApplication(app *models.K8sApplication, resources PortainerApplicationResources) {
if app.ID == "" || app.Name == "" {
return
}
if len(resources.Services) > 0 {
updateApplicationWithService(app, resources.Services)
}
if len(resources.HorizontalPodAutoscalers) > 0 {
updateApplicationWithHorizontalPodAutoscaler(app, resources.HorizontalPodAutoscalers)
}
}
// hasNoScheduledPods checks if a workload has completely failed to schedule any pods
// it checks for no replicas desired, i.e. nothing to schedule and see if any pods are running
// if any pods exist at all (even if not ready), it returns false
func hasNoScheduledPods(obj interface{}) bool {
switch resource := obj.(type) {
case appsv1.Deployment:
if resource.Status.Replicas > 0 {
return false
}
return resource.Status.ReadyReplicas == 0 && resource.Status.AvailableReplicas == 0
case appsv1.StatefulSet:
if resource.Status.Replicas > 0 {
return false
}
return resource.Status.ReadyReplicas == 0 && resource.Status.CurrentReplicas == 0
case appsv1.DaemonSet:
if resource.Status.CurrentNumberScheduled > 0 || resource.Status.NumberMisscheduled > 0 {
return false
}
return resource.Status.NumberReady == 0 && resource.Status.DesiredNumberScheduled > 0
default:
return false
}
}