mirror of https://github.com/portainer/portainer
feat(support): collect system info bundle to assist support troubleshooting [r8s-157] (#154)
Malcolm Lockyer
GitHub
parent
17648d12fe
commit
783ab253af
File diff suppressed because it is too large
Load Diff
|
|
@ -672,6 +672,7 @@
|
|||
{
|
||||
"Docker": {
|
||||
"ContainerCount": 0,
|
||||
"DiagnosticsData": {},
|
||||
"DockerSnapshotRaw": {
|
||||
"Containers": null,
|
||||
"Images": null,
|
||||
|
|
|
|||
|
|
@ -1,20 +1,9 @@
|
|||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
dockerclient "github.com/portainer/portainer/api/docker/client"
|
||||
"github.com/portainer/portainer/api/docker/consts"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
_container "github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/volume"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/portainer/portainer/pkg/snapshot"
|
||||
)
|
||||
|
||||
// Snapshotter represents a service used to create environment(endpoint) snapshots
|
||||
|
|
@ -37,247 +26,5 @@ func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*p
|
|||
}
|
||||
defer cli.Close()
|
||||
|
||||
return snapshot(cli, endpoint)
|
||||
}
|
||||
|
||||
func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) {
|
||||
if _, err := cli.Ping(context.Background()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
snapshot := &portainer.DockerSnapshot{
|
||||
StackCount: 0,
|
||||
}
|
||||
|
||||
if err := snapshotInfo(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine information")
|
||||
}
|
||||
|
||||
if snapshot.Swarm {
|
||||
if err := snapshotSwarmServices(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm services")
|
||||
}
|
||||
|
||||
if err := snapshotNodes(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm nodes")
|
||||
}
|
||||
}
|
||||
|
||||
if err := snapshotContainers(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot containers")
|
||||
}
|
||||
|
||||
if err := snapshotImages(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot images")
|
||||
}
|
||||
|
||||
if err := snapshotVolumes(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot volumes")
|
||||
}
|
||||
|
||||
if err := snapshotNetworks(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot networks")
|
||||
}
|
||||
|
||||
if err := snapshotVersion(snapshot, cli); err != nil {
|
||||
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine version")
|
||||
}
|
||||
|
||||
snapshot.Time = time.Now().Unix()
|
||||
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func snapshotInfo(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
info, err := cli.Info(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.Swarm = info.Swarm.ControlAvailable
|
||||
snapshot.DockerVersion = info.ServerVersion
|
||||
snapshot.TotalCPU = info.NCPU
|
||||
snapshot.TotalMemory = info.MemTotal
|
||||
snapshot.SnapshotRaw.Info = info
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var nanoCpus int64
|
||||
var totalMem int64
|
||||
|
||||
for _, node := range nodes {
|
||||
nanoCpus += node.Description.Resources.NanoCPUs
|
||||
totalMem += node.Description.Resources.MemoryBytes
|
||||
}
|
||||
|
||||
snapshot.TotalCPU = int(nanoCpus / 1e9)
|
||||
snapshot.TotalMemory = totalMem
|
||||
snapshot.NodeCount = len(nodes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
stacks := make(map[string]struct{})
|
||||
|
||||
services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, service := range services {
|
||||
for k, v := range service.Spec.Labels {
|
||||
if k == "com.docker.stack.namespace" {
|
||||
stacks[v] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
snapshot.ServiceCount = len(services)
|
||||
snapshot.StackCount += len(stacks)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stacks := make(map[string]struct{})
|
||||
gpuUseSet := make(map[string]struct{})
|
||||
gpuUseAll := false
|
||||
|
||||
for _, container := range containers {
|
||||
if container.State == "running" {
|
||||
// Snapshot GPUs
|
||||
response, err := cli.ContainerInspect(context.Background(), container.ID)
|
||||
if err != nil {
|
||||
// Inspect a container will fail when the container runs on a different
|
||||
// Swarm node, so it is better to log the error instead of return error
|
||||
// when the Swarm mode is enabled
|
||||
if !snapshot.Swarm {
|
||||
return err
|
||||
} else {
|
||||
if !strings.Contains(err.Error(), "No such container") {
|
||||
return err
|
||||
}
|
||||
// It is common to have containers running on different Swarm nodes,
|
||||
// so we just log the error in the debug level
|
||||
log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
|
||||
}
|
||||
} else {
|
||||
var gpuOptions *_container.DeviceRequest = nil
|
||||
for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests {
|
||||
if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" {
|
||||
gpuOptions = &deviceRequest
|
||||
}
|
||||
}
|
||||
|
||||
if gpuOptions != nil {
|
||||
if gpuOptions.Count == -1 {
|
||||
gpuUseAll = true
|
||||
}
|
||||
|
||||
for _, id := range gpuOptions.DeviceIDs {
|
||||
gpuUseSet[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range container.Labels {
|
||||
if k == consts.ComposeStackNameLabel {
|
||||
stacks[v] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gpuUseList := make([]string, 0, len(gpuUseSet))
|
||||
for gpuUse := range gpuUseSet {
|
||||
gpuUseList = append(gpuUseList, gpuUse)
|
||||
}
|
||||
|
||||
snapshot.GpuUseAll = gpuUseAll
|
||||
snapshot.GpuUseList = gpuUseList
|
||||
|
||||
stats := CalculateContainerStats(containers)
|
||||
|
||||
snapshot.ContainerCount = stats.Total
|
||||
snapshot.RunningContainerCount = stats.Running
|
||||
snapshot.StoppedContainerCount = stats.Stopped
|
||||
snapshot.HealthyContainerCount = stats.Healthy
|
||||
snapshot.UnhealthyContainerCount = stats.Unhealthy
|
||||
snapshot.StackCount += len(stacks)
|
||||
|
||||
for _, container := range containers {
|
||||
snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.ImageCount = len(images)
|
||||
snapshot.SnapshotRaw.Images = images
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.VolumeCount = len(volumes.Volumes)
|
||||
snapshot.SnapshotRaw.Volumes = volumes
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotNetworks(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.SnapshotRaw.Networks = networks
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotVersion(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
version, err := cli.ServerVersion(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.SnapshotRaw.Version = version
|
||||
snapshot.IsPodman = isPodman(version)
|
||||
return nil
|
||||
}
|
||||
|
||||
// isPodman checks if the version is for Podman by checking if any of the components contain "podman".
|
||||
// If it's podman, a component name should be "Podman Engine"
|
||||
func isPodman(version types.Version) bool {
|
||||
for _, component := range version.Components {
|
||||
if strings.Contains(strings.ToLower(component.Name), "podman") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return snapshot.CreateDockerSnapshot(cli)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +1,9 @@
|
|||
package kubernetes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/kubernetes/cli"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"github.com/portainer/portainer/pkg/snapshot"
|
||||
)
|
||||
|
||||
type Snapshotter struct {
|
||||
|
|
@ -30,55 +24,5 @@ func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*p
|
|||
return nil, err
|
||||
}
|
||||
|
||||
return snapshot(client, endpoint)
|
||||
}
|
||||
|
||||
func snapshot(cli *kubernetes.Clientset, endpoint *portainer.Endpoint) (*portainer.KubernetesSnapshot, error) {
|
||||
res := cli.RESTClient().Get().AbsPath("/healthz").Do(context.TODO())
|
||||
if res.Error() != nil {
|
||||
return nil, res.Error()
|
||||
}
|
||||
|
||||
snapshot := &portainer.KubernetesSnapshot{}
|
||||
|
||||
err := snapshotVersion(snapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Str("endpoint", endpoint.Name).Err(err).Msg("unable to snapshot cluster version")
|
||||
}
|
||||
|
||||
err = snapshotNodes(snapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Str("endpoint", endpoint.Name).Err(err).Msg("unable to snapshot cluster nodes")
|
||||
}
|
||||
|
||||
snapshot.Time = time.Now().Unix()
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func snapshotVersion(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
||||
versionInfo, err := cli.ServerVersion()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.KubernetesVersion = versionInfo.GitVersion
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotNodes(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
||||
nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var totalCPUs, totalMemory int64
|
||||
for _, node := range nodeList.Items {
|
||||
totalCPUs += node.Status.Capacity.Cpu().Value()
|
||||
totalMemory += node.Status.Capacity.Memory().Value()
|
||||
}
|
||||
|
||||
snapshot.TotalCPU = totalCPUs
|
||||
snapshot.TotalMemory = totalMemory
|
||||
snapshot.NodeCount = len(nodeList.Items)
|
||||
return nil
|
||||
return snapshot.CreateKubernetesSnapshot(client)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -185,6 +185,16 @@ type (
|
|||
// CustomTemplatePlatform represents a custom template platform
|
||||
CustomTemplatePlatform int
|
||||
|
||||
// DiagnosticsData represents the diagnostics data for an environment
|
||||
// this contains the logs, telnet, traceroute, dns and proxy information
|
||||
// which will be part of the DockerSnapshot and KubernetesSnapshot structs
|
||||
DiagnosticsData struct {
|
||||
Log string `json:"Log,omitempty"`
|
||||
Telnet map[string]string `json:"Telnet,omitempty"`
|
||||
DNS map[string]string `json:"DNS,omitempty"`
|
||||
Proxy map[string]string `json:"Proxy,omitempty"`
|
||||
}
|
||||
|
||||
// DockerHub represents all the required information to connect and use the
|
||||
// Docker Hub
|
||||
DockerHub struct {
|
||||
|
|
@ -217,6 +227,7 @@ type (
|
|||
GpuUseAll bool `json:"GpuUseAll"`
|
||||
GpuUseList []string `json:"GpuUseList"`
|
||||
IsPodman bool `json:"IsPodman"`
|
||||
DiagnosticsData *DiagnosticsData `json:"DiagnosticsData"`
|
||||
}
|
||||
|
||||
// DockerContainerSnapshot is an extent of Docker's Container struct
|
||||
|
|
@ -636,11 +647,12 @@ type (
|
|||
|
||||
// KubernetesSnapshot represents a snapshot of a specific Kubernetes environment(endpoint) at a specific time
|
||||
KubernetesSnapshot struct {
|
||||
Time int64 `json:"Time"`
|
||||
KubernetesVersion string `json:"KubernetesVersion"`
|
||||
NodeCount int `json:"NodeCount"`
|
||||
TotalCPU int64 `json:"TotalCPU"`
|
||||
TotalMemory int64 `json:"TotalMemory"`
|
||||
Time int64 `json:"Time"`
|
||||
KubernetesVersion string `json:"KubernetesVersion"`
|
||||
NodeCount int `json:"NodeCount"`
|
||||
TotalCPU int64 `json:"TotalCPU"`
|
||||
TotalMemory int64 `json:"TotalMemory"`
|
||||
DiagnosticsData *DiagnosticsData `json:"DiagnosticsData"`
|
||||
}
|
||||
|
||||
// KubernetesConfiguration represents the configuration of a Kubernetes environment(endpoint)
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -10,6 +10,7 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2 v1.24.1
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.16
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.24.1
|
||||
github.com/aws/smithy-go v1.19.0
|
||||
github.com/cbroglie/mustache v1.4.0
|
||||
github.com/compose-spec/compose-go/v2 v2.0.2
|
||||
github.com/containers/image/v5 v5.30.1
|
||||
|
|
@ -41,7 +42,7 @@ require (
|
|||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/rs/zerolog v1.29.0
|
||||
github.com/segmentio/encoding v0.3.6
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/urfave/negroni v1.0.0
|
||||
github.com/viney-shih/go-lock v1.1.1
|
||||
go.etcd.io/bbolt v1.3.10
|
||||
|
|
@ -86,7 +87,6 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
|
||||
github.com/aws/smithy-go v1.19.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/buger/goterm v1.0.4 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -618,8 +618,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
|||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
|
||||
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
|
||||
github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
package edge
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// GetPortainerURLFromEdgeKey returns the portainer URL from an edge key
|
||||
// format: <portainer_instance_url>|<tunnel_server_addr>|<tunnel_server_fingerprint>|<endpoint_id>
|
||||
func GetPortainerURLFromEdgeKey(edgeKey string) (string, error) {
|
||||
decodedKey, err := base64.RawStdEncoding.DecodeString(edgeKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
keyInfo := strings.Split(string(decodedKey), "|")
|
||||
|
||||
if len(keyInfo) != 4 {
|
||||
return "", errors.New("invalid key format")
|
||||
}
|
||||
|
||||
_, err = strconv.Atoi(keyInfo[3])
|
||||
if err != nil {
|
||||
return "", errors.New("invalid key format")
|
||||
}
|
||||
|
||||
return keyInfo[0], nil
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package edge
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetPortainerURLFromEdgeKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
edgeKey string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "ValidEdgeKey",
|
||||
edgeKey: "aHR0cHM6Ly9wb3J0YWluZXIuaW98cG9ydGFpbmVyLmlvOjgwMDB8YXNkZnwx",
|
||||
expected: "https://portainer.io",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := GetPortainerURLFromEdgeKey(tt.edgeKey)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,14 @@
|
|||
package endpoints
|
||||
|
||||
import portainer "github.com/portainer/portainer/api"
|
||||
import (
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
)
|
||||
|
||||
// IsRegularAgentEndpoint returns true if this is a regular agent endpoint
|
||||
func IsRegularAgentEndpoint(endpoint *portainer.Endpoint) bool {
|
||||
return endpoint.Type == portainer.AgentOnDockerEnvironment ||
|
||||
endpoint.Type == portainer.AgentOnKubernetesEnvironment
|
||||
}
|
||||
|
||||
// IsEdgeEndpoint returns true if this is an Edge endpoint
|
||||
func IsEdgeEndpoint(endpoint *portainer.Endpoint) bool {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,50 @@ import (
|
|||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsRegularAgentEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint *portainer.Endpoint
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "AgentOnDockerEnvironment",
|
||||
endpoint: &portainer.Endpoint{
|
||||
Type: portainer.AgentOnDockerEnvironment,
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "AgentOnKubernetesEnvironment",
|
||||
endpoint: &portainer.Endpoint{
|
||||
Type: portainer.AgentOnKubernetesEnvironment,
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "EdgeAgentOnDockerEnvironment",
|
||||
endpoint: &portainer.Endpoint{
|
||||
Type: portainer.EdgeAgentOnDockerEnvironment,
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "EdgeAgentOnKubernetesEnvironment",
|
||||
endpoint: &portainer.Endpoint{
|
||||
Type: portainer.EdgeAgentOnKubernetesEnvironment,
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := IsRegularAgentEndpoint(tt.endpoint)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEdgeEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
package networking
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/encoding/json"
|
||||
)
|
||||
|
||||
// ProbeDNSConnection probes a DNS connection and returns a JSON string with the DNS lookup status and IP addresses.
|
||||
// ignores errors for the dns lookup since we want to know if the host is reachable
|
||||
func ProbeDNSConnection(url string) string {
|
||||
_, host, _ := parseURL(url)
|
||||
result := map[string]interface{}{
|
||||
"operation": "dns lookup",
|
||||
"remote_address": host,
|
||||
"connected_at": time.Now().Format(time.RFC3339),
|
||||
"status": "dns lookup successful",
|
||||
"resolved_ips": []net.IP{},
|
||||
}
|
||||
|
||||
ipAddresses, err := net.LookupIP(host)
|
||||
if err != nil {
|
||||
result["status"] = fmt.Sprintf("dns lookup failed: %s", err)
|
||||
} else {
|
||||
result["resolved_ips"] = ipAddresses
|
||||
}
|
||||
|
||||
jsonData, _ := json.Marshal(result)
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
// ProbeTelnetConnection probes a telnet connection and returns a JSON string with the telnet connection status, local and remote addresses.
|
||||
// ignores errors for the telnet connection since we want to know if the host is reachable
|
||||
func ProbeTelnetConnection(url string) string {
|
||||
network, host, port := parseURL(url)
|
||||
if network == "https" || network == "http" {
|
||||
network = "tcp"
|
||||
}
|
||||
|
||||
address := fmt.Sprintf("%s:%s", host, port)
|
||||
result := map[string]string{
|
||||
"operation": "telnet connection",
|
||||
"local_address": "unknown",
|
||||
"remote_address": "unknown",
|
||||
"network": network,
|
||||
"status": "connected to " + address,
|
||||
"connected_at": time.Now().Format(time.RFC3339),
|
||||
}
|
||||
|
||||
connection, err := net.DialTimeout(network, address, 5*time.Second)
|
||||
if err != nil {
|
||||
result["status"] = fmt.Sprintf("failed to connect to %s: %s", address, err)
|
||||
} else {
|
||||
defer connection.Close()
|
||||
result["local_address"] = connection.LocalAddr().String()
|
||||
result["remote_address"] = connection.RemoteAddr().String()
|
||||
}
|
||||
|
||||
jsonData, _ := json.Marshal(result)
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
// DetectProxy probes a target URL and returns a JSON string with the proxy detection status, local and remote addresses.
|
||||
// ignores errors for the http request since we want to know if the host is reachable
|
||||
func DetectProxy(url string) string {
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
result := map[string]string{
|
||||
"operation": "proxy detection",
|
||||
"local_address": "unknown",
|
||||
"remote_address": "unknown",
|
||||
"network": "https",
|
||||
"status": "no proxy detected",
|
||||
"connected_at": time.Now().Format(time.RFC3339),
|
||||
}
|
||||
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
result["status"] = fmt.Sprintf("failed to make request: %s", err)
|
||||
} else {
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.Request != nil {
|
||||
result["local_address"] = resp.Request.Host
|
||||
result["remote_address"] = resp.Request.RemoteAddr
|
||||
}
|
||||
|
||||
if resp.Header.Get("Via") != "" || resp.Header.Get("X-Forwarded-For") != "" || resp.Header.Get("Proxy-Connection") != "" {
|
||||
result["status"] = "proxy detected via headers"
|
||||
} else if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
|
||||
cert := resp.TLS.PeerCertificates[0]
|
||||
if cert.IsCA || strings.Contains(strings.ToLower(cert.Issuer.CommonName), "proxy") {
|
||||
result["status"] = "proxy detected via certificate"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
jsonData, _ := json.Marshal(result)
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
// parseURL parses a raw URL and returns the network, host and port
|
||||
// it also ensures the network is tcp and the port is set to the default for the network
|
||||
func parseURL(rawURL string) (network, host, port string) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
network = u.Scheme
|
||||
host = u.Hostname()
|
||||
port = u.Port()
|
||||
|
||||
if port == "" {
|
||||
if network == "https" {
|
||||
port = "443"
|
||||
} else if network == "http" {
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
|
||||
return network, host, port
|
||||
}
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
package networking
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Response structs for each function
|
||||
type dnsResponse struct {
|
||||
Operation string `json:"operation"`
|
||||
ResolvedIPs []string `json:"resolved_ips"`
|
||||
RemoteAddr string `json:"remote_address"`
|
||||
ConnectedAt string `json:"connected_at"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
type telnetResponse struct {
|
||||
Operation string `json:"operation"`
|
||||
LocalAddr string `json:"local_address"`
|
||||
RemoteAddr string `json:"remote_address"`
|
||||
Network string `json:"network"`
|
||||
Status string `json:"status"`
|
||||
ConnectedAt string `json:"connected_at"`
|
||||
}
|
||||
|
||||
type proxyResponse struct {
|
||||
Operation string `json:"operation"`
|
||||
LocalAddr string `json:"local_address"`
|
||||
RemoteAddr string `json:"remote_address"`
|
||||
Network string `json:"network"`
|
||||
Status string `json:"status"`
|
||||
ConnectedAt string `json:"connected_at"`
|
||||
}
|
||||
|
||||
func TestProbeDNSConnection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
host string
|
||||
wantSuccess bool
|
||||
statusContains string
|
||||
}{
|
||||
{
|
||||
name: "Valid domain",
|
||||
host: "https://api.portainer.io",
|
||||
wantSuccess: true,
|
||||
statusContains: "dns lookup successful",
|
||||
},
|
||||
{
|
||||
name: "Invalid domain",
|
||||
host: "https://nonexistent.domain.invalid",
|
||||
wantSuccess: false,
|
||||
statusContains: "dns lookup failed",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
response := ProbeDNSConnection(tt.host)
|
||||
|
||||
var result dnsResponse
|
||||
if err := json.Unmarshal([]byte(response), &result); err != nil {
|
||||
t.Fatalf("Invalid JSON response: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(result.Status, tt.statusContains) {
|
||||
t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeTelnetConnection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
wantSuccess bool
|
||||
statusContains string
|
||||
}{
|
||||
{
|
||||
name: "Valid connection",
|
||||
url: "https://api.portainer.io",
|
||||
wantSuccess: true,
|
||||
statusContains: "connected to",
|
||||
},
|
||||
{
|
||||
name: "Invalid port",
|
||||
url: "https://api.portainer.io:99999",
|
||||
wantSuccess: false,
|
||||
statusContains: "failed to connect",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
response := ProbeTelnetConnection(tt.url)
|
||||
|
||||
var result telnetResponse
|
||||
if err := json.Unmarshal([]byte(response), &result); err != nil {
|
||||
t.Fatalf("Invalid JSON response: %v", err)
|
||||
}
|
||||
|
||||
validateCommonFields(t, result.Operation, "telnet connection", result.ConnectedAt)
|
||||
if !strings.Contains(result.Status, tt.statusContains) {
|
||||
t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectProxy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
wantSuccess bool
|
||||
statusContains string
|
||||
}{
|
||||
{
|
||||
name: "Valid URL",
|
||||
url: "https://api.portainer.io",
|
||||
wantSuccess: true,
|
||||
statusContains: "proxy",
|
||||
},
|
||||
{
|
||||
name: "Invalid URL",
|
||||
url: "https://nonexistent.domain.invalid",
|
||||
wantSuccess: false,
|
||||
statusContains: "failed to make request",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
response := DetectProxy(tt.url)
|
||||
|
||||
var result proxyResponse
|
||||
if err := json.Unmarshal([]byte(response), &result); err != nil {
|
||||
t.Fatalf("Invalid JSON response: %v", err)
|
||||
}
|
||||
|
||||
validateCommonFields(t, result.Operation, "proxy detection", result.ConnectedAt)
|
||||
if result.Network != "https" {
|
||||
t.Errorf("Expected network https, got %s", result.Network)
|
||||
}
|
||||
if !strings.Contains(result.Status, tt.statusContains) {
|
||||
t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to validate common fields across all responses
|
||||
func validateCommonFields(t *testing.T, operation, expectedOperation, connectedAt string) {
|
||||
t.Helper()
|
||||
|
||||
if operation != expectedOperation {
|
||||
t.Errorf("Expected operation '%s', got '%s'", expectedOperation, operation)
|
||||
}
|
||||
|
||||
if _, err := time.Parse(time.RFC3339, connectedAt); err != nil {
|
||||
t.Errorf("Invalid connected_at timestamp: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,372 @@
|
|||
package snapshot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/encoding/json"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
_container "github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/image"
|
||||
"github.com/docker/docker/api/types/volume"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/docker/docker/pkg/stdcopy"
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/docker/consts"
|
||||
edgeutils "github.com/portainer/portainer/pkg/edge"
|
||||
networkingutils "github.com/portainer/portainer/pkg/networking"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func CreateDockerSnapshot(cli *client.Client) (*portainer.DockerSnapshot, error) {
|
||||
if _, err := cli.Ping(context.Background()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dockerSnapshot := &portainer.DockerSnapshot{
|
||||
StackCount: 0,
|
||||
}
|
||||
|
||||
err := dockerSnapshotInfo(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot engine information")
|
||||
}
|
||||
|
||||
if dockerSnapshot.Swarm {
|
||||
err = dockerSnapshotSwarmServices(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot Swarm services")
|
||||
}
|
||||
|
||||
err = dockerSnapshotNodes(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot Swarm nodes")
|
||||
}
|
||||
}
|
||||
|
||||
err = dockerSnapshotContainers(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot containers")
|
||||
}
|
||||
|
||||
err = dockerSnapshotImages(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot images")
|
||||
}
|
||||
|
||||
err = dockerSnapshotVolumes(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot volumes")
|
||||
}
|
||||
|
||||
err = dockerSnapshotNetworks(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot networks")
|
||||
}
|
||||
|
||||
err = dockerSnapshotVersion(dockerSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot engine version")
|
||||
}
|
||||
|
||||
dockerSnapshot.Time = time.Now().Unix()
|
||||
|
||||
return dockerSnapshot, nil
|
||||
}
|
||||
|
||||
func dockerSnapshotInfo(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
info, err := cli.Info(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.Swarm = info.Swarm.ControlAvailable
|
||||
snapshot.DockerVersion = info.ServerVersion
|
||||
snapshot.TotalCPU = info.NCPU
|
||||
snapshot.TotalMemory = info.MemTotal
|
||||
snapshot.SnapshotRaw.Info = info
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dockerSnapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var nanoCpus int64
|
||||
var totalMem int64
|
||||
|
||||
for _, node := range nodes {
|
||||
nanoCpus += node.Description.Resources.NanoCPUs
|
||||
totalMem += node.Description.Resources.MemoryBytes
|
||||
}
|
||||
|
||||
snapshot.TotalCPU = int(nanoCpus / 1e9)
|
||||
snapshot.TotalMemory = totalMem
|
||||
snapshot.NodeCount = len(nodes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dockerSnapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
stacks := make(map[string]struct{})
|
||||
|
||||
services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, service := range services {
|
||||
for k, v := range service.Spec.Labels {
|
||||
if k == "com.docker.stack.namespace" {
|
||||
stacks[v] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
snapshot.ServiceCount = len(services)
|
||||
snapshot.StackCount += len(stacks)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dockerSnapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stacks := make(map[string]struct{})
|
||||
gpuUseSet := make(map[string]struct{})
|
||||
gpuUseAll := false
|
||||
|
||||
for _, container := range containers {
|
||||
if container.State == "running" {
|
||||
// Snapshot GPUs
|
||||
response, err := cli.ContainerInspect(context.Background(), container.ID)
|
||||
if err != nil {
|
||||
// Inspect a container will fail when the container runs on a different
|
||||
// Swarm node, so it is better to log the error instead of return error
|
||||
// when the Swarm mode is enabled
|
||||
if !snapshot.Swarm {
|
||||
return err
|
||||
} else {
|
||||
if !strings.Contains(err.Error(), "No such container") {
|
||||
return err
|
||||
}
|
||||
// It is common to have containers running on different Swarm nodes,
|
||||
// so we just log the error in the debug level
|
||||
log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes")
|
||||
}
|
||||
} else {
|
||||
var gpuOptions *_container.DeviceRequest = nil
|
||||
for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests {
|
||||
if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" {
|
||||
gpuOptions = &deviceRequest
|
||||
}
|
||||
}
|
||||
|
||||
if gpuOptions != nil {
|
||||
if gpuOptions.Count == -1 {
|
||||
gpuUseAll = true
|
||||
}
|
||||
|
||||
for _, id := range gpuOptions.DeviceIDs {
|
||||
gpuUseSet[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range container.Labels {
|
||||
if k == consts.ComposeStackNameLabel {
|
||||
stacks[v] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gpuUseList := make([]string, 0, len(gpuUseSet))
|
||||
for gpuUse := range gpuUseSet {
|
||||
gpuUseList = append(gpuUseList, gpuUse)
|
||||
}
|
||||
|
||||
snapshot.GpuUseAll = gpuUseAll
|
||||
snapshot.GpuUseList = gpuUseList
|
||||
|
||||
stats := calculateContainerStats(containers)
|
||||
|
||||
snapshot.ContainerCount = stats.Total
|
||||
snapshot.RunningContainerCount = stats.Running
|
||||
snapshot.StoppedContainerCount = stats.Stopped
|
||||
snapshot.HealthyContainerCount = stats.Healthy
|
||||
snapshot.UnhealthyContainerCount = stats.Unhealthy
|
||||
snapshot.StackCount += len(stacks)
|
||||
|
||||
for _, container := range containers {
|
||||
snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dockerSnapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
images, err := cli.ImageList(context.Background(), image.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.ImageCount = len(images)
|
||||
snapshot.SnapshotRaw.Images = images
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dockerSnapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.VolumeCount = len(volumes.Volumes)
|
||||
snapshot.SnapshotRaw.Volumes = volumes
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dockerSnapshotNetworks(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.SnapshotRaw.Networks = networks
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dockerSnapshotVersion(snapshot *portainer.DockerSnapshot, cli *client.Client) error {
|
||||
version, err := cli.ServerVersion(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.SnapshotRaw.Version = version
|
||||
snapshot.IsPodman = isPodman(version)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DockerSnapshotDiagnostics returns the diagnostics data for the agent
|
||||
func DockerSnapshotDiagnostics(cli *client.Client, edgeKey string) (*portainer.DiagnosticsData, error) {
|
||||
containerID := os.Getenv("HOSTNAME")
|
||||
snapshot := &portainer.DockerSnapshot{
|
||||
DiagnosticsData: &portainer.DiagnosticsData{
|
||||
DNS: make(map[string]string),
|
||||
Telnet: make(map[string]string),
|
||||
},
|
||||
}
|
||||
|
||||
err := dockerSnapshotContainerErrorLogs(snapshot, cli, containerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if edgeKey != "" {
|
||||
url, err := edgeutils.GetPortainerURLFromEdgeKey(edgeKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get portainer URL from edge key: %w", err)
|
||||
}
|
||||
|
||||
snapshot.DiagnosticsData.DNS["edge-to-portainer"] = networkingutils.ProbeDNSConnection(url)
|
||||
snapshot.DiagnosticsData.Telnet["edge-to-portainer"] = networkingutils.ProbeTelnetConnection(url)
|
||||
}
|
||||
|
||||
return snapshot.DiagnosticsData, nil
|
||||
}
|
||||
|
||||
// DockerSnapshotContainerErrorLogs returns the 5 most recent error logs of the agent container
|
||||
// this will primarily be used for agent snapshot
|
||||
func dockerSnapshotContainerErrorLogs(snapshot *portainer.DockerSnapshot, cli *client.Client, containerId string) error {
|
||||
if containerId == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
rd, err := cli.ContainerLogs(context.Background(), containerId, container.LogsOptions{
|
||||
ShowStdout: false,
|
||||
ShowStderr: true,
|
||||
Tail: "5",
|
||||
Timestamps: true,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get container logs: %w", err)
|
||||
}
|
||||
defer rd.Close()
|
||||
|
||||
var stdOut, stdErr bytes.Buffer
|
||||
_, err = stdcopy.StdCopy(&stdErr, &stdOut, rd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy error logs: %w", err)
|
||||
}
|
||||
|
||||
var logs []map[string]string
|
||||
jsonLogs, err := json.Marshal(logs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal logs to JSON: %w", err)
|
||||
}
|
||||
|
||||
snapshot.DiagnosticsData.Log = string(jsonLogs)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isPodman checks if the version is for Podman by checking if any of the components contain "podman".
|
||||
// If it's podman, a component name should be "Podman Engine"
|
||||
func isPodman(version types.Version) bool {
|
||||
for _, component := range version.Components {
|
||||
if strings.Contains(strings.ToLower(component.Name), "podman") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type ContainerStats struct {
|
||||
Running int
|
||||
Stopped int
|
||||
Healthy int
|
||||
Unhealthy int
|
||||
Total int
|
||||
}
|
||||
|
||||
func calculateContainerStats(containers []types.Container) ContainerStats {
|
||||
var running, stopped, healthy, unhealthy int
|
||||
for _, container := range containers {
|
||||
switch container.State {
|
||||
case "running":
|
||||
running++
|
||||
case "healthy":
|
||||
running++
|
||||
healthy++
|
||||
case "unhealthy":
|
||||
running++
|
||||
unhealthy++
|
||||
case "exited", "stopped":
|
||||
stopped++
|
||||
}
|
||||
}
|
||||
|
||||
return ContainerStats{
|
||||
Running: running,
|
||||
Stopped: stopped,
|
||||
Healthy: healthy,
|
||||
Unhealthy: unhealthy,
|
||||
Total: len(containers),
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
package snapshot
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
package snapshot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/encoding/json"
|
||||
|
||||
"github.com/aws/smithy-go/ptr"
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
edgeutils "github.com/portainer/portainer/pkg/edge"
|
||||
networkingutils "github.com/portainer/portainer/pkg/networking"
|
||||
"github.com/rs/zerolog/log"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
func CreateKubernetesSnapshot(cli *kubernetes.Clientset) (*portainer.KubernetesSnapshot, error) {
|
||||
kubernetesSnapshot := &portainer.KubernetesSnapshot{}
|
||||
|
||||
err := kubernetesSnapshotVersion(kubernetesSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot cluster version")
|
||||
}
|
||||
|
||||
err = kubernetesSnapshotNodes(kubernetesSnapshot, cli)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to snapshot cluster nodes")
|
||||
}
|
||||
|
||||
kubernetesSnapshot.Time = time.Now().Unix()
|
||||
return kubernetesSnapshot, nil
|
||||
}
|
||||
|
||||
func kubernetesSnapshotVersion(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
||||
versionInfo, err := cli.ServerVersion()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshot.KubernetesVersion = versionInfo.GitVersion
|
||||
return nil
|
||||
}
|
||||
|
||||
func kubernetesSnapshotNodes(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error {
|
||||
nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var totalCPUs, totalMemory int64
|
||||
for _, node := range nodeList.Items {
|
||||
totalCPUs += node.Status.Capacity.Cpu().Value()
|
||||
totalMemory += node.Status.Capacity.Memory().Value()
|
||||
}
|
||||
|
||||
snapshot.TotalCPU = totalCPUs
|
||||
snapshot.TotalMemory = totalMemory
|
||||
snapshot.NodeCount = len(nodeList.Items)
|
||||
return nil
|
||||
}
|
||||
|
||||
// KubernetesSnapshotDiagnostics returns the diagnostics data for the agent
|
||||
func KubernetesSnapshotDiagnostics(cli *kubernetes.Clientset, edgeKey string) (*portainer.DiagnosticsData, error) {
|
||||
podID := os.Getenv("HOSTNAME")
|
||||
snapshot := &portainer.KubernetesSnapshot{
|
||||
DiagnosticsData: &portainer.DiagnosticsData{
|
||||
DNS: make(map[string]string),
|
||||
Telnet: make(map[string]string),
|
||||
},
|
||||
}
|
||||
|
||||
err := kubernetesSnapshotPodErrorLogs(snapshot, cli, "portainer", podID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to snapshot pod error logs: %w", err)
|
||||
}
|
||||
|
||||
if edgeKey != "" {
|
||||
url, err := edgeutils.GetPortainerURLFromEdgeKey(edgeKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get portainer URL from edge key: %w", err)
|
||||
}
|
||||
|
||||
snapshot.DiagnosticsData.DNS["edge-to-portainer"] = networkingutils.ProbeDNSConnection(url)
|
||||
snapshot.DiagnosticsData.Telnet["edge-to-portainer"] = networkingutils.ProbeTelnetConnection(url)
|
||||
}
|
||||
|
||||
return snapshot.DiagnosticsData, nil
|
||||
}
|
||||
|
||||
// KubernetesSnapshotPodErrorLogs returns 0 to 10 lines of the most recent error logs of the agent container
|
||||
// this will primarily be used for agent snapshot
|
||||
func kubernetesSnapshotPodErrorLogs(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset, namespace, podID string) error {
|
||||
if namespace == "" || podID == "" {
|
||||
return errors.New("both namespace and podID are required to capture pod error logs in the snapshot")
|
||||
}
|
||||
|
||||
logsStream, err := cli.CoreV1().Pods(namespace).GetLogs(podID, &corev1.PodLogOptions{TailLines: ptr.Int64(10), Timestamps: true}).Stream(context.TODO())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to stream logs: %w", err)
|
||||
}
|
||||
defer logsStream.Close()
|
||||
|
||||
logBytes, err := io.ReadAll(logsStream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read error logs: %w", err)
|
||||
}
|
||||
|
||||
logs := filterLogsByPattern(logBytes, []string{"error", "err", "level=error", "exception", "fatal", "panic"})
|
||||
|
||||
jsonLogs, err := json.Marshal(logs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal logs: %w", err)
|
||||
}
|
||||
snapshot.DiagnosticsData.Log = string(jsonLogs)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// filterLogsByPattern filters the logs by the given patterns and returns a list of logs that match the patterns
|
||||
// the logs are returned as a list of maps with the keys "timestamp" and "message"
|
||||
func filterLogsByPattern(logBytes []byte, patterns []string) []map[string]string {
|
||||
logs := []map[string]string{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(logBytes)), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if parts := strings.SplitN(line, " ", 2); len(parts) == 2 {
|
||||
messageLower := strings.ToLower(parts[1])
|
||||
for _, pattern := range patterns {
|
||||
if strings.Contains(messageLower, pattern) {
|
||||
logs = append(logs, map[string]string{
|
||||
"timestamp": parts[0],
|
||||
"message": parts[1],
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return logs
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
package snapshot
|
||||
Loading…
Reference in New Issue