diff --git a/api/datastore/test_data/input_24.json b/api/datastore/test_data/input_24.json index a4f837477..e89d59d7c 100644 --- a/api/datastore/test_data/input_24.json +++ b/api/datastore/test_data/input_24.json @@ -43,6 +43,12 @@ "PublicURL": "", "Snapshots": [ { + "DiagnosticsData": { + "Log": "", + "DNS": {}, + "Proxy": {}, + "Telnet": {} + }, "DockerVersion": "20.10.13", "HealthyContainerCount": 0, "ImageCount": 9, @@ -69,7 +75,9 @@ "maintainer": "NGINX Docker Maintainers \u003cdocker-maint@nginx.com\u003e" }, "Mounts": [], - "Names": ["/nginx_redis-master_1"], + "Names": [ + "/nginx_redis-master_1" + ], "NetworkSettings": { "Networks": { "nginx_default": { @@ -125,7 +133,9 @@ "kompose.service.type": "LoadBalancer" }, "Mounts": [], - "Names": ["/redis_frontend_1"], + "Names": [ + "/redis_frontend_1" + ], "NetworkSettings": { "Networks": { "redis_default": { @@ -191,7 +201,9 @@ "Type": "volume" } ], - "Names": ["/redis_redis-master_1"], + "Names": [ + "/redis_redis-master_1" + ], "NetworkSettings": { "Networks": { "redis_default": { @@ -257,7 +269,9 @@ "Type": "volume" } ], - "Names": ["/redis_redis-slave_1"], + "Names": [ + "/redis_redis-slave_1" + ], "NetworkSettings": { "Networks": { "redis_default": { @@ -316,7 +330,9 @@ "Type": "volume" } ], - "Names": ["/httpd"], + "Names": [ + "/httpd" + ], "NetworkSettings": { "Networks": { "bridge": { @@ -361,8 +377,13 @@ "Id": "sha256:86a511f3915ac296298506d2caf827e9597483bf84115bbe4e1707829de8534a", "Labels": null, "ParentId": "sha256:d03d4e751a04005e130d20ca4f22b3074b1ffbfdcf6b59a5d85727e267d31d98", - "RepoDigests": ["prabhat/ubuntu@sha256:bfe1371854e7e0e28517a041bfda08ecb3b345738c62092bfdf04f0513c27219"], - "RepoTags": ["ubuntu-prabhat:latest", "prabhat/ubuntu:latest"], + "RepoDigests": [ + "prabhat/ubuntu@sha256:bfe1371854e7e0e28517a041bfda08ecb3b345738c62092bfdf04f0513c27219" + ], + "RepoTags": [ + "ubuntu-prabhat:latest", + "prabhat/ubuntu:latest" + ], "SharedSize": -1, "Size": 753642080, "VirtualSize": 753642080 @@ -373,8 +394,12 @@ "Id": "sha256:ff0fea8310f3957d9b1e6ba494f3e4b63cb348c76160c6c15578e65995ffaa87", "Labels": null, "ParentId": "", - "RepoDigests": ["ubuntu@sha256:bea6d19168bbfd6af8d77c2cc3c572114eb5d113e6f422573c93cb605a0e2ffb"], - "RepoTags": ["ubuntu:latest"], + "RepoDigests": [ + "ubuntu@sha256:bea6d19168bbfd6af8d77c2cc3c572114eb5d113e6f422573c93cb605a0e2ffb" + ], + "RepoTags": [ + "ubuntu:latest" + ], "SharedSize": -1, "Size": 72759731, "VirtualSize": 72759731 @@ -385,8 +410,12 @@ "Id": "sha256:6b8e87fff1072470bbfc957a735e7e46007177864a7f61bd9e0f5872d3d7b4a5", "Labels": null, "ParentId": "", - "RepoDigests": ["httpd@sha256:73496cbfc473872dd185154a3b96faa4407d773e893c6a7b9d8f977c331bc45d"], - "RepoTags": ["httpd:latest"], + "RepoDigests": [ + "httpd@sha256:73496cbfc473872dd185154a3b96faa4407d773e893c6a7b9d8f977c331bc45d" + ], + "RepoTags": [ + "httpd:latest" + ], "SharedSize": -1, "Size": 143974476, "VirtualSize": 143974476 @@ -397,7 +426,9 @@ "Id": "sha256:a4ca82e34b45e34c0a8bffe4e974983a528e8beca6030c1f9d17ac7f96c7847f", "Labels": null, "ParentId": "", - "RepoDigests": ["portainer/agent@sha256:ca1a51a745f2490cf5345883dd8c5f6a953a15251ab95af246ca9e2fb3436dde"], + "RepoDigests": [ + "portainer/agent@sha256:ca1a51a745f2490cf5345883dd8c5f6a953a15251ab95af246ca9e2fb3436dde" + ], "RepoTags": null, "SharedSize": -1, "Size": 154347153, @@ -415,7 +446,10 @@ "nginx@sha256:1c13bc6de5dfca749c377974146ac05256791ca2fe1979fc8e8278bf0121d285", "prabhat/nginx@sha256:2468d48e476b6a079eb646e87620f96ce1818ac0c5b3a8450532cea64b3421f4" ], - "RepoTags": ["nginx:latest", "prabhat/nginx:latest"], + "RepoTags": [ + "nginx:latest", + "prabhat/nginx:latest" + ], "SharedSize": -1, "Size": 141505630, "VirtualSize": 141505630 @@ -426,7 +460,9 @@ "Id": "sha256:6d1ef012b5674ad8a127ecfa9b5e6f5178d171b90ee462846974177fd9bdd39f", "Labels": null, "ParentId": "", - "RepoDigests": ["alpine@sha256:8421d9a84432575381bfabd248f1eb56f3aa21d9d7cd2511583c68c9b7511d10"], + "RepoDigests": [ + "alpine@sha256:8421d9a84432575381bfabd248f1eb56f3aa21d9d7cd2511583c68c9b7511d10" + ], "RepoTags": null, "SharedSize": -1, "Size": 4206494, @@ -438,8 +474,12 @@ "Id": "sha256:e2b3e8542af735080e6bda06873ce666e2319eea353884a88e45f3c9ef996846", "Labels": {}, "ParentId": "", - "RepoDigests": ["gcr.io/google-samples/gb-frontend@sha256:d44e7d7491a537f822e7fe8615437e4a8a08f3a7a1d7d4cb9066b92f7556ba6d"], - "RepoTags": ["gcr.io/google-samples/gb-frontend:v4"], + "RepoDigests": [ + "gcr.io/google-samples/gb-frontend@sha256:d44e7d7491a537f822e7fe8615437e4a8a08f3a7a1d7d4cb9066b92f7556ba6d" + ], + "RepoTags": [ + "gcr.io/google-samples/gb-frontend:v4" + ], "SharedSize": -1, "Size": 512161546, "VirtualSize": 512161546 @@ -450,8 +490,12 @@ "Id": "sha256:5f026ddffa27f011242781f7f2498538334e173869e7fe757008881fb48180b6", "Labels": null, "ParentId": "", - "RepoDigests": ["gcr.io/google_samples/gb-redisslave@sha256:90f62695e641e1a27d1a5e0bbb8b622205a48e18311b51b0da419ffad24b9016"], - "RepoTags": ["gcr.io/google_samples/gb-redisslave:v1"], + "RepoDigests": [ + "gcr.io/google_samples/gb-redisslave@sha256:90f62695e641e1a27d1a5e0bbb8b622205a48e18311b51b0da419ffad24b9016" + ], + "RepoTags": [ + "gcr.io/google_samples/gb-redisslave:v1" + ], "SharedSize": -1, "Size": 109508753, "VirtualSize": 109508753 @@ -462,8 +506,12 @@ "Id": "sha256:e5e67996c442f903cda78dd983ea6e94bb4e542950fd2eba666b44cbd303df42", "Labels": null, "ParentId": "", - "RepoDigests": ["k8s.gcr.io/redis@sha256:f066bcf26497fbc55b9bf0769cb13a35c0afa2aa42e737cc46b7fb04b23a2f25"], - "RepoTags": ["k8s.gcr.io/redis:e2e"], + "RepoDigests": [ + "k8s.gcr.io/redis@sha256:f066bcf26497fbc55b9bf0769cb13a35c0afa2aa42e737cc46b7fb04b23a2f25" + ], + "RepoTags": [ + "k8s.gcr.io/redis:e2e" + ], "SharedSize": -1, "Size": 419003740, "VirtualSize": 419003740 @@ -493,10 +541,22 @@ "DockerRootDir": "/var/lib/docker", "Driver": "overlay2", "DriverStatus": [ - ["Backing Filesystem", "extfs"], - ["Supports d_type", "true"], - ["Native Overlay Diff", "true"], - ["userxattr", "false"] + [ + "Backing Filesystem", + "extfs" + ], + [ + "Supports d_type", + "true" + ], + [ + "Native Overlay Diff", + "true" + ], + [ + "userxattr", + "false" + ] ], "ExperimentalBuild": false, "GenericResources": null, @@ -532,9 +592,29 @@ "PidsLimit": true, "Plugins": { "Authorization": null, - "Log": ["awslogs", "fluentd", "gcplogs", "gelf", "journald", "json-file", "local", "logentries", "splunk", "syslog"], - "Network": ["bridge", "host", "ipvlan", "macvlan", "null", "overlay"], - "Volume": ["local"] + "Log": [ + "awslogs", + "fluentd", + "gcplogs", + "gelf", + "journald", + "json-file", + "local", + "logentries", + "splunk", + "syslog" + ], + "Network": [ + "bridge", + "host", + "ipvlan", + "macvlan", + "null", + "overlay" + ], + "Volume": [ + "local" + ] }, "RegistryConfig": { "AllowNondistributableArtifactsCIDRs": [], @@ -547,7 +627,9 @@ "Secure": true } }, - "InsecureRegistryCIDRs": ["127.0.0.0/8"], + "InsecureRegistryCIDRs": [ + "127.0.0.0/8" + ], "Mirrors": [] }, "RuncCommit": { @@ -565,7 +647,10 @@ "path": "runc" } }, - "SecurityOptions": ["name=apparmor", "name=seccomp,profile=default"], + "SecurityOptions": [ + "name=apparmor", + "name=seccomp,profile=default" + ], "ServerVersion": "20.10.13", "SwapLimit": true, "Swarm": { @@ -1481,12 +1566,16 @@ { "Id": 1, "administrator_only": false, - "categories": ["docker"], + "categories": [ + "docker" + ], "description": "Docker image registry", "image": "registry:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/registry.png", "platform": "linux", - "ports": ["5000/tcp"], + "ports": [ + "5000/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1503,12 +1592,17 @@ { "Id": 2, "administrator_only": false, - "categories": ["webserver"], + "categories": [ + "webserver" + ], "description": "High performance web server", "image": "nginx:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/nginx.png", "platform": "linux", - "ports": ["80/tcp", "443/tcp"], + "ports": [ + "80/tcp", + "443/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1528,12 +1622,16 @@ { "Id": 3, "administrator_only": false, - "categories": ["webserver"], + "categories": [ + "webserver" + ], "description": "Open-source HTTP server", "image": "httpd:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/httpd.png", "platform": "linux", - "ports": ["80/tcp"], + "ports": [ + "80/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1550,12 +1648,18 @@ { "Id": 4, "administrator_only": false, - "categories": ["webserver"], + "categories": [ + "webserver" + ], "description": "HTTP/2 web server with automatic HTTPS", "image": "abiosoft/caddy:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/caddy.png", "platform": "linux", - "ports": ["80/tcp", "443/tcp", "2015/tcp"], + "ports": [ + "80/tcp", + "443/tcp", + "2015/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1572,7 +1676,9 @@ { "Id": 5, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "The most popular open-source database", "env": [ { @@ -1583,7 +1689,9 @@ "image": "mysql:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/mysql.png", "platform": "linux", - "ports": ["3306/tcp"], + "ports": [ + "3306/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1600,7 +1708,9 @@ { "Id": 6, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "Performance beyond MySQL", "env": [ { @@ -1611,7 +1721,9 @@ "image": "mariadb:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/mariadb.png", "platform": "linux", - "ports": ["3306/tcp"], + "ports": [ + "3306/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1628,7 +1740,9 @@ { "Id": 7, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "The most advanced open-source database", "env": [ { @@ -1643,7 +1757,9 @@ "image": "postgres:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/postgres.png", "platform": "linux", - "ports": ["5432/tcp"], + "ports": [ + "5432/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1660,12 +1776,16 @@ { "Id": 8, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "Open-source document-oriented database", "image": "mongo:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/mongo.png", "platform": "linux", - "ports": ["27017/tcp"], + "ports": [ + "27017/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1682,13 +1802,18 @@ { "Id": 9, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "command": "start --insecure", "description": "An open-source, survivable, strongly consistent, scale-out SQL database", "image": "cockroachdb/cockroach:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/cockroachdb.png", "platform": "linux", - "ports": ["26257/tcp", "8080/tcp"], + "ports": [ + "26257/tcp", + "8080/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1705,12 +1830,17 @@ { "Id": 10, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "An open-source distributed SQL database", "image": "crate:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/cratedb.png", "platform": "linux", - "ports": ["4200/tcp", "4300/tcp"], + "ports": [ + "4200/tcp", + "4300/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1727,12 +1857,17 @@ { "Id": 11, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "Open-source search and analytics engine", "image": "elasticsearch:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/elasticsearch.png", "platform": "linux", - "ports": ["9200/tcp", "9300/tcp"], + "ports": [ + "9200/tcp", + "9300/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1749,13 +1884,20 @@ { "Id": 12, "administrator_only": false, - "categories": ["development", "project-management"], + "categories": [ + "development", + "project-management" + ], "description": "Open-source end-to-end software development platform", "image": "gitlab/gitlab-ce:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/gitlab_ce.png", "note": "Default username is \u003cb\u003eroot\u003c/b\u003e. Check the \u003ca href=\"https://docs.gitlab.com/omnibus/docker/README.html#after-starting-a-container\" target=\"_blank\"\u003eGitlab documentation\u003c/a\u003e to get started.", "platform": "linux", - "ports": ["80/tcp", "443/tcp", "22/tcp"], + "ports": [ + "80/tcp", + "443/tcp", + "22/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1778,7 +1920,9 @@ { "Id": 13, "administrator_only": false, - "categories": ["storage"], + "categories": [ + "storage" + ], "command": "server /data", "description": "A distributed object storage server built for cloud applications and devops", "env": [ @@ -1794,7 +1938,9 @@ "image": "minio/minio:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/minio.png", "platform": "linux", - "ports": ["9000/tcp"], + "ports": [ + "9000/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1814,7 +1960,9 @@ { "Id": 14, "administrator_only": false, - "categories": ["storage"], + "categories": [ + "storage" + ], "description": "Standalone AWS S3 protocol server", "env": [ { @@ -1829,7 +1977,9 @@ "image": "scality/s3server", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/scality-s3.png", "platform": "linux", - "ports": ["8000/tcp"], + "ports": [ + "8000/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1849,7 +1999,9 @@ { "Id": 15, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "Microsoft SQL Server on Linux", "env": [ { @@ -1864,7 +2016,9 @@ "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/microsoft.png", "note": "Password needs to include at least 8 characters including uppercase, lowercase letters, base-10 digits and/or non-alphanumeric symbols.", "platform": "linux", - "ports": ["1433/tcp"], + "ports": [ + "1433/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1876,7 +2030,9 @@ { "Id": 16, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "Microsoft SQL Server Developer for Windows containers", "env": [ { @@ -1891,7 +2047,9 @@ "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/microsoft.png", "note": "Password needs to include at least 8 characters including uppercase, lowercase letters, base-10 digits and/or non-alphanumeric symbols.", "platform": "windows", - "ports": ["1433/tcp"], + "ports": [ + "1433/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1908,7 +2066,9 @@ { "Id": 17, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "Microsoft SQL Server Express for Windows containers", "env": [ { @@ -1923,7 +2083,9 @@ "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/microsoft.png", "note": "Password needs to include at least 8 characters including uppercase, lowercase letters, base-10 digits and/or non-alphanumeric symbols.", "platform": "windows", - "ports": ["1433/tcp"], + "ports": [ + "1433/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -1940,12 +2102,16 @@ { "Id": 18, "administrator_only": false, - "categories": ["serverless"], + "categories": [ + "serverless" + ], "description": "Open-source serverless computing platform", "image": "iron/functions:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/ironfunctions.png", "platform": "linux", - "ports": ["8080/tcp"], + "ports": [ + "8080/tcp" + ], "privileged": true, "repository": { "stackfile": "", @@ -1963,7 +2129,9 @@ { "Id": 19, "administrator_only": false, - "categories": ["serverless"], + "categories": [ + "serverless" + ], "description": "Open-source user interface for IronFunctions", "env": [ { @@ -1974,7 +2142,9 @@ "image": "iron/functions-ui:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/ironfunctions.png", "platform": "linux", - "ports": ["4000/tcp"], + "ports": [ + "4000/tcp" + ], "privileged": true, "repository": { "stackfile": "", @@ -1992,12 +2162,16 @@ { "Id": 20, "administrator_only": false, - "categories": ["search-engine"], + "categories": [ + "search-engine" + ], "description": "Open-source enterprise search platform", "image": "solr:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/solr.png", "platform": "linux", - "ports": ["8983/tcp"], + "ports": [ + "8983/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2014,12 +2188,16 @@ { "Id": 21, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "Open-source in-memory data structure store", "image": "redis:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/redis.png", "platform": "linux", - "ports": ["6379/tcp"], + "ports": [ + "6379/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2036,12 +2214,17 @@ { "Id": 22, "administrator_only": false, - "categories": ["messaging"], + "categories": [ + "messaging" + ], "description": "Highly reliable enterprise messaging system", "image": "rabbitmq:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/rabbitmq.png", "platform": "linux", - "ports": ["5671/tcp", "5672/tcp"], + "ports": [ + "5671/tcp", + "5672/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2058,13 +2241,17 @@ { "Id": 23, "administrator_only": false, - "categories": ["blog"], + "categories": [ + "blog" + ], "description": "Free and open-source blogging platform", "image": "ghost:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/ghost.png", "note": "Access the blog management interface under \u003ccode\u003e/ghost/\u003c/code\u003e.", "platform": "linux", - "ports": ["2368/tcp"], + "ports": [ + "2368/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2081,13 +2268,22 @@ { "Id": 24, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "WebOps platform and hosting control panel", "image": "plesk/plesk:preview", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/plesk.png", "note": "Default credentials: admin / changeme", "platform": "linux", - "ports": ["21/tcp", "80/tcp", "443/tcp", "8880/tcp", "8443/tcp", "8447/tcp"], + "ports": [ + "21/tcp", + "80/tcp", + "443/tcp", + "8880/tcp", + "8443/tcp", + "8447/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2099,7 +2295,9 @@ { "Id": 25, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "Another free and open-source CMS", "env": [ { @@ -2114,7 +2312,9 @@ "image": "joomla:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/joomla.png", "platform": "linux", - "ports": ["80/tcp"], + "ports": [ + "80/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2131,12 +2331,16 @@ { "Id": 26, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "Open-source content management framework", "image": "drupal:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/drupal.png", "platform": "linux", - "ports": ["80/tcp"], + "ports": [ + "80/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2153,12 +2357,16 @@ { "Id": 27, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "A free and open-source CMS built on top of Zope", "image": "plone:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/plone.png", "platform": "linux", - "ports": ["8080/tcp"], + "ports": [ + "8080/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2175,12 +2383,18 @@ { "Id": 28, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "Open-source e-commerce platform", "image": "alankent/gsd:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/magento.png", "platform": "linux", - "ports": ["80/tcp", "3000/tcp", "3001/tcp"], + "ports": [ + "80/tcp", + "3000/tcp", + "3001/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2197,7 +2411,10 @@ { "Id": 29, "administrator_only": false, - "categories": ["Log Management", "Monitoring"], + "categories": [ + "Log Management", + "Monitoring" + ], "description": "Collect logs, metrics and docker events", "env": [ { @@ -2231,7 +2448,9 @@ { "Id": 30, "administrator_only": false, - "categories": ["Monitoring"], + "categories": [ + "Monitoring" + ], "description": "Collect events and metrics", "env": [ { @@ -2270,7 +2489,9 @@ { "Id": 31, "administrator_only": false, - "categories": ["marketing"], + "categories": [ + "marketing" + ], "description": "Open-source marketing automation platform", "env": [ { @@ -2285,7 +2506,9 @@ "image": "mautic/mautic:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/mautic.png", "platform": "linux", - "ports": ["80/tcp"], + "ports": [ + "80/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2302,7 +2525,9 @@ { "Id": 32, "administrator_only": false, - "categories": ["streaming"], + "categories": [ + "streaming" + ], "description": "Streaming media server", "env": [ { @@ -2317,7 +2542,12 @@ "image": "sameersbn/wowza:4.1.2-8", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/wowza.png", "platform": "linux", - "ports": ["1935/tcp", "8086/tcp", "8087/tcp", "8088/tcp"], + "ports": [ + "1935/tcp", + "8086/tcp", + "8087/tcp", + "8088/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2334,7 +2564,9 @@ { "Id": 33, "administrator_only": false, - "categories": ["continuous-integration"], + "categories": [ + "continuous-integration" + ], "description": "Open-source continuous integration tool", "env": [ { @@ -2345,7 +2577,10 @@ "image": "jenkins/jenkins:lts", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/jenkins.png", "platform": "linux", - "ports": ["8080/tcp", "50000/tcp"], + "ports": [ + "8080/tcp", + "50000/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2362,12 +2597,16 @@ { "Id": 34, "administrator_only": false, - "categories": ["project-management"], + "categories": [ + "project-management" + ], "description": "Open-source project management tool", "image": "redmine:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/redmine.png", "platform": "linux", - "ports": ["3000/tcp"], + "ports": [ + "3000/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2384,7 +2623,9 @@ { "Id": 35, "administrator_only": false, - "categories": ["project-management"], + "categories": [ + "project-management" + ], "description": "Open-source business apps", "env": [ { @@ -2403,7 +2644,9 @@ "image": "odoo:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/odoo.png", "platform": "linux", - "ports": ["8069/tcp"], + "ports": [ + "8069/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2423,13 +2666,20 @@ { "Id": 36, "administrator_only": false, - "categories": ["backup"], + "categories": [ + "backup" + ], "description": "Open-source network backup", "image": "cfstras/urbackup", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/urbackup.png", "note": "This application web interface is exposed on the port 55414 inside the container.", "platform": "linux", - "ports": ["55413/tcp", "55414/tcp", "55415/tcp", "35622/tcp"], + "ports": [ + "55413/tcp", + "55414/tcp", + "55415/tcp", + "35622/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2446,14 +2696,19 @@ { "Id": 37, "administrator_only": false, - "categories": ["filesystem", "storage"], + "categories": [ + "filesystem", + "storage" + ], "command": "--port 80 --database /data/database.db --scope /srv", "description": "A web file manager", "image": "filebrowser/filebrowser:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/filebrowser.png", "note": "Default credentials: admin/admin", "platform": "linux", - "ports": ["80/tcp"], + "ports": [ + "80/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2473,7 +2728,9 @@ { "Id": 38, "administrator_only": false, - "categories": ["development"], + "categories": [ + "development" + ], "description": "ColdFusion (CFML) CLI", "env": [ { @@ -2483,7 +2740,10 @@ "image": "ortussolutions/commandbox:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/ortussolutions-commandbox.png", "platform": "linux", - "ports": ["8080/tcp", "8443/tcp"], + "ports": [ + "8080/tcp", + "8443/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2495,7 +2755,9 @@ { "Id": 39, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "Open-source modular CMS", "env": [ { @@ -2511,7 +2773,10 @@ "image": "ortussolutions/contentbox:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/ortussolutions-contentbox.png", "platform": "linux", - "ports": ["8080/tcp", "8443/tcp"], + "ports": [ + "8080/tcp", + "8443/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2531,7 +2796,9 @@ { "Id": 40, "administrator_only": false, - "categories": ["portainer"], + "categories": [ + "portainer" + ], "description": "Manage all the resources in your Swarm cluster", "image": "", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/portainer.png", @@ -2548,7 +2815,9 @@ { "Id": 41, "administrator_only": false, - "categories": ["serverless"], + "categories": [ + "serverless" + ], "description": "Serverless functions made simple", "image": "", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/openfaas.png", @@ -2566,7 +2835,9 @@ { "Id": 42, "administrator_only": false, - "categories": ["serverless"], + "categories": [ + "serverless" + ], "description": "Open-source serverless computing platform", "image": "", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/ironfunctions.png", @@ -2583,7 +2854,9 @@ { "Id": 43, "administrator_only": false, - "categories": ["database"], + "categories": [ + "database" + ], "description": "CockroachDB cluster", "image": "", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/cockroachdb.png", @@ -2600,7 +2873,9 @@ { "Id": 44, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "Wordpress setup with a MySQL database", "env": [ { @@ -2624,7 +2899,9 @@ { "Id": 45, "administrator_only": false, - "categories": ["CMS"], + "categories": [ + "CMS" + ], "description": "Wordpress setup with a MySQL database", "env": [ { @@ -2648,7 +2925,9 @@ { "Id": 46, "administrator_only": false, - "categories": ["OPS"], + "categories": [ + "OPS" + ], "description": "Microsoft Operations Management Suite Linux agent.", "env": [ { @@ -2676,7 +2955,10 @@ { "Id": 47, "administrator_only": false, - "categories": ["Log Management", "Monitoring"], + "categories": [ + "Log Management", + "Monitoring" + ], "description": "Collect logs, metrics and docker events", "env": [ { @@ -2702,7 +2984,9 @@ { "Id": 48, "administrator_only": false, - "categories": ["Monitoring"], + "categories": [ + "Monitoring" + ], "description": "Collect events and metrics", "env": [ { @@ -2724,12 +3008,16 @@ { "Id": 49, "administrator_only": false, - "categories": ["docker"], + "categories": [ + "docker" + ], "description": "Sonatype Nexus3 registry manager", "image": "sonatype/nexus3:latest", "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/sonatype.png", "platform": "linux", - "ports": ["8081/tcp"], + "ports": [ + "8081/tcp" + ], "repository": { "stackfile": "", "url": "" @@ -2802,4 +3090,4 @@ "version": { "DB_VERSION": 24 } -} +} \ No newline at end of file diff --git a/api/datastore/test_data/output_24_to_latest.json b/api/datastore/test_data/output_24_to_latest.json index 3ec817231..322de72a8 100644 --- a/api/datastore/test_data/output_24_to_latest.json +++ b/api/datastore/test_data/output_24_to_latest.json @@ -672,6 +672,7 @@ { "Docker": { "ContainerCount": 0, + "DiagnosticsData": {}, "DockerSnapshotRaw": { "Containers": null, "Images": null, diff --git a/api/docker/snapshot.go b/api/docker/snapshot.go index d380cc8ee..a488d72e1 100644 --- a/api/docker/snapshot.go +++ b/api/docker/snapshot.go @@ -1,20 +1,9 @@ package docker import ( - "context" - "strings" - "time" - portainer "github.com/portainer/portainer/api" dockerclient "github.com/portainer/portainer/api/docker/client" - "github.com/portainer/portainer/api/docker/consts" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - _container "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/volume" - "github.com/docker/docker/client" - "github.com/rs/zerolog/log" + "github.com/portainer/portainer/pkg/snapshot" ) // Snapshotter represents a service used to create environment(endpoint) snapshots @@ -37,247 +26,5 @@ func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*p } defer cli.Close() - return snapshot(cli, endpoint) -} - -func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) { - if _, err := cli.Ping(context.Background()); err != nil { - return nil, err - } - - snapshot := &portainer.DockerSnapshot{ - StackCount: 0, - } - - if err := snapshotInfo(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine information") - } - - if snapshot.Swarm { - if err := snapshotSwarmServices(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm services") - } - - if err := snapshotNodes(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm nodes") - } - } - - if err := snapshotContainers(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot containers") - } - - if err := snapshotImages(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot images") - } - - if err := snapshotVolumes(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot volumes") - } - - if err := snapshotNetworks(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot networks") - } - - if err := snapshotVersion(snapshot, cli); err != nil { - log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine version") - } - - snapshot.Time = time.Now().Unix() - - return snapshot, nil -} - -func snapshotInfo(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - info, err := cli.Info(context.Background()) - if err != nil { - return err - } - - snapshot.Swarm = info.Swarm.ControlAvailable - snapshot.DockerVersion = info.ServerVersion - snapshot.TotalCPU = info.NCPU - snapshot.TotalMemory = info.MemTotal - snapshot.SnapshotRaw.Info = info - - return nil -} - -func snapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{}) - if err != nil { - return err - } - - var nanoCpus int64 - var totalMem int64 - - for _, node := range nodes { - nanoCpus += node.Description.Resources.NanoCPUs - totalMem += node.Description.Resources.MemoryBytes - } - - snapshot.TotalCPU = int(nanoCpus / 1e9) - snapshot.TotalMemory = totalMem - snapshot.NodeCount = len(nodes) - - return nil -} - -func snapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - stacks := make(map[string]struct{}) - - services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{}) - if err != nil { - return err - } - - for _, service := range services { - for k, v := range service.Spec.Labels { - if k == "com.docker.stack.namespace" { - stacks[v] = struct{}{} - } - } - } - - snapshot.ServiceCount = len(services) - snapshot.StackCount += len(stacks) - - return nil -} - -func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true}) - if err != nil { - return err - } - - stacks := make(map[string]struct{}) - gpuUseSet := make(map[string]struct{}) - gpuUseAll := false - - for _, container := range containers { - if container.State == "running" { - // Snapshot GPUs - response, err := cli.ContainerInspect(context.Background(), container.ID) - if err != nil { - // Inspect a container will fail when the container runs on a different - // Swarm node, so it is better to log the error instead of return error - // when the Swarm mode is enabled - if !snapshot.Swarm { - return err - } else { - if !strings.Contains(err.Error(), "No such container") { - return err - } - // It is common to have containers running on different Swarm nodes, - // so we just log the error in the debug level - log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes") - } - } else { - var gpuOptions *_container.DeviceRequest = nil - for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests { - if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" { - gpuOptions = &deviceRequest - } - } - - if gpuOptions != nil { - if gpuOptions.Count == -1 { - gpuUseAll = true - } - - for _, id := range gpuOptions.DeviceIDs { - gpuUseSet[id] = struct{}{} - } - } - } - } - - for k, v := range container.Labels { - if k == consts.ComposeStackNameLabel { - stacks[v] = struct{}{} - } - } - } - - gpuUseList := make([]string, 0, len(gpuUseSet)) - for gpuUse := range gpuUseSet { - gpuUseList = append(gpuUseList, gpuUse) - } - - snapshot.GpuUseAll = gpuUseAll - snapshot.GpuUseList = gpuUseList - - stats := CalculateContainerStats(containers) - - snapshot.ContainerCount = stats.Total - snapshot.RunningContainerCount = stats.Running - snapshot.StoppedContainerCount = stats.Stopped - snapshot.HealthyContainerCount = stats.Healthy - snapshot.UnhealthyContainerCount = stats.Unhealthy - snapshot.StackCount += len(stacks) - - for _, container := range containers { - snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container}) - } - - return nil -} - -func snapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - images, err := cli.ImageList(context.Background(), types.ImageListOptions{}) - if err != nil { - return err - } - - snapshot.ImageCount = len(images) - snapshot.SnapshotRaw.Images = images - - return nil -} - -func snapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{}) - if err != nil { - return err - } - - snapshot.VolumeCount = len(volumes.Volumes) - snapshot.SnapshotRaw.Volumes = volumes - - return nil -} - -func snapshotNetworks(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{}) - if err != nil { - return err - } - - snapshot.SnapshotRaw.Networks = networks - - return nil -} - -func snapshotVersion(snapshot *portainer.DockerSnapshot, cli *client.Client) error { - version, err := cli.ServerVersion(context.Background()) - if err != nil { - return err - } - - snapshot.SnapshotRaw.Version = version - snapshot.IsPodman = isPodman(version) - return nil -} - -// isPodman checks if the version is for Podman by checking if any of the components contain "podman". -// If it's podman, a component name should be "Podman Engine" -func isPodman(version types.Version) bool { - for _, component := range version.Components { - if strings.Contains(strings.ToLower(component.Name), "podman") { - return true - } - } - return false + return snapshot.CreateDockerSnapshot(cli) } diff --git a/api/kubernetes/snapshot.go b/api/kubernetes/snapshot.go index 39c4b1b8d..fe3f50554 100644 --- a/api/kubernetes/snapshot.go +++ b/api/kubernetes/snapshot.go @@ -1,15 +1,9 @@ package kubernetes import ( - "context" - "time" - portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/kubernetes/cli" - - "github.com/rs/zerolog/log" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" + "github.com/portainer/portainer/pkg/snapshot" ) type Snapshotter struct { @@ -30,55 +24,5 @@ func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*p return nil, err } - return snapshot(client, endpoint) -} - -func snapshot(cli *kubernetes.Clientset, endpoint *portainer.Endpoint) (*portainer.KubernetesSnapshot, error) { - res := cli.RESTClient().Get().AbsPath("/healthz").Do(context.TODO()) - if res.Error() != nil { - return nil, res.Error() - } - - snapshot := &portainer.KubernetesSnapshot{} - - err := snapshotVersion(snapshot, cli) - if err != nil { - log.Warn().Str("endpoint", endpoint.Name).Err(err).Msg("unable to snapshot cluster version") - } - - err = snapshotNodes(snapshot, cli) - if err != nil { - log.Warn().Str("endpoint", endpoint.Name).Err(err).Msg("unable to snapshot cluster nodes") - } - - snapshot.Time = time.Now().Unix() - return snapshot, nil -} - -func snapshotVersion(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error { - versionInfo, err := cli.ServerVersion() - if err != nil { - return err - } - - snapshot.KubernetesVersion = versionInfo.GitVersion - return nil -} - -func snapshotNodes(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error { - nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - - var totalCPUs, totalMemory int64 - for _, node := range nodeList.Items { - totalCPUs += node.Status.Capacity.Cpu().Value() - totalMemory += node.Status.Capacity.Memory().Value() - } - - snapshot.TotalCPU = totalCPUs - snapshot.TotalMemory = totalMemory - snapshot.NodeCount = len(nodeList.Items) - return nil + return snapshot.CreateKubernetesSnapshot(client) } diff --git a/api/portainer.go b/api/portainer.go index f7f4668b9..0b14d6462 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -185,6 +185,16 @@ type ( // CustomTemplatePlatform represents a custom template platform CustomTemplatePlatform int + // DiagnosticsData represents the diagnostics data for an environment + // this contains the logs, telnet, traceroute, dns and proxy information + // which will be part of the DockerSnapshot and KubernetesSnapshot structs + DiagnosticsData struct { + Log string `json:"Log,omitempty"` + Telnet map[string]string `json:"Telnet,omitempty"` + DNS map[string]string `json:"DNS,omitempty"` + Proxy map[string]string `json:"Proxy,omitempty"` + } + // DockerHub represents all the required information to connect and use the // Docker Hub DockerHub struct { @@ -217,6 +227,7 @@ type ( GpuUseAll bool `json:"GpuUseAll"` GpuUseList []string `json:"GpuUseList"` IsPodman bool `json:"IsPodman"` + DiagnosticsData *DiagnosticsData `json:"DiagnosticsData"` } // DockerContainerSnapshot is an extent of Docker's Container struct @@ -636,11 +647,12 @@ type ( // KubernetesSnapshot represents a snapshot of a specific Kubernetes environment(endpoint) at a specific time KubernetesSnapshot struct { - Time int64 `json:"Time"` - KubernetesVersion string `json:"KubernetesVersion"` - NodeCount int `json:"NodeCount"` - TotalCPU int64 `json:"TotalCPU"` - TotalMemory int64 `json:"TotalMemory"` + Time int64 `json:"Time"` + KubernetesVersion string `json:"KubernetesVersion"` + NodeCount int `json:"NodeCount"` + TotalCPU int64 `json:"TotalCPU"` + TotalMemory int64 `json:"TotalMemory"` + DiagnosticsData *DiagnosticsData `json:"DiagnosticsData"` } // KubernetesConfiguration represents the configuration of a Kubernetes environment(endpoint) diff --git a/go.mod b/go.mod index 5b2a31813..426cabc39 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.24.1 github.com/aws/aws-sdk-go-v2/credentials v1.16.16 github.com/aws/aws-sdk-go-v2/service/ecr v1.24.1 + github.com/aws/smithy-go v1.19.0 github.com/cbroglie/mustache v1.4.0 github.com/compose-spec/compose-go/v2 v2.0.2 github.com/containers/image/v5 v5.30.1 @@ -41,7 +42,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/rs/zerolog v1.29.0 github.com/segmentio/encoding v0.3.6 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/urfave/negroni v1.0.0 github.com/viney-shih/go-lock v1.1.1 go.etcd.io/bbolt v1.3.10 @@ -86,7 +87,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect - github.com/aws/smithy-go v1.19.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/goterm v1.0.4 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index 05e1c52ef..084703ece 100644 --- a/go.sum +++ b/go.sum @@ -618,8 +618,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c= diff --git a/pkg/edge/utils.go b/pkg/edge/utils.go new file mode 100644 index 000000000..cef1ae9d9 --- /dev/null +++ b/pkg/edge/utils.go @@ -0,0 +1,30 @@ +package edge + +import ( + "encoding/base64" + "errors" + "strconv" + "strings" +) + +// GetPortainerURLFromEdgeKey returns the portainer URL from an edge key +// format: ||| +func GetPortainerURLFromEdgeKey(edgeKey string) (string, error) { + decodedKey, err := base64.RawStdEncoding.DecodeString(edgeKey) + if err != nil { + return "", err + } + + keyInfo := strings.Split(string(decodedKey), "|") + + if len(keyInfo) != 4 { + return "", errors.New("invalid key format") + } + + _, err = strconv.Atoi(keyInfo[3]) + if err != nil { + return "", errors.New("invalid key format") + } + + return keyInfo[0], nil +} diff --git a/pkg/edge/utils_test.go b/pkg/edge/utils_test.go new file mode 100644 index 000000000..292bc39d4 --- /dev/null +++ b/pkg/edge/utils_test.go @@ -0,0 +1,29 @@ +package edge + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetPortainerURLFromEdgeKey(t *testing.T) { + tests := []struct { + name string + edgeKey string + expected string + }{ + { + name: "ValidEdgeKey", + edgeKey: "aHR0cHM6Ly9wb3J0YWluZXIuaW98cG9ydGFpbmVyLmlvOjgwMDB8YXNkZnwx", + expected: "https://portainer.io", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := GetPortainerURLFromEdgeKey(tt.edgeKey) + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/endpoints/utils.go b/pkg/endpoints/utils.go index 8c4ad7ce8..e32b56fea 100644 --- a/pkg/endpoints/utils.go +++ b/pkg/endpoints/utils.go @@ -1,6 +1,14 @@ package endpoints -import portainer "github.com/portainer/portainer/api" +import ( + portainer "github.com/portainer/portainer/api" +) + +// IsRegularAgentEndpoint returns true if this is a regular agent endpoint +func IsRegularAgentEndpoint(endpoint *portainer.Endpoint) bool { + return endpoint.Type == portainer.AgentOnDockerEnvironment || + endpoint.Type == portainer.AgentOnKubernetesEnvironment +} // IsEdgeEndpoint returns true if this is an Edge endpoint func IsEdgeEndpoint(endpoint *portainer.Endpoint) bool { diff --git a/pkg/endpoints/utils_test.go b/pkg/endpoints/utils_test.go index b69686ffe..47ef2db8d 100644 --- a/pkg/endpoints/utils_test.go +++ b/pkg/endpoints/utils_test.go @@ -7,6 +7,50 @@ import ( "github.com/stretchr/testify/assert" ) +func TestIsRegularAgentEndpoint(t *testing.T) { + tests := []struct { + name string + endpoint *portainer.Endpoint + expected bool + }{ + { + name: "AgentOnDockerEnvironment", + endpoint: &portainer.Endpoint{ + Type: portainer.AgentOnDockerEnvironment, + }, + expected: true, + }, + { + name: "AgentOnKubernetesEnvironment", + endpoint: &portainer.Endpoint{ + Type: portainer.AgentOnKubernetesEnvironment, + }, + expected: true, + }, + { + name: "EdgeAgentOnDockerEnvironment", + endpoint: &portainer.Endpoint{ + Type: portainer.EdgeAgentOnDockerEnvironment, + }, + expected: false, + }, + { + name: "EdgeAgentOnKubernetesEnvironment", + endpoint: &portainer.Endpoint{ + Type: portainer.EdgeAgentOnKubernetesEnvironment, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := IsRegularAgentEndpoint(tt.endpoint) + assert.Equal(t, tt.expected, result) + }) + } +} + func TestIsEdgeEndpoint(t *testing.T) { tests := []struct { name string diff --git a/pkg/networking/diagnostics.go b/pkg/networking/diagnostics.go new file mode 100644 index 000000000..2c0e19f1d --- /dev/null +++ b/pkg/networking/diagnostics.go @@ -0,0 +1,136 @@ +package networking + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/segmentio/encoding/json" +) + +// ProbeDNSConnection probes a DNS connection and returns a JSON string with the DNS lookup status and IP addresses. +// ignores errors for the dns lookup since we want to know if the host is reachable +func ProbeDNSConnection(url string) string { + _, host, _ := parseURL(url) + result := map[string]interface{}{ + "operation": "dns lookup", + "remote_address": host, + "connected_at": time.Now().Format(time.RFC3339), + "status": "dns lookup successful", + "resolved_ips": []net.IP{}, + } + + ipAddresses, err := net.LookupIP(host) + if err != nil { + result["status"] = fmt.Sprintf("dns lookup failed: %s", err) + } else { + result["resolved_ips"] = ipAddresses + } + + jsonData, _ := json.Marshal(result) + return string(jsonData) +} + +// ProbeTelnetConnection probes a telnet connection and returns a JSON string with the telnet connection status, local and remote addresses. +// ignores errors for the telnet connection since we want to know if the host is reachable +func ProbeTelnetConnection(url string) string { + network, host, port := parseURL(url) + if network == "https" || network == "http" { + network = "tcp" + } + + address := fmt.Sprintf("%s:%s", host, port) + result := map[string]string{ + "operation": "telnet connection", + "local_address": "unknown", + "remote_address": "unknown", + "network": network, + "status": "connected to " + address, + "connected_at": time.Now().Format(time.RFC3339), + } + + connection, err := net.DialTimeout(network, address, 5*time.Second) + if err != nil { + result["status"] = fmt.Sprintf("failed to connect to %s: %s", address, err) + } else { + defer connection.Close() + result["local_address"] = connection.LocalAddr().String() + result["remote_address"] = connection.RemoteAddr().String() + } + + jsonData, _ := json.Marshal(result) + return string(jsonData) +} + +// DetectProxy probes a target URL and returns a JSON string with the proxy detection status, local and remote addresses. +// ignores errors for the http request since we want to know if the host is reachable +func DetectProxy(url string) string { + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + Timeout: 10 * time.Second, + } + + result := map[string]string{ + "operation": "proxy detection", + "local_address": "unknown", + "remote_address": "unknown", + "network": "https", + "status": "no proxy detected", + "connected_at": time.Now().Format(time.RFC3339), + } + + resp, err := client.Get(url) + if err != nil { + result["status"] = fmt.Sprintf("failed to make request: %s", err) + } else { + defer resp.Body.Close() + + if resp.Request != nil { + result["local_address"] = resp.Request.Host + result["remote_address"] = resp.Request.RemoteAddr + } + + if resp.Header.Get("Via") != "" || resp.Header.Get("X-Forwarded-For") != "" || resp.Header.Get("Proxy-Connection") != "" { + result["status"] = "proxy detected via headers" + } else if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 { + cert := resp.TLS.PeerCertificates[0] + if cert.IsCA || strings.Contains(strings.ToLower(cert.Issuer.CommonName), "proxy") { + result["status"] = "proxy detected via certificate" + } + } + } + + jsonData, _ := json.Marshal(result) + return string(jsonData) +} + +// parseURL parses a raw URL and returns the network, host and port +// it also ensures the network is tcp and the port is set to the default for the network +func parseURL(rawURL string) (network, host, port string) { + u, err := url.Parse(rawURL) + if err != nil { + return "", "", "" + } + + network = u.Scheme + host = u.Hostname() + port = u.Port() + + if port == "" { + if network == "https" { + port = "443" + } else if network == "http" { + port = "80" + } + } + + return network, host, port +} diff --git a/pkg/networking/diagnostics_test.go b/pkg/networking/diagnostics_test.go new file mode 100644 index 000000000..33c270be4 --- /dev/null +++ b/pkg/networking/diagnostics_test.go @@ -0,0 +1,164 @@ +package networking + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// Response structs for each function +type dnsResponse struct { + Operation string `json:"operation"` + ResolvedIPs []string `json:"resolved_ips"` + RemoteAddr string `json:"remote_address"` + ConnectedAt string `json:"connected_at"` + Status string `json:"status"` +} + +type telnetResponse struct { + Operation string `json:"operation"` + LocalAddr string `json:"local_address"` + RemoteAddr string `json:"remote_address"` + Network string `json:"network"` + Status string `json:"status"` + ConnectedAt string `json:"connected_at"` +} + +type proxyResponse struct { + Operation string `json:"operation"` + LocalAddr string `json:"local_address"` + RemoteAddr string `json:"remote_address"` + Network string `json:"network"` + Status string `json:"status"` + ConnectedAt string `json:"connected_at"` +} + +func TestProbeDNSConnection(t *testing.T) { + tests := []struct { + name string + host string + wantSuccess bool + statusContains string + }{ + { + name: "Valid domain", + host: "https://api.portainer.io", + wantSuccess: true, + statusContains: "dns lookup successful", + }, + { + name: "Invalid domain", + host: "https://nonexistent.domain.invalid", + wantSuccess: false, + statusContains: "dns lookup failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + response := ProbeDNSConnection(tt.host) + + var result dnsResponse + if err := json.Unmarshal([]byte(response), &result); err != nil { + t.Fatalf("Invalid JSON response: %v", err) + } + + if !strings.Contains(result.Status, tt.statusContains) { + t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status) + } + }) + } +} + +func TestProbeTelnetConnection(t *testing.T) { + tests := []struct { + name string + url string + wantSuccess bool + statusContains string + }{ + { + name: "Valid connection", + url: "https://api.portainer.io", + wantSuccess: true, + statusContains: "connected to", + }, + { + name: "Invalid port", + url: "https://api.portainer.io:99999", + wantSuccess: false, + statusContains: "failed to connect", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + response := ProbeTelnetConnection(tt.url) + + var result telnetResponse + if err := json.Unmarshal([]byte(response), &result); err != nil { + t.Fatalf("Invalid JSON response: %v", err) + } + + validateCommonFields(t, result.Operation, "telnet connection", result.ConnectedAt) + if !strings.Contains(result.Status, tt.statusContains) { + t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status) + } + }) + } +} + +func TestDetectProxy(t *testing.T) { + tests := []struct { + name string + url string + wantSuccess bool + statusContains string + }{ + { + name: "Valid URL", + url: "https://api.portainer.io", + wantSuccess: true, + statusContains: "proxy", + }, + { + name: "Invalid URL", + url: "https://nonexistent.domain.invalid", + wantSuccess: false, + statusContains: "failed to make request", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + response := DetectProxy(tt.url) + + var result proxyResponse + if err := json.Unmarshal([]byte(response), &result); err != nil { + t.Fatalf("Invalid JSON response: %v", err) + } + + validateCommonFields(t, result.Operation, "proxy detection", result.ConnectedAt) + if result.Network != "https" { + t.Errorf("Expected network https, got %s", result.Network) + } + if !strings.Contains(result.Status, tt.statusContains) { + t.Errorf("Status should contain '%s', got: %s", tt.statusContains, result.Status) + } + }) + } +} + +// Helper function to validate common fields across all responses +func validateCommonFields(t *testing.T, operation, expectedOperation, connectedAt string) { + t.Helper() + + if operation != expectedOperation { + t.Errorf("Expected operation '%s', got '%s'", expectedOperation, operation) + } + + if _, err := time.Parse(time.RFC3339, connectedAt); err != nil { + t.Errorf("Invalid connected_at timestamp: %v", err) + } +} diff --git a/pkg/snapshot/docker.go b/pkg/snapshot/docker.go new file mode 100644 index 000000000..9b0a89aa2 --- /dev/null +++ b/pkg/snapshot/docker.go @@ -0,0 +1,372 @@ +package snapshot + +import ( + "bytes" + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/segmentio/encoding/json" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + _container "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/volume" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/docker/consts" + edgeutils "github.com/portainer/portainer/pkg/edge" + networkingutils "github.com/portainer/portainer/pkg/networking" + "github.com/rs/zerolog/log" +) + +func CreateDockerSnapshot(cli *client.Client) (*portainer.DockerSnapshot, error) { + if _, err := cli.Ping(context.Background()); err != nil { + return nil, err + } + + dockerSnapshot := &portainer.DockerSnapshot{ + StackCount: 0, + } + + err := dockerSnapshotInfo(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot engine information") + } + + if dockerSnapshot.Swarm { + err = dockerSnapshotSwarmServices(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot Swarm services") + } + + err = dockerSnapshotNodes(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot Swarm nodes") + } + } + + err = dockerSnapshotContainers(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot containers") + } + + err = dockerSnapshotImages(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot images") + } + + err = dockerSnapshotVolumes(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot volumes") + } + + err = dockerSnapshotNetworks(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot networks") + } + + err = dockerSnapshotVersion(dockerSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot engine version") + } + + dockerSnapshot.Time = time.Now().Unix() + + return dockerSnapshot, nil +} + +func dockerSnapshotInfo(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + info, err := cli.Info(context.Background()) + if err != nil { + return err + } + + snapshot.Swarm = info.Swarm.ControlAvailable + snapshot.DockerVersion = info.ServerVersion + snapshot.TotalCPU = info.NCPU + snapshot.TotalMemory = info.MemTotal + snapshot.SnapshotRaw.Info = info + + return nil +} + +func dockerSnapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{}) + if err != nil { + return err + } + + var nanoCpus int64 + var totalMem int64 + + for _, node := range nodes { + nanoCpus += node.Description.Resources.NanoCPUs + totalMem += node.Description.Resources.MemoryBytes + } + + snapshot.TotalCPU = int(nanoCpus / 1e9) + snapshot.TotalMemory = totalMem + snapshot.NodeCount = len(nodes) + + return nil +} + +func dockerSnapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + stacks := make(map[string]struct{}) + + services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{}) + if err != nil { + return err + } + + for _, service := range services { + for k, v := range service.Spec.Labels { + if k == "com.docker.stack.namespace" { + stacks[v] = struct{}{} + } + } + } + + snapshot.ServiceCount = len(services) + snapshot.StackCount += len(stacks) + + return nil +} + +func dockerSnapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: true}) + if err != nil { + return err + } + + stacks := make(map[string]struct{}) + gpuUseSet := make(map[string]struct{}) + gpuUseAll := false + + for _, container := range containers { + if container.State == "running" { + // Snapshot GPUs + response, err := cli.ContainerInspect(context.Background(), container.ID) + if err != nil { + // Inspect a container will fail when the container runs on a different + // Swarm node, so it is better to log the error instead of return error + // when the Swarm mode is enabled + if !snapshot.Swarm { + return err + } else { + if !strings.Contains(err.Error(), "No such container") { + return err + } + // It is common to have containers running on different Swarm nodes, + // so we just log the error in the debug level + log.Debug().Str("container", container.ID).Err(err).Msg("unable to inspect container in other Swarm nodes") + } + } else { + var gpuOptions *_container.DeviceRequest = nil + for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests { + if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" { + gpuOptions = &deviceRequest + } + } + + if gpuOptions != nil { + if gpuOptions.Count == -1 { + gpuUseAll = true + } + + for _, id := range gpuOptions.DeviceIDs { + gpuUseSet[id] = struct{}{} + } + } + } + } + + for k, v := range container.Labels { + if k == consts.ComposeStackNameLabel { + stacks[v] = struct{}{} + } + } + } + + gpuUseList := make([]string, 0, len(gpuUseSet)) + for gpuUse := range gpuUseSet { + gpuUseList = append(gpuUseList, gpuUse) + } + + snapshot.GpuUseAll = gpuUseAll + snapshot.GpuUseList = gpuUseList + + stats := calculateContainerStats(containers) + + snapshot.ContainerCount = stats.Total + snapshot.RunningContainerCount = stats.Running + snapshot.StoppedContainerCount = stats.Stopped + snapshot.HealthyContainerCount = stats.Healthy + snapshot.UnhealthyContainerCount = stats.Unhealthy + snapshot.StackCount += len(stacks) + + for _, container := range containers { + snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container}) + } + + return nil +} + +func dockerSnapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + images, err := cli.ImageList(context.Background(), image.ListOptions{}) + if err != nil { + return err + } + + snapshot.ImageCount = len(images) + snapshot.SnapshotRaw.Images = images + + return nil +} + +func dockerSnapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + volumes, err := cli.VolumeList(context.Background(), volume.ListOptions{}) + if err != nil { + return err + } + + snapshot.VolumeCount = len(volumes.Volumes) + snapshot.SnapshotRaw.Volumes = volumes + + return nil +} + +func dockerSnapshotNetworks(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{}) + if err != nil { + return err + } + + snapshot.SnapshotRaw.Networks = networks + + return nil +} + +func dockerSnapshotVersion(snapshot *portainer.DockerSnapshot, cli *client.Client) error { + version, err := cli.ServerVersion(context.Background()) + if err != nil { + return err + } + + snapshot.SnapshotRaw.Version = version + snapshot.IsPodman = isPodman(version) + return nil +} + +// DockerSnapshotDiagnostics returns the diagnostics data for the agent +func DockerSnapshotDiagnostics(cli *client.Client, edgeKey string) (*portainer.DiagnosticsData, error) { + containerID := os.Getenv("HOSTNAME") + snapshot := &portainer.DockerSnapshot{ + DiagnosticsData: &portainer.DiagnosticsData{ + DNS: make(map[string]string), + Telnet: make(map[string]string), + }, + } + + err := dockerSnapshotContainerErrorLogs(snapshot, cli, containerID) + if err != nil { + return nil, err + } + + if edgeKey != "" { + url, err := edgeutils.GetPortainerURLFromEdgeKey(edgeKey) + if err != nil { + return nil, fmt.Errorf("failed to get portainer URL from edge key: %w", err) + } + + snapshot.DiagnosticsData.DNS["edge-to-portainer"] = networkingutils.ProbeDNSConnection(url) + snapshot.DiagnosticsData.Telnet["edge-to-portainer"] = networkingutils.ProbeTelnetConnection(url) + } + + return snapshot.DiagnosticsData, nil +} + +// DockerSnapshotContainerErrorLogs returns the 5 most recent error logs of the agent container +// this will primarily be used for agent snapshot +func dockerSnapshotContainerErrorLogs(snapshot *portainer.DockerSnapshot, cli *client.Client, containerId string) error { + if containerId == "" { + return nil + } + + rd, err := cli.ContainerLogs(context.Background(), containerId, container.LogsOptions{ + ShowStdout: false, + ShowStderr: true, + Tail: "5", + Timestamps: true, + }) + if err != nil { + return fmt.Errorf("failed to get container logs: %w", err) + } + defer rd.Close() + + var stdOut, stdErr bytes.Buffer + _, err = stdcopy.StdCopy(&stdErr, &stdOut, rd) + if err != nil { + return fmt.Errorf("failed to copy error logs: %w", err) + } + + var logs []map[string]string + jsonLogs, err := json.Marshal(logs) + if err != nil { + return fmt.Errorf("failed to marshal logs to JSON: %w", err) + } + + snapshot.DiagnosticsData.Log = string(jsonLogs) + + return nil +} + +// isPodman checks if the version is for Podman by checking if any of the components contain "podman". +// If it's podman, a component name should be "Podman Engine" +func isPodman(version types.Version) bool { + for _, component := range version.Components { + if strings.Contains(strings.ToLower(component.Name), "podman") { + return true + } + } + return false +} + +type ContainerStats struct { + Running int + Stopped int + Healthy int + Unhealthy int + Total int +} + +func calculateContainerStats(containers []types.Container) ContainerStats { + var running, stopped, healthy, unhealthy int + for _, container := range containers { + switch container.State { + case "running": + running++ + case "healthy": + running++ + healthy++ + case "unhealthy": + running++ + unhealthy++ + case "exited", "stopped": + stopped++ + } + } + + return ContainerStats{ + Running: running, + Stopped: stopped, + Healthy: healthy, + Unhealthy: unhealthy, + Total: len(containers), + } +} diff --git a/pkg/snapshot/docker_test.go b/pkg/snapshot/docker_test.go new file mode 100644 index 000000000..8df14bc39 --- /dev/null +++ b/pkg/snapshot/docker_test.go @@ -0,0 +1 @@ +package snapshot diff --git a/pkg/snapshot/kubernetes.go b/pkg/snapshot/kubernetes.go new file mode 100644 index 000000000..d8e550e84 --- /dev/null +++ b/pkg/snapshot/kubernetes.go @@ -0,0 +1,150 @@ +package snapshot + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/segmentio/encoding/json" + + "github.com/aws/smithy-go/ptr" + portainer "github.com/portainer/portainer/api" + edgeutils "github.com/portainer/portainer/pkg/edge" + networkingutils "github.com/portainer/portainer/pkg/networking" + "github.com/rs/zerolog/log" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +func CreateKubernetesSnapshot(cli *kubernetes.Clientset) (*portainer.KubernetesSnapshot, error) { + kubernetesSnapshot := &portainer.KubernetesSnapshot{} + + err := kubernetesSnapshotVersion(kubernetesSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot cluster version") + } + + err = kubernetesSnapshotNodes(kubernetesSnapshot, cli) + if err != nil { + log.Warn().Err(err).Msg("unable to snapshot cluster nodes") + } + + kubernetesSnapshot.Time = time.Now().Unix() + return kubernetesSnapshot, nil +} + +func kubernetesSnapshotVersion(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error { + versionInfo, err := cli.ServerVersion() + if err != nil { + return err + } + + snapshot.KubernetesVersion = versionInfo.GitVersion + return nil +} + +func kubernetesSnapshotNodes(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset) error { + nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + + var totalCPUs, totalMemory int64 + for _, node := range nodeList.Items { + totalCPUs += node.Status.Capacity.Cpu().Value() + totalMemory += node.Status.Capacity.Memory().Value() + } + + snapshot.TotalCPU = totalCPUs + snapshot.TotalMemory = totalMemory + snapshot.NodeCount = len(nodeList.Items) + return nil +} + +// KubernetesSnapshotDiagnostics returns the diagnostics data for the agent +func KubernetesSnapshotDiagnostics(cli *kubernetes.Clientset, edgeKey string) (*portainer.DiagnosticsData, error) { + podID := os.Getenv("HOSTNAME") + snapshot := &portainer.KubernetesSnapshot{ + DiagnosticsData: &portainer.DiagnosticsData{ + DNS: make(map[string]string), + Telnet: make(map[string]string), + }, + } + + err := kubernetesSnapshotPodErrorLogs(snapshot, cli, "portainer", podID) + if err != nil { + return nil, fmt.Errorf("failed to snapshot pod error logs: %w", err) + } + + if edgeKey != "" { + url, err := edgeutils.GetPortainerURLFromEdgeKey(edgeKey) + if err != nil { + return nil, fmt.Errorf("failed to get portainer URL from edge key: %w", err) + } + + snapshot.DiagnosticsData.DNS["edge-to-portainer"] = networkingutils.ProbeDNSConnection(url) + snapshot.DiagnosticsData.Telnet["edge-to-portainer"] = networkingutils.ProbeTelnetConnection(url) + } + + return snapshot.DiagnosticsData, nil +} + +// KubernetesSnapshotPodErrorLogs returns 0 to 10 lines of the most recent error logs of the agent container +// this will primarily be used for agent snapshot +func kubernetesSnapshotPodErrorLogs(snapshot *portainer.KubernetesSnapshot, cli *kubernetes.Clientset, namespace, podID string) error { + if namespace == "" || podID == "" { + return errors.New("both namespace and podID are required to capture pod error logs in the snapshot") + } + + logsStream, err := cli.CoreV1().Pods(namespace).GetLogs(podID, &corev1.PodLogOptions{TailLines: ptr.Int64(10), Timestamps: true}).Stream(context.TODO()) + if err != nil { + return fmt.Errorf("failed to stream logs: %w", err) + } + defer logsStream.Close() + + logBytes, err := io.ReadAll(logsStream) + if err != nil { + return fmt.Errorf("failed to read error logs: %w", err) + } + + logs := filterLogsByPattern(logBytes, []string{"error", "err", "level=error", "exception", "fatal", "panic"}) + + jsonLogs, err := json.Marshal(logs) + if err != nil { + return fmt.Errorf("failed to marshal logs: %w", err) + } + snapshot.DiagnosticsData.Log = string(jsonLogs) + + return nil +} + +// filterLogsByPattern filters the logs by the given patterns and returns a list of logs that match the patterns +// the logs are returned as a list of maps with the keys "timestamp" and "message" +func filterLogsByPattern(logBytes []byte, patterns []string) []map[string]string { + logs := []map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(logBytes)), "\n") { + if line == "" { + continue + } + + if parts := strings.SplitN(line, " ", 2); len(parts) == 2 { + messageLower := strings.ToLower(parts[1]) + for _, pattern := range patterns { + if strings.Contains(messageLower, pattern) { + logs = append(logs, map[string]string{ + "timestamp": parts[0], + "message": parts[1], + }) + break + } + } + } + } + + return logs +} diff --git a/pkg/snapshot/kubernetes_test.go b/pkg/snapshot/kubernetes_test.go new file mode 100644 index 000000000..8df14bc39 --- /dev/null +++ b/pkg/snapshot/kubernetes_test.go @@ -0,0 +1 @@ +package snapshot