k3s/tests/docker/lazypull/lazypull_test.go

package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"testing"

	tester "github.com/k3s-io/k3s/tests/docker"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
)

var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
var config *tester.TestConfig

func Test_DockerLazyPull(t *testing.T) {
	flag.Parse()
	RegisterFailHandler(Fail)
	RunSpecs(t, "LazyPull Docker Test Suite")
}

var _ = Describe("LazyPull Tests", Ordered, func() {

	Context("Setup Cluster", func() {
		It("should provision servers", func() {
			var err error
			config, err = tester.NewTestConfig(*k3sImage)
			Expect(err).NotTo(HaveOccurred())

			Expect(os.Setenv("SERVER_ARGS", "--snapshotter=stargz")).To(Succeed())
			Expect(config.ProvisionServers(1)).To(Succeed())
			Eventually(func() error {
				return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
			}, "60s", "5s").Should(Succeed())
			Eventually(func() error {
				return tester.NodesReady(config.KubeconfigFile)
			}, "40s", "5s").Should(Succeed())
		})
	})

	Context("Use Snapshot Container", func() {
		It("should apply local storage volume", func() {
			const snapshotTestManifest = "../resources/snapshot-test.yaml"

			// Apply the manifest
			cmd := fmt.Sprintf("kubectl apply -f %s --kubeconfig=%s", snapshotTestManifest, config.KubeconfigFile)
			_, err := tester.RunCommand(cmd)
			Expect(err).NotTo(HaveOccurred(), "failed to apply volume test manifest")
		})
		It("should have the pod come up", func() {
			Eventually(func() (bool, error) {
				return tester.PodReady("stargz-snapshot-test", "default", config.KubeconfigFile)
			}, "30s", "5s").Should(BeTrue())
		})
		var topLayer string
		It("extracts the topmost layer of the container", func() {
			Eventually(func() (string, error) {
				var err error
				topLayer, err = getTopmostLayer(config.Servers[0].Name, "stargz-snapshot-test")
				topLayer = strings.TrimSpace(topLayer)
				return topLayer, err
			}, "30s", "5s").ShouldNot(BeEmpty())
			fmt.Println("Topmost layer: ", topLayer)
		})
		It("checks all layers are remote snapshots", func() {
			Expect(lookLayers(config.Servers[0].Name, topLayer)).To(Succeed())
		})
	})
})

var failed bool
var _ = AfterEach(func() {
	failed = failed || CurrentSpecReport().Failed()
})

var _ = AfterSuite(func() {
	if config != nil && !failed {
		config.Cleanup()
	}
})

func lookLayers(node, layer string) error {
	remoteSnapshotLabel := "containerd.io/snapshot/remote"
	layersNum := 0
	var err error
	for layersNum = 0; layersNum < 100; layersNum++ {
		// We use RunCommand instead of RunCmdOnDocker because we pipe the output to jq
		cmd := fmt.Sprintf("docker exec -i %s ctr --namespace=k8s.io snapshot --snapshotter=stargz info %s | jq -r '.Parent'", node, layer)
		layer, err = tester.RunCommand(cmd)
		if err != nil {
			return fmt.Errorf("failed to get parent layer: %v", err)
		}
		layer = strings.TrimSpace(layer)
		// If the layer is null, we have reached the topmost layer
		if layer == "null" {
			break
		}
		cmd = fmt.Sprintf("docker exec -i %s ctr --namespace=k8s.io snapshots --snapshotter=stargz info %s | jq -r '.Labels.\"%s\"'", node, layer, remoteSnapshotLabel)
		label, err := tester.RunCommand(cmd)
		if err != nil {
			return fmt.Errorf("failed to get layer label: %v", err)
		}
		label = strings.TrimSpace(label)
		fmt.Printf("Checking layer %s : %s\n", layer, label)
		if label == "null" {
			return fmt.Errorf("layer %s isn't remote snapshot", layer)
		}
	}

	if layersNum == 0 {
		return fmt.Errorf("cannot get layers")
	} else if layersNum >= 100 {
		return fmt.Errorf("testing image contains too many layers > 100")
	}

	return nil
}

func getTopmostLayer(node, container string) (string, error) {
	var targetContainer string
	cmd := fmt.Sprintf("docker exec -i %s ctr --namespace=k8s.io c ls -q labels.\"io.kubernetes.container.name\"==\"%s\" | sed -n 1p", node, container)
	targetContainer, _ = tester.RunCommand(cmd)
	targetContainer = strings.TrimSpace(targetContainer)
	fmt.Println("targetContainer: ", targetContainer)
	if targetContainer == "" {
		return "", fmt.Errorf("failed to get target container")
	}
	cmd = fmt.Sprintf("docker exec -i %s ctr --namespace=k8s.io c info %s | jq -r '.SnapshotKey'", node, targetContainer)
	layer, err := tester.RunCommand(cmd)
	if err != nil {
		return "", fmt.Errorf("failed to get topmost layer: %v", err)
	}
	return strings.TrimSpace(layer), nil
}