package kubeflags import ( "strings" "testing" testutil "github.com/k3s-io/k3s/tests/integration" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pkg/errors" ) var server *testutil.K3sServer var serverArgs = []string{"--cluster-init", "--kube-apiserver-arg", "advertise-port=1234", "--kube-controller-manager-arg", "allocate-node-cidrs=false", "--kube-scheduler-arg", "allow-metric-labels=metric1,label1='v3'", "--kube-cloud-controller-manager-arg", "allocate-node-cidrs=false", "--kubelet-arg", "address=127.0.0.1", "--kube-proxy-arg", "cluster-cidr=127.0.0.1/16", } var testLock int var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error testLock, err = testutil.K3sTestLock() Expect(err).ToNot(HaveOccurred()) server, err = testutil.K3sStartServer(serverArgs...) Expect(err).ToNot(HaveOccurred()) } }) var _ = Describe("create a new cluster with kube-* flags", Ordered, func() { BeforeEach(func() { if testutil.IsExistingServer() { Skip("Test needs k3s server with: " + strings.Join(serverArgs, " ")) } }) When("should print the args on the console", func() { It("should find cloud-controller-manager starting", func() { Eventually(func() error { match, err := testutil.SearchK3sLog(server, "Running cloud-controller-manager --allocate-node-cidrs=false") if err != nil { return err } if match { return nil } return errors.New("error finding cloud-controller-manager") }, "30s", "2s").Should(Succeed()) }) It("should find kube-scheduler starting", func() { Eventually(func() error { match, err := testutil.SearchK3sLog(server, "Running kube-scheduler --allow-metric-labels=metric1,label1='v3'") if err != nil { return err } if match { return nil } return errors.New("error finding kube-scheduler") }, "30s", "2s").Should(Succeed()) }) It("should find kube-apiserver starting", func() { Eventually(func() error { match, err := testutil.SearchK3sLog(server, "Running kube-apiserver --advertise-port=1234") if err != nil { return err } if match { return nil } return errors.New("error finding kube-apiserver") }, "30s", "2s").Should(Succeed()) }) It("should find kube-controller-manager starting", func() { Eventually(func() error { match, err := testutil.SearchK3sLog(server, "Running kube-controller-manager --allocate-node-cidrs=false") if err != nil { return err } if match { return nil } return errors.New("error finding kube-controller-manager") }, "30s", "2s").Should(Succeed()) }) It("should find kubelet starting", func() { Eventually(func() error { match, err := testutil.SearchK3sLog(server, "Running kubelet --address=127.0.0.1") if err != nil { return err } if match { return nil } return errors.New("error finding kubelet") }, "120s", "15s").Should(Succeed()) }) It("should find kube-proxy starting", func() { Eventually(func() error { match, err := testutil.SearchK3sLog(server, "Running kube-proxy --cluster-cidr=127.0.0.1/16") if err != nil { return err } if match { return nil } return errors.New("error finding kube-proxy") }, "300s", "15s").Should(Succeed()) }) When("server is setup without kube-proxy or cloud-controller-manager ", func() { It("kills previous server and clean logs", func() { Expect(testutil.K3sKillServer(server)).To(Succeed()) }) It("start up with disabled kube-proxy and cloud controller", func() { var err error localServerArgs := []string{ "--cluster-init", "--disable-cloud-controller=true", "--disable-kube-proxy=true", } server, err = testutil.K3sStartServer(localServerArgs...) Expect(err).ToNot(HaveOccurred()) // Pods should not be healthy without kube-proxy Consistently(func() error { return testutil.K3sDefaultDeployments() }, "100s", "5s").Should(HaveOccurred()) }) It("should not find kube-proxy starting", func() { Consistently(func() error { match, err := testutil.SearchK3sLog(server, "Running kube-proxy") if err != nil { return err } if !match { return nil } return errors.New("found kube-proxy starting") }, "100s", "5s").Should(Succeed()) }) /* The flag --disable-cloud-controller doesn't stop ccm from running, it appends -cloud-node and -cloud-node-lifecycle to the end of the --controllers flag https://github.com/k3s-io/k3s/blob/master/docs/adrs/servicelb-ccm.md */ It("should find cloud-controller-manager starting with"+ "\"--cloud-node,--cloud-node-lifecycle,--secure-port=0\" flags ", func() { Eventually(func() error { match, err := testutil.SearchK3sLog(server, "Running cloud-controller-manager --allocate-node-cidrs=true "+ "--authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/cloud-controller.kubeconfig "+ "--authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/cloud-controller.kubeconfig --bind-address=127.0.0.1 "+ "--cloud-config=/var/lib/rancher/k3s/server/etc/cloud-config.yaml --cloud-provider=k3s --cluster-cidr=10.42.0.0/16 "+ "--configure-cloud-routes=false --controllers=*,-route,-cloud-node,-cloud-node-lifecycle "+ "--feature-gates=CloudDualStackNodeIPs=true --kubeconfig=/var/lib/rancher/k3s/server/cred/cloud-controller.kubeconfig "+ "--leader-elect-resource-name=k3s-cloud-controller-manager --node-status-update-frequency=1m0s --profiling=false --secure-port=0") if err != nil { return err } if match { return nil } return errors.New("found cloud-controller-manager starting with wrong flags") }, "30s", "2s").Should(Succeed()) }) It("kills previous server and clean logs", func() { Expect(testutil.K3sKillServer(server)).To(Succeed()) }) It("start up with no problems and fully disabled cloud controller", func() { var err error localServerArgs := []string{ "--cluster-init", "--disable-cloud-controller=true", "--disable=servicelb", } server, err = testutil.K3sStartServer(localServerArgs...) Expect(err).ToNot(HaveOccurred()) Eventually(func() error { return testutil.K3sDefaultDeployments() }, "180s", "5s").Should(Succeed()) }) It("should not find cloud-controller-manager starting", func() { Consistently(func() error { match, err := testutil.SearchK3sLog(server, "Running cloud-controller-manager") if err != nil { return err } if !match { return nil } return errors.New("found cloud-controller-manager starting") }, "100s", "5s").Should(Succeed()) }) }) }) }) var failed bool var _ = AfterEach(func() { failed = failed || CurrentSpecReport().Failed() }) var _ = AfterSuite(func() { if !testutil.IsExistingServer() { if failed { testutil.K3sSaveLog(server, false) } Expect(testutil.K3sKillServer(server)).To(Succeed()) Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) } }) func Test_IntegrationEtcdSnapshot(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Etcd Snapshot Suite") }