Merge pull request #1466 from galal-hussein/traefik_to_nginx

Replace traefik with nginx

manifests/nginx.yaml

@@ -0,0 +1,14 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: nginx-ingress
+  namespace: kube-system
+spec:
+  chart: https://%{KUBERNETES_API}%/static/charts/nginx-ingress-1.33.0.tgz
+  set:
+    rbac.create: "true"
+    controller.service.enableHttps: "true"
+    controller.metrics.enabled: "true"
+    controller.publishService.enabled: "true"
+    controller.image.repository: "%{NGINX_IMAGE}"
+

manifests/traefik.yaml

@@ -1,12 +0,0 @@
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: traefik
-  namespace: kube-system
-spec:
-  chart: https://%{KUBERNETES_API}%/static/charts/traefik-1.81.0.tgz
-  set:
-    rbac.enabled: "true"
-    ssl.enabled: "true"
-    metrics.prometheus.enabled: "true"
-    kubernetes.ingressEndpoint.useDefaultPublishedService: "true"

pkg/cli/cmds/server.go

@@ -194,7 +194,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 			},
 			cli.StringSliceFlag{
 				Name:  "disable",
-				Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: coredns, servicelb, traefik, local-storage, metrics-server)",
+				Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: coredns, servicelb, nginx, local-storage, metrics-server)",
 			},
 			cli.BoolFlag{
 				Name:        "disable-scheduler",

pkg/deploy/controller.go

@@ -115,6 +115,9 @@ func (w *watcher) listFilesIn(base string, force bool) error {
 
 	var errs []error
 	for _, path := range keys {
+		if w.skipNginx(path) {
+			continue
+		}
 		if shouldDisableService(base, path, w.disables) {
 			if err := w.delete(path); err != nil {
 				errs = append(errs, errors2.Wrapf(err, "failed to delete %s", path))
@@ -337,3 +340,14 @@ func shouldDisableService(base, fileName string, disables map[string]bool) bool
 	}
 	return false
 }
+
+func (w *watcher) skipNginx(path string) bool {
+	name := name(path)
+	if name == "nginx" {
+		addon, err := w.addonCache.Get(ns, "traefik")
+		if err == nil && addon != nil {
+			return true
+		}
+	}
+	return false
+}

pkg/deploy/zz_generated_bindata.go

@@ -10,8 +10,8 @@
 // manifests/metrics-server/metrics-server-deployment.yaml
 // manifests/metrics-server/metrics-server-service.yaml
 // manifests/metrics-server/resource-reader.yaml
+// manifests/nginx.yaml
 // manifests/rolebindings.yaml
-// manifests/traefik.yaml
 package deploy
 
 import (
@@ -288,42 +288,42 @@ func metricsServerResourceReaderYaml() (*asset, error) {
 	return a, nil
 }
 
-var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x92\x31\x6f\xe3\x30\x0c\x85\x77\xfd\x0a\x21\xbb\x72\x38\xdc\x72\xf0\xd8\x0e\xdd\x03\xb4\x3b\x6d\xb3\x09\x6b\x59\x14\x48\x2a\x41\xfb\xeb\x0b\xa7\x6e\x82\xa4\x76\x90\xb4\xdd\x24\x41\x7c\x1f\x1f\xf9\x20\xd3\x13\x8a\x12\xa7\xca\x4b\x0d\xcd\x12\x8a\x6d\x58\xe8\x0d\x8c\x38\x2d\xbb\xff\xba\x24\xfe\xb3\xfd\xeb\x3a\x4a\x6d\xe5\xef\x63\x51\x43\x59\x71\xc4\x3b\x4a\x2d\xa5\xb5\xeb\xd1\xa0\x05\x83\xca\x79\x9f\xa0\xc7\xca\x77\xa5\xc6\x00\x99\x14\x65\x8b\x12\x86\x6b\x44\x0b\xd0\xf6\x94\x9c\x70\xc4\x15\x3e\x0f\xbf\x21\xd3\x83\x70\xc9\x17\xc8\xce\xfb\x2f\xe0\x03\x47\x5f\xd5\xb0\xaf\x0e\xfa\x99\x46\x86\x96\xfa\x05\x1b\xd3\xca\x85\x9b\x20\x8f\x8a\x32\xe3\xc2\xb9\x10\x82\xfb\xfe\xb4\x26\xc6\xf4\xd9\xfe\x3f\x0d\x0d\x27\x13\x8e\x11\xc5\x49\x89\x78\xd2\xb8\x0e\x15\xc1\x2f\x16\xce\x7b\x41\xe5\x22\x0d\x8e\x6f\x89\x5b\x54\xe7\xfd\x16\xa5\x1e\x9f\xd6\x68\x57\xd6\x42\x8f\x9a\xa1\x39\x17\x88\xa4\xb6\x3f\xec\xc0\x9a\xcd\x84\x56\x42\xdb\xb1\x74\x94\xd6\xa3\xdf\x29\xf1\x8f\x3f\x99\x23\x35\x74\x33\x61\x42\x10\x53\x9b\x99\x92\xe9\xfe\x96\xb9\x9d\xd3\x1c\xfc\x1f\xb5\x7f\xb8\xb4\xf9\x88\xcf\xec\xee\xf7\xb3\x7d\x0a\x38\x06\x7b\xf0\x78\x1d\xe3\x2c\xdc\x97\x01\xef\x01\x00\x00\xff\xff\x46\xd3\x6d\x9d\x0f\x04\x00\x00")
+var _nginxYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x74\xcd\xc1\x6a\x02\x31\x10\xc6\xf1\x7b\x9e\x62\x58\xf0\x68\xb6\xe2\x2d\x37\x5b\x16\x95\x52\x29\xb5\x2d\xbd\xc9\x6c\x1c\x76\x83\x49\x36\xcc\x8c\x52\x2b\xbe\x7b\x51\xbc\x78\xf0\xfa\xf1\x9b\xff\x60\x09\xdf\xc4\x12\x86\xec\xa0\xa7\x98\xac\x47\xd5\x48\x36\x0c\xf5\x61\x62\x76\x21\x6f\x1d\x2c\x28\xa6\x97\x1e\x59\x4d\x22\xc5\x2d\x2a\x3a\x03\x90\x31\x91\x83\xdc\x85\xfc\x3b\x0e\xb9\x63\x12\xb9\xad\x52\xd0\x93\x83\xdd\xbe\xa5\xb1\x1c\x45\x29\x19\x29\xe4\x2f\x47\xfe\x92\x71\xd0\xab\x16\x71\x75\x3d\x3a\xbd\x7e\x3d\x37\x1f\xab\xe6\xb3\x59\x6f\x66\xef\xcb\xf3\xa8\x16\x45\x0d\xbe\xbe\x42\xa9\xef\xf2\xe3\x89\x9d\x4e\xed\x93\xd5\xee\xcf\x00\x08\xe9\xa5\x08\xc0\x2d\x7a\xeb\x99\x50\xc9\x41\xa5\xbc\xa7\xea\xba\xfb\x21\x2b\x0f\x31\x12\x5b\x21\x3e\x04\x4f\x96\x32\xb6\x91\x16\xd7\xef\x0f\x68\x22\xe5\xe0\xe5\x46\xb7\x8f\x58\xd9\xb7\x31\x48\xbf\xbe\x0b\x3f\xd4\x21\x61\x47\x96\xa9\x0c\x12\x74\xe0\xa3\x83\x6a\x74\x5a\xcd\x97\xab\x9f\xcd\xf2\x6d\x36\x6f\xce\x95\x31\xff\x01\x00\x00\xff\xff\x71\x90\x7d\xf9\x8a\x01\x00\x00")
 
-func rolebindingsYamlBytes() ([]byte, error) {
+func nginxYamlBytes() ([]byte, error) {
 	return bindataRead(
-		_rolebindingsYaml,
-		"rolebindings.yaml",
+		_nginxYaml,
+		"nginx.yaml",
 	)
 }
 
-func rolebindingsYaml() (*asset, error) {
-	bytes, err := rolebindingsYamlBytes()
+func nginxYaml() (*asset, error) {
+	bytes, err := nginxYamlBytes()
 	if err != nil {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "rolebindings.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	info := bindataFileInfo{name: "nginx.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
 
-var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x6c\x8f\xcd\x6a\xc3\x30\x10\x84\xef\x7e\x8a\x25\x90\x63\xe4\xe6\x56\x74\xeb\x8f\xa1\xa5\x50\x42\xd3\xf6\x5a\xd6\xf2\x24\x16\x91\x64\xa1\x5d\x05\xda\xd2\x77\x2f\x0e\x3e\xe6\xb8\x3b\x1f\x1f\x33\x9c\xfd\x27\x8a\xf8\x29\x59\x1a\x11\xa2\x71\xac\x1a\x60\xfc\xd4\x9e\xb7\xcd\xc9\xa7\xc1\xd2\x13\x42\x7c\x18\xb9\x68\x13\xa1\x3c\xb0\xb2\x6d\x88\x12\x47\x58\xd2\xc2\x38\xf8\xd3\x72\x4b\x66\x07\x4b\xa7\xda\x63\x23\xdf\xa2\x88\x8d\x64\xb8\x19\x77\xb3\xc0\xd2\xa8\x9a\xc5\xb6\xed\xfa\xf7\xe5\xe3\xbe\x7b\x7b\xed\xde\xbb\xfd\xd7\xdd\xee\xf9\x6f\xdd\x8a\xb2\x7a\xd7\x5e\x40\x69\x17\xf1\x66\x6b\x6e\xb7\xe6\xc6\xe8\xf1\xa7\x21\x12\xe8\xec\x22\x2a\x3d\x3b\x83\xc4\x7d\xc0\x60\x69\xa5\xa5\x62\x75\x09\x44\xc2\xd5\x7f\x84\x16\xef\xc4\xe4\x32\x45\xe8\x88\x2a\x57\xb1\xb9\x79\x49\x50\x88\xf1\xe9\x58\x20\xd2\xa5\x21\x4f\x3e\xa9\xa9\x82\x47\x1c\xb8\x06\xdd\xd5\x3e\x78\x19\x31\xec\x51\xce\x7e\x1e\xbc\x18\xfe\x03\x00\x00\xff\xff\xb5\x07\xd7\x40\x4d\x01\x00\x00")
+var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x92\x31\x6f\xe3\x30\x0c\x85\x77\xfd\x0a\x21\xbb\x72\x38\xdc\x72\xf0\xd8\x0e\xdd\x03\xb4\x3b\x6d\xb3\x09\x6b\x59\x14\x48\x2a\x41\xfb\xeb\x0b\xa7\x6e\x82\xa4\x76\x90\xb4\xdd\x24\x41\x7c\x1f\x1f\xf9\x20\xd3\x13\x8a\x12\xa7\xca\x4b\x0d\xcd\x12\x8a\x6d\x58\xe8\x0d\x8c\x38\x2d\xbb\xff\xba\x24\xfe\xb3\xfd\xeb\x3a\x4a\x6d\xe5\xef\x63\x51\x43\x59\x71\xc4\x3b\x4a\x2d\xa5\xb5\xeb\xd1\xa0\x05\x83\xca\x79\x9f\xa0\xc7\xca\x77\xa5\xc6\x00\x99\x14\x65\x8b\x12\x86\x6b\x44\x0b\xd0\xf6\x94\x9c\x70\xc4\x15\x3e\x0f\xbf\x21\xd3\x83\x70\xc9\x17\xc8\xce\xfb\x2f\xe0\x03\x47\x5f\xd5\xb0\xaf\x0e\xfa\x99\x46\x86\x96\xfa\x05\x1b\xd3\xca\x85\x9b\x20\x8f\x8a\x32\xe3\xc2\xb9\x10\x82\xfb\xfe\xb4\x26\xc6\xf4\xd9\xfe\x3f\x0d\x0d\x27\x13\x8e\x11\xc5\x49\x89\x78\xd2\xb8\x0e\x15\xc1\x2f\x16\xce\x7b\x41\xe5\x22\x0d\x8e\x6f\x89\x5b\x54\xe7\xfd\x16\xa5\x1e\x9f\xd6\x68\x57\xd6\x42\x8f\x9a\xa1\x39\x17\x88\xa4\xb6\x3f\xec\xc0\x9a\xcd\x84\x56\x42\xdb\xb1\x74\x94\xd6\xa3\xdf\x29\xf1\x8f\x3f\x99\x23\x35\x74\x33\x61\x42\x10\x53\x9b\x99\x92\xe9\xfe\x96\xb9\x9d\xd3\x1c\xfc\x1f\xb5\x7f\xb8\xb4\xf9\x88\xcf\xec\xee\xf7\xb3\x7d\x0a\x38\x06\x7b\xf0\x78\x1d\xe3\x2c\xdc\x97\x01\xef\x01\x00\x00\xff\xff\x46\xd3\x6d\x9d\x0f\x04\x00\x00")
 
-func traefikYamlBytes() ([]byte, error) {
+func rolebindingsYamlBytes() ([]byte, error) {
 	return bindataRead(
-		_traefikYaml,
-		"traefik.yaml",
+		_rolebindingsYaml,
+		"rolebindings.yaml",
 	)
 }
 
-func traefikYaml() (*asset, error) {
-	bytes, err := traefikYamlBytes()
+func rolebindingsYaml() (*asset, error) {
+	bytes, err := rolebindingsYamlBytes()
 	if err != nil {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "traefik.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	info := bindataFileInfo{name: "rolebindings.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
 	a := &asset{bytes: bytes, info: info}
 	return a, nil
 }
@@ -390,8 +390,8 @@ var _bindata = map[string]func() (*asset, error){
 	"metrics-server/metrics-server-deployment.yaml": metricsServerMetricsServerDeploymentYaml,
 	"metrics-server/metrics-server-service.yaml":    metricsServerMetricsServerServiceYaml,
 	"metrics-server/resource-reader.yaml":           metricsServerResourceReaderYaml,
+	"nginx.yaml":                                    nginxYaml,
 	"rolebindings.yaml":                             rolebindingsYaml,
-	"traefik.yaml":                                  traefikYaml,
 }
 
 // AssetDir returns the file names below a certain
@@ -447,8 +447,8 @@ var _bintree = &bintree{nil, map[string]*bintree{
 		"metrics-server-service.yaml":    &bintree{metricsServerMetricsServerServiceYaml, map[string]*bintree{}},
 		"resource-reader.yaml":           &bintree{metricsServerResourceReaderYaml, map[string]*bintree{}},
 	}},
+	"nginx.yaml":        &bintree{nginxYaml, map[string]*bintree{}},
 	"rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}},
-	"traefik.yaml":      &bintree{traefikYaml, map[string]*bintree{}},
 }}
 
 // RestoreAsset restores an asset under the given directory

pkg/server/server.go

@@ -9,6 +9,7 @@ import (
 	net2 "net"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -33,7 +34,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/net"
 )
 
-const MasterRoleLabelKey = "node-role.kubernetes.io/master"
+const (
+	MasterRoleLabelKey = "node-role.kubernetes.io/master"
+	NginxIngressImage  = "quay.io/kubernetes-ingress-controller/nginx-ingress-controller"
+)
 
 func resolveDataDir(dataDir string) (string, error) {
 	dataDir, err := datadir.Resolve(dataDir)
@@ -170,6 +174,7 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
 		"%{CLUSTER_DNS}%":                controlConfig.ClusterDNS.String(),
 		"%{CLUSTER_DOMAIN}%":             controlConfig.ClusterDomain,
 		"%{DEFAULT_LOCAL_STORAGE_PATH}%": controlConfig.DefaultLocalStoragePath,
+		"%{NGINX_IMAGE}":                 nginxImage(),
 	}
 
 	if err := deploy.Stage(dataDir, templateVars, controlConfig.Skips); err != nil {
@@ -410,3 +415,11 @@ func setMasterRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
 	}
 	return nil
 }
+
+func nginxImage() string {
+	nginxImage := NginxIngressImage
+	if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" {
+		nginxImage = fmt.Sprintf("%s-%s", nginxImage, runtime.GOARCH)
+	}
+	return nginxImage
+}

scripts/airgap/image-list.txt

@@ -1,5 +1,4 @@
 docker.io/coredns/coredns:1.6.3
-docker.io/library/traefik:1.7.19
 docker.io/rancher/klipper-helm:v0.2.3
 docker.io/rancher/klipper-lb:v0.1.2
 docker.io/rancher/local-path-provisioner:v0.0.11

scripts/download

@@ -5,7 +5,7 @@ cd $(dirname $0)/..
 . ./scripts/version.sh
 
 ROOT_VERSION=v0.3.0
-TRAEFIK_VERSION=1.81.0
+NGINX_VERSION=1.33.0
 CHARTS_DIR=build/static/charts
 
 mkdir -p ${CHARTS_DIR}
@@ -17,7 +17,7 @@ for target in iptables iptables-save iptables-restore ip6tables ip6tables-save i
 done
 mkdir -p bin/aux && rm bin/mount && ln -sf ../busybox bin/aux/mount
 
-TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz
-curl -sfL https://kubernetes-charts.storage.googleapis.com/${TRAEFIK_FILE} -o ${CHARTS_DIR}/${TRAEFIK_FILE}
+NGINX_FILE=nginx-ingress-${NGINX_VERSION}.tgz
+curl -sfL https://kubernetes-charts.storage.googleapis.com/${NGINX_FILE} -o ${CHARTS_DIR}/${NGINX_FILE}
 
 cp scripts/wg-add.sh bin/aux/

scripts/package-airgap

@@ -7,4 +7,12 @@ cd $(dirname $0)/..
 
 images=$(cat scripts/airgap/image-list.txt)
 xargs -n1 docker pull <<< "${images}"
-docker save ${images} -o dist/artifacts/k3s-airgap-images-${ARCH}.tar 
+# Add exception for nginx
+NGINX_IMAGE_VERSION="0.30.0"
+NGINX_IMAGE="quay.io/kubernetes-ingress-controller/nginx-ingress-controller"
+ADD_ARCH=""
+if [ "$ARCH" == "arm" ] || [ "$ARCH" == "arm64" ]; then
+  ADD_ARCH="-$ARCH"
+fi
+docker pull "$NGINX_IMAGE$ADD_ARCH:$NGINX_IMAGE_VERSION"
+docker save ${images} -o dist/artifacts/k3s-airgap-images-${ARCH}.tar