From da16869555775cf17d4d97ffaf8a13b70bc738c2 Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Thu, 28 Oct 2021 22:38:22 +0200 Subject: [PATCH] Update to v1.19.16 (#4352) * Update to v1.19.16 Signed-off-by: galal-hussein * Update to v1.19.16 Signed-off-by: galal-hussein --- go.mod | 62 ++++----- go.sum | 86 ++++++------ vendor/k8s.io/client-go/pkg/version/base.go | 6 +- vendor/k8s.io/component-base/version/base.go | 6 +- .../pkg/kubelet/logs/container_log_manager.go | 7 +- .../noderesources/balanced_allocation.go | 12 +- .../pkg/volume/vsphere_volume/attacher.go | 1 - .../legacy-cloud-providers/vsphere/BUILD | 4 + .../vsphere/nodemanager.go | 11 ++ .../vsphere/vclib/utils.go | 9 -- .../legacy-cloud-providers/vsphere/vsphere.go | 30 ++++- .../vsphere/vsphere_util.go | 124 +++++++++++++++++- .../vsphere/vsphere_volume_map.go | 113 ++++++++++++++++ vendor/modules.txt | 92 ++++++------- 14 files changed, 414 insertions(+), 149 deletions(-) create mode 100644 vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_volume_map.go diff --git a/go.mod b/go.mod index 337d34889e..68c28049fa 100644 --- a/go.mod +++ b/go.mod @@ -36,31 +36,31 @@ replace ( google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 google.golang.org/grpc => google.golang.org/grpc v1.27.1 gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.2.2 - k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.15-k3s1 - k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.15-k3s1 - k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.15-k3s1 - k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.15-k3s1 - k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.15-k3s1 - k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.15-k3s1 - k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.15-k3s1 - k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.15-k3s1 - k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.15-k3s1 - k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.15-k3s1 - k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.15-k3s1 - k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.15-k3s1 - k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.15-k3s1 - k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.15-k3s1 - k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.15-k3s1 - k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.15-k3s1 - k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.15-k3s1 - k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.15-k3s1 - k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.19.15-k3s1 - k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.15-k3s1 - k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.15-k3s1 - k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.19.15-k3s1 - k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.19.15-k3s1 - k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.19.15-k3s1 - k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.19.15-k3s1 + k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.16-k3s1 + k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.16-k3s1 + k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.16-k3s1 + k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.16-k3s1 + k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.16-k3s1 + k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.16-k3s1 + k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.16-k3s1 + k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.16-k3s1 + k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.16-k3s1 + k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.16-k3s1 + k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.16-k3s1 + k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.16-k3s1 + k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.16-k3s1 + k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.16-k3s1 + k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.16-k3s1 + k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.16-k3s1 + k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.16-k3s1 + k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.16-k3s1 + k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.19.16-k3s1 + k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.16-k3s1 + k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.16-k3s1 + k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.19.16-k3s1 + k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.19.16-k3s1 + k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.19.16-k3s1 + k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.19.16-k3s1 mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34 ) @@ -113,15 +113,15 @@ require ( golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 google.golang.org/grpc v1.33.2 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.19.15 - k8s.io/apimachinery v0.19.15 - k8s.io/apiserver v0.19.15 + k8s.io/api v0.19.16 + k8s.io/apimachinery v0.19.16 + k8s.io/apiserver v0.19.16 k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible - k8s.io/cloud-provider v0.19.15 - k8s.io/component-base v0.19.15 + k8s.io/cloud-provider v0.19.16 + k8s.io/component-base v0.19.16 k8s.io/cri-api v0.20.1 k8s.io/klog v1.0.0 - k8s.io/kubernetes v1.19.15 + k8s.io/kubernetes v1.19.16 k8s.io/utils v0.0.0-20200729134348-d5654de09c73 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 166bc23710..b28c8200cf 100644 --- a/go.sum +++ b/go.sum @@ -478,49 +478,49 @@ github.com/k3s-io/helm-controller v0.10.8 h1:O7zoqUBp3W+6+nRCUWNiAoQMzOX6xw9IsBD github.com/k3s-io/helm-controller v0.10.8/go.mod h1:nZP8FH3KZrNNUf5r+SwwiMR63HS6lxdHdpHijgPfF74= github.com/k3s-io/kine v0.6.2 h1:1aJTPfB8HG4exqMKFVE5H0z4bepF05tJHtYNXotWXa4= github.com/k3s-io/kine v0.6.2/go.mod h1:rzCs93+rQHZGOiewMd84PDrER92QeZ6eeHbWkfEy4+w= -github.com/k3s-io/kubernetes v1.19.15-k3s1 h1:A1RJfAmS3JinAk0s9TJD5W82wnuuLMC8P88UZutZPQE= -github.com/k3s-io/kubernetes v1.19.15-k3s1/go.mod h1:HD/Q62CbvekHttYG9YffbrTHx54dkZCnF7t3oJCzxOA= -github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.15-k3s1 h1:jDAamlpMbWBKgr7RBCYLEyUFjXFxehzrvuEoHxzXwDw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.15-k3s1/go.mod h1:UcjwtWbtWvJiBBYsrqZFSZyK9yfDQvyyOaitZREMNNI= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.15-k3s1 h1:HLTzkYFQf56sB9nAXm6+cjcS5hFcifxGa+eZf4Qi7sk= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.15-k3s1/go.mod h1:WgS2vy/M8pNm2Ym++3tDRbJprDEFUJ/eqkzdSHS5j+8= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.15-k3s1 h1:Y7gTGV8XcjnLlznxO6EZLU1f3neqr4gm69kTFPu1tI0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.15-k3s1/go.mod h1:sr8wM2JwYaIf+oYmE1Iv8YztGo79Kd52yrmWlisz0cw= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.15-k3s1 h1:cmUfZmlLjMztUJ0txBklhM96xaGQYZ3vagKKo3Ql8+U= -github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.15-k3s1/go.mod h1:RGWpAhYP+eF18BA1b7/Gf2l/NFnlhVMrUgo3wdKNZW0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.15-k3s1 h1:c9VAPTWK+E4fGuKtvBTmbPzEkIgoN6LAX8LNP4wb160= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.15-k3s1/go.mod h1:twd45pbv7psOvyGxI8eABhpeoXWW3bCX6aB5NVS6TXs= -github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.15-k3s1 h1:VbD6msu2nceRQj5jwRKWSv3lD0j2MTchgFpi+1lMvWA= -github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.15-k3s1/go.mod h1:mFyJ7Q7VaffgOf2fBYWE3u6tI6ORgNH3y86VrIPy++Y= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.15-k3s1 h1:fX5/q0XvcWY1ZPox96YP/BwrAgX6sJi4CLkD6W+93i4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.15-k3s1/go.mod h1:vXBe7m69RSxIR/m6bm820O+WYUJHP9OXtEyTZf3twRo= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.15-k3s1 h1:BzxNh45impklUwhmwABqPzwKYrizCMrCzQRqNpWDF+o= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.15-k3s1/go.mod h1:kEUR4nHaGTacDcHrY2P4IriUdykXxPuwBg7picxa+gk= -github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.15-k3s1 h1:Z+bMLSOZX2j/qmZzwNmk/jpGBWLa/VXrOi1l+sKqRL0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.15-k3s1/go.mod h1:ybBf1oAbFmYzk+mZE78A7W5GStT+ypahDaWnjlHrkGQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.15-k3s1 h1:fmLp/ilzl3Q8tvEaDiAfucTemoROm5Vs+gC+eBRqeJE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.15-k3s1/go.mod h1:jR+bJp7erYNUmcS7lWDm404aFVonltWE56LV8CuqKyg= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.15-k3s1 h1:5FaRGDTx8/43s3v/hV50fFhAvHPEtngAvLQd3Gi42e0= -github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.15-k3s1/go.mod h1:u/3V4xqf6jIp7eOZRZwzmiXUsRTt+XoKjivv6O9gZAg= -github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.15-k3s1 h1:0BOk679c/h92w7QngNerhBDnKPyAmBEkVjkjuFTqWzY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.15-k3s1/go.mod h1:adA9bBHXD+K7tPn7kTpjQ3VcUzK6PFgAdNEofr4fEx4= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.15-k3s1 h1:7s9aSG9b64nHxiitUmDBEk9wPL9Zxn3rDDUCb0cMMzY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.15-k3s1/go.mod h1:s5+c+ujL0WM82kzxP4CedFV+vn31pD7WJRM9+GYTk7Q= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.15-k3s1 h1:BcUSzO87JBF9fFTQv0UnwrBfI4q/lMLJXLiavz4OtIc= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.15-k3s1/go.mod h1:6CzB8GMVD+ZlCXZjHiqccHpC9FFlRTCz+rHd176Lxfc= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.15-k3s1 h1:wBcdViVubLJyErTuCMpBg/2cg8KRUwGTxdrLiW6yVcY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.15-k3s1/go.mod h1:PWMBqO9xuXWJS8REJ8QWiouJzbiOwVVVT81ZTwYb2Nk= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.15-k3s1 h1:Z1n1V9UyrUQc0F4foxuFl5l/5+CpdRCwBgw1mAfQV6c= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.15-k3s1/go.mod h1:23iT4D9thFRxYGZ9DOa7zQ4ZyrFHyp+Bz5dXiruHNwI= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.15-k3s1 h1:aMtyY2q3TH+tpxrlqhVfZVN2D1/9su/p5pAZuk+u8iE= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.15-k3s1/go.mod h1:UBqQ69FVXyP8Wpzo+Ni5LUZhwjzJCPSb+6eG8naeDKM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.15-k3s1 h1:e4pXeqg7F2lsrFwq9u70gKCEumsBQiRDSUGl85B5jrQ= -github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.15-k3s1/go.mod h1:fuR1PhKU79SVjnVW43hQGrj6FKfNEZQ3iFU0IZCYFbY= -github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.15-k3s1 h1:6EA6qFhHwrXvhR+1OqeoAyjGpvm3+U76Sx4VRKLq/yM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.15-k3s1/go.mod h1:iv4u51XYDkRdyvp7BBP+KuQ+ZHjEjoCECFVzMJBDGZA= -github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.15-k3s1 h1:Qn054HXGQqwktMGLdnmXZfKnZqpJ+jfTCYbav21HwGU= -github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.15-k3s1/go.mod h1:6b8BmSxdcCMv3dXmS5BWv5biBmAX9BLmylPttQ0T+iM= -github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.19.15-k3s1/go.mod h1:tl3x2SPSVsIqnioD4z87jXFemilbRh1EYDm3KirMsjI= +github.com/k3s-io/kubernetes v1.19.16-k3s1 h1:2mgwqyhs4W8PyDlJUwiY3g54yrtpGkOTfKNZFK8FMQw= +github.com/k3s-io/kubernetes v1.19.16-k3s1/go.mod h1:HD/Q62CbvekHttYG9YffbrTHx54dkZCnF7t3oJCzxOA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.16-k3s1 h1:2XGBKqK13zEBp61XrV5n+5K+5iSBZt+1fuioCGcuPGM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.16-k3s1/go.mod h1:UcjwtWbtWvJiBBYsrqZFSZyK9yfDQvyyOaitZREMNNI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.16-k3s1 h1:iNhmC3umVy8rz3sV+GWsp4456nPc9uIdKIk3xsmoIYg= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.16-k3s1/go.mod h1:WgS2vy/M8pNm2Ym++3tDRbJprDEFUJ/eqkzdSHS5j+8= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.16-k3s1 h1:MR19qcTKM89xAnRY4VthmS53w6I2b/2D4plgOyp/qbE= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.16-k3s1/go.mod h1:sr8wM2JwYaIf+oYmE1Iv8YztGo79Kd52yrmWlisz0cw= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.16-k3s1 h1:0i8W2WrTGjdqv5f88k26CheRhq0NtbUHTZPcOuSx1bQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.16-k3s1/go.mod h1:RGWpAhYP+eF18BA1b7/Gf2l/NFnlhVMrUgo3wdKNZW0= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.16-k3s1 h1:OK4sdktPrcMMF+yCZLylp9VzjZb/fd1gUVCaL7QXRaY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.16-k3s1/go.mod h1:twd45pbv7psOvyGxI8eABhpeoXWW3bCX6aB5NVS6TXs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.16-k3s1 h1:GXItOz+wn9OhAl/lxZEJPMTqSx4CEwXNp3Z7noD0NoM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.16-k3s1/go.mod h1:mFyJ7Q7VaffgOf2fBYWE3u6tI6ORgNH3y86VrIPy++Y= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.16-k3s1 h1:HIE75qbdgxMEVlH2PtseUjW/6ZLgRd4+90yQeZnNS0Q= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.16-k3s1/go.mod h1:vXBe7m69RSxIR/m6bm820O+WYUJHP9OXtEyTZf3twRo= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.16-k3s1 h1:+Eo2nsHUC+yxhRgBwhX4EfdQ1hjxpgKFVTfEnelreLs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.16-k3s1/go.mod h1:kEUR4nHaGTacDcHrY2P4IriUdykXxPuwBg7picxa+gk= +github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.16-k3s1 h1:9ZAkU+P9UmikjFm7CvPdBsb09zFYTvI/zlQnaNdWK9k= +github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.16-k3s1/go.mod h1:ybBf1oAbFmYzk+mZE78A7W5GStT+ypahDaWnjlHrkGQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.16-k3s1 h1:A6o6hVR58uwwgVt+b2Y2j1oyY0eTeYA9o1mHmmLLYM4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.16-k3s1/go.mod h1:jR+bJp7erYNUmcS7lWDm404aFVonltWE56LV8CuqKyg= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.16-k3s1 h1:Wh//jbY24IHG5gODhRfxH+KJ4umKliOoAkEK/58GHDQ= +github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.16-k3s1/go.mod h1:u/3V4xqf6jIp7eOZRZwzmiXUsRTt+XoKjivv6O9gZAg= +github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.16-k3s1 h1:PQPM3UK+J+jmI99ZQrgizyORFFfEi7l6CD5WHFO9EkY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.16-k3s1/go.mod h1:adA9bBHXD+K7tPn7kTpjQ3VcUzK6PFgAdNEofr4fEx4= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.16-k3s1 h1:wTV1/cLr0GmvhIoDE9ddPwtArbn2pRPvZNdMxZCgz1M= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.16-k3s1/go.mod h1:s5+c+ujL0WM82kzxP4CedFV+vn31pD7WJRM9+GYTk7Q= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.16-k3s1 h1:8yNmQoY1Qlb33nXcO4+k3F8V5Y0HXH9GZwbJjctMAbo= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.16-k3s1/go.mod h1:6CzB8GMVD+ZlCXZjHiqccHpC9FFlRTCz+rHd176Lxfc= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.16-k3s1 h1:YIyuFlB5gE+0JDIWeoaiJf+PYkOPURzxhyQK+nY2XiI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.16-k3s1/go.mod h1:PWMBqO9xuXWJS8REJ8QWiouJzbiOwVVVT81ZTwYb2Nk= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.16-k3s1 h1:bg5Svyv69VhbezdCQ4ZLxES+Mbl6CI/pKVicgeC0/qA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.16-k3s1/go.mod h1:23iT4D9thFRxYGZ9DOa7zQ4ZyrFHyp+Bz5dXiruHNwI= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.16-k3s1 h1:viencGw49FD87sKi5FsRzCefcShyF7VBJP6eE5an5hs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.16-k3s1/go.mod h1:UBqQ69FVXyP8Wpzo+Ni5LUZhwjzJCPSb+6eG8naeDKM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.16-k3s1 h1:QfZzB8qJoS/1PNurBo8vSGoKlrXxSnvL9TnktcU3n88= +github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.16-k3s1/go.mod h1:fuR1PhKU79SVjnVW43hQGrj6FKfNEZQ3iFU0IZCYFbY= +github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.16-k3s1 h1:BzAw2RzlLx2g8ng4Nc/OMBQBfZz6QgkFZCv6GlpX1Sk= +github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.16-k3s1/go.mod h1:iv4u51XYDkRdyvp7BBP+KuQ+ZHjEjoCECFVzMJBDGZA= +github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.16-k3s1 h1:B1Q3cuchyuxBxCnvqvCLlyuleCVi+gVIalTbdx9w5Vs= +github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.16-k3s1/go.mod h1:6b8BmSxdcCMv3dXmS5BWv5biBmAX9BLmylPttQ0T+iM= +github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.19.16-k3s1/go.mod h1:tl3x2SPSVsIqnioD4z87jXFemilbRh1EYDm3KirMsjI= github.com/karrick/godirwalk v1.7.5 h1:VbzFqwXwNbAZoA6W5odrLr+hKK197CcENcPh6E/gJ0M= github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= diff --git a/vendor/k8s.io/client-go/pkg/version/base.go b/vendor/k8s.io/client-go/pkg/version/base.go index ebfb16e1d7..8b3b4abbe5 100644 --- a/vendor/k8s.io/client-go/pkg/version/base.go +++ b/vendor/k8s.io/client-go/pkg/version/base.go @@ -3,8 +3,8 @@ package version var ( gitMajor = "1" gitMinor = "19" - gitVersion = "v1.19.15-k3s1" - gitCommit = "a14bf227db31db89a172864ca2a934ff1c637eef" + gitVersion = "v1.19.16-k3s1" + gitCommit = "0c01d3bc4234649f6d071c70081997779f974d9c" gitTreeState = "clean" - buildDate = "2021-09-16T15:13:53Z" + buildDate = "2021-10-28T17:29:36Z" ) diff --git a/vendor/k8s.io/component-base/version/base.go b/vendor/k8s.io/component-base/version/base.go index ebfb16e1d7..8b3b4abbe5 100644 --- a/vendor/k8s.io/component-base/version/base.go +++ b/vendor/k8s.io/component-base/version/base.go @@ -3,8 +3,8 @@ package version var ( gitMajor = "1" gitMinor = "19" - gitVersion = "v1.19.15-k3s1" - gitCommit = "a14bf227db31db89a172864ca2a934ff1c637eef" + gitVersion = "v1.19.16-k3s1" + gitCommit = "0c01d3bc4234649f6d071c70081997779f974d9c" gitTreeState = "clean" - buildDate = "2021-09-16T15:13:53Z" + buildDate = "2021-10-28T17:29:36Z" ) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/logs/container_log_manager.go b/vendor/k8s.io/kubernetes/pkg/kubelet/logs/container_log_manager.go index 56e4f31169..d746d258f0 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/logs/container_log_manager.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/logs/container_log_manager.go @@ -138,9 +138,6 @@ func parseMaxSize(size string) (int64, error) { if !ok { return 0, fmt.Errorf("invalid max log size") } - if maxSize < 0 { - return 0, fmt.Errorf("negative max log size %d", maxSize) - } return maxSize, nil } @@ -161,6 +158,10 @@ func NewContainerLogManager(runtimeService internalapi.RuntimeService, osInterfa if err != nil { return nil, fmt.Errorf("failed to parse container log max size %q: %v", maxSize, err) } + // Negative number means to disable container log rotation + if parsedMaxSize < 0 { + return NewStubContainerLogManager(), nil + } // policy LogRotatePolicy return &containerLogManager{ osInterface: osInterface, diff --git a/vendor/k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go b/vendor/k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go index 3da8884333..95495e6d18 100644 --- a/vendor/k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go +++ b/vendor/k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go @@ -80,16 +80,12 @@ func NewBalancedAllocation(_ runtime.Object, h framework.FrameworkHandle) (frame // todo: use resource weights in the scorer function func balancedResourceScorer(requested, allocable resourceToValueMap, includeVolumes bool, requestedVolumes int, allocatableVolumes int) int64 { - // This to find a node which has most balanced CPU, memory and volume usage. cpuFraction := fractionOfCapacity(requested[v1.ResourceCPU], allocable[v1.ResourceCPU]) memoryFraction := fractionOfCapacity(requested[v1.ResourceMemory], allocable[v1.ResourceMemory]) - // fractions might be greater than 1 because pods with no requests get minimum - // values. - if cpuFraction > 1 { - cpuFraction = 1 - } - if memoryFraction > 1 { - memoryFraction = 1 + // This to find a node which has most balanced CPU, memory and volume usage. + if cpuFraction >= 1 || memoryFraction >= 1 { + // if requested >= capacity, the corresponding host should never be preferred. + return 0 } if includeVolumes && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && allocatableVolumes > 0 { diff --git a/vendor/k8s.io/kubernetes/pkg/volume/vsphere_volume/attacher.go b/vendor/k8s.io/kubernetes/pkg/volume/vsphere_volume/attacher.go index 178d8172f9..5ca699fca1 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/vsphere_volume/attacher.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/vsphere_volume/attacher.go @@ -275,7 +275,6 @@ func (plugin *vsphereVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, // Detach the given device from the given node. func (detacher *vsphereVMDKDetacher) Detach(volumeName string, nodeName types.NodeName) error { - volPath := getVolPathfromVolumeName(volumeName) attached, newVolumePath, err := detacher.vsphereVolumes.DiskIsAttached(volPath, nodeName) if err != nil { diff --git a/vendor/k8s.io/legacy-cloud-providers/vsphere/BUILD b/vendor/k8s.io/legacy-cloud-providers/vsphere/BUILD index 6dd239dd9c..7811251a4f 100644 --- a/vendor/k8s.io/legacy-cloud-providers/vsphere/BUILD +++ b/vendor/k8s.io/legacy-cloud-providers/vsphere/BUILD @@ -18,6 +18,7 @@ go_library( "vsphere_util_linux.go", "vsphere_util_unsupported.go", "vsphere_util_windows.go", + "vsphere_volume_map.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/vsphere", importpath = "k8s.io/legacy-cloud-providers/vsphere", @@ -32,6 +33,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/node/helpers:go_default_library", + "//staging/src/k8s.io/cloud-provider/volume/errors:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers:go_default_library", @@ -55,6 +57,7 @@ go_test( "credentialmanager_test.go", "vsphere_test.go", "vsphere_util_test.go", + "vsphere_volume_map_test.go", ], embed = [":go_default_library"], deps = [ @@ -71,6 +74,7 @@ go_test( "//vendor/github.com/vmware/govmomi:go_default_library", "//vendor/github.com/vmware/govmomi/find:go_default_library", "//vendor/github.com/vmware/govmomi/lookup/simulator:go_default_library", + "//vendor/github.com/vmware/govmomi/object:go_default_library", "//vendor/github.com/vmware/govmomi/property:go_default_library", "//vendor/github.com/vmware/govmomi/simulator:go_default_library", "//vendor/github.com/vmware/govmomi/simulator/vpx:go_default_library", diff --git a/vendor/k8s.io/legacy-cloud-providers/vsphere/nodemanager.go b/vendor/k8s.io/legacy-cloud-providers/vsphere/nodemanager.go index 196f776599..5fa1385012 100644 --- a/vendor/k8s.io/legacy-cloud-providers/vsphere/nodemanager.go +++ b/vendor/k8s.io/legacy-cloud-providers/vsphere/nodemanager.go @@ -311,6 +311,17 @@ func (nm *NodeManager) GetNodeDetails() ([]NodeDetails, error) { return nodeDetails, nil } +// GetNodeNames returns list of nodes that are known to vsphere cloudprovider. +// These are typically nodes that make up k8s cluster. +func (nm *NodeManager) GetNodeNames() []k8stypes.NodeName { + nodes := nm.getNodes() + var nodeNameList []k8stypes.NodeName + for _, node := range nodes { + nodeNameList = append(nodeNameList, k8stypes.NodeName(node.Name)) + } + return nodeNameList +} + func (nm *NodeManager) refreshNodes() (errList []error) { for nodeName := range nm.getNodes() { nodeInfo, err := nm.getRefreshedNodeInfo(convertToK8sType(nodeName)) diff --git a/vendor/k8s.io/legacy-cloud-providers/vsphere/vclib/utils.go b/vendor/k8s.io/legacy-cloud-providers/vsphere/vclib/utils.go index 8ff97771a9..86dd0ea6d5 100644 --- a/vendor/k8s.io/legacy-cloud-providers/vsphere/vclib/utils.go +++ b/vendor/k8s.io/legacy-cloud-providers/vsphere/vclib/utils.go @@ -24,7 +24,6 @@ import ( "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" "k8s.io/klog/v2" @@ -181,14 +180,6 @@ func IsInvalidCredentialsError(err error) bool { return isInvalidCredentialsError } -// VerifyVolumePathsForVM verifies if the volume paths (volPaths) are attached to VM. -func VerifyVolumePathsForVM(vmMo mo.VirtualMachine, volPaths []string, nodeName string, nodeVolumeMap map[string]map[string]bool) { - // Verify if the volume paths are present on the VM backing virtual disk devices - vmDevices := object.VirtualDeviceList(vmMo.Config.Hardware.Device) - VerifyVolumePathsForVMDevices(vmDevices, volPaths, nodeName, nodeVolumeMap) - -} - // VerifyVolumePathsForVMDevices verifies if the volume paths (volPaths) are attached to VM. func VerifyVolumePathsForVMDevices(vmDevices object.VirtualDeviceList, volPaths []string, nodeName string, nodeVolumeMap map[string]map[string]bool) { volPathsMap := make(map[string]bool) diff --git a/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere.go b/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere.go index 7376d04ca8..ad06b98f79 100644 --- a/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere.go +++ b/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "net" "net/url" "os" @@ -48,6 +49,7 @@ import ( "k8s.io/client-go/tools/cache" cloudprovider "k8s.io/cloud-provider" nodehelpers "k8s.io/cloud-provider/node/helpers" + volerr "k8s.io/cloud-provider/volume/errors" volumehelpers "k8s.io/cloud-provider/volume/helpers" "k8s.io/klog/v2" @@ -95,6 +97,7 @@ type VSphere struct { hostName string // Maps the VSphere IP address to VSphereInstance vsphereInstanceMap map[string]*VSphereInstance + vsphereVolumeMap *VsphereVolumeMap // Responsible for managing discovery of k8s node, their location etc. nodeManager *NodeManager vmUUID string @@ -542,6 +545,7 @@ func buildVSphereFromConfig(cfg VSphereConfig) (*VSphere, error) { nodeInfoMap: make(map[string]*NodeInfo), registeredNodes: make(map[string]*v1.Node), }, + vsphereVolumeMap: NewVsphereVolumeMap(), isSecretInfoProvided: isSecretInfoProvided, isSecretManaged: !cfg.Global.SecretNotManaged, cfg: &cfg, @@ -950,6 +954,20 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyName string, nodeN } } klog.V(4).Infof("AttachDisk executed for node %s and volume %s with diskUUID %s. Err: %s", convertToString(nodeName), vmDiskPath, diskUUID, err) + if err != nil { + // if attach failed, we should check if disk is attached somewhere else. This can happen for several reasons + // and throwing a dangling volume error here will allow attach-detach controller to detach disk from a node + // where it is not needed. + existingNode, ok := vs.vsphereVolumeMap.CheckForVolume(vmDiskPath) + if ok { + attached, newVolumePath, diskAttachedError := vs.DiskIsAttached(vmDiskPath, existingNode) + // if disk is attached somewhere else then we can throw a dangling error + if diskAttachedError == nil && attached && (nodeName != existingNode) { + klog.V(3).Infof("found dangling volume %s to node %s", vmDiskPath, existingNode) + return "", volerr.NewDanglingError(err.Error(), existingNode, newVolumePath) + } + } + } vclib.RecordvSphereMetric(vclib.OperationAttachVolume, requestTime, err) return diskUUID, err } @@ -1083,6 +1101,7 @@ func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (b // 5b. If VMs are removed from vSphere inventory they are ignored. func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) (map[k8stypes.NodeName]map[string]bool, error) { disksAreAttachedInternal := func(nodeVolumes map[k8stypes.NodeName][]string) (map[k8stypes.NodeName]map[string]bool, error) { + vs.vsphereVolumeMap.StartDiskVerification() // disksAreAttach checks whether disks are attached to the nodes. // Returns nodes that need to be retried if retry is true @@ -1194,8 +1213,17 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) for nodeName, volPaths := range attached { disksAttached[convertToK8sType(nodeName)] = volPaths } + } - klog.V(4).Infof("DisksAreAttach successfully executed. result: %+v", attached) + klog.V(4).Infof("DisksAreAttached successfully executed. result: %+v", attached) + // There could be nodes in cluster which do not have any pods with vsphere volumes running on them + // such nodes won't be part of nodeVolumes map because attach-detach controller does not keep track + // such nodes. But such nodes may still have dangling volumes on them and hence we need to scan all the + // remaining nodes which weren't scanned by code previously. + vs.BuildMissingVolumeNodeMap(ctx) + // any volume which we could not verify will be removed from the map. + vs.vsphereVolumeMap.RemoveUnverified() + klog.V(4).Infof("current node volume map is: %+v", vs.vsphereVolumeMap.volumeNodeMap) return disksAttached, nil } requestTime := time.Now() diff --git a/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_util.go b/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_util.go index a207867289..f8eb7a86c0 100644 --- a/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_util.go +++ b/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_util.go @@ -26,6 +26,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "github.com/vmware/govmomi/find" @@ -569,6 +570,7 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN return nodesToRetry, err } klog.V(4).Infof("Verifying Volume Paths by devices for node %s and VM %s", nodeName, nodeInfo.vm) + vs.vsphereVolumeMap.Add(nodeName, devices) vclib.VerifyVolumePathsForVMDevices(devices, nodeVolumes[nodeName], convertToString(nodeName), attached) } } @@ -599,11 +601,131 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN } nodeUUID = strings.ToLower(nodeUUID) klog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %v", nodeName, nodeUUID, vmMoMap) - vclib.VerifyVolumePathsForVM(vmMoMap[nodeUUID], nodeVolumes[nodeName], convertToString(nodeName), attached) + vmMo := vmMoMap[nodeUUID] + vmDevices := object.VirtualDeviceList(vmMo.Config.Hardware.Device) + vs.vsphereVolumeMap.Add(nodeName, vmDevices) + vclib.VerifyVolumePathsForVMDevices(vmDevices, nodeVolumes[nodeName], convertToString(nodeName), attached) } return nodesToRetry, nil } +// BuildMissingVolumeNodeMap builds a map of volumes and nodes which are not known to attach detach controller. +// There could be nodes in cluster which do not have any pods with vsphere volumes running on them +// such nodes won't be part of disk verification check because attach-detach controller does not keep track +// such nodes. But such nodes may still have dangling volumes on them and hence we need to scan all the +// remaining nodes which weren't scanned by code previously. +func (vs *VSphere) BuildMissingVolumeNodeMap(ctx context.Context) { + nodeNames := vs.nodeManager.GetNodeNames() + // Segregate nodes according to VC-DC + dcNodes := make(map[string][]k8stypes.NodeName) + + for _, nodeName := range nodeNames { + // if given node is not in node volume map + if !vs.vsphereVolumeMap.CheckForNode(nodeName) { + nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName) + if err != nil { + klog.V(4).Infof("Failed to get node info: %+v. err: %+v", nodeInfo.vm, err) + continue + } + vcDC := nodeInfo.vcServer + nodeInfo.dataCenter.String() + dcNodes[vcDC] = append(dcNodes[vcDC], nodeName) + } + } + + var wg sync.WaitGroup + + for _, nodeNames := range dcNodes { + // Start go routines per VC-DC to check disks are attached + wg.Add(1) + go func(nodes []k8stypes.NodeName) { + err := vs.checkNodeDisks(ctx, nodes) + if err != nil { + klog.Errorf("Failed to check disk attached for nodes: %+v. err: %+v", nodes, err) + } + wg.Done() + }(nodeNames) + } + wg.Wait() +} + +func (vs *VSphere) checkNodeDisks(ctx context.Context, nodeNames []k8stypes.NodeName) error { + var vmList []*vclib.VirtualMachine + var nodeInfo NodeInfo + var err error + + for _, nodeName := range nodeNames { + nodeInfo, err = vs.nodeManager.GetNodeInfo(nodeName) + if err != nil { + return err + } + vmList = append(vmList, nodeInfo.vm) + } + + // Making sure session is valid + _, err = vs.getVSphereInstanceForServer(nodeInfo.vcServer, ctx) + if err != nil { + return err + } + + // If any of the nodes are not present property collector query will fail for entire operation + vmMoList, err := nodeInfo.dataCenter.GetVMMoList(ctx, vmList, []string{"config.hardware.device", "name", "config.uuid"}) + if err != nil { + if vclib.IsManagedObjectNotFoundError(err) { + klog.V(4).Infof("checkNodeDisks: ManagedObjectNotFound for property collector query for nodes: %+v vms: %+v", nodeNames, vmList) + // Property Collector Query failed + // VerifyVolumePaths per VM + for _, nodeName := range nodeNames { + nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName) + if err != nil { + return err + } + devices, err := nodeInfo.vm.VirtualMachine.Device(ctx) + if err != nil { + if vclib.IsManagedObjectNotFoundError(err) { + klog.V(4).Infof("checkNodeDisks: ManagedObjectNotFound for Kubernetes node: %s with vSphere Virtual Machine reference: %v", nodeName, nodeInfo.vm) + continue + } + return err + } + klog.V(4).Infof("Verifying Volume Paths by devices for node %s and VM %s", nodeName, nodeInfo.vm) + vs.vsphereVolumeMap.Add(nodeName, devices) + } + return nil + } + return err + } + + vmMoMap := make(map[string]mo.VirtualMachine) + for _, vmMo := range vmMoList { + if vmMo.Config == nil { + klog.Errorf("Config is not available for VM: %q", vmMo.Name) + continue + } + klog.V(9).Infof("vmMoMap vmname: %q vmuuid: %s", vmMo.Name, strings.ToLower(vmMo.Config.Uuid)) + vmMoMap[strings.ToLower(vmMo.Config.Uuid)] = vmMo + } + + klog.V(9).Infof("vmMoMap: +%v", vmMoMap) + + for _, nodeName := range nodeNames { + node, err := vs.nodeManager.GetNode(nodeName) + if err != nil { + return err + } + nodeUUID, err := GetNodeUUID(&node) + if err != nil { + klog.Errorf("Node Discovery failed to get node uuid for node %s with error: %v", node.Name, err) + return err + } + nodeUUID = strings.ToLower(nodeUUID) + klog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %v", nodeName, nodeUUID, vmMoMap) + vmMo := vmMoMap[nodeUUID] + vmDevices := object.VirtualDeviceList(vmMo.Config.Hardware.Device) + vs.vsphereVolumeMap.Add(nodeName, vmDevices) + } + return nil +} + func (vs *VSphere) GetNodeNameFromProviderID(providerID string) (string, error) { var nodeName string nodes, err := vs.nodeManager.GetNodeDetails() diff --git a/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_volume_map.go b/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_volume_map.go new file mode 100644 index 0000000000..6895bf9028 --- /dev/null +++ b/vendor/k8s.io/legacy-cloud-providers/vsphere/vsphere_volume_map.go @@ -0,0 +1,113 @@ +// +build !providerless + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "sync" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + k8stypes "k8s.io/apimachinery/pkg/types" +) + +type volumePath string + +type nodeVolumeStatus struct { + nodeName k8stypes.NodeName + verified bool +} + +// VsphereVolumeMap stores last known state of node and volume mapping +type VsphereVolumeMap struct { + volumeNodeMap map[volumePath]nodeVolumeStatus + nodeMap map[k8stypes.NodeName]bool + lock sync.RWMutex +} + +func NewVsphereVolumeMap() *VsphereVolumeMap { + return &VsphereVolumeMap{ + volumeNodeMap: map[volumePath]nodeVolumeStatus{}, + nodeMap: map[k8stypes.NodeName]bool{}, + } +} + +// StartDiskVerification marks all known volumes as unverified so as +// disks which aren't verified can be removed at the end of verification process +func (vsphereVolume *VsphereVolumeMap) StartDiskVerification() { + vsphereVolume.lock.Lock() + defer vsphereVolume.lock.Unlock() + for k, v := range vsphereVolume.volumeNodeMap { + v.verified = false + vsphereVolume.volumeNodeMap[k] = v + } + // reset nodeMap to empty so that any node we could not verify via usual verification process + // can still be verified. + vsphereVolume.nodeMap = map[k8stypes.NodeName]bool{} +} + +// CheckForVolume verifies if disk is attached to some node in the cluster. +// This check is not definitive and should be followed up by separate verification. +func (vsphereVolume *VsphereVolumeMap) CheckForVolume(path string) (k8stypes.NodeName, bool) { + vsphereVolume.lock.RLock() + defer vsphereVolume.lock.RUnlock() + vPath := volumePath(path) + ns, ok := vsphereVolume.volumeNodeMap[vPath] + if ok { + return ns.nodeName, true + } + return "", false +} + +// CheckForNode returns true if given node has already been processed by volume +// verification mechanism. This is used to skip verifying attached disks on nodes +// which were previously verified. +func (vsphereVolume *VsphereVolumeMap) CheckForNode(nodeName k8stypes.NodeName) bool { + vsphereVolume.lock.RLock() + defer vsphereVolume.lock.RUnlock() + _, ok := vsphereVolume.nodeMap[nodeName] + return ok +} + +// Add all devices found on a node to the device map +func (vsphereVolume *VsphereVolumeMap) Add(node k8stypes.NodeName, vmDevices object.VirtualDeviceList) { + vsphereVolume.lock.Lock() + defer vsphereVolume.lock.Unlock() + for _, device := range vmDevices { + if vmDevices.TypeName(device) == "VirtualDisk" { + virtualDevice := device.GetVirtualDevice() + if backing, ok := virtualDevice.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { + filename := volumePath(backing.FileName) + vsphereVolume.volumeNodeMap[filename] = nodeVolumeStatus{node, true} + vsphereVolume.nodeMap[node] = true + } + } + } +} + +// RemoveUnverified will remove any device which we could not verify to be attached to a node. +func (vsphereVolume *VsphereVolumeMap) RemoveUnverified() { + vsphereVolume.lock.Lock() + defer vsphereVolume.lock.Unlock() + for k, v := range vsphereVolume.volumeNodeMap { + if !v.verified { + delete(vsphereVolume.volumeNodeMap, k) + delete(vsphereVolume.nodeMap, v.nodeName) + } + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 91fed33837..bbc6fdea9d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1378,7 +1378,7 @@ gopkg.in/warnings.v0 gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 gopkg.in/yaml.v3 -# k8s.io/api v0.19.15 => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.15-k3s1 +# k8s.io/api v0.19.16 => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.16-k3s1 ## explicit k8s.io/api/admission/v1 k8s.io/api/admission/v1beta1 @@ -1424,7 +1424,7 @@ k8s.io/api/settings/v1alpha1 k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 -# k8s.io/apiextensions-apiserver v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.15-k3s1 +# k8s.io/apiextensions-apiserver v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.16-k3s1 k8s.io/apiextensions-apiserver/pkg/apihelpers k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install @@ -1464,7 +1464,7 @@ k8s.io/apiextensions-apiserver/pkg/generated/openapi k8s.io/apiextensions-apiserver/pkg/registry/customresource k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition -# k8s.io/apimachinery v0.19.15 => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.15-k3s1 +# k8s.io/apimachinery v0.19.16 => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.16-k3s1 ## explicit k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -1528,7 +1528,7 @@ k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/netutil k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.19.15 => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.15-k3s1 +# k8s.io/apiserver v0.19.16 => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.16-k3s1 ## explicit k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/admission/configuration @@ -1656,7 +1656,7 @@ k8s.io/apiserver/plugin/pkg/audit/webhook k8s.io/apiserver/plugin/pkg/authenticator/token/oidc k8s.io/apiserver/plugin/pkg/authenticator/token/webhook k8s.io/apiserver/plugin/pkg/authorizer/webhook -# k8s.io/cli-runtime v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.15-k3s1 +# k8s.io/cli-runtime v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.16-k3s1 k8s.io/cli-runtime/pkg/genericclioptions k8s.io/cli-runtime/pkg/kustomize k8s.io/cli-runtime/pkg/kustomize/k8sdeps @@ -1669,7 +1669,7 @@ k8s.io/cli-runtime/pkg/kustomize/k8sdeps/transformer/patch k8s.io/cli-runtime/pkg/kustomize/k8sdeps/validator k8s.io/cli-runtime/pkg/printers k8s.io/cli-runtime/pkg/resource -# k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.15-k3s1 +# k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.16-k3s1 ## explicit k8s.io/client-go/discovery k8s.io/client-go/discovery/cached @@ -1908,7 +1908,7 @@ k8s.io/client-go/util/jsonpath k8s.io/client-go/util/keyutil k8s.io/client-go/util/retry k8s.io/client-go/util/workqueue -# k8s.io/cloud-provider v0.19.15 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.15-k3s1 +# k8s.io/cloud-provider v0.19.16 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.16-k3s1 ## explicit k8s.io/cloud-provider k8s.io/cloud-provider/api @@ -1921,13 +1921,13 @@ k8s.io/cloud-provider/service/helpers k8s.io/cloud-provider/volume k8s.io/cloud-provider/volume/errors k8s.io/cloud-provider/volume/helpers -# k8s.io/cluster-bootstrap v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.15-k3s1 +# k8s.io/cluster-bootstrap v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.16-k3s1 k8s.io/cluster-bootstrap/token/api k8s.io/cluster-bootstrap/token/jws k8s.io/cluster-bootstrap/token/util k8s.io/cluster-bootstrap/util/secrets k8s.io/cluster-bootstrap/util/tokens -# k8s.io/code-generator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.15-k3s1 +# k8s.io/code-generator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.16-k3s1 k8s.io/code-generator/cmd/client-gen/args k8s.io/code-generator/cmd/client-gen/generators k8s.io/code-generator/cmd/client-gen/generators/fake @@ -1942,7 +1942,7 @@ k8s.io/code-generator/cmd/lister-gen/args k8s.io/code-generator/cmd/lister-gen/generators k8s.io/code-generator/pkg/namer k8s.io/code-generator/pkg/util -# k8s.io/component-base v0.19.15 => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.15-k3s1 +# k8s.io/component-base v0.19.16 => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.16-k3s1 ## explicit k8s.io/component-base/cli/flag k8s.io/component-base/cli/globalflag @@ -1966,11 +1966,11 @@ k8s.io/component-base/metrics/testutil k8s.io/component-base/term k8s.io/component-base/version k8s.io/component-base/version/verflag -# k8s.io/cri-api v0.20.1 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.15-k3s1 +# k8s.io/cri-api v0.20.1 => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.16-k3s1 ## explicit k8s.io/cri-api/pkg/apis k8s.io/cri-api/pkg/apis/runtime/v1alpha2 -# k8s.io/csi-translation-lib v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.15-k3s1 +# k8s.io/csi-translation-lib v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.16-k3s1 k8s.io/csi-translation-lib k8s.io/csi-translation-lib/plugins # k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14 @@ -1988,7 +1988,7 @@ k8s.io/heapster/metrics/api/v1/types k8s.io/klog # k8s.io/klog/v2 v2.2.0 k8s.io/klog/v2 -# k8s.io/kube-aggregator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.15-k3s1 +# k8s.io/kube-aggregator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.16-k3s1 k8s.io/kube-aggregator/pkg/apis/apiregistration k8s.io/kube-aggregator/pkg/apis/apiregistration/install k8s.io/kube-aggregator/pkg/apis/apiregistration/v1 @@ -2016,7 +2016,7 @@ k8s.io/kube-aggregator/pkg/controllers/status k8s.io/kube-aggregator/pkg/registry/apiservice k8s.io/kube-aggregator/pkg/registry/apiservice/etcd k8s.io/kube-aggregator/pkg/registry/apiservice/rest -# k8s.io/kube-controller-manager v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.15-k3s1 +# k8s.io/kube-controller-manager v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.16-k3s1 k8s.io/kube-controller-manager/config/v1alpha1 # k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 k8s.io/kube-openapi/pkg/aggregator @@ -2027,13 +2027,13 @@ k8s.io/kube-openapi/pkg/schemaconv k8s.io/kube-openapi/pkg/util k8s.io/kube-openapi/pkg/util/proto k8s.io/kube-openapi/pkg/util/proto/validation -# k8s.io/kube-proxy v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.15-k3s1 +# k8s.io/kube-proxy v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.16-k3s1 k8s.io/kube-proxy/config/v1alpha1 -# k8s.io/kube-scheduler v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.15-k3s1 +# k8s.io/kube-scheduler v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.16-k3s1 k8s.io/kube-scheduler/config/v1 k8s.io/kube-scheduler/config/v1beta1 k8s.io/kube-scheduler/extender/v1 -# k8s.io/kubectl v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.15-k3s1 +# k8s.io/kubectl v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.16-k3s1 k8s.io/kubectl/pkg/apps k8s.io/kubectl/pkg/cmd k8s.io/kubectl/pkg/cmd/annotate @@ -2109,11 +2109,11 @@ k8s.io/kubectl/pkg/util/storage k8s.io/kubectl/pkg/util/templates k8s.io/kubectl/pkg/util/term k8s.io/kubectl/pkg/validation -# k8s.io/kubelet v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.15-k3s1 +# k8s.io/kubelet v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.16-k3s1 k8s.io/kubelet/config/v1beta1 k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1 k8s.io/kubelet/pkg/apis/pluginregistration/v1 -# k8s.io/kubernetes v1.19.15 => github.com/k3s-io/kubernetes v1.19.15-k3s1 +# k8s.io/kubernetes v1.19.16 => github.com/k3s-io/kubernetes v1.19.16-k3s1 ## explicit k8s.io/kubernetes/cmd/cloud-controller-manager/app k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config @@ -2852,7 +2852,7 @@ k8s.io/kubernetes/third_party/forked/gonum/graph k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear k8s.io/kubernetes/third_party/forked/gonum/graph/simple k8s.io/kubernetes/third_party/forked/gonum/graph/traverse -# k8s.io/legacy-cloud-providers v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.15-k3s1 +# k8s.io/legacy-cloud-providers v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.16-k3s1 k8s.io/legacy-cloud-providers/aws k8s.io/legacy-cloud-providers/azure k8s.io/legacy-cloud-providers/azure/auth @@ -2894,7 +2894,7 @@ k8s.io/legacy-cloud-providers/openstack k8s.io/legacy-cloud-providers/vsphere k8s.io/legacy-cloud-providers/vsphere/vclib k8s.io/legacy-cloud-providers/vsphere/vclib/diskmanagers -# k8s.io/metrics v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.15-k3s1 +# k8s.io/metrics v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.16-k3s1 k8s.io/metrics/pkg/apis/custom_metrics k8s.io/metrics/pkg/apis/custom_metrics/v1beta1 k8s.io/metrics/pkg/apis/custom_metrics/v1beta2 @@ -2995,29 +2995,29 @@ vbom.ml/util/sortorder # google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 # google.golang.org/grpc => google.golang.org/grpc v1.27.1 # gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.2.2 -# k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.15-k3s1 -# k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.15-k3s1 -# k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.15-k3s1 -# k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.15-k3s1 -# k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.15-k3s1 -# k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.15-k3s1 -# k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.15-k3s1 -# k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.15-k3s1 -# k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.15-k3s1 -# k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.15-k3s1 -# k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.15-k3s1 -# k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.15-k3s1 -# k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.15-k3s1 -# k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.15-k3s1 -# k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.15-k3s1 -# k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.15-k3s1 -# k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.15-k3s1 -# k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.15-k3s1 -# k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.19.15-k3s1 -# k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.15-k3s1 -# k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.15-k3s1 -# k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.19.15-k3s1 -# k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.19.15-k3s1 -# k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.19.15-k3s1 -# k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.19.15-k3s1 +# k8s.io/api => github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.19.16-k3s1 +# k8s.io/apiextensions-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.19.16-k3s1 +# k8s.io/apimachinery => github.com/k3s-io/kubernetes/staging/src/k8s.io/apimachinery v1.19.16-k3s1 +# k8s.io/apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/apiserver v1.19.16-k3s1 +# k8s.io/cli-runtime => github.com/k3s-io/kubernetes/staging/src/k8s.io/cli-runtime v1.19.16-k3s1 +# k8s.io/client-go => github.com/k3s-io/kubernetes/staging/src/k8s.io/client-go v1.19.16-k3s1 +# k8s.io/cloud-provider => github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.19.16-k3s1 +# k8s.io/cluster-bootstrap => github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.19.16-k3s1 +# k8s.io/code-generator => github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.19.16-k3s1 +# k8s.io/component-base => github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.19.16-k3s1 +# k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.19.16-k3s1 +# k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.19.16-k3s1 +# k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.19.16-k3s1 +# k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.19.16-k3s1 +# k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.19.16-k3s1 +# k8s.io/kube-scheduler => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-scheduler v1.19.16-k3s1 +# k8s.io/kubectl => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubectl v1.19.16-k3s1 +# k8s.io/kubelet => github.com/k3s-io/kubernetes/staging/src/k8s.io/kubelet v1.19.16-k3s1 +# k8s.io/kubernetes => github.com/k3s-io/kubernetes v1.19.16-k3s1 +# k8s.io/legacy-cloud-providers => github.com/k3s-io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.19.16-k3s1 +# k8s.io/metrics => github.com/k3s-io/kubernetes/staging/src/k8s.io/metrics v1.19.16-k3s1 +# k8s.io/node-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/node-api v1.19.16-k3s1 +# k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.19.16-k3s1 +# k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.19.16-k3s1 +# k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.19.16-k3s1 # mvdan.cc/unparam => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34