Merge pull request #57613 from mtaufen/explicit-kubelet-flags

Automatic merge from submit-queue (batch tested with PRs 57733, 57613, 57953). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Explicit kubelet flags

The Kubelet was using the global flag set.
Libraries also often use the global flag set.
There are flags in the Kubelet's usage statement for which the Kubelet does not control registration.
This is bad, we must have full control of our command-line API.
This PR puts a stop to it.

I worked through the Kubelet's usage printout and tracked down the flags. 

In the below list, flags with checkboxes are one of:
- explicitly **registered** in this PR
- thought about and outright **rejected**
- thought about and **registered-deprecated** for legacy reasons

For the others we need to figure out whether they should be **registered** or **rejected** or **registered-deprecated**. 

### cadvisor:
grep commands, for reference:
```
# flag definitions in cadvisor
git grep -E "\"application_metrics_count_limit\"|\"boot_id_file\"|\"container_hints\"|\"containerd\"|\"docker\"|\"docker_env_metadata_whitelist\"|\"docker_only\"|\"docker_root\"|\"docker-tls\"|\"docker-tls-ca\"|\"docker-tls-cert\"|\"docker-tls-key\"|\"enable_load_reader\"|\"event_storage_age_limit\"|\"event_storage_event_limit\"|\"global_housekeeping_interval\"|\"housekeeping_interval\"|\"log_cadvisor_usage\"|\"machine_id_file\"|\"storage_driver_buffer_duration\"|\"storage_driver_db\"|\"storage_driver_host\"|\"storage_driver_password\"|\"storage_driver_secure\"|\"storage_driver_table\"|\"storage_driver_user\"" -- vendor/github.com/google/cadvisor
# flag invocations
git grep -E "\--application_metrics_count_limit|--boot_id_file|--container_hints|--containerd|--docker|--docker_env_metadata_whitelist|--docker_only|--docker_root|--docker_tls|--docker_tls_ca|--docker_tls_cert|--docker_tls_key|--enable_load_reader|--event_storage_age_limit|--event_storage_event_limit|--global_housekeeping_interval|--housekeeping_interval|--log_cadvisor_usage|--machine_id_file|--storage_driver_buffer_duration|--storage_driver_db|--storage_driver_host|--storage_driver_password|--storage_driver_secure|--storage_driver_table|--storage_driver_user" -- ':(exclude)pkg/generated/bindata.go' ':(exclude)Godeps' ':(exclude)CHANGELOG*' ':(exclude)vendor'
# normalized flag invocations
git grep -E "\--application-metrics-count-limit|--boot-id-file|--container-hints|--containerd|--docker|--docker-env-metadata-whitelist|--docker-only|--docker-root|--docker-tls|--docker-tls-ca|--docker-tls-cert|--docker-tls-key|--enable-load-reader|--event-storage-age-limit|--event-storage-event-limit|--global-housekeeping-interval|--housekeeping-interval|--log-cadvisor-usage|--machine-id-file|--storage-driver-buffer-duration|--storage-driver-db|--storage-driver-host|--storage-driver-password|--storage-driver-secure|--storage-driver-table|--storage-driver-user" -- ':(exclude)pkg/generated/bindata.go' ':(exclude)Godeps' ':(exclude)CHANGELOG*' ':(exclude)vendor'
# flag names
git grep -E "application_metrics_count_limit|boot_id_file|container_hints|containerd|docker|docker_env_metadata_whitelist|docker_only|docker_root|docker_tls|docker_tls_ca|docker_tls_cert|docker_tls_key|enable_load_reader|event_storage_age_limit|event_storage_event_limit|global_housekeeping_interval|housekeeping_interval|log_cadvisor_usage|machine_id_file|storage_driver_buffer_duration|storage_driver_db|storage_driver_host|storage_driver_password|storage_driver_secure|storage_driver_table|storage_driver_user"  ':(exclude)pkg/generated/bindata.go' ':(exclude)Godeps' ':(exclude)CHANGELOG*' ':(exclude)vendor'
# normalized flag names
git grep -E "application-metrics-count-limit|boot-id-file|container-hints|containerd|docker|docker-env-metadata-whitelist|docker-only|docker-root|docker-tls|docker-tls-ca|docker-tls-cert|docker-tls-key|enable-load-reader|event-storage-age-limit|event-storage-event-limit|global-housekeeping-interval|housekeeping-interval|log-cadvisor-usage|machine-id-file|storage-driver-buffer-duration|storage-driver-db|storage-driver-host|storage-driver-password|storage-driver-secure|storage-driver-table|storage-driver-user"  ':(exclude)pkg/generated/bindata.go' ':(exclude)Godeps' ':(exclude)CHANGELOG*' ':(exclude)vendor'
# normalized and underscore names combined
git grep -E "application_metrics_count_limit|boot_id_file|container_hints|containerd|docker|docker_env_metadata_whitelist|docker_only|docker_root|docker_tls|docker_tls_ca|docker_tls_cert|docker_tls_key|enable_load_reader|event_storage_age_limit|event_storage_event_limit|global_housekeeping_interval|housekeeping_interval|log_cadvisor_usage|machine_id_file|storage_driver_buffer_duration|storage_driver_db|storage_driver_host|storage_driver_password|storage_driver_secure|storage_driver_table|storage_driver_user|application-metrics-count-limit|boot-id-file|container-hints|containerd|docker-env-metadata-whitelist|docker-only|docker-root|docker-tls|docker-tls-ca|docker-tls-cert|docker-tls-key|enable-load-reader|event-storage-age-limit|event-storage-event-limit|global-housekeeping-interval|housekeeping-interval|log-cadvisor-usage|machine-id-file|storage-driver-buffer-duration|storage-driver-db|storage-driver-host|storage-driver-password|storage-driver-secure|storage-driver-table|storage-driver-user"  ':(exclude)pkg/generated/bindata.go' ':(exclude)Godeps' ':(exclude)CHANGELOG*' ':(exclude)vendor'
```
- [x]      --docker-root (**registered** - this is used in `cluster/saltbase/salt/kubelet/default`)
- [x]      --housekeeping-interval (**registered** - e2e node tests rely on this (`test/e2e_node/resource_collector.go`))
- [x]      --application-metrics-count-limit (**registered-deprecated** - temporarily register for legacy)
- [x]      --boot-id-file (**registered-deprecated** - temporarily register for legacy)
- [x]      --container-hints (**registered-deprecated** - temporarily register for legacy)
- [x]      --containerd (**registered-deprecated** - temporarily register for legacy)
- [x]      --docker (**registered-deprecated** - temporarily register for legacy)
- [x]      --docker-env-metadata-whitelist (**registered-deprecated** - temporarily register for legacy)
- [x]      --docker-only (**registered-deprecated** - temporarily register for legacy)
- [x]      --docker-tls (**registered-deprecated** - temporarily register for legacy)
- [x]      --docker-tls-ca (**registered-deprecated** - temporarily register for legacy)
- [x]      --docker-tls-cert (**registered-deprecated** - temporarily register for legacy)
- [x]      --docker-tls-key (**registered-deprecated** - temporarily register for legacy)
- [x]      --enable-load-reader (**registered-deprecated** - temporarily register for legacy)
- [x]      --event-storage-age-limit (**registered-deprecated** - the Kubelet overrides the default via the global flagset (`pkg/kubelet/cadvisor/cadvisor_linux.go`), but nothing else in core repo provides)
- [x]      --event-storage-event-limit (**registered-deprecated** - the Kubelet overrides the default via the global flagset (`pkg/kubelet/cadvisor/cadvisor_linux.go`), but nothing else in core repo provides)
- [x]      --global-housekeeping-interval (**registered-deprecated** - temporarily register for legacy)
- [x]      --log-cadvisor-usage (**registered-deprecated** - temporarily register for legacy)
- [x]      --machine-id-file (**registered-deprecated** - temporarily register for legacy)
- [x]      --storage-driver-user (**registered-deprecated** - temporarily register for legacy)
- [x]      --storage-driver-password (**registered-deprecated** - temporarily register for legacy)
- [x]      --storage-driver-host (**registered-deprecated** - temporarily register for legacy)
- [x]      --storage-driver-db (**registered-deprecated** - temporarily register for legacy)
- [x]      --storage-driver-table (**registered-deprecated** - temporarily register for legacy)
- [x]      --storage-driver-secure (**registered-deprecated** - temporarily register for legacy)
- [x]      --storage-driver-buffer-duration (**registered-deprecated** - temporarily register for legacy)

### pkg/apiserver/util/logs:
- [x]      --log-flush-frequency (**registered** - like the glog flags, this is probably useful)

### pkg/credentialprovider/azure/azure_credentials.go:
- [x]      --azure-container-registry-config (**registered** - This isn't quite as straightforward as --google-json-key, because the file it points to isn't static. For now we will just register, and we will deprecate it when there is an alternative. See below comments.)

### pkg/credentialprovider/gcp/jwt.go:
- [x]      --google-json-key (**registered-deprecated** - This is really old legacy stuff to allow kubelets to authenticate with gcr (see: d5e0054eb0). See @liggit's below comment for what should be used instead.)

### pkg/cloudprovider/providers/gce/gce_loadbalancer.go:
- [x]      --cloud-provider-gce-lb-src-cidrs (**rejected** - Kubelet doesn't need to know about the cidrs that were opened in the firewall for the load balancer)

### glog:
I registered all of these, since this logging library is used pretty much everywhere in the Kubelet, and all of its toggles are probably useful.
- [x]      --logtostderr (**registered**)
- [x]      --alsologtostderr (**registered**)
- [x]  -v, --v (**registered**)
- [x]      --stderrthreshold (**registered**)
- [x]      --vmodule (**registered**)
- [x]      --log-backtrace-at (**registered**)
- [x]      --log-dir (**registered**)

### verflag:
This is how you get the Kubelet's version, absolutely necessary to register this.
- [x]      --version (**registered**)

```release-note
The Kubelet now explicitly registers all of its command-line flags with an internal flagset, which prevents flags from third party libraries from unintentionally leaking into the Kubelet's command-line API. Many unintentionally leaked flags are now marked deprecated, so that users have a chance to migrate away from them before they are removed. One previously leaked flag, --cloud-provider-gce-lb-src-cidrs, was entirely removed from the Kubelet's command-line API, because it is irrelevant to Kubelet operation.
```

Thanks to @liggitt for being surprised that we didn't already do this.
Thanks to @dashpole for realizing cadvisor flags were leaked after seeing #55863.
Thanks to @tallclair who recognized this problem a long time ago in #19432.
pull/6/head
Kubernetes Submit Queue 2018-01-09 21:51:37 -08:00 committed by GitHub
commit 5e444bbe12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 231 additions and 13 deletions

View File

@ -24,6 +24,7 @@ go_library(
"//pkg/client/metrics/prometheus:go_default_library",
"//pkg/version/prometheus:go_default_library",
"//pkg/version/verflag:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",

View File

@ -10,12 +10,15 @@ go_library(
name = "go_default_library",
srcs = [
"container_runtime.go",
"globalflags.go",
"options.go",
],
importpath = "k8s.io/kubernetes/cmd/kubelet/app/options",
deps = [
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/core:go_default_library",
"//pkg/credentialprovider/azure:go_default_library",
"//pkg/credentialprovider/gcp:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
@ -24,10 +27,20 @@ go_library(
"//pkg/kubelet/config:go_default_library",
"//pkg/kubelet/types:go_default_library",
"//pkg/util/taints:go_default_library",
"//pkg/version/verflag:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/google/cadvisor/container/common:go_default_library",
"//vendor/github.com/google/cadvisor/container/containerd:go_default_library",
"//vendor/github.com/google/cadvisor/container/docker:go_default_library",
"//vendor/github.com/google/cadvisor/container/raw:go_default_library",
"//vendor/github.com/google/cadvisor/machine:go_default_library",
"//vendor/github.com/google/cadvisor/manager:go_default_library",
"//vendor/github.com/google/cadvisor/storage:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
],
)

View File

@ -0,0 +1,167 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"flag"
"fmt"
"os"
"strings"
"github.com/spf13/pflag"
// libs that provide registration functions
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/kubernetes/pkg/version/verflag"
// ensure libs have a chance to globally register their flags
_ "github.com/golang/glog"
_ "github.com/google/cadvisor/container/common"
_ "github.com/google/cadvisor/container/containerd"
_ "github.com/google/cadvisor/container/docker"
_ "github.com/google/cadvisor/container/raw"
_ "github.com/google/cadvisor/machine"
_ "github.com/google/cadvisor/manager"
_ "github.com/google/cadvisor/storage"
_ "k8s.io/kubernetes/pkg/credentialprovider/azure"
_ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
)
// AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
// against the global flagsets from "flag" and "github.com/spf13/pflag".
// We do this in order to prevent unwanted flags from leaking into the Kubelet's flagset.
func AddGlobalFlags(fs *pflag.FlagSet) {
addGlogFlags(fs)
addCadvisorFlags(fs)
verflag.AddFlags(fs)
logs.AddFlags(fs)
}
// normalize replaces underscores with hyphens
// we should always use hyphens instead of underscores when registering kubelet flags
func normalize(s string) string {
return strings.Replace(s, "_", "-", -1)
}
// register adds a flag to local that targets the Value associated with the Flag named globalName in global
func register(global *flag.FlagSet, local *pflag.FlagSet, globalName string) {
if f := global.Lookup(globalName); f != nil {
f.Name = normalize(f.Name)
local.AddFlag(pflag.PFlagFromGoFlag(f))
} else {
panic(fmt.Sprintf("failed to find flag in global flagset (flag): %s", globalName))
}
}
// pflagRegister adds a flag to local that targets the Value associated with the Flag named globalName in global
func pflagRegister(global, local *pflag.FlagSet, globalName string) {
if f := global.Lookup(globalName); f != nil {
f.Name = normalize(f.Name)
local.AddFlag(f)
} else {
panic(fmt.Sprintf("failed to find flag in global flagset (pflag): %s", globalName))
}
}
// registerDeprecated registers the flag with register, and then marks it deprecated
func registerDeprecated(global *flag.FlagSet, local *pflag.FlagSet, globalName, deprecated string) {
register(global, local, globalName)
local.Lookup(normalize(globalName)).Deprecated = deprecated
}
// pflagRegisterDeprecated registers the flag with pflagRegister, and then marks it deprecated
func pflagRegisterDeprecated(global, local *pflag.FlagSet, globalName, deprecated string) {
pflagRegister(global, local, globalName)
local.Lookup(normalize(globalName)).Deprecated = deprecated
}
// addCredentialProviderFlags adds flags from k8s.io/kubernetes/pkg/credentialprovider
func addCredentialProviderFlags(fs *pflag.FlagSet) {
// lookup flags in global flag set and re-register the values with our flagset
global := pflag.CommandLine
local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
// Note this is deprecated in the library that provides it, so we just allow that deprecation
// notice to pass through our registration here.
pflagRegister(global, local, "google-json-key")
// TODO(#58034): This is not a static file, so it's not quite as straightforward as --google-json-key.
// We need to figure out how ACR users can dynamically provide pull credentials before we can deprecate this.
pflagRegister(global, local, "azure-container-registry-config")
fs.AddFlagSet(local)
}
// addGlogFlags adds flags from github.com/golang/glog
func addGlogFlags(fs *pflag.FlagSet) {
// lookup flags in global flag set and re-register the values with our flagset
global := flag.CommandLine
local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
register(global, local, "logtostderr")
register(global, local, "alsologtostderr")
register(global, local, "v")
register(global, local, "stderrthreshold")
register(global, local, "vmodule")
register(global, local, "log_backtrace_at")
register(global, local, "log_dir")
fs.AddFlagSet(local)
}
// addCadvisorFlags adds flags from cadvisor
func addCadvisorFlags(fs *pflag.FlagSet) {
// lookup flags in global flag set and re-register the values with our flagset
global := flag.CommandLine
local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
// These flags were also implicit from cadvisor, but are actually used by something in the core repo:
// TODO(mtaufen): This one is stil used by our salt, but for heaven's sake it's even deprecated in cadvisor
register(global, local, "docker_root")
// e2e node tests rely on this
register(global, local, "housekeeping_interval")
// These flags were implicit from cadvisor, and are mistakes that should be registered deprecated:
const deprecated = "This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed."
registerDeprecated(global, local, "application_metrics_count_limit", deprecated)
registerDeprecated(global, local, "boot_id_file", deprecated)
registerDeprecated(global, local, "container_hints", deprecated)
registerDeprecated(global, local, "containerd", deprecated)
registerDeprecated(global, local, "docker", deprecated)
registerDeprecated(global, local, "docker_env_metadata_whitelist", deprecated)
registerDeprecated(global, local, "docker_only", deprecated)
registerDeprecated(global, local, "docker-tls", deprecated)
registerDeprecated(global, local, "docker-tls-ca", deprecated)
registerDeprecated(global, local, "docker-tls-cert", deprecated)
registerDeprecated(global, local, "docker-tls-key", deprecated)
registerDeprecated(global, local, "enable_load_reader", deprecated)
registerDeprecated(global, local, "event_storage_age_limit", deprecated)
registerDeprecated(global, local, "event_storage_event_limit", deprecated)
registerDeprecated(global, local, "global_housekeeping_interval", deprecated)
registerDeprecated(global, local, "log_cadvisor_usage", deprecated)
registerDeprecated(global, local, "machine_id_file", deprecated)
registerDeprecated(global, local, "storage_driver_user", deprecated)
registerDeprecated(global, local, "storage_driver_password", deprecated)
registerDeprecated(global, local, "storage_driver_host", deprecated)
registerDeprecated(global, local, "storage_driver_db", deprecated)
registerDeprecated(global, local, "storage_driver_table", deprecated)
registerDeprecated(global, local, "storage_driver_secure", deprecated)
registerDeprecated(global, local, "storage_driver_buffer_duration", deprecated)
// finally, add cadvisor flags to the provided flagset
fs.AddFlagSet(local)
}

View File

@ -24,6 +24,7 @@ import (
"fmt"
"os"
"github.com/golang/glog"
"github.com/spf13/pflag"
utilfeature "k8s.io/apiserver/pkg/util/feature"
@ -36,25 +37,43 @@ import (
"k8s.io/kubernetes/pkg/version/verflag"
)
func parseFlagSet(fs *pflag.FlagSet, args []string) error {
if err := fs.Parse(args); err != nil {
return err
}
fs.VisitAll(func(flag *pflag.Flag) {
glog.V(2).Infof("FLAG: --%s=%q", flag.Name, flag.Value)
})
return nil
}
func die(err error) {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
func main() {
// construct KubeletFlags object and register command line flags mapping
kubeletFlags := options.NewKubeletFlags()
kubeletFlags.AddFlags(pflag.CommandLine)
fs := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
// set the normalize func, similar to k8s.io/apiserver/pkg/util/flag/flags.go:InitFlags
fs.SetNormalizeFunc(flag.WordSepNormalizeFunc)
// explicitly add flags from libs that register global flags
options.AddGlobalFlags(fs)
// construct KubeletConfiguration object and register command line flags mapping
// register kubelet flags
kubeletFlags := options.NewKubeletFlags()
kubeletFlags.AddFlags(fs)
// register kubelet config flags
defaultConfig, err := options.NewKubeletConfiguration()
if err != nil {
die(err)
}
options.AddKubeletConfigFlags(pflag.CommandLine, defaultConfig)
options.AddKubeletConfigFlags(fs, defaultConfig)
// parse the command line flags into the respective objects
flag.InitFlags()
// parse flags
if err := parseFlagSet(fs, os.Args[1:]); err != nil {
die(err)
}
// initialize logging and defer flush
logs.InitLogs()

View File

@ -34,7 +34,7 @@ import (
)
var flagConfigFile = pflag.String("azure-container-registry-config", "",
"Path to the file container Azure container registry configuration information.")
"Path to the file containing Azure container registry configuration information.")
const dummyRegistryEmail = "name@contoso.com"

View File

@ -31,10 +31,11 @@ import (
const (
storageReadOnlyScope = "https://www.googleapis.com/auth/devstorage.read_only"
jwtFileFlagName = "google-json-key"
)
var (
flagJwtFile = pflag.String("google-json-key", "",
flagJwtFile = pflag.String(jwtFileFlagName, "",
"The Google Cloud Platform Service Account JSON Key to use for authentication.")
)
@ -49,6 +50,9 @@ type jwtProvider struct {
// init registers the various means by which credentials may
// be resolved on GCP.
func init() {
pflag.CommandLine.MarkDeprecated(jwtFileFlagName, "Will be removed in a future version. "+
"To maintain node-level authentication, credentials should instead be included in a docker "+
"config.json file, located inside the Kubelet's --root-dir.")
credentialprovider.RegisterCredentialProvider("google-jwt-key",
&credentialprovider.CachingDockerConfigProvider{
Provider: &jwtProvider{

View File

@ -85,10 +85,16 @@ func Version(name string, value versionValue, usage string) *versionValue {
return p
}
const versionFlagName = "version"
var (
versionFlag = Version("version", VersionFalse, "Print version information and quit")
versionFlag = Version(versionFlagName, VersionFalse, "Print version information and quit")
)
func AddFlags(fs *flag.FlagSet) {
fs.AddFlag(flag.Lookup(versionFlagName))
}
// PrintAndExitIfRequested will check if the -version flag was passed
// and, if so, print the version and exit.
func PrintAndExitIfRequested() {

View File

@ -26,13 +26,21 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
)
var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum number of seconds between log flushes")
const logFlushFreqFlagName = "log-flush-frequency"
var logFlushFreq = pflag.Duration(logFlushFreqFlagName, 5*time.Second, "Maximum number of seconds between log flushes")
// TODO(thockin): This is temporary until we agree on log dirs and put those into each cmd.
func init() {
flag.Set("logtostderr", "true")
}
// AddFlags registers this package's flags on arbitrary FlagSets, such that they point to the
// same value as the global flags.
func AddFlags(fs *pflag.FlagSet) {
fs.AddFlag(pflag.Lookup(logFlushFreqFlagName))
}
// GlogWriter serves as a bridge between the standard log package and the glog package.
type GlogWriter struct{}

View File

@ -48,7 +48,6 @@ go_library(
"//pkg/api/v1/pod:go_default_library",
"//pkg/cloudprovider/providers/azure:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/kubectl/util/logs:go_default_library",
"//pkg/version:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
@ -71,6 +70,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)

View File

@ -32,10 +32,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/util/logs"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
"k8s.io/kubernetes/pkg/kubectl/util/logs"
"k8s.io/kubernetes/pkg/version"
commontest "k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"