diff --git a/pkg/registry/rbac/rest/BUILD b/pkg/registry/rbac/rest/BUILD index 7fa5d4e4c4..e04d4579a0 100644 --- a/pkg/registry/rbac/rest/BUILD +++ b/pkg/registry/rbac/rest/BUILD @@ -33,6 +33,7 @@ go_library( "//pkg/registry/rbac/rolebinding/etcd:go_default_library", "//pkg/registry/rbac/rolebinding/policybased:go_default_library", "//pkg/util/runtime:go_default_library", + "//pkg/util/wait:go_default_library", "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", "//vendor:github.com/golang/glog", ], diff --git a/pkg/registry/rbac/rest/storage_rbac.go b/pkg/registry/rbac/rest/storage_rbac.go index 0ab6b7a57e..c318ad5e2e 100644 --- a/pkg/registry/rbac/rest/storage_rbac.go +++ b/pkg/registry/rbac/rest/storage_rbac.go @@ -19,6 +19,7 @@ package rest import ( "fmt" "sync" + "time" "github.com/golang/glog" @@ -43,6 +44,7 @@ import ( rolebindingetcd "k8s.io/kubernetes/pkg/registry/rbac/rolebinding/etcd" rolebindingpolicybased "k8s.io/kubernetes/pkg/registry/rbac/rolebinding/policybased" utilruntime "k8s.io/kubernetes/pkg/util/runtime" + "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" ) @@ -114,50 +116,54 @@ func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStart } func PostStartHook(hookContext genericapiserver.PostStartHookContext) error { - clientset, err := rbacclient.NewForConfig(hookContext.LoopbackClientConfig) - if err != nil { - utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err)) - return nil - } - - existingClusterRoles, err := clientset.ClusterRoles().List(api.ListOptions{}) - if err != nil { - utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err)) - return nil - } - // if clusterroles already exist, then assume we don't have work to do because we've already - // initialized or another API server has started this task - if len(existingClusterRoles.Items) > 0 { - return nil - } - - for _, clusterRole := range append(bootstrappolicy.ClusterRoles(), bootstrappolicy.ControllerRoles()...) { - if _, err := clientset.ClusterRoles().Create(&clusterRole); err != nil { - // don't fail on failures, try to create as many as you can + // intializing roles is really important. On some e2e runs, we've seen cases where etcd is down when the server + // starts, the roles don't initialize, and nothing works. + err := wait.Poll(1*time.Second, 30*time.Second, func() (done bool, err error) { + clientset, err := rbacclient.NewForConfig(hookContext.LoopbackClientConfig) + if err != nil { utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err)) - continue + return false, nil } - glog.Infof("Created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name) - } - existingClusterRoleBindings, err := clientset.ClusterRoleBindings().List(api.ListOptions{}) - if err != nil { - utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err)) - return nil - } - // if clusterrolebindings already exist, then assume we don't have work to do because we've already - // initialized or another API server has started this task - if len(existingClusterRoleBindings.Items) > 0 { - return nil - } + existingClusterRoles, err := clientset.ClusterRoles().List(api.ListOptions{}) + if err != nil { + utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err)) + return false, nil + } + // only initialized on empty etcd + if len(existingClusterRoles.Items) == 0 { + for _, clusterRole := range append(bootstrappolicy.ClusterRoles(), bootstrappolicy.ControllerRoles()...) { + if _, err := clientset.ClusterRoles().Create(&clusterRole); err != nil { + // don't fail on failures, try to create as many as you can + utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err)) + continue + } + glog.Infof("Created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name) + } + } - for _, clusterRoleBinding := range append(bootstrappolicy.ClusterRoleBindings(), bootstrappolicy.ControllerRoleBindings()...) { - if _, err := clientset.ClusterRoleBindings().Create(&clusterRoleBinding); err != nil { - // don't fail on failures, try to create as many as you can + existingClusterRoleBindings, err := clientset.ClusterRoleBindings().List(api.ListOptions{}) + if err != nil { utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err)) - continue + return false, nil } - glog.Infof("Created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) + // only initialized on empty etcd + if len(existingClusterRoleBindings.Items) == 0 { + for _, clusterRoleBinding := range append(bootstrappolicy.ClusterRoleBindings(), bootstrappolicy.ControllerRoleBindings()...) { + if _, err := clientset.ClusterRoleBindings().Create(&clusterRoleBinding); err != nil { + // don't fail on failures, try to create as many as you can + utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err)) + continue + } + glog.Infof("Created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name) + } + } + + return true, nil + }) + // if we're never able to make it through intialization, kill the API server + if err != nil { + return fmt.Errorf("unable to initialize roles: %v", err) } return nil