From 163844cb138ac0d74a059c8e24529fc73be0ae63 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Tue, 15 Jan 2019 16:16:26 -0500
Subject: [PATCH 01/12] Added ability to execute subsets of cli tests

---
 build/root/Makefile                           |   1 -
 test/cmd/README.md                            |  44 +
 .../cmd/kubeadm.sh                            |  19 +-
 test/cmd/legacy-script.sh                     | 787 +++++++++---------
 4 files changed, 458 insertions(+), 393 deletions(-)
 create mode 100644 test/cmd/README.md
 rename hack/make-rules/test-kubeadm-cmd.sh => test/cmd/kubeadm.sh (63%)

diff --git a/build/root/Makefile b/build/root/Makefile
index 6a86d1ea45..b464e75638 100644
--- a/build/root/Makefile
+++ b/build/root/Makefile
@@ -291,7 +291,6 @@ test-cmd:
 	@echo "$$TEST_CMD_HELP_INFO"
 else
 test-cmd: generated_files
-	hack/make-rules/test-kubeadm-cmd.sh
 	hack/make-rules/test-cmd.sh
 endif
 
diff --git a/test/cmd/README.md b/test/cmd/README.md
new file mode 100644
index 0000000000..f9572e929d
--- /dev/null
+++ b/test/cmd/README.md
@@ -0,0 +1,44 @@
+# Kubernetes Command-Line Integration Test Suite
+
+This document describes how Kubernetes should interact with the Kubernetes command-line integration test suite.
+
+## Running Tests
+
+### All Tests
+
+To run this entire suite, execute `make test-cmd` from the top level.  This will import each file containing tests functions
+
+### Specific Tests
+
+To run a subset of tests( e.g. `run_deployment_test` and `run_impersonation_test`), execute `make test-cmd WHAT="deployment impersonation"`.  Running specific
+tests will not try and validate any required resources are available on the server.
+
+## Adding Tests
+
+Test functions need to have the format `run_*_test` so they can executed individually.  Once a test has been added, insert a section in `legacy-script.sh` like
+
+```bash
+######################
+# Replica Sets       #
+######################
+
+if kube::test::if_supports_resource "${replicasets}" ; then
+    record_command run_rs_tests
+fi
+```
+
+Be sure to validate any supported resouces required for the test by using the `kue::test::if_supports_resource` function. 
+
+
+### New File
+
+If the test resides in a new file, source the file in the top of the `legacy-script.sh` file by adding a new line in
+```bash
+source "${KUBE_ROOT}/test/cmd/apply.sh"
+source "${KUBE_ROOT}/test/cmd/apps.sh"
+source "${KUBE_ROOT}/test/cmd/authorization.sh"
+source "${KUBE_ROOT}/test/cmd/batch.sh"
+...
+```
+
+Please keep the order of the source list alphabetical.
diff --git a/hack/make-rules/test-kubeadm-cmd.sh b/test/cmd/kubeadm.sh
similarity index 63%
rename from hack/make-rules/test-kubeadm-cmd.sh
rename to test/cmd/kubeadm.sh
index 86d82b0892..40fb0ef1c1 100755
--- a/hack/make-rules/test-kubeadm-cmd.sh
+++ b/test/cmd/kubeadm.sh
@@ -18,15 +18,20 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
-source "${KUBE_ROOT}/hack/lib/init.sh"
+run_kubeadm_tests() {
+  set -o nounset
+  set -o errexit
+  KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
-KUBEADM_PATH="${KUBEADM_PATH:=$(kube::realpath "${KUBE_ROOT}")/cluster/kubeadm.sh}"
+  KUBEADM_PATH="${KUBEADM_PATH:=$(kube::realpath "${KUBE_ROOT}")/cluster/kubeadm.sh}"
 
-# If testing a different version of kubeadm than the current build, you can
-# comment this out to save yourself from needlessly building here.
-make -C "${KUBE_ROOT}" WHAT=cmd/kubeadm
+  # If testing a different version of kubeadm than the current build, you can
+  # comment this out to save yourself from needlessly building here.
+  make -C "${KUBE_ROOT}" WHAT=cmd/kubeadm
 
-make -C "${KUBE_ROOT}" test \
+  make -C "${KUBE_ROOT}" test \
   WHAT=k8s.io/kubernetes/cmd/kubeadm/test/cmd \
   KUBE_TEST_ARGS="--kubeadm-path '${KUBEADM_PATH}'"
+  set +o nounset
+  set +o errexit
+}
\ No newline at end of file
diff --git a/test/cmd/legacy-script.sh b/test/cmd/legacy-script.sh
index 819a5452d6..bc50a781b6 100755
--- a/test/cmd/legacy-script.sh
+++ b/test/cmd/legacy-script.sh
@@ -20,6 +20,15 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+if [ ! -z "$WHAT" ]; then #ccheck not empty
+    for pkg in ${WHAT}
+    do 
+      echo ${pkg}
+    done
+fi
+exit 0
+
+
 # Set locale to ensure english responses from kubectl commands
 export LANG=C
 
@@ -40,6 +49,7 @@ source "${KUBE_ROOT}/test/cmd/discovery.sh"
 source "${KUBE_ROOT}/test/cmd/generic-resources.sh"
 source "${KUBE_ROOT}/test/cmd/get.sh"
 source "${KUBE_ROOT}/test/cmd/initializers.sh"
+source "${KUBE_ROOT}/test/cmd/kubeadm.sh"
 source "${KUBE_ROOT}/test/cmd/kubeconfig.sh"
 source "${KUBE_ROOT}/test/cmd/node-management.sh"
 source "${KUBE_ROOT}/test/cmd/old-print.sh"
@@ -384,455 +394,462 @@ runTests() {
     kubectl get "${kube_flags[@]}" -f hack/testdata/kubernetes-service.yaml
   fi
 
-  #########################
-  # Kubectl version #
-  #########################
+  if [ ! -z "$WHAT" ]; then #ccheck not empty
+    for pkg in ${WHAT}
+    do 
+      record_command run_${pkg}_tests
+    done
+  else
 
-  record_command run_kubectl_version_tests
+    #########################
+    # Kubectl version #
+    #########################
 
-  #######################
-  # kubectl config set #
-  #######################
+    record_command run_kubectl_version_tests
 
-  record_command run_kubectl_config_set_tests
+    #######################
+    # kubectl config set #
+    #######################
 
-  #######################
-  # kubectl local proxy #
-  #######################
+    record_command run_kubectl_config_set_tests
 
-  record_command run_kubectl_local_proxy_tests
+    #######################
+    # kubectl local proxy #
+    #######################
 
-  #########################
-  # RESTMapper evaluation #
-  #########################
+    record_command run_kubectl_local_proxy_tests
 
-  record_command run_RESTMapper_evaluation_tests
+    #########################
+    # RESTMapper evaluation #
+    #########################
 
-  # find all resources
-  kubectl "${kube_flags[@]}" api-resources
-  # find all namespaced resources that support list by name and get them
-  kubectl "${kube_flags[@]}" api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl "${kube_flags[@]}" get -o name
+    record_command run_RESTMapper_evaluation_tests
 
-  ################
-  # Cluster Role #
-  ################
+    # find all resources
+    kubectl "${kube_flags[@]}" api-resources
+    # find all namespaced resources that support list by name and get them
+    kubectl "${kube_flags[@]}" api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl "${kube_flags[@]}" get -o name
 
-  if kube::test::if_supports_resource "${clusterroles}" ; then
-    record_command run_clusterroles_tests
-  fi
+    ################
+    # Cluster Role #
+    ################
 
-  ########
-  # Role #
-  ########
-  if kube::test::if_supports_resource "${roles}" ; then
-      record_command run_role_tests
-  fi
-
-  #########################
-  # Assert short name     #
-  #########################
-
-  record_command run_assert_short_name_tests
-
-  #########################
-  # Assert categories     #
-  #########################
-
-  ## test if a category is exported during discovery
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_assert_categories_tests
-  fi
-
-  ###########################
-  # POD creation / deletion #
-  ###########################
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_pod_tests
-  fi
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_save_config_tests
-  fi
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_kubectl_create_error_tests
-  fi
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_kubectl_apply_tests
-    record_command run_kubectl_run_tests
-    record_command run_kubectl_create_filter_tests
-  fi
-
-  if kube::test::if_supports_resource "${deployments}" ; then
-    record_command run_kubectl_apply_deployments_tests
-  fi
-
-  ################
-  # Kubectl diff #
-  ################
-  record_command run_kubectl_diff_tests
-  record_command run_kubectl_diff_same_names
-
-  ###############
-  # Kubectl get #
-  ###############
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_kubectl_get_tests
-    record_command run_kubectl_old_print_tests
-  fi
-
-
-  ######################
-  # Create             #
-  ######################
-  if kube::test::if_supports_resource "${secrets}" ; then
-    record_command run_create_secret_tests
-  fi
-
-  ##################
-  # Global timeout #
-  ##################
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_kubectl_request_timeout_tests
-  fi
-
-  #####################################
-  # CustomResourceDefinitions         #
-  #####################################
-
-  # customresourcedefinitions cleanup after themselves.
-  if kube::test::if_supports_resource "${customresourcedefinitions}" ; then
-    record_command run_crd_tests
-  fi
-
-  #################
-  # Run cmd w img #
-  #################
-
-  if kube::test::if_supports_resource "${deployments}" ; then
-    record_command run_cmd_with_img_tests
-  fi
-
-
-  #####################################
-  # Recursive Resources via directory #
-  #####################################
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    run_recursive_resources_tests
-  fi
-
-
-  ##############
-  # Namespaces #
-  ##############
-  if kube::test::if_supports_resource "${namespaces}" ; then
-    record_command run_namespace_tests
-  fi
-
-
-  ###########
-  # Secrets #
-  ###########
-  if kube::test::if_supports_resource "${namespaces}" ; then
-    if kube::test::if_supports_resource "${secrets}" ; then
-      record_command run_secrets_test
+    if kube::test::if_supports_resource "${clusterroles}" ; then
+      record_command run_clusterroles_tests
     fi
-  fi
 
-
-  ######################
-  # ConfigMap          #
-  ######################
-
-  if kube::test::if_supports_resource "${namespaces}"; then
-    if kube::test::if_supports_resource "${configmaps}" ; then
-      record_command run_configmap_tests
+    ########
+    # Role #
+    ########
+    if kube::test::if_supports_resource "${roles}" ; then
+        record_command run_role_tests
     fi
-  fi
 
-  ####################
-  # Client Config    #
-  ####################
+    #########################
+    # Assert short name     #
+    #########################
 
-  record_command run_client_config_tests
+    record_command run_assert_short_name_tests
 
-  ####################
-  # Service Accounts #
-  ####################
+    #########################
+    # Assert categories     #
+    #########################
 
-  if kube::test::if_supports_resource "${namespaces}" && kube::test::if_supports_resource "${serviceaccounts}" ; then
-    record_command run_service_accounts_tests
-  fi
-
-  ####################
-  # Job              #
-  ####################
-
-  if kube::test::if_supports_resource "${job}" ; then
-    record_command run_job_tests
-    record_command run_create_job_tests
-  fi
-
-  #################
-  # Pod templates #
-  #################
-
-  if kube::test::if_supports_resource "${podtemplates}" ; then
-    record_command run_pod_templates_tests
-  fi
-
-  ############
-  # Services #
-  ############
-
-  if kube::test::if_supports_resource "${services}" ; then
-    record_command run_service_tests
-  fi
-
-  ##################
-  # DaemonSets     #
-  ##################
-
-  if kube::test::if_supports_resource "${daemonsets}" ; then
-    record_command run_daemonset_tests
-    if kube::test::if_supports_resource "${controllerrevisions}"; then
-      record_command run_daemonset_history_tests
+    ## test if a category is exported during discovery
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_assert_categories_tests
     fi
-  fi
 
-  ###########################
-  # Replication controllers #
-  ###########################
+    ###########################
+    # POD creation / deletion #
+    ###########################
 
-  if kube::test::if_supports_resource "${namespaces}" ; then
-    if kube::test::if_supports_resource "${replicationcontrollers}" ; then
-      record_command run_rc_tests
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_pod_tests
     fi
-  fi
 
-  ######################
-  # Deployments       #
-  ######################
-
-  if kube::test::if_supports_resource "${deployments}" ; then
-    record_command run_deployment_tests
-  fi
-
-  ######################
-  # Replica Sets       #
-  ######################
-
-  if kube::test::if_supports_resource "${replicasets}" ; then
-    record_command run_rs_tests
-  fi
-
-  #################
-  # Stateful Sets #
-  #################
-
-  if kube::test::if_supports_resource "${statefulsets}" ; then
-    record_command run_stateful_set_tests
-    if kube::test::if_supports_resource "${controllerrevisions}"; then
-      record_command run_statefulset_history_tests
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_save_config_tests
     fi
-  fi
 
-  ######################
-  # Lists              #
-  ######################
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_kubectl_create_error_tests
+    fi
+
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_kubectl_apply_tests
+      record_command run_kubectl_run_tests
+      record_command run_kubectl_create_filter_tests
+    fi
 
-  if kube::test::if_supports_resource "${services}" ; then
     if kube::test::if_supports_resource "${deployments}" ; then
-      record_command run_lists_tests
+      record_command run_kubectl_apply_deployments_tests
     fi
-  fi
 
+    ################
+    # Kubectl diff #
+    ################
+    record_command run_kubectl_diff_tests
+    record_command run_kubectl_diff_same_names
 
-  ######################
-  # Multiple Resources #
-  ######################
-  if kube::test::if_supports_resource "${services}" ; then
-    if kube::test::if_supports_resource "${replicationcontrollers}" ; then
-      record_command run_multi_resources_tests
+    ###############
+    # Kubectl get #
+    ###############
+
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_kubectl_get_tests
+      record_command run_kubectl_old_print_tests
     fi
-  fi
-
-  ######################
-  # Persistent Volumes #
-  ######################
-
-  if kube::test::if_supports_resource "${persistentvolumes}" ; then
-    record_command run_persistent_volumes_tests
-  fi
-
-  ############################
-  # Persistent Volume Claims #
-  ############################
-
-  if kube::test::if_supports_resource "${persistentvolumeclaims}" ; then
-    record_command run_persistent_volume_claims_tests
-  fi
-
-  ############################
-  # Storage Classes #
-  ############################
-
-  if kube::test::if_supports_resource "${storageclass}" ; then
-    record_command run_storage_class_tests
-  fi
-
-  #########
-  # Nodes #
-  #########
-
-  if kube::test::if_supports_resource "${nodes}" ; then
-    record_command run_nodes_tests
-  fi
 
 
-  ########################
-  # authorization.k8s.io #
-  ########################
+    ######################
+    # Create             #
+    ######################
+    if kube::test::if_supports_resource "${secrets}" ; then
+      record_command run_create_secret_tests
+    fi
 
-  if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
-    record_command run_authorization_tests
-  fi
+    ##################
+    # Global timeout #
+    ##################
 
-  # kubectl auth can-i
-  # kube-apiserver is started with authorization mode AlwaysAllow, so kubectl can-i always returns yes
-  if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
-    output_message=$(kubectl auth can-i '*' '*' 2>&1 "${kube_flags[@]}")
-    kube::test::if_has_string "${output_message}" "yes"
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_kubectl_request_timeout_tests
+    fi
 
-    output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}")
-    kube::test::if_has_string "${output_message}" "yes"
+    #####################################
+    # CustomResourceDefinitions         #
+    #####################################
 
-    output_message=$(kubectl auth can-i get invalid_resource 2>&1 "${kube_flags[@]}")
-    kube::test::if_has_string "${output_message}" "the server doesn't have a resource type"
+    # customresourcedefinitions cleanup after themselves.
+    if kube::test::if_supports_resource "${customresourcedefinitions}" ; then
+      record_command run_crd_tests
+    fi
 
-    output_message=$(kubectl auth can-i get /logs/ 2>&1 "${kube_flags[@]}")
-    kube::test::if_has_string "${output_message}" "yes"
+    #################
+    # Run cmd w img #
+    #################
 
-    output_message=$(! kubectl auth can-i get /logs/ --subresource=log 2>&1 "${kube_flags[@]}")
-    kube::test::if_has_string "${output_message}" "subresource can not be used with NonResourceURL"
+    if kube::test::if_supports_resource "${deployments}" ; then
+      record_command run_cmd_with_img_tests
+    fi
 
-    output_message=$(kubectl auth can-i list jobs.batch/bar -n foo --quiet 2>&1 "${kube_flags[@]}")
-    kube::test::if_empty_string "${output_message}"
 
-    output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}"; echo $?)
-    kube::test::if_has_string "${output_message}" '0'
+    #####################################
+    # Recursive Resources via directory #
+    #####################################
 
-    output_message=$(kubectl auth can-i get pods --subresource=log --quiet 2>&1 "${kube_flags[@]}"; echo $?)
-    kube::test::if_has_string "${output_message}" '0'
-  fi
+    if kube::test::if_supports_resource "${pods}" ; then
+      run_recursive_resources_tests
+    fi
 
-  # kubectl auth reconcile
-  if kube::test::if_supports_resource "${clusterroles}" ; then
-    kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-resource-plus.yaml
-    kube::test::get_object_assert 'rolebindings -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-RB:'
-    kube::test::get_object_assert 'roles -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-R:'
-    kube::test::get_object_assert 'clusterrolebindings -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CRB:'
-    kube::test::get_object_assert 'clusterroles -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CR:'
 
-    failure_message=$(! kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-v1beta1.yaml 2>&1 )
-    kube::test::if_has_string "${failure_message}" 'only rbac.authorization.k8s.io/v1 is supported'
+    ##############
+    # Namespaces #
+    ##############
+    if kube::test::if_supports_resource "${namespaces}" ; then
+      record_command run_namespace_tests
+    fi
 
-    kubectl delete "${kube_flags[@]}" rolebindings,role,clusterroles,clusterrolebindings -n some-other-random -l test-cmd=auth
-  fi
 
-  #####################
-  # Retrieve multiple #
-  #####################
+    ###########
+    # Secrets #
+    ###########
+    if kube::test::if_supports_resource "${namespaces}" ; then
+      if kube::test::if_supports_resource "${secrets}" ; then
+        record_command run_secrets_test
+      fi
+    fi
+
+
+    ######################
+    # ConfigMap          #
+    ######################
+
+    if kube::test::if_supports_resource "${namespaces}"; then
+      if kube::test::if_supports_resource "${configmaps}" ; then
+        record_command run_configmap_tests
+      fi
+    fi
+
+    ####################
+    # Client Config    #
+    ####################
+
+    record_command run_client_config_tests
+
+    ####################
+    # Service Accounts #
+    ####################
+
+    if kube::test::if_supports_resource "${namespaces}" && kube::test::if_supports_resource "${serviceaccounts}" ; then
+      record_command run_service_accounts_tests
+    fi
+
+    ####################
+    # Job              #
+    ####################
+
+    if kube::test::if_supports_resource "${job}" ; then
+      record_command run_job_tests
+      record_command run_create_job_tests
+    fi
+
+    #################
+    # Pod templates #
+    #################
+
+    if kube::test::if_supports_resource "${podtemplates}" ; then
+      record_command run_pod_templates_tests
+    fi
+
+    ############
+    # Services #
+    ############
 
-  if kube::test::if_supports_resource "${nodes}" ; then
     if kube::test::if_supports_resource "${services}" ; then
-      record_command run_retrieve_multiple_tests
+      record_command run_service_tests
     fi
-  fi
 
+    ##################
+    # DaemonSets     #
+    ##################
 
-  #####################
-  # Resource aliasing #
-  #####################
-
-  if kube::test::if_supports_resource "${services}" ; then
-    if kube::test::if_supports_resource "${replicationcontrollers}" ; then
-      record_command run_resource_aliasing_tests
+    if kube::test::if_supports_resource "${daemonsets}" ; then
+      record_command run_daemonset_tests
+      if kube::test::if_supports_resource "${controllerrevisions}"; then
+        record_command run_daemonset_history_tests
+      fi
     fi
-  fi
 
-  ###########
-  # Explain #
-  ###########
+    ###########################
+    # Replication controllers #
+    ###########################
 
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_kubectl_explain_tests
-  fi
+    if kube::test::if_supports_resource "${namespaces}" ; then
+      if kube::test::if_supports_resource "${replicationcontrollers}" ; then
+        record_command run_rc_tests
+      fi
+    fi
+
+    ######################
+    # Deployments       #
+    ######################
+
+    if kube::test::if_supports_resource "${deployments}" ; then
+      record_command run_deployment_tests
+    fi
+
+    ######################
+    # Replica Sets       #
+    ######################
+
+    if kube::test::if_supports_resource "${replicasets}" ; then
+      record_command run_rs_tests
+    fi
+
+    #################
+    # Stateful Sets #
+    #################
+
+    if kube::test::if_supports_resource "${statefulsets}" ; then
+      record_command run_stateful_set_tests
+      if kube::test::if_supports_resource "${controllerrevisions}"; then
+        record_command run_statefulset_history_tests
+      fi
+    fi
+
+    ######################
+    # Lists              #
+    ######################
+
+    if kube::test::if_supports_resource "${services}" ; then
+      if kube::test::if_supports_resource "${deployments}" ; then
+        record_command run_lists_tests
+      fi
+    fi
 
 
-  ###########
-  # Swagger #
-  ###########
+    ######################
+    # Multiple Resources #
+    ######################
+    if kube::test::if_supports_resource "${services}" ; then
+      if kube::test::if_supports_resource "${replicationcontrollers}" ; then
+        record_command run_multi_resources_tests
+      fi
+    fi
 
-  record_command run_swagger_tests
+    ######################
+    # Persistent Volumes #
+    ######################
 
-  #####################
-  # Kubectl --sort-by #
-  #####################
+    if kube::test::if_supports_resource "${persistentvolumes}" ; then
+      record_command run_persistent_volumes_tests
+    fi
 
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_kubectl_sort_by_tests
-  fi
+    ############################
+    # Persistent Volume Claims #
+    ############################
 
-  ############################
-  # Kubectl --all-namespaces #
-  ############################
+    if kube::test::if_supports_resource "${persistentvolumeclaims}" ; then
+      record_command run_persistent_volume_claims_tests
+    fi
+
+    ############################
+    # Storage Classes #
+    ############################
+
+    if kube::test::if_supports_resource "${storageclass}" ; then
+      record_command run_storage_class_tests
+    fi
+
+    #########
+    # Nodes #
+    #########
 
-  if kube::test::if_supports_resource "${pods}" ; then
     if kube::test::if_supports_resource "${nodes}" ; then
-      record_command run_kubectl_all_namespace_tests
+      record_command run_nodes_tests
     fi
+
+
+    ########################
+    # authorization.k8s.io #
+    ########################
+
+    if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
+      record_command run_authorization_tests
+    fi
+
+    # kubectl auth can-i
+    # kube-apiserver is started with authorization mode AlwaysAllow, so kubectl can-i always returns yes
+    if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
+      output_message=$(kubectl auth can-i '*' '*' 2>&1 "${kube_flags[@]}")
+      kube::test::if_has_string "${output_message}" "yes"
+
+      output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}")
+      kube::test::if_has_string "${output_message}" "yes"
+
+      output_message=$(kubectl auth can-i get invalid_resource 2>&1 "${kube_flags[@]}")
+      kube::test::if_has_string "${output_message}" "the server doesn't have a resource type"
+
+      output_message=$(kubectl auth can-i get /logs/ 2>&1 "${kube_flags[@]}")
+      kube::test::if_has_string "${output_message}" "yes"
+
+      output_message=$(! kubectl auth can-i get /logs/ --subresource=log 2>&1 "${kube_flags[@]}")
+      kube::test::if_has_string "${output_message}" "subresource can not be used with NonResourceURL"
+
+      output_message=$(kubectl auth can-i list jobs.batch/bar -n foo --quiet 2>&1 "${kube_flags[@]}")
+      kube::test::if_empty_string "${output_message}"
+
+      output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}"; echo $?)
+      kube::test::if_has_string "${output_message}" '0'
+
+      output_message=$(kubectl auth can-i get pods --subresource=log --quiet 2>&1 "${kube_flags[@]}"; echo $?)
+      kube::test::if_has_string "${output_message}" '0'
+    fi
+
+    # kubectl auth reconcile
+    if kube::test::if_supports_resource "${clusterroles}" ; then
+      kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-resource-plus.yaml
+      kube::test::get_object_assert 'rolebindings -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-RB:'
+      kube::test::get_object_assert 'roles -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-R:'
+      kube::test::get_object_assert 'clusterrolebindings -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CRB:'
+      kube::test::get_object_assert 'clusterroles -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CR:'
+
+      failure_message=$(! kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-v1beta1.yaml 2>&1 )
+      kube::test::if_has_string "${failure_message}" 'only rbac.authorization.k8s.io/v1 is supported'
+
+      kubectl delete "${kube_flags[@]}" rolebindings,role,clusterroles,clusterrolebindings -n some-other-random -l test-cmd=auth
+    fi
+
+    #####################
+    # Retrieve multiple #
+    #####################
+
+    if kube::test::if_supports_resource "${nodes}" ; then
+      if kube::test::if_supports_resource "${services}" ; then
+        record_command run_retrieve_multiple_tests
+      fi
+    fi
+
+
+    #####################
+    # Resource aliasing #
+    #####################
+
+    if kube::test::if_supports_resource "${services}" ; then
+      if kube::test::if_supports_resource "${replicationcontrollers}" ; then
+        record_command run_resource_aliasing_tests
+      fi
+    fi
+
+    ###########
+    # Explain #
+    ###########
+
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_kubectl_explain_tests
+    fi
+
+
+    ###########
+    # Swagger #
+    ###########
+
+    record_command run_swagger_tests
+
+    #####################
+    # Kubectl --sort-by #
+    #####################
+
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_kubectl_sort_by_tests
+    fi
+
+    ############################
+    # Kubectl --all-namespaces #
+    ############################
+
+    if kube::test::if_supports_resource "${pods}" ; then
+      if kube::test::if_supports_resource "${nodes}" ; then
+        record_command run_kubectl_all_namespace_tests
+      fi
+    fi
+
+    ######################
+    # kubectl --template #
+    ######################
+
+    if kube::test::if_supports_resource "${pods}" ; then
+      record_command run_template_output_tests
+    fi
+
+    ################
+    # Certificates #
+    ################
+
+    if kube::test::if_supports_resource "${csr}" ; then
+      record_command run_certificates_tests
+    fi
+
+    ######################
+    # Cluster Management #
+    ######################
+    if kube::test::if_supports_resource "${nodes}" ; then
+      record_command run_cluster_management_tests
+    fi
+
+    ###########
+    # Plugins #
+    ###########
+
+    record_command run_plugins_tests
+
+    #################
+    # Impersonation #
+    #################
+    record_command run_impersonation_tests
   fi
-
-  ######################
-  # kubectl --template #
-  ######################
-
-  if kube::test::if_supports_resource "${pods}" ; then
-    record_command run_template_output_tests
-  fi
-
-  ################
-  # Certificates #
-  ################
-
-  if kube::test::if_supports_resource "${csr}" ; then
-    record_command run_certificates_tests
-  fi
-
-  ######################
-  # Cluster Management #
-  ######################
-  if kube::test::if_supports_resource "${nodes}" ; then
-    record_command run_cluster_management_tests
-  fi
-
-  ###########
-  # Plugins #
-  ###########
-
-  record_command run_plugins_tests
-
-  #################
-  # Impersonation #
-  #################
-  record_command run_impersonation_tests
-
   kube::test::clear_all
 
   if [[ -n "${foundError}" ]]; then

From f47e40412586c40b5f5286834f733a78caa53494 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Tue, 15 Jan 2019 16:21:59 -0500
Subject: [PATCH 02/12] removed debugging code

---
 test/cmd/legacy-script.sh | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/test/cmd/legacy-script.sh b/test/cmd/legacy-script.sh
index bc50a781b6..768ae20a05 100755
--- a/test/cmd/legacy-script.sh
+++ b/test/cmd/legacy-script.sh
@@ -20,15 +20,6 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [ ! -z "$WHAT" ]; then #ccheck not empty
-    for pkg in ${WHAT}
-    do 
-      echo ${pkg}
-    done
-fi
-exit 0
-
-
 # Set locale to ensure english responses from kubectl commands
 export LANG=C
 

From ba901aac5ff3b5e707a12f25f242a133b414f1b3 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Wed, 23 Jan 2019 05:37:09 -0500
Subject: [PATCH 03/12] modified hack scripts to referenced moved kubeadm.sh
 file.

---
 hack/.shellcheck_failures | 2 +-
 hack/make-rules/BUILD     | 8 --------
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures
index 41c604bc4e..af0746cec8 100644
--- a/hack/.shellcheck_failures
+++ b/hack/.shellcheck_failures
@@ -114,7 +114,6 @@
 ./hack/make-rules/test-cmd.sh
 ./hack/make-rules/test-e2e-node.sh
 ./hack/make-rules/test-integration.sh
-./hack/make-rules/test-kubeadm-cmd.sh
 ./hack/make-rules/test.sh
 ./hack/make-rules/update.sh
 ./hack/make-rules/verify.sh
@@ -234,6 +233,7 @@
 ./test/cmd/generic-resources.sh
 ./test/cmd/get.sh
 ./test/cmd/initializers.sh
+./test/cmd/kubeadm.sh
 ./test/cmd/legacy-script.sh
 ./test/cmd/node-management.sh
 ./test/cmd/old-print.sh
diff --git a/hack/make-rules/BUILD b/hack/make-rules/BUILD
index fa982fcef3..3b68b5d129 100644
--- a/hack/make-rules/BUILD
+++ b/hack/make-rules/BUILD
@@ -41,14 +41,6 @@ sh_binary(
     ],
 )
 
-sh_binary(
-    name = "test-kubeadm-cmd",
-    srcs = ["test-kubeadm-cmd.sh"],
-    deps = [
-        "//hack/lib",
-    ],
-)
-
 sh_binary(
     name = "build",
     srcs = ["build.sh"],

From 5d766bdb2418e9ced21d925ce53f3fa840054c72 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Wed, 23 Jan 2019 07:01:55 -0500
Subject: [PATCH 04/12] Removed resetting KUBE_ROOT

---
 test/cmd/kubeadm.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/cmd/kubeadm.sh b/test/cmd/kubeadm.sh
index 40fb0ef1c1..aae65bf974 100755
--- a/test/cmd/kubeadm.sh
+++ b/test/cmd/kubeadm.sh
@@ -21,7 +21,6 @@ set -o pipefail
 run_kubeadm_tests() {
   set -o nounset
   set -o errexit
-  KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
   KUBEADM_PATH="${KUBEADM_PATH:=$(kube::realpath "${KUBE_ROOT}")/cluster/kubeadm.sh}"
 

From fc4bdd9e9d876d07173ffd2843e3c9877267cfc6 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Thu, 24 Jan 2019 19:44:51 -0500
Subject: [PATCH 05/12] Removed test/kubeadm.sh from shellcheck failure list
 since it now passes

---
 hack/.shellcheck_failures | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hack/.shellcheck_failures b/hack/.shellcheck_failures
index af0746cec8..ae8fa4ced0 100644
--- a/hack/.shellcheck_failures
+++ b/hack/.shellcheck_failures
@@ -233,7 +233,6 @@
 ./test/cmd/generic-resources.sh
 ./test/cmd/get.sh
 ./test/cmd/initializers.sh
-./test/cmd/kubeadm.sh
 ./test/cmd/legacy-script.sh
 ./test/cmd/node-management.sh
 ./test/cmd/old-print.sh

From a844df60f80d756c4eaf13ef00f2f0ce5548af18 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Thu, 24 Jan 2019 19:54:28 -0500
Subject: [PATCH 06/12] Fixed spacing in readme

---
 test/cmd/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/cmd/README.md b/test/cmd/README.md
index f9572e929d..c003ce0b78 100644
--- a/test/cmd/README.md
+++ b/test/cmd/README.md
@@ -10,7 +10,7 @@ To run this entire suite, execute `make test-cmd` from the top level.  This will
 
 ### Specific Tests
 
-To run a subset of tests( e.g. `run_deployment_test` and `run_impersonation_test`), execute `make test-cmd WHAT="deployment impersonation"`.  Running specific
+To run a subset of tests (e.g. `run_deployment_test` and `run_impersonation_test`), execute `make test-cmd WHAT="deployment impersonation"`.  Running specific
 tests will not try and validate any required resources are available on the server.
 
 ## Adding Tests

From eecd88b450f1d10d77be807d1964647984f5af9a Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Sat, 26 Jan 2019 09:06:40 -0500
Subject: [PATCH 07/12] fixed merge

---
 test/cmd/legacy-script.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/cmd/legacy-script.sh b/test/cmd/legacy-script.sh
index 768ae20a05..da430a17fd 100755
--- a/test/cmd/legacy-script.sh
+++ b/test/cmd/legacy-script.sh
@@ -39,7 +39,6 @@ source "${KUBE_ROOT}/test/cmd/diff.sh"
 source "${KUBE_ROOT}/test/cmd/discovery.sh"
 source "${KUBE_ROOT}/test/cmd/generic-resources.sh"
 source "${KUBE_ROOT}/test/cmd/get.sh"
-source "${KUBE_ROOT}/test/cmd/initializers.sh"
 source "${KUBE_ROOT}/test/cmd/kubeadm.sh"
 source "${KUBE_ROOT}/test/cmd/kubeconfig.sh"
 source "${KUBE_ROOT}/test/cmd/node-management.sh"

From aeb92ee754bb29c79baafda2674a26b9bb0fe3cc Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Tue, 29 Jan 2019 13:10:55 -0500
Subject: [PATCH 08/12] Source hack/lib/init.sh for running make test-cmd

---
 build/root/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build/root/Makefile b/build/root/Makefile
index b464e75638..72dcae6f95 100644
--- a/build/root/Makefile
+++ b/build/root/Makefile
@@ -291,6 +291,7 @@ test-cmd:
 	@echo "$$TEST_CMD_HELP_INFO"
 else
 test-cmd: generated_files
+	source hack/lib/init.sh
 	hack/make-rules/test-cmd.sh
 endif
 

From 8772e263156cdad8f50dd7edf85d619b6b0316bc Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Fri, 1 Feb 2019 13:53:23 -0500
Subject: [PATCH 09/12] Fixed variable check for new WHAT command

---
 test/cmd/legacy-script.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/cmd/legacy-script.sh b/test/cmd/legacy-script.sh
index da430a17fd..cfa6e54c50 100755
--- a/test/cmd/legacy-script.sh
+++ b/test/cmd/legacy-script.sh
@@ -384,13 +384,12 @@ runTests() {
     kubectl get "${kube_flags[@]}" -f hack/testdata/kubernetes-service.yaml
   fi
 
-  if [ ! -z "$WHAT" ]; then #ccheck not empty
+   if [[ -n "${WHAT-}" ]]; then
     for pkg in ${WHAT}
     do 
       record_command run_${pkg}_tests
     done
   else
-
     #########################
     # Kubectl version #
     #########################

From 30c78c7e1c38e7b9a71fd2b7b29ca7ca62ad4532 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Thu, 21 Feb 2019 06:15:13 -0500
Subject: [PATCH 10/12] Feedback from review

---
 test/cmd/README.md        |   4 +-
 test/cmd/legacy-script.sh | 878 +++++++++++++++++++-------------------
 2 files changed, 444 insertions(+), 438 deletions(-)

diff --git a/test/cmd/README.md b/test/cmd/README.md
index c003ce0b78..d5146c12e0 100644
--- a/test/cmd/README.md
+++ b/test/cmd/README.md
@@ -1,6 +1,6 @@
 # Kubernetes Command-Line Integration Test Suite
 
-This document describes how Kubernetes should interact with the Kubernetes command-line integration test suite.
+This document describes how you can use the Kubernetes command-line integration test-suite.
 
 ## Running Tests
 
@@ -27,7 +27,7 @@ if kube::test::if_supports_resource "${replicasets}" ; then
 fi
 ```
 
-Be sure to validate any supported resouces required for the test by using the `kue::test::if_supports_resource` function. 
+Be sure to validate any supported resouces required for the test by using the `kube::test::if_supports_resource` function. 
 
 
 ### New File
diff --git a/test/cmd/legacy-script.sh b/test/cmd/legacy-script.sh
index b6aeee2722..f58584e6b7 100755
--- a/test/cmd/legacy-script.sh
+++ b/test/cmd/legacy-script.sh
@@ -386,479 +386,485 @@ runTests() {
     kubectl get "${kube_flags[@]}" -f hack/testdata/kubernetes-service.yaml
   fi
 
+  cleanup(){
+     kube::test::clear_all
+    if [[ -n "${foundError}" ]]; then
+      echo "FAILED TESTS: ""${foundError}"
+      exit 1
+    fi
+  }
+
    if [[ -n "${WHAT-}" ]]; then
     for pkg in ${WHAT}
     do 
       record_command run_${pkg}_tests
     done
-  else
-    #########################
-    # Kubectl version #
-    #########################
+    cleanup
+    return
+  fi
 
-    record_command run_kubectl_version_tests
+  #########################
+  # Kubectl version #
+  #########################
 
-    #######################
-    # kubectl config set #
-    #######################
+  record_command run_kubectl_version_tests
 
-    record_command run_kubectl_config_set_tests
+  #######################
+  # kubectl config set #
+  #######################
 
-    #######################
-    # kubectl local proxy #
-    #######################
+  record_command run_kubectl_config_set_tests
 
-    record_command run_kubectl_local_proxy_tests
+  #######################
+  # kubectl local proxy #
+  #######################
 
-    #########################
-    # RESTMapper evaluation #
-    #########################
+  record_command run_kubectl_local_proxy_tests
 
-    record_command run_RESTMapper_evaluation_tests
+  #########################
+  # RESTMapper evaluation #
+  #########################
 
-    # find all resources
-    kubectl "${kube_flags[@]}" api-resources
-    # find all namespaced resources that support list by name and get them
-    kubectl "${kube_flags[@]}" api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl "${kube_flags[@]}" get -o name
+  record_command run_RESTMapper_evaluation_tests
 
-    ################
-    # Cluster Role #
-    ################
+  # find all resources
+  kubectl "${kube_flags[@]}" api-resources
+  # find all namespaced resources that support list by name and get them
+  kubectl "${kube_flags[@]}" api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl "${kube_flags[@]}" get -o name
 
-    if kube::test::if_supports_resource "${clusterroles}" ; then
-      record_command run_clusterroles_tests
-    fi
+  ################
+  # Cluster Role #
+  ################
 
-    ########
-    # Role #
-    ########
-    if kube::test::if_supports_resource "${roles}" ; then
-        record_command run_role_tests
-    fi
+  if kube::test::if_supports_resource "${clusterroles}" ; then
+    record_command run_clusterroles_tests
+  fi
 
-    #########################
-    # Assert short name     #
-    #########################
+  ########
+  # Role #
+  ########
+  if kube::test::if_supports_resource "${roles}" ; then
+      record_command run_role_tests
+  fi
 
-    record_command run_assert_short_name_tests
+  #########################
+  # Assert short name     #
+  #########################
 
-    #########################
-    # Assert categories     #
-    #########################
+  record_command run_assert_short_name_tests
 
-    ## test if a category is exported during discovery
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_assert_categories_tests
-    fi
+  #########################
+  # Assert categories     #
+  #########################
 
-    ###########################
-    # POD creation / deletion #
-    ###########################
+  ## test if a category is exported during discovery
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_assert_categories_tests
+  fi
 
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_pod_tests
-    fi
+  ###########################
+  # POD creation / deletion #
+  ###########################
 
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_save_config_tests
-    fi
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_pod_tests
+  fi
 
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_kubectl_create_error_tests
-    fi
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_save_config_tests
+  fi
 
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_kubectl_apply_tests
-      record_command run_kubectl_run_tests
-      record_command run_kubectl_create_filter_tests
-    fi
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_kubectl_create_error_tests
+  fi
 
-    if kube::test::if_supports_resource "${deployments}" ; then
-      record_command run_kubectl_apply_deployments_tests
-    fi
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_kubectl_apply_tests
+    record_command run_kubectl_run_tests
+    record_command run_kubectl_create_filter_tests
+  fi
 
-    ################
-    # Kubectl diff #
-    ################
-    record_command run_kubectl_diff_tests
-    record_command run_kubectl_diff_same_names
+  if kube::test::if_supports_resource "${deployments}" ; then
+    record_command run_kubectl_apply_deployments_tests
+  fi
 
-    ###############
-    # Kubectl get #
-    ###############
+  ################
+  # Kubectl diff #
+  ################
+  record_command run_kubectl_diff_tests
+  record_command run_kubectl_diff_same_names
 
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_kubectl_get_tests
-      record_command run_kubectl_old_print_tests
-    fi
+  ###############
+  # Kubectl get #
+  ###############
+
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_kubectl_get_tests
+    record_command run_kubectl_old_print_tests
+  fi
 
 
-    ######################
-    # Create             #
-    ######################
+  ######################
+  # Create             #
+  ######################
+  if kube::test::if_supports_resource "${secrets}" ; then
+    record_command run_create_secret_tests
+  fi
+
+  ######################
+  # Delete             #
+  ######################
+  if kube::test::if_supports_resource "${configmaps}" ; then
+    record_command run_kubectl_delete_allnamespaces_tests
+  fi
+
+  ##################
+  # Global timeout #
+  ##################
+
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_kubectl_request_timeout_tests
+  fi
+
+  #####################################
+  # CustomResourceDefinitions         #
+  #####################################
+
+  # customresourcedefinitions cleanup after themselves.
+  if kube::test::if_supports_resource "${customresourcedefinitions}" ; then
+    record_command run_crd_tests
+  fi
+
+  #################
+  # Run cmd w img #
+  #################
+
+  if kube::test::if_supports_resource "${deployments}" ; then
+    record_command run_cmd_with_img_tests
+  fi
+
+
+  #####################################
+  # Recursive Resources via directory #
+  #####################################
+
+  if kube::test::if_supports_resource "${pods}" ; then
+    run_recursive_resources_tests
+  fi
+
+
+  ##############
+  # Namespaces #
+  ##############
+  if kube::test::if_supports_resource "${namespaces}" ; then
+    record_command run_namespace_tests
+  fi
+
+
+  ###########
+  # Secrets #
+  ###########
+  if kube::test::if_supports_resource "${namespaces}" ; then
     if kube::test::if_supports_resource "${secrets}" ; then
-      record_command run_create_secret_tests
+      record_command run_secrets_test
     fi
+  fi
 
-    ######################
-    # Delete             #
-    ######################
+
+  ######################
+  # ConfigMap          #
+  ######################
+
+  if kube::test::if_supports_resource "${namespaces}"; then
     if kube::test::if_supports_resource "${configmaps}" ; then
-      record_command run_kubectl_delete_allnamespaces_tests
+      record_command run_configmap_tests
     fi
-
-    ##################
-    # Global timeout #
-    ##################
-
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_kubectl_request_timeout_tests
-    fi
-
-    #####################################
-    # CustomResourceDefinitions         #
-    #####################################
-
-    # customresourcedefinitions cleanup after themselves.
-    if kube::test::if_supports_resource "${customresourcedefinitions}" ; then
-      record_command run_crd_tests
-    fi
-
-    #################
-    # Run cmd w img #
-    #################
-
-    if kube::test::if_supports_resource "${deployments}" ; then
-      record_command run_cmd_with_img_tests
-    fi
-
-
-    #####################################
-    # Recursive Resources via directory #
-    #####################################
-
-    if kube::test::if_supports_resource "${pods}" ; then
-      run_recursive_resources_tests
-    fi
-
-
-    ##############
-    # Namespaces #
-    ##############
-    if kube::test::if_supports_resource "${namespaces}" ; then
-      record_command run_namespace_tests
-    fi
-
-
-    ###########
-    # Secrets #
-    ###########
-    if kube::test::if_supports_resource "${namespaces}" ; then
-      if kube::test::if_supports_resource "${secrets}" ; then
-        record_command run_secrets_test
-      fi
-    fi
-
-
-    ######################
-    # ConfigMap          #
-    ######################
-
-    if kube::test::if_supports_resource "${namespaces}"; then
-      if kube::test::if_supports_resource "${configmaps}" ; then
-        record_command run_configmap_tests
-      fi
-    fi
-
-    ####################
-    # Client Config    #
-    ####################
-
-    record_command run_client_config_tests
-
-    ####################
-    # Service Accounts #
-    ####################
-
-    if kube::test::if_supports_resource "${namespaces}" && kube::test::if_supports_resource "${serviceaccounts}" ; then
-      record_command run_service_accounts_tests
-    fi
-
-    ####################
-    # Job              #
-    ####################
-
-    if kube::test::if_supports_resource "${job}" ; then
-      record_command run_job_tests
-      record_command run_create_job_tests
-    fi
-
-    #################
-    # Pod templates #
-    #################
-
-    if kube::test::if_supports_resource "${podtemplates}" ; then
-      record_command run_pod_templates_tests
-    fi
-
-    ############
-    # Services #
-    ############
-
-    if kube::test::if_supports_resource "${services}" ; then
-      record_command run_service_tests
-    fi
-
-    ##################
-    # DaemonSets     #
-    ##################
-
-    if kube::test::if_supports_resource "${daemonsets}" ; then
-      record_command run_daemonset_tests
-      if kube::test::if_supports_resource "${controllerrevisions}"; then
-        record_command run_daemonset_history_tests
-      fi
-    fi
-
-    ###########################
-    # Replication controllers #
-    ###########################
-
-    if kube::test::if_supports_resource "${namespaces}" ; then
-      if kube::test::if_supports_resource "${replicationcontrollers}" ; then
-        record_command run_rc_tests
-      fi
-    fi
-
-    ######################
-    # Deployments       #
-    ######################
-
-    if kube::test::if_supports_resource "${deployments}" ; then
-      record_command run_deployment_tests
-    fi
-
-    ######################
-    # Replica Sets       #
-    ######################
-
-    if kube::test::if_supports_resource "${replicasets}" ; then
-      record_command run_rs_tests
-    fi
-
-    #################
-    # Stateful Sets #
-    #################
-
-    if kube::test::if_supports_resource "${statefulsets}" ; then
-      record_command run_stateful_set_tests
-      if kube::test::if_supports_resource "${controllerrevisions}"; then
-        record_command run_statefulset_history_tests
-      fi
-    fi
-
-    ######################
-    # Lists              #
-    ######################
-
-    if kube::test::if_supports_resource "${services}" ; then
-      if kube::test::if_supports_resource "${deployments}" ; then
-        record_command run_lists_tests
-      fi
-    fi
-
-
-    ######################
-    # Multiple Resources #
-    ######################
-    if kube::test::if_supports_resource "${services}" ; then
-      if kube::test::if_supports_resource "${replicationcontrollers}" ; then
-        record_command run_multi_resources_tests
-      fi
-    fi
-
-    ######################
-    # Persistent Volumes #
-    ######################
-
-    if kube::test::if_supports_resource "${persistentvolumes}" ; then
-      record_command run_persistent_volumes_tests
-    fi
-
-    ############################
-    # Persistent Volume Claims #
-    ############################
-
-    if kube::test::if_supports_resource "${persistentvolumeclaims}" ; then
-      record_command run_persistent_volume_claims_tests
-    fi
-
-    ############################
-    # Storage Classes #
-    ############################
-
-    if kube::test::if_supports_resource "${storageclass}" ; then
-      record_command run_storage_class_tests
-    fi
-
-    #########
-    # Nodes #
-    #########
-
-    if kube::test::if_supports_resource "${nodes}" ; then
-      record_command run_nodes_tests
-    fi
-
-
-    ########################
-    # authorization.k8s.io #
-    ########################
-
-    if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
-      record_command run_authorization_tests
-    fi
-
-    # kubectl auth can-i
-    # kube-apiserver is started with authorization mode AlwaysAllow, so kubectl can-i always returns yes
-    if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
-      output_message=$(kubectl auth can-i '*' '*' 2>&1 "${kube_flags[@]}")
-      kube::test::if_has_string "${output_message}" "yes"
-
-      output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}")
-      kube::test::if_has_string "${output_message}" "yes"
-
-      output_message=$(kubectl auth can-i get invalid_resource 2>&1 "${kube_flags[@]}")
-      kube::test::if_has_string "${output_message}" "the server doesn't have a resource type"
-
-      output_message=$(kubectl auth can-i get /logs/ 2>&1 "${kube_flags[@]}")
-      kube::test::if_has_string "${output_message}" "yes"
-
-      output_message=$(! kubectl auth can-i get /logs/ --subresource=log 2>&1 "${kube_flags[@]}")
-      kube::test::if_has_string "${output_message}" "subresource can not be used with NonResourceURL"
-
-      output_message=$(kubectl auth can-i list jobs.batch/bar -n foo --quiet 2>&1 "${kube_flags[@]}")
-      kube::test::if_empty_string "${output_message}"
-
-      output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}"; echo $?)
-      kube::test::if_has_string "${output_message}" '0'
-
-      output_message=$(kubectl auth can-i get pods --subresource=log --quiet 2>&1 "${kube_flags[@]}"; echo $?)
-      kube::test::if_has_string "${output_message}" '0'
-    fi
-
-    # kubectl auth reconcile
-    if kube::test::if_supports_resource "${clusterroles}" ; then
-      kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-resource-plus.yaml
-      kube::test::get_object_assert 'rolebindings -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-RB:'
-      kube::test::get_object_assert 'roles -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-R:'
-      kube::test::get_object_assert 'clusterrolebindings -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CRB:'
-      kube::test::get_object_assert 'clusterroles -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CR:'
-
-      failure_message=$(! kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-v1beta1.yaml 2>&1 )
-      kube::test::if_has_string "${failure_message}" 'only rbac.authorization.k8s.io/v1 is supported'
-
-      kubectl delete "${kube_flags[@]}" rolebindings,role,clusterroles,clusterrolebindings -n some-other-random -l test-cmd=auth
-    fi
-
-    #####################
-    # Retrieve multiple #
-    #####################
-
-    if kube::test::if_supports_resource "${nodes}" ; then
-      if kube::test::if_supports_resource "${services}" ; then
-        record_command run_retrieve_multiple_tests
-      fi
-    fi
-
-
-    #####################
-    # Resource aliasing #
-    #####################
-
-    if kube::test::if_supports_resource "${services}" ; then
-      if kube::test::if_supports_resource "${replicationcontrollers}" ; then
-        record_command run_resource_aliasing_tests
-      fi
-    fi
-
-    ###########
-    # Explain #
-    ###########
-
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_kubectl_explain_tests
-    fi
-
-
-    ###########
-    # Swagger #
-    ###########
-
-    record_command run_swagger_tests
-
-    #####################
-    # Kubectl --sort-by #
-    #####################
-
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_kubectl_sort_by_tests
-    fi
-
-    ############################
-    # Kubectl --all-namespaces #
-    ############################
-
-    if kube::test::if_supports_resource "${pods}" ; then
-      if kube::test::if_supports_resource "${nodes}" ; then
-        record_command run_kubectl_all_namespace_tests
-      fi
-    fi
-
-    ######################
-    # kubectl --template #
-    ######################
-
-    if kube::test::if_supports_resource "${pods}" ; then
-      record_command run_template_output_tests
-    fi
-
-    ################
-    # Certificates #
-    ################
-
-    if kube::test::if_supports_resource "${csr}" ; then
-      record_command run_certificates_tests
-    fi
-
-    ######################
-    # Cluster Management #
-    ######################
-    if kube::test::if_supports_resource "${nodes}" ; then
-      record_command run_cluster_management_tests
-    fi
-
-    ###########
-    # Plugins #
-    ###########
-
-    record_command run_plugins_tests
-
-
-    #################
-    # Impersonation #
-    #################
-    record_command run_impersonation_tests
- 
-    ####################
-    # kubectl wait     #
-    ####################
-
-    record_command run_wait_tests
   fi
-  kube::test::clear_all
 
-  if [[ -n "${foundError}" ]]; then
-    echo "FAILED TESTS: ""${foundError}"
-    exit 1
+  ####################
+  # Client Config    #
+  ####################
+
+  record_command run_client_config_tests
+
+  ####################
+  # Service Accounts #
+  ####################
+
+  if kube::test::if_supports_resource "${namespaces}" && kube::test::if_supports_resource "${serviceaccounts}" ; then
+    record_command run_service_accounts_tests
   fi
+
+  ####################
+  # Job              #
+  ####################
+
+  if kube::test::if_supports_resource "${job}" ; then
+    record_command run_job_tests
+    record_command run_create_job_tests
+  fi
+
+  #################
+  # Pod templates #
+  #################
+
+  if kube::test::if_supports_resource "${podtemplates}" ; then
+    record_command run_pod_templates_tests
+  fi
+
+  ############
+  # Services #
+  ############
+
+  if kube::test::if_supports_resource "${services}" ; then
+    record_command run_service_tests
+  fi
+
+  ##################
+  # DaemonSets     #
+  ##################
+
+  if kube::test::if_supports_resource "${daemonsets}" ; then
+    record_command run_daemonset_tests
+    if kube::test::if_supports_resource "${controllerrevisions}"; then
+      record_command run_daemonset_history_tests
+    fi
+  fi
+
+  ###########################
+  # Replication controllers #
+  ###########################
+
+  if kube::test::if_supports_resource "${namespaces}" ; then
+    if kube::test::if_supports_resource "${replicationcontrollers}" ; then
+      record_command run_rc_tests
+    fi
+  fi
+
+  ######################
+  # Deployments       #
+  ######################
+
+  if kube::test::if_supports_resource "${deployments}" ; then
+    record_command run_deployment_tests
+  fi
+
+  ######################
+  # Replica Sets       #
+  ######################
+
+  if kube::test::if_supports_resource "${replicasets}" ; then
+    record_command run_rs_tests
+  fi
+
+  #################
+  # Stateful Sets #
+  #################
+
+  if kube::test::if_supports_resource "${statefulsets}" ; then
+    record_command run_stateful_set_tests
+    if kube::test::if_supports_resource "${controllerrevisions}"; then
+      record_command run_statefulset_history_tests
+    fi
+  fi
+
+  ######################
+  # Lists              #
+  ######################
+
+  if kube::test::if_supports_resource "${services}" ; then
+    if kube::test::if_supports_resource "${deployments}" ; then
+      record_command run_lists_tests
+    fi
+  fi
+
+
+  ######################
+  # Multiple Resources #
+  ######################
+  if kube::test::if_supports_resource "${services}" ; then
+    if kube::test::if_supports_resource "${replicationcontrollers}" ; then
+      record_command run_multi_resources_tests
+    fi
+  fi
+
+  ######################
+  # Persistent Volumes #
+  ######################
+
+  if kube::test::if_supports_resource "${persistentvolumes}" ; then
+    record_command run_persistent_volumes_tests
+  fi
+
+  ############################
+  # Persistent Volume Claims #
+  ############################
+
+  if kube::test::if_supports_resource "${persistentvolumeclaims}" ; then
+    record_command run_persistent_volume_claims_tests
+  fi
+
+  ############################
+  # Storage Classes #
+  ############################
+
+  if kube::test::if_supports_resource "${storageclass}" ; then
+    record_command run_storage_class_tests
+  fi
+
+  #########
+  # Nodes #
+  #########
+
+  if kube::test::if_supports_resource "${nodes}" ; then
+    record_command run_nodes_tests
+  fi
+
+
+  ########################
+  # authorization.k8s.io #
+  ########################
+
+  if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
+    record_command run_authorization_tests
+  fi
+
+  # kubectl auth can-i
+  # kube-apiserver is started with authorization mode AlwaysAllow, so kubectl can-i always returns yes
+  if kube::test::if_supports_resource "${subjectaccessreviews}" ; then
+    output_message=$(kubectl auth can-i '*' '*' 2>&1 "${kube_flags[@]}")
+    kube::test::if_has_string "${output_message}" "yes"
+
+    output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}")
+    kube::test::if_has_string "${output_message}" "yes"
+
+    output_message=$(kubectl auth can-i get invalid_resource 2>&1 "${kube_flags[@]}")
+    kube::test::if_has_string "${output_message}" "the server doesn't have a resource type"
+
+    output_message=$(kubectl auth can-i get /logs/ 2>&1 "${kube_flags[@]}")
+    kube::test::if_has_string "${output_message}" "yes"
+
+    output_message=$(! kubectl auth can-i get /logs/ --subresource=log 2>&1 "${kube_flags[@]}")
+    kube::test::if_has_string "${output_message}" "subresource can not be used with NonResourceURL"
+
+    output_message=$(kubectl auth can-i list jobs.batch/bar -n foo --quiet 2>&1 "${kube_flags[@]}")
+    kube::test::if_empty_string "${output_message}"
+
+    output_message=$(kubectl auth can-i get pods --subresource=log 2>&1 "${kube_flags[@]}"; echo $?)
+    kube::test::if_has_string "${output_message}" '0'
+
+    output_message=$(kubectl auth can-i get pods --subresource=log --quiet 2>&1 "${kube_flags[@]}"; echo $?)
+    kube::test::if_has_string "${output_message}" '0'
+  fi
+
+  # kubectl auth reconcile
+  if kube::test::if_supports_resource "${clusterroles}" ; then
+    kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-resource-plus.yaml
+    kube::test::get_object_assert 'rolebindings -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-RB:'
+    kube::test::get_object_assert 'roles -n some-other-random -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-R:'
+    kube::test::get_object_assert 'clusterrolebindings -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CRB:'
+    kube::test::get_object_assert 'clusterroles -l test-cmd=auth' "{{range.items}}{{$id_field}}:{{end}}" 'testing-CR:'
+
+    failure_message=$(! kubectl auth reconcile "${kube_flags[@]}" -f test/fixtures/pkg/kubectl/cmd/auth/rbac-v1beta1.yaml 2>&1 )
+    kube::test::if_has_string "${failure_message}" 'only rbac.authorization.k8s.io/v1 is supported'
+
+    kubectl delete "${kube_flags[@]}" rolebindings,role,clusterroles,clusterrolebindings -n some-other-random -l test-cmd=auth
+  fi
+
+  #####################
+  # Retrieve multiple #
+  #####################
+
+  if kube::test::if_supports_resource "${nodes}" ; then
+    if kube::test::if_supports_resource "${services}" ; then
+      record_command run_retrieve_multiple_tests
+    fi
+  fi
+
+
+  #####################
+  # Resource aliasing #
+  #####################
+
+  if kube::test::if_supports_resource "${services}" ; then
+    if kube::test::if_supports_resource "${replicationcontrollers}" ; then
+      record_command run_resource_aliasing_tests
+    fi
+  fi
+
+  ###########
+  # Explain #
+  ###########
+
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_kubectl_explain_tests
+  fi
+
+
+  ###########
+  # Swagger #
+  ###########
+
+  record_command run_swagger_tests
+
+  #####################
+  # Kubectl --sort-by #
+  #####################
+
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_kubectl_sort_by_tests
+  fi
+
+  ############################
+  # Kubectl --all-namespaces #
+  ############################
+
+  if kube::test::if_supports_resource "${pods}" ; then
+    if kube::test::if_supports_resource "${nodes}" ; then
+      record_command run_kubectl_all_namespace_tests
+    fi
+  fi
+
+  ######################
+  # kubectl --template #
+  ######################
+
+  if kube::test::if_supports_resource "${pods}" ; then
+    record_command run_template_output_tests
+  fi
+
+  ################
+  # Certificates #
+  ################
+
+  if kube::test::if_supports_resource "${csr}" ; then
+    record_command run_certificates_tests
+  fi
+
+  ######################
+  # Cluster Management #
+  ######################
+  if kube::test::if_supports_resource "${nodes}" ; then
+    record_command run_cluster_management_tests
+  fi
+
+  ###########
+  # Plugins #
+  ###########
+
+  record_command run_plugins_tests
+
+
+  #################
+  # Impersonation #
+  #################
+  record_command run_impersonation_tests
+
+  ####################
+  # kubectl wait     #
+  ####################
+
+  record_command run_wait_tests
+
+  cleanup
 }

From a502b96326fa30b8c7db9bb3884b88dedbe11ca4 Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Fri, 22 Feb 2019 21:01:54 -0500
Subject: [PATCH 11/12] changed cleanup function name to not overwrite actualy
 cleanup() function

---
 build/root/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/build/root/Makefile b/build/root/Makefile
index 72dcae6f95..b464e75638 100644
--- a/build/root/Makefile
+++ b/build/root/Makefile
@@ -291,7 +291,6 @@ test-cmd:
 	@echo "$$TEST_CMD_HELP_INFO"
 else
 test-cmd: generated_files
-	source hack/lib/init.sh
 	hack/make-rules/test-cmd.sh
 endif
 

From d4df8879f31dc4e021446296bb471f1b253c153a Mon Sep 17 00:00:00 2001
From: Thomas Runyon <runyontr@gmail.com>
Date: Fri, 22 Feb 2019 21:23:53 -0500
Subject: [PATCH 12/12] call kube:test:clear_all during cleanup

---
 test/cmd/legacy-script.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/cmd/legacy-script.sh b/test/cmd/legacy-script.sh
index f58584e6b7..bf3e871bef 100755
--- a/test/cmd/legacy-script.sh
+++ b/test/cmd/legacy-script.sh
@@ -386,8 +386,8 @@ runTests() {
     kubectl get "${kube_flags[@]}" -f hack/testdata/kubernetes-service.yaml
   fi
 
-  cleanup(){
-     kube::test::clear_all
+  cleanup_tests(){
+    kube::test::clear_all
     if [[ -n "${foundError}" ]]; then
       echo "FAILED TESTS: ""${foundError}"
       exit 1
@@ -399,7 +399,7 @@ runTests() {
     do 
       record_command run_${pkg}_tests
     done
-    cleanup
+    cleanup_tests
     return
   fi
 
@@ -866,5 +866,5 @@ runTests() {
 
   record_command run_wait_tests
 
-  cleanup
+  cleanup_tests
 }