diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go
index da7787285a..ec09c91f88 100644
--- a/pkg/cloudprovider/providers/aws/aws.go
+++ b/pkg/cloudprovider/providers/aws/aws.go
@@ -4212,18 +4212,39 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin
 		// Note that this is annoying: the load balancer disappears from the API immediately, but it is still
 		// deleting in the background.  We get a DependencyViolation until the load balancer has deleted itself
 
+		var loadBalancerSGs = aws.StringValueSlice(lb.SecurityGroups)
+
+		describeRequest := &ec2.DescribeSecurityGroupsInput{}
+		filters := []*ec2.Filter{
+			newEc2Filter("group-id", loadBalancerSGs...),
+		}
+		describeRequest.Filters = c.tagging.addFilters(filters)
+		response, err := c.ec2.DescribeSecurityGroups(describeRequest)
+		if err != nil {
+			return fmt.Errorf("Error querying security groups for ELB: %q", err)
+		}
+
 		// Collect the security groups to delete
 		securityGroupIDs := map[string]struct{}{}
-		for _, securityGroupID := range lb.SecurityGroups {
-			if *securityGroupID == c.cfg.Global.ElbSecurityGroup {
+
+		for _, sg := range response {
+			sgID := aws.StringValue(sg.GroupId)
+
+			if sgID == c.cfg.Global.ElbSecurityGroup {
 				//We don't want to delete a security group that was defined in the Cloud Configurationn.
 				continue
 			}
-			if aws.StringValue(securityGroupID) == "" {
+			if sgID == "" {
 				klog.Warning("Ignoring empty security group in ", service.Name)
 				continue
 			}
-			securityGroupIDs[*securityGroupID] = struct{}{}
+
+			if !c.tagging.hasClusterTag(sg.Tags) {
+				klog.Warning("Ignoring security group with no cluster tag in", service.Name)
+				continue
+			}
+
+			securityGroupIDs[sgID] = struct{}{}
 		}
 
 		// Loop through and try to delete them