diff --git a/pkg/proxy/ipvs/graceful_termination.go b/pkg/proxy/ipvs/graceful_termination.go
index f6a8ca1a2a..2ec005afc8 100644
--- a/pkg/proxy/ipvs/graceful_termination.go
+++ b/pkg/proxy/ipvs/graceful_termination.go
@@ -164,7 +164,8 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e
 	}
 	for _, rs := range rss {
 		if rsToDelete.RealServer.Equal(rs) {
-			if rs.ActiveConn != 0 {
+			// Don't delete TCP RS with Active Connections or UDP RS (ActiveConn is always 0 for UDP)
+			if rs.ActiveConn != 0 || (rsToDelete.VirtualServer.Protocol == "UDP" && rs.InactiveConn != 0) {
 				return false, nil
 			}
 			klog.Infof("Deleting rs: %s", rsToDelete.String())