diff --git a/pkg/scheduler/cache/node_info.go b/pkg/scheduler/cache/node_info.go
index 31be774578..2cad6573fa 100644
--- a/pkg/scheduler/cache/node_info.go
+++ b/pkg/scheduler/cache/node_info.go
@@ -400,8 +400,13 @@ func (n *NodeInfo) Clone() *NodeInfo {
 		clone.pods = append([]*v1.Pod(nil), n.pods...)
 	}
 	if len(n.usedPorts) > 0 {
-		for k, v := range n.usedPorts {
-			clone.usedPorts[k] = v
+		// util.HostPortInfo is a map-in-map struct
+		// make sure it's deep copied
+		for ip, portMap := range n.usedPorts {
+			clone.usedPorts[ip] = make(map[util.ProtocolPort]struct{})
+			for protocolPort, v := range portMap {
+				clone.usedPorts[ip][protocolPort] = v
+			}
 		}
 	}
 	if len(n.podsWithAffinity) > 0 {
diff --git a/pkg/scheduler/cache/node_info_test.go b/pkg/scheduler/cache/node_info_test.go
index cf3eae51d9..0bfbbb19a5 100644
--- a/pkg/scheduler/cache/node_info_test.go
+++ b/pkg/scheduler/cache/node_info_test.go
@@ -495,6 +495,7 @@ func TestNodeInfoClone(t *testing.T) {
 		ni := test.nodeInfo.Clone()
 		// Modify the field to check if the result is a clone of the origin one.
 		test.nodeInfo.generation += 10
+		test.nodeInfo.usedPorts.Remove("127.0.0.1", "TCP", 80)
 		if !reflect.DeepEqual(test.expected, ni) {
 			t.Errorf("expected: %#v, got: %#v", test.expected, ni)
 		}