diff --git a/app/proxyman/inbound/worker.go b/app/proxyman/inbound/worker.go
index afa09aa4..e431c732 100644
--- a/app/proxyman/inbound/worker.go
+++ b/app/proxyman/inbound/worker.go
@@ -210,8 +210,10 @@ func (w *udpWorker) getConnection(id connID) (*udpConn, bool) {
 
 func (w *udpWorker) callback(b *buf.Buffer, source net.Destination, originalDest net.Destination) {
 	id := connID{
-		src:  source,
-		dest: originalDest,
+		src: source,
+	}
+	if originalDest.IsValid() {
+		id.dest = originalDest
 	}
 	conn, existing := w.getConnection(id)
 	select {