diff --git a/app/proxyman/inbound/worker.go b/app/proxyman/inbound/worker.go
index 40c8bc15..c89a0ce2 100644
--- a/app/proxyman/inbound/worker.go
+++ b/app/proxyman/inbound/worker.go
@@ -162,7 +162,6 @@ type udpConn struct {
 	uplink           stats.Counter
 	downlink         stats.Counter
 	inactive         bool
-	cancel           context.CancelFunc
 }
 
 func (c *udpConn) setInactive() {
@@ -205,9 +204,6 @@ func (c *udpConn) Write(buf []byte) (int, error) {
 }
 
 func (c *udpConn) Close() error {
-	if c.cancel != nil {
-		c.cancel()
-	}
 	common.Must(c.done.Close())
 	common.Must(common.Close(c.writer))
 	return nil
@@ -264,7 +260,6 @@ func (w *udpWorker) getConnection(id connID) (*udpConn, bool) {
 	defer w.Unlock()
 
 	if conn, found := w.activeConn[id]; found && !conn.done.Done() {
-		conn.updateActivity()
 		return conn, true
 	}
 
@@ -312,8 +307,7 @@ func (w *udpWorker) callback(b *buf.Buffer, source net.Destination, originalDest
 		common.Must(w.checker.Start())
 
 		go func() {
-			ctx, cancel := context.WithCancel(w.ctx)
-			conn.cancel = cancel
+			ctx := w.ctx
 			sid := session.NewID()
 			ctx = c.ContextWithID(ctx, sid)
 
diff --git a/proxy/shadowsocks/server.go b/proxy/shadowsocks/server.go
index ec022084..356868e4 100644
--- a/proxy/shadowsocks/server.go
+++ b/proxy/shadowsocks/server.go
@@ -128,7 +128,6 @@ func (s *Server) handleUDPPayload(ctx context.Context, conn stat.Connection, dis
 
 		conn.Write(data.Bytes())
 	})
-	defer udpServer.RemoveRay()
 
 	inbound := session.InboundFromContext(ctx)
 	var dest *net.Destination
diff --git a/proxy/socks/server.go b/proxy/socks/server.go
index 8a159fad..dd6f3953 100644
--- a/proxy/socks/server.go
+++ b/proxy/socks/server.go
@@ -245,15 +245,13 @@ func (s *Server) handleUDPPayload(ctx context.Context, conn stat.Connection, dis
 		udpMessage, err := EncodeUDPPacket(request, payload.Bytes())
 		payload.Release()
 
+		defer udpMessage.Release()
 		if err != nil {
 			errors.LogWarningInner(ctx, err, "failed to write UDP response")
-			return
 		}
-		defer udpMessage.Release()
 
 		conn.Write(udpMessage.Bytes())
 	})
-	defer udpServer.RemoveRay()
 
 	inbound := session.InboundFromContext(ctx)
 	if inbound != nil && inbound.Source.IsValid() {
diff --git a/proxy/trojan/server.go b/proxy/trojan/server.go
index 0ce14408..44662ac3 100644
--- a/proxy/trojan/server.go
+++ b/proxy/trojan/server.go
@@ -259,7 +259,6 @@ func (s *Server) handleUDPPayload(ctx context.Context, clientReader *PacketReade
 			errors.LogWarningInner(ctx, err, "failed to write response")
 		}
 	})
-	defer udpServer.RemoveRay()
 
 	inbound := session.InboundFromContext(ctx)
 	user := inbound.User
diff --git a/proxy/vless/inbound/inbound.go b/proxy/vless/inbound/inbound.go
index dfa8d470..9d8b9506 100644
--- a/proxy/vless/inbound/inbound.go
+++ b/proxy/vless/inbound/inbound.go
@@ -206,10 +206,7 @@ func (h *Handler) Process(ctx context.Context, network net.Network, connection s
 
 	first := buf.FromBytes(make([]byte, buf.Size))
 	first.Clear()
-	firstLen, errR := first.ReadFrom(connection)
-	if errR != nil {
-		return errR
-	}
+	firstLen, _ := first.ReadFrom(connection)
 	errors.LogInfo(ctx, "firstLen = ", firstLen)
 
 	reader := &buf.BufferedReader{