diff --git a/protocol.go b/protocol.go
index 9e21e31..79f2694 100644
--- a/protocol.go
+++ b/protocol.go
@@ -104,6 +104,7 @@ func (p *Conn) RemoteAddr() net.Addr {
 	p.once.Do(func() {
 		if err := p.checkPrefix(); err != nil && err != io.EOF {
 			log.Printf("[ERR] Failed to read proxy prefix: %v", err)
+			p.Close()
 		}
 	})
 	if p.srcAddr != nil {