diff --git a/config_quic.go b/config_quic.go
index cd3eaee..3f397cb 100644
--- a/config_quic.go
+++ b/config_quic.go
@@ -127,11 +127,6 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t
 		close(ch)
 	}()
 
-	udpConn, err := net.ListenUDP("udp", nil)
-	if err != nil {
-		return nil, err
-	}
-
 	for _, addr := range addrs {
 		go func(addr string) {
 			defer wg.Done()
@@ -140,6 +135,11 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t
 				ch <- &parallelDialerResult{conn: nil, err: err}
 				return
 			}
+			udpConn, err := net.ListenUDP("udp", nil)
+			if err != nil {
+				ch <- &parallelDialerResult{conn: nil, err: err}
+				return
+			}
 			conn, err := quic.DialEarly(ctx, udpConn, remoteAddr, tlsCfg, cfg)
 			select {
 			case ch <- &parallelDialerResult{conn: conn, err: err}:
@@ -147,6 +147,9 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t
 				if conn != nil {
 					conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeNoError), "")
 				}
+				if udpConn != nil {
+					udpConn.Close()
+				}
 			}
 		}(addr)
 	}