diff --git a/lib/ngtcp2_conn.c b/lib/ngtcp2_conn.c
index 6226fc32eb24b17141cb214b12cb887e908a5a98..c571ff10b0d30fcb3b2931c539d3f3e9d8c9b314 100644
--- a/lib/ngtcp2_conn.c
+++ b/lib/ngtcp2_conn.c
@@ -6279,6 +6279,13 @@ static int conn_recv_handshake_done(ngtcp2_conn *conn, ngtcp2_tstamp ts) {
 
   conn_discard_handshake_state(conn, ts);
 
+  if (conn->remote.transport_params.preferred_address_present) {
+    rv = conn_select_preferred_addr(conn);
+    if (rv != 0) {
+      return rv;
+    }
+  }
+
   if (conn->callbacks.handshake_confirmed) {
     rv = conn->callbacks.handshake_confirmed(conn, conn->user_data);
     if (rv != 0) {
@@ -7826,13 +7833,6 @@ static ngtcp2_ssize conn_write_handshake(ngtcp2_conn *conn, uint8_t *dest,
       return (ngtcp2_ssize)rv;
     }
 
-    if (conn->remote.transport_params.preferred_address_present) {
-      rv = conn_select_preferred_addr(conn);
-      if (rv != 0) {
-        return rv;
-      }
-    }
-
     cstat->bytes_sent += (size_t)res;
 
     return res;