diff --git a/picoquic/packet.c b/picoquic/packet.c
index 2784dfc7c515e24cdccb0fdbfb2232a5ef56d2c4..98c5b215919e6b5fc472119dc6b1581b4e804e4b 100644
--- a/picoquic/packet.c
+++ b/picoquic/packet.c
@@ -833,7 +833,8 @@ int picoquic_incoming_initial(
 
     /* Logic to test the retry token.
      * TODO: this should probably be implemented as a callback */
-    if ((*pcnx)->quic->flags&picoquic_context_check_token &&
+    if (((*pcnx)->quic->flags&picoquic_context_check_token) &&
+        (*pcnx)->cnx_state == picoquic_state_server_init &&
         ((*pcnx)->quic->flags&picoquic_context_server_busy) == 0) {
         uint8_t * base;
         size_t len;