diff --git a/lib/includes/ngtcp2/ngtcp2.h b/lib/includes/ngtcp2/ngtcp2.h
index 98e5181790bde41352b6b9c2f9c0f62fdaa5622f..1e77be4f1c2d03fc038247d14d8a9207d8025e9c 100644
--- a/lib/includes/ngtcp2/ngtcp2.h
+++ b/lib/includes/ngtcp2/ngtcp2.h
@@ -331,7 +331,8 @@ NGTCP2_EXTERN ssize_t ngtcp2_pkt_decode_hd(ngtcp2_pkt_hd *dest,
 
 NGTCP2_EXTERN ssize_t ngtcp2_pkt_decode_frame(ngtcp2_frame *dest,
                                               const uint8_t *payload,
-                                              size_t payloadlen);
+                                              size_t payloadlen,
+                                              uint64_t max_rx_pkt_num);
 
 /**
  * @function
diff --git a/lib/ngtcp2_conn.c b/lib/ngtcp2_conn.c
index 40a01010f9af5abd6f8f82efc3389ef861cdd9b3..8666a8f32dec65e10fe860ed5aa9a5443ee50fc8 100644
--- a/lib/ngtcp2_conn.c
+++ b/lib/ngtcp2_conn.c
@@ -586,6 +586,8 @@ static int ngtcp2_conn_recv_cleartext(ngtcp2_conn *conn, uint8_t exptype,
   pkt += nread;
   pktlen -= (size_t)nread;
 
+  hd.pkt_num = ngtcp2_pkt_adjust_pkt_num(conn->max_rx_pkt_num, hd.pkt_num, 32);
+
   rv = conn_call_recv_pkt(conn, &hd);
   if (rv != 0) {
     return rv;
@@ -615,7 +617,7 @@ static int ngtcp2_conn_recv_cleartext(ngtcp2_conn *conn, uint8_t exptype,
   }
 
   for (; pktlen;) {
-    nread = ngtcp2_pkt_decode_frame(&fr, pkt, pktlen);
+    nread = ngtcp2_pkt_decode_frame(&fr, pkt, pktlen, conn->max_rx_pkt_num);
     if (nread < 0) {
       return (int)nread;
     }
diff --git a/lib/ngtcp2_conn.h b/lib/ngtcp2_conn.h
index aff0f7cdb855b642290595b844d893ccf533de5b..ebbb39690e1d8a3181babb5f868ad5b82d333919 100644
--- a/lib/ngtcp2_conn.h
+++ b/lib/ngtcp2_conn.h
@@ -96,6 +96,7 @@ struct ngtcp2_conn {
   ngtcp2_strm strm0;
   uint64_t conn_id;
   uint64_t next_tx_pkt_num;
+  uint64_t max_rx_pkt_num;
   ngtcp2_mem *mem;
   void *user_data;
   uint32_t version;
diff --git a/lib/ngtcp2_pkt.c b/lib/ngtcp2_pkt.c
index 73b3bf5d1fe048b56b791c4c186d9de71c69c982..e42d669a30a5c86bdf11858f1556538f5c4c8d02 100644
--- a/lib/ngtcp2_pkt.c
+++ b/lib/ngtcp2_pkt.c
@@ -245,7 +245,7 @@ ssize_t ngtcp2_pkt_encode_hd_short(uint8_t *out, size_t outlen,
 static int has_mask(uint8_t b, uint8_t mask) { return (b & mask) == mask; }
 
 ssize_t ngtcp2_pkt_decode_frame(ngtcp2_frame *dest, const uint8_t *payload,
-                                size_t payloadlen) {
+                                size_t payloadlen, uint64_t max_rx_pkt_num) {
   uint8_t type;
 
   if (payloadlen == 0) {
@@ -259,7 +259,8 @@ ssize_t ngtcp2_pkt_decode_frame(ngtcp2_frame *dest, const uint8_t *payload,
   }
 
   if (has_mask(type, NGTCP2_FRAME_ACK)) {
-    return ngtcp2_pkt_decode_ack_frame(&dest->ack, payload, payloadlen);
+    return ngtcp2_pkt_decode_ack_frame(&dest->ack, payload, payloadlen,
+                                       max_rx_pkt_num);
   }
 
   switch (type) {
@@ -402,7 +403,8 @@ ssize_t ngtcp2_pkt_decode_stream_frame(ngtcp2_stream *dest,
 }
 
 ssize_t ngtcp2_pkt_decode_ack_frame(ngtcp2_ack *dest, const uint8_t *payload,
-                                    size_t payloadlen) {
+                                    size_t payloadlen,
+                                    uint64_t max_rx_pkt_num) {
   uint8_t type;
   size_t num_blks = 0;
   size_t num_ts;
@@ -482,16 +484,19 @@ ssize_t ngtcp2_pkt_decode_ack_frame(ngtcp2_ack *dest, const uint8_t *payload,
 
   switch (lalen) {
   case 1:
-    dest->largest_ack = *p;
+    dest->largest_ack = ngtcp2_pkt_adjust_pkt_num(max_rx_pkt_num, *p, 8);
     break;
   case 2:
-    dest->largest_ack = ngtcp2_get_uint16(p);
+    dest->largest_ack =
+        ngtcp2_pkt_adjust_pkt_num(max_rx_pkt_num, ngtcp2_get_uint16(p), 16);
     break;
   case 4:
-    dest->largest_ack = ngtcp2_get_uint32(p);
+    dest->largest_ack =
+        ngtcp2_pkt_adjust_pkt_num(max_rx_pkt_num, ngtcp2_get_uint32(p), 32);
     break;
   case 6:
-    dest->largest_ack = ngtcp2_get_uint48(p);
+    dest->largest_ack =
+        ngtcp2_pkt_adjust_pkt_num(max_rx_pkt_num, ngtcp2_get_uint48(p), 48);
     break;
   }
 
@@ -849,3 +854,18 @@ size_t ngtcp2_pkt_decode_version_negotiation(uint32_t *dest,
 
   return payloadlen / sizeof(uint32_t);
 }
+
+uint64_t ngtcp2_pkt_adjust_pkt_num(uint64_t max_pkt_num, uint64_t pkt_num,
+                                   size_t n) {
+  uint64_t k = max_pkt_num + 1;
+  uint64_t u = k & ~((1llu << n) - 1);
+  uint64_t a = u | pkt_num;
+  uint64_t b = (u + (1llu << n)) | pkt_num;
+  uint64_t a1 = k < a ? a - k : k - a;
+  uint64_t b1 = k < b ? b - k : k - b;
+
+  if (a1 < b1) {
+    return a;
+  }
+  return b;
+}
diff --git a/lib/ngtcp2_pkt.h b/lib/ngtcp2_pkt.h
index abf61ebaf4ff137533574c0f9958621888a2b1dc..930f369eacb6492b6a68c54c0e8d0efad5358ba3 100644
--- a/lib/ngtcp2_pkt.h
+++ b/lib/ngtcp2_pkt.h
@@ -157,7 +157,7 @@ ssize_t ngtcp2_pkt_decode_stream_frame(ngtcp2_stream *dest,
  *     Payload is too short to include ACK frame
  */
 ssize_t ngtcp2_pkt_decode_ack_frame(ngtcp2_ack *dest, const uint8_t *payload,
-                                    size_t payloadlen);
+                                    size_t payloadlen, uint64_t max_rx_pkt_num);
 
 /*
  * ngtcp2_pkt_decode_padding_frame decodes contiguous PADDING frames
@@ -261,4 +261,19 @@ ssize_t ngtcp2_pkt_encode_ack_frame(uint8_t *out, size_t outlen,
 ssize_t ngtcp2_pkt_encode_padding_frame(uint8_t *out, size_t outlen,
                                         const ngtcp2_padding *fr);
 
+/**
+ * ngtcp2_pkt_adjust_pkt_num find the full 64 bits packet number for
+ * |pkt_num|, which is expected to be least significant |n| bits.  The
+ * |max_pkt_num| is the highest successfully authenticated packet
+ * number.
+ */
+uint64_t ngtcp2_pkt_adjust_pkt_num(uint64_t max_pkt_num, uint64_t pkt_num,
+                                   size_t n);
+
+/**
+ * ngtcp2_pkt_adjust_ack_pkt_num adjusts all packet numbers in |ack|
+ * using the maximum packet number |max_pkt_num| received so far.
+ */
+void ngtcp2_pkt_adjust_ack_pkt_num(ngtcp2_ack *ack, uint64_t max_pkt_num);
+
 #endif /* NGTCP2_PKT_H */
diff --git a/tests/main.c b/tests/main.c
index dd3c60437f2be99aef7ce96afe2507e24ea43e4d..0698f61c9597ed0af054716fb276604f49ad541a 100644
--- a/tests/main.c
+++ b/tests/main.c
@@ -68,6 +68,8 @@ int main() {
                    test_ngtcp2_pkt_encode_stream_frame) ||
       !CU_add_test(pSuite, "pkt_encode_ack_frame",
                    test_ngtcp2_pkt_encode_ack_frame) ||
+      !CU_add_test(pSuite, "pkt_adjust_pkt_num",
+                   test_ngtcp2_pkt_adjust_pkt_num) ||
       !CU_add_test(pSuite, "upe_encode", test_ngtcp2_upe_encode) ||
       !CU_add_test(pSuite, "upe_encode_version_negotiation",
                    test_ngtcp2_upe_encode_version_negotiation)) {
diff --git a/tests/ngtcp2_pkt_test.c b/tests/ngtcp2_pkt_test.c
index af70820e61776864a09322c792a47b3e2a3b60a5..eea3cf0b4735facecb47a75f3481a651cb96a21f 100644
--- a/tests/ngtcp2_pkt_test.c
+++ b/tests/ngtcp2_pkt_test.c
@@ -269,10 +269,11 @@ void test_ngtcp2_pkt_decode_ack_frame(void) {
 
   CU_ASSERT(expectedlen == buflen);
 
-  rv = ngtcp2_pkt_decode_ack_frame(&fr.ack, buf, buflen);
+  rv = ngtcp2_pkt_decode_ack_frame(&fr.ack, buf, buflen, 0);
 
   CU_ASSERT((ssize_t)expectedlen == rv);
   CU_ASSERT(0xf1f2f3f4f5f6llu == fr.ack.largest_ack);
+  fprintf(stderr, "%016lx\n", fr.ack.largest_ack);
 }
 
 void test_ngtcp2_pkt_decode_padding_frame(void) {
@@ -468,7 +469,7 @@ void test_ngtcp2_pkt_encode_ack_frame(void) {
 
   CU_ASSERT((ssize_t)framelen == rv);
 
-  rv = ngtcp2_pkt_decode_ack_frame(&nfr.ack, buf, framelen);
+  rv = ngtcp2_pkt_decode_ack_frame(&nfr.ack, buf, framelen, 0);
 
   CU_ASSERT((ssize_t)framelen == rv);
   CU_ASSERT(fr.type == nfr.type);
@@ -497,7 +498,7 @@ void test_ngtcp2_pkt_encode_ack_frame(void) {
 
   CU_ASSERT((ssize_t)framelen == rv);
 
-  rv = ngtcp2_pkt_decode_ack_frame(&nfr.ack, buf, framelen);
+  rv = ngtcp2_pkt_decode_ack_frame(&nfr.ack, buf, framelen, 0);
 
   CU_ASSERT((ssize_t)framelen == rv);
   CU_ASSERT(fr.type == nfr.type);
@@ -514,3 +515,11 @@ void test_ngtcp2_pkt_encode_ack_frame(void) {
 
   memset(&nfr, 0, sizeof(nfr));
 }
+
+void test_ngtcp2_pkt_adjust_pkt_num(void) {
+  CU_ASSERT(0xaa831f94llu ==
+            ngtcp2_pkt_adjust_pkt_num(0xaa82f30ellu, 0x1f94, 16));
+
+  CU_ASSERT(0x01ff == ngtcp2_pkt_adjust_pkt_num(0x0100, 0xff, 1));
+  CU_ASSERT(0x02ff == ngtcp2_pkt_adjust_pkt_num(0x01ff, 0xff, 1));
+}
diff --git a/tests/ngtcp2_pkt_test.h b/tests/ngtcp2_pkt_test.h
index 6a8f42b074c1c82720d9fe96f0288fbef21fab08..83f5d9687e90fcd1e18307bb9fa91a185f7fe520 100644
--- a/tests/ngtcp2_pkt_test.h
+++ b/tests/ngtcp2_pkt_test.h
@@ -36,5 +36,6 @@ void test_ngtcp2_pkt_decode_ack_frame(void);
 void test_ngtcp2_pkt_decode_padding_frame(void);
 void test_ngtcp2_pkt_encode_stream_frame(void);
 void test_ngtcp2_pkt_encode_ack_frame(void);
+void test_ngtcp2_pkt_adjust_pkt_num(void);
 
 #endif /* NGTCP2_PKT_TEST_H */
diff --git a/tests/ngtcp2_upe_test.c b/tests/ngtcp2_upe_test.c
index 0ce888e67aeabbaded4b23c23b64e6a2dbf3d8f0..b0d71624021f6bd7732eba273e4c90f7699f5a6b 100644
--- a/tests/ngtcp2_upe_test.c
+++ b/tests/ngtcp2_upe_test.c
@@ -105,7 +105,7 @@ void test_ngtcp2_upe_encode(void) {
   pktlen -= (size_t)nread;
 
   /* Read first STREAM frame */
-  nread = ngtcp2_pkt_decode_frame(&ns, out, pktlen);
+  nread = ngtcp2_pkt_decode_frame(&ns, out, pktlen, 0);
 
   CU_ASSERT(nread > 0);
   CU_ASSERT(s1.type == ns.type);
@@ -118,7 +118,7 @@ void test_ngtcp2_upe_encode(void) {
   pktlen -= (size_t)nread;
 
   /* Read second STREAM frame */
-  nread = ngtcp2_pkt_decode_frame(&ns, out, pktlen);
+  nread = ngtcp2_pkt_decode_frame(&ns, out, pktlen, 0);
 
   CU_ASSERT(nread > 0);
   CU_ASSERT(s2.type == ns.type);
@@ -131,7 +131,7 @@ void test_ngtcp2_upe_encode(void) {
   pktlen -= (size_t)nread;
 
   /* Read PADDING frames to the end */
-  nread = ngtcp2_pkt_decode_frame(&ns, out, pktlen);
+  nread = ngtcp2_pkt_decode_frame(&ns, out, pktlen, 0);
 
   CU_ASSERT(nread == (ssize_t)pktlen);
   CU_ASSERT(NGTCP2_FRAME_PADDING == ns.type);