diff --git a/examples/Makefile.am b/examples/Makefile.am index 16a1d2431168e3e0444296b8c2060a2cdb7be348..2ec35f1abc24f0ebad6eceeb6fec93272b480446 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -45,7 +45,7 @@ client_SOURCES = client.cc client.h \ debug.cc debug.h \ util.cc util.h \ keylog.cc keylog.h \ - shared.h \ + shared.cc shared.h \ crypto_openssl.cc \ crypto.cc @@ -54,7 +54,7 @@ server_SOURCES = server.cc server.h \ debug.cc debug.h \ util.cc util.h \ keylog.cc keylog.h \ - shared.h \ + shared.cc shared.h \ crypto_openssl.cc \ crypto.cc \ http.cc http.h diff --git a/examples/client.cc b/examples/client.cc index d6aa48333fb0bce9fbb4818ec8ee42566ecd77af..ecc6e82d2f0ab55fd1800a2f7573281031f62444 100644 --- a/examples/client.cc +++ b/examples/client.cc @@ -42,6 +42,8 @@ #include <openssl/bio.h> #include <openssl/err.h> +#include <http-parser/http_parser.h> + #include "client.h" #include "network.h" #include "debug.h" @@ -71,19 +73,10 @@ Buffer::Buffer(size_t datalen) : buf(datalen), begin(buf.data()), head(begin), tail(begin) {} Buffer::Buffer() : begin(buf.data()), head(begin), tail(begin) {} -Stream::Stream(int64_t stream_id) - : stream_id(stream_id), - streambuf_idx(0), - tx_stream_offset(0), - should_send_fin(false) {} +Stream::Stream(int64_t stream_id) : stream_id(stream_id) {} Stream::~Stream() {} -void Stream::buffer_file() { - streambuf.emplace_back(config.data, config.data + config.datalen); - should_send_fin = true; -} - namespace { int key_cb(SSL *ssl, int name, const unsigned char *secret, size_t secretlen, void *arg) { @@ -341,16 +334,6 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) { } } // namespace -namespace { -void stdin_readcb(struct ev_loop *loop, ev_io *w, int revents) { - auto c = static_cast<Client *>(w->data); - - if (c->send_interactive_input()) { - c->disconnect(); - } -} -} // namespace - namespace { void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { auto c = static_cast<Client *>(w->data); @@ -429,27 +412,24 @@ Client::Client(struct ev_loop *loop, SSL_CTX *ssl_ctx) ssl_ctx_(ssl_ctx), ssl_(nullptr), fd_(-1), - datafd_(-1), chandshake_idx_(0), tx_crypto_offset_(0), nsread_(0), conn_(nullptr), + httpconn_(nullptr), addr_(nullptr), hs_crypto_ctx_{}, crypto_ctx_{}, + last_error_{QUICErrorType::Transport, 0}, sendbuf_{NGTCP2_MAX_PKTLEN_IPV4}, - last_stream_id_(-1), nstreams_done_(0), nkey_update_(0), version_(0), - tls_alert_(0), resumption_(false) { ev_io_init(&wev_, writecb, 0, EV_WRITE); ev_io_init(&rev_, readcb, 0, EV_READ); - ev_io_init(&stdinrev_, stdin_readcb, 0, EV_READ); wev_.data = this; rev_.data = this; - stdinrev_.data = this; ev_timer_init(&timer_, timeoutcb, 0., config.timeout); timer_.data = this; ev_timer_init(&rttimer_, retransmitcb, 0., 0.); @@ -467,9 +447,7 @@ Client::~Client() { close(); } -void Client::disconnect() { disconnect(0); } - -void Client::disconnect(int liberr) { +void Client::disconnect() { config.tx_loss_prob = 0; ev_timer_stop(loop_, &key_update_timer_); @@ -477,17 +455,21 @@ void Client::disconnect(int liberr) { ev_timer_stop(loop_, &rttimer_); ev_timer_stop(loop_, &timer_); - ev_io_stop(loop_, &stdinrev_); ev_io_stop(loop_, &rev_); ev_signal_stop(loop_, &sigintev_); - handle_error(liberr); + handle_error(); } void Client::close() { ev_io_stop(loop_, &wev_); + if (httpconn_) { + nghttp3_conn_del(httpconn_); + httpconn_ = nullptr; + } + if (conn_) { ngtcp2_conn_del(conn_); conn_ = nullptr; @@ -548,8 +530,13 @@ int recv_stream_data(ngtcp2_conn *conn, int64_t stream_id, int fin, if (!config.quiet) { debug::print_stream_data(stream_id, data, datalen); } - ngtcp2_conn_extend_max_stream_offset(conn, stream_id, datalen); - ngtcp2_conn_extend_max_offset(conn, datalen); + + auto c = static_cast<Client *>(user_data); + + if (c->recv_stream_data(stream_id, fin, data, datalen) != 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + return 0; } } // namespace @@ -569,7 +556,7 @@ int acked_stream_data_offset(ngtcp2_conn *conn, int64_t stream_id, uint64_t offset, size_t datalen, void *user_data, void *stream_user_data) { auto c = static_cast<Client *>(user_data); - if (c->remove_tx_stream_data(stream_id, offset, datalen) != 0) { + if (c->acked_stream_data_offset(stream_id, datalen) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; @@ -591,6 +578,10 @@ int handshake_completed(ngtcp2_conn *conn, void *user_data) { c->start_key_update_timer(); } + if (c->setup_httpconn() != 0) { + return NGHTTP3_ERR_CALLBACK_FAILURE; + } + return 0; } } // namespace @@ -619,6 +610,18 @@ int stream_close(ngtcp2_conn *conn, int64_t stream_id, uint16_t app_error_code, } } // namespace +namespace { +int stream_reset(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size, + uint16_t app_error_code, void *user_data, + void *stream_user_data) { + auto c = static_cast<Client *>(user_data); + + c->on_stream_reset(stream_id); + + return 0; +} +} // namespace + namespace { int extend_max_streams_bidi(ngtcp2_conn *conn, uint64_t max_streams, void *user_data) { @@ -875,8 +878,7 @@ int Client::init_ssl() { } int Client::init(int fd, const Address &local_addr, const Address &remote_addr, - const char *addr, const char *port, int datafd, - uint32_t version) { + const char *addr, const char *port, uint32_t version) { int rv; local_addr_ = local_addr; @@ -894,7 +896,6 @@ int Client::init(int fd, const Address &local_addr, const Address &remote_addr, } fd_ = fd; - datafd_ = datafd; addr_ = addr; port_ = port; version_ = version; @@ -915,9 +916,9 @@ int Client::init(int fd, const Address &local_addr, const Address &remote_addr, do_decrypt, do_in_hp_mask, do_hp_mask, - recv_stream_data, + ::recv_stream_data, acked_crypto_offset, - acked_stream_data_offset, + ::acked_stream_data_offset, nullptr, // stream_open stream_close, nullptr, // recv_stateless_reset @@ -930,7 +931,7 @@ int Client::init(int fd, const Address &local_addr, const Address &remote_addr, ::update_key, path_validation, ::select_preferred_address, - nullptr, // stream_reset + stream_reset, nullptr, // extend_max_remote_streams_bidi, nullptr, // extend_max_remote_streams_uni, }; @@ -959,7 +960,7 @@ int Client::init(int fd, const Address &local_addr, const Address &remote_addr, settings.max_stream_data_uni = 256_k; settings.max_data = 1_m; settings.max_streams_bidi = 1; - settings.max_streams_uni = 1; + settings.max_streams_uni = 3; settings.idle_timeout = config.timeout; auto path = ngtcp2_path{ @@ -1212,7 +1213,8 @@ int Client::feed_data(const sockaddr *sa, socklen_t salen, uint8_t *data, util::timestamp(loop_)); if (rv != 0) { std::cerr << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv) << std::endl; - disconnect(rv); + last_error_ = quicErrorTransport(rv); + disconnect(); return -1; } return 0; @@ -1228,7 +1230,8 @@ int Client::do_handshake_read_once(const ngtcp2_path *path, const uint8_t *data, if (rv < 0) { std::cerr << "ngtcp2_conn_read_handshake: " << ngtcp2_strerror(rv) << std::endl; - disconnect(rv); + last_error_ = quicErrorTransport(rv); + disconnect(); return -1; } @@ -1241,7 +1244,8 @@ ssize_t Client::do_handshake_write_once() { if (nwrite < 0) { std::cerr << "ngtcp2_conn_write_handshake: " << ngtcp2_strerror(nwrite) << std::endl; - disconnect(nwrite); + last_error_ = quicErrorTransport(nwrite); + disconnect(); return -1; } @@ -1344,7 +1348,8 @@ int Client::on_write(bool retransmit) { auto rv = send_packet(); if (rv != NETWORK_ERR_OK) { if (rv != NETWORK_ERR_SEND_NON_FATAL) { - disconnect(NGTCP2_ERR_INTERNAL); + last_error_ = quicErrorTransport(NGTCP2_ERR_INTERNAL); + disconnect(); } return rv; } @@ -1358,7 +1363,8 @@ int Client::on_write(bool retransmit) { if (rv != 0) { std::cerr << "ngtcp2_conn_on_loss_detection_timer: " << ngtcp2_strerror(rv) << std::endl; - disconnect(NGTCP2_ERR_INTERNAL); + last_error_ = quicErrorTransport(NGTCP2_ERR_INTERNAL); + disconnect(); return -1; } } @@ -1376,7 +1382,8 @@ int Client::on_write(bool retransmit) { max_pktlen_, util::timestamp(loop_)); if (n < 0) { std::cerr << "ngtcp2_conn_write_pkt: " << ngtcp2_strerror(n) << std::endl; - disconnect(n); + last_error_ = quicErrorTransport(n); + disconnect(); return -1; } if (n == 0) { @@ -1400,6 +1407,9 @@ int Client::on_write(bool retransmit) { if (!retransmit) { auto rv = write_streams(); if (rv != 0) { + if (rv == NETWORK_ERR_SEND_NON_FATAL) { + schedule_retransmit(); + } return rv; } } @@ -1409,148 +1419,216 @@ int Client::on_write(bool retransmit) { } int Client::write_streams() { - for (auto &p : streams_) { - auto &stream = p.second; - auto &streambuf = stream->streambuf; - auto &streambuf_idx = stream->streambuf_idx; - - for (auto it = std::begin(streambuf) + streambuf_idx; - it != std::end(streambuf); ++it) { - auto &v = *it; - auto fin = stream->should_send_fin && it + 1 == std::end(streambuf); - auto rv = on_write_stream(stream->stream_id, fin, v); - if (rv != 0) { - if (rv == NETWORK_ERR_SEND_NON_FATAL) { - schedule_retransmit(); - return 0; - } - return rv; - } - if (v.size() > 0) { - break; - } - ++streambuf_idx; - } - } - - return 0; -} + std::array<nghttp3_vec, 16> vec; + int rv; -int Client::on_write_stream(int64_t stream_id, uint8_t fin, Buffer &data) { - ssize_t ndatalen; + if (ngtcp2_conn_get_max_data_left(conn_) == 0) { + return 0; + } - PathStorage path; for (;;) { - auto n = ngtcp2_conn_write_stream( - conn_, &path.path, sendbuf_.wpos(), max_pktlen_, &ndatalen, stream_id, - fin, data.rpos(), data.size(), util::timestamp(loop_)); - if (n < 0) { - switch (n) { - case NGTCP2_ERR_EARLY_DATA_REJECTED: - case NGTCP2_ERR_STREAM_DATA_BLOCKED: - case NGTCP2_ERR_STREAM_SHUT_WR: - case NGTCP2_ERR_STREAM_NOT_FOUND: // This means that stream is - // closed. - return 0; - } - std::cerr << "ngtcp2_conn_write_stream: " << ngtcp2_strerror(n) + int64_t stream_id; + int fin; + + auto sveccnt = nghttp3_conn_writev_stream(httpconn_, &stream_id, &fin, + vec.data(), vec.size()); + if (sveccnt < 0) { + std::cerr << "nghttp3_conn_writev_stream: " << nghttp3_strerror(sveccnt) << std::endl; - disconnect(n); + last_error_ = quicErrorApplication(sveccnt); + disconnect(); return -1; } - if (n == 0) { - return 0; - } - - if (ndatalen > 0) { - data.seek(ndatalen); + if (sveccnt == 0) { + break; } - sendbuf_.push(n); + ssize_t ndatalen; + PathStorage path; + auto v = vec.data(); + auto vcnt = static_cast<size_t>(sveccnt); + for (;;) { + auto nwrite = ngtcp2_conn_writev_stream( + conn_, &path.path, sendbuf_.wpos(), max_pktlen_, &ndatalen, stream_id, + fin, reinterpret_cast<const ngtcp2_vec *>(v), vcnt, + util::timestamp(loop_)); + if (nwrite < 0) { + auto should_break = false; + switch (nwrite) { + case NGTCP2_ERR_STREAM_DATA_BLOCKED: + if (ngtcp2_conn_get_max_data_left(conn_) == 0) { + return 0; + } + + rv = nghttp3_conn_block_stream(httpconn_, stream_id); + if (rv != 0) { + std::cerr << "nghttp3_conn_block_stream: " << nghttp3_strerror(rv) + << std::endl; + last_error_ = quicErrorApplication(rv); + disconnect(); + return -1; + } + should_break = true; + break; + case NGTCP2_ERR_EARLY_DATA_REJECTED: + case NGTCP2_ERR_STREAM_SHUT_WR: + case NGTCP2_ERR_STREAM_NOT_FOUND: // This means that stream is + // closed. + should_break = true; + break; + } - update_remote_addr(&path.path.remote); + if (should_break) { + break; + } - auto rv = send_packet(); - if (rv != NETWORK_ERR_OK) { - return rv; - } + std::cerr << "ngtcp2_conn_write_stream: " << ngtcp2_strerror(nwrite) + << std::endl; + last_error_ = quicErrorTransport(nwrite); + disconnect(); + return -1; + } - if (data.size() == 0) { - break; - } - } + if (nwrite == 0) { + // We are congestion limited. + return 0; + } - return 0; -} + sendbuf_.push(nwrite); -int Client::write_0rtt_streams() { - for (auto &p : streams_) { - auto &stream = p.second; - auto &streambuf = stream->streambuf; - auto &streambuf_idx = stream->streambuf_idx; - for (auto it = std::begin(streambuf) + streambuf_idx; - it != std::end(streambuf); ++it) { - auto &v = *it; - auto fin = stream->should_send_fin && it + 1 == std::end(streambuf); - auto rv = on_write_0rtt_stream(stream->stream_id, fin, v); - if (rv != 0) { - if (rv == NETWORK_ERR_SEND_NON_FATAL) { - schedule_retransmit(); - return 0; + if (ndatalen > 0) { + rv = nghttp3_conn_add_write_offset(httpconn_, stream_id, ndatalen); + if (rv != 0) { + std::cerr << "nghttp3_conn_add_write_offset: " << nghttp3_strerror(rv) + << std::endl; + last_error_ = quicErrorApplication(rv); + disconnect(); + return -1; } + + nghttp3_vec_consume(&v, &vcnt, ndatalen); + } + + update_remote_addr(&path.path.remote); + + auto rv = send_packet(); + if (rv != NETWORK_ERR_OK) { return rv; } - if (v.size() > 0) { + + if (nghttp3_vec_empty(v, vcnt)) { break; } - ++streambuf_idx; } } return 0; } -int Client::on_write_0rtt_stream(int64_t stream_id, uint8_t fin, Buffer &data) { - ssize_t ndatalen; +int Client::write_0rtt_streams() { + if (!httpconn_) { + return 0; + } + + std::array<nghttp3_vec, 16> vec; + int rv; + + if (ngtcp2_conn_get_max_data_left(conn_) == 0) { + return 0; + } for (;;) { - ngtcp2_vec datav{const_cast<uint8_t *>(data.rpos()), data.size()}; - auto n = ngtcp2_conn_client_write_handshake( - conn_, sendbuf_.wpos(), max_pktlen_, &ndatalen, stream_id, fin, &datav, - 1, util::timestamp(loop_)); - if (n < 0) { - switch (n) { - case NGTCP2_ERR_EARLY_DATA_REJECTED: - case NGTCP2_ERR_STREAM_DATA_BLOCKED: - case NGTCP2_ERR_STREAM_SHUT_WR: - case NGTCP2_ERR_STREAM_NOT_FOUND: // This means that stream is - // closed. - return 0; - } - std::cerr << "ngtcp2_conn_client_write_handshake: " << ngtcp2_strerror(n) + int64_t stream_id; + int fin; + + auto sveccnt = nghttp3_conn_writev_stream(httpconn_, &stream_id, &fin, + vec.data(), vec.size()); + if (sveccnt < 0) { + std::cerr << "nghttp3_conn_writev_stream: " << nghttp3_strerror(sveccnt) << std::endl; - disconnect(n); + last_error_ = quicErrorApplication(sveccnt); + disconnect(); return -1; } - if (n == 0) { - return 0; + if (sveccnt == 0) { + break; } - if (ndatalen > 0) { - data.seek(ndatalen); - } + ssize_t ndatalen; + auto v = vec.data(); + auto vcnt = static_cast<size_t>(sveccnt); + for (;;) { + auto nwrite = ngtcp2_conn_client_write_handshake( + conn_, sendbuf_.wpos(), max_pktlen_, &ndatalen, stream_id, fin, + reinterpret_cast<const ngtcp2_vec *>(v), vcnt, + util::timestamp(loop_)); + if (nwrite < 0) { + auto should_break = false; + switch (nwrite) { + case NGTCP2_ERR_STREAM_DATA_BLOCKED: + if (ngtcp2_conn_get_max_data_left(conn_) == 0) { + return 0; + } + + rv = nghttp3_conn_block_stream(httpconn_, stream_id); + if (rv != 0) { + std::cerr << "nghttp3_conn_block_stream: " << nghttp3_strerror(rv) + << std::endl; + last_error_ = quicErrorApplication(rv); + disconnect(); + return -1; + } + should_break = true; + break; + case NGTCP2_ERR_EARLY_DATA_REJECTED: + case NGTCP2_ERR_STREAM_SHUT_WR: + case NGTCP2_ERR_STREAM_NOT_FOUND: // This means that stream is + // closed. + should_break = true; + break; + } - sendbuf_.push(n); + if (should_break) { + break; + } - auto rv = send_packet(); - if (rv != NETWORK_ERR_OK) { - return rv; - } + std::cerr << "ngtcp2_conn_write_stream: " << ngtcp2_strerror(nwrite) + << std::endl; + last_error_ = quicErrorTransport(nwrite); + disconnect(); + return -1; + } - if (data.size() == 0) { - break; + if (nwrite == 0) { + // We are congestion limited. + return 0; + } + + sendbuf_.push(nwrite); + + if (ndatalen > 0) { + rv = nghttp3_conn_add_write_offset(httpconn_, stream_id, ndatalen); + if (rv != 0) { + std::cerr << "nghttp3_conn_add_write_offset: " << nghttp3_strerror(rv) + << std::endl; + last_error_ = quicErrorApplication(rv); + disconnect(); + return -1; + } + + nghttp3_vec_consume(&v, &vcnt, ndatalen); + } + + auto rv = send_packet(); + if (rv != NETWORK_ERR_OK) { + return rv; + } + + if (nghttp3_vec_empty(v, vcnt)) { + break; + } } } @@ -1951,83 +2029,7 @@ int Client::send_packet() { return NETWORK_ERR_OK; } -int Client::start_interactive_input() { - int rv; - - std::cerr << "Interactive session started. Hit Ctrl-D to end the session." - << std::endl; - - ev_io_set(&stdinrev_, datafd_, EV_READ); - ev_io_start(loop_, &stdinrev_); - - int64_t stream_id; - - rv = ngtcp2_conn_open_bidi_stream(conn_, &stream_id, nullptr); - if (rv != 0) { - std::cerr << "ngtcp2_conn_open_bidi_stream: " << ngtcp2_strerror(rv) - << std::endl; - if (rv == NGTCP2_ERR_STREAM_ID_BLOCKED) { - return 0; - } - return -1; - } - - std::cerr << "The stream " << stream_id << " has opened." << std::endl; - - last_stream_id_ = stream_id; - - auto stream = std::make_unique<Stream>(stream_id); - - streams_.emplace(stream_id, std::move(stream)); - - return 0; -} - -int Client::send_interactive_input() { - ssize_t nread; - std::array<uint8_t, 1_k> buf; - - while ((nread = read(datafd_, buf.data(), buf.size())) == -1 && - errno == EINTR) - ; - if (nread == -1) { - return stop_interactive_input(); - } - if (nread == 0) { - return stop_interactive_input(); - } - - // TODO fix this - assert(!streams_.empty()); - - auto &stream = streams_[last_stream_id_]; - - stream->streambuf.emplace_back(buf.data(), nread); - - ev_feed_event(loop_, &wev_, EV_WRITE); - - return 0; -} - -int Client::stop_interactive_input() { - assert(!streams_.empty()); - - auto &stream = (*std::begin(streams_)).second; - - stream->should_send_fin = true; - if (stream->streambuf.empty()) { - stream->streambuf.emplace_back(); - } - ev_io_stop(loop_, &stdinrev_); - - std::cerr << "Interactive session has ended." << std::endl; - - ev_feed_event(loop_, &wev_, EV_WRITE); - - return 0; -} - -int Client::handle_error(int liberr) { +int Client::handle_error() { if (!conn_ || ngtcp2_conn_is_in_closing_period(conn_)) { return 0; } @@ -2035,29 +2037,33 @@ int Client::handle_error(int liberr) { sendbuf_.reset(); assert(sendbuf_.left() >= max_pktlen_); - if (liberr == NGTCP2_ERR_RECV_VERSION_NEGOTIATION) { + if (last_error_.type == QUICErrorType::TransportVersionNegotiation) { return 0; } - uint16_t err_code; - if (tls_alert_) { - err_code = NGTCP2_CRYPTO_ERROR | tls_alert_; - } else { - err_code = ngtcp2_err_infer_quic_transport_error_code(liberr); - } - PathStorage path; - auto n = ngtcp2_conn_write_connection_close(conn_, &path.path, - sendbuf_.wpos(), max_pktlen_, - err_code, util::timestamp(loop_)); - if (n < 0) { - std::cerr << "ngtcp2_conn_write_connection_close: " << ngtcp2_strerror(n) - << std::endl; - return -1; + if (last_error_.type == QUICErrorType::Transport) { + auto n = ngtcp2_conn_write_connection_close( + conn_, &path.path, sendbuf_.wpos(), max_pktlen_, last_error_.code, + util::timestamp(loop_)); + if (n < 0) { + std::cerr << "ngtcp2_conn_write_connection_close: " << ngtcp2_strerror(n) + << std::endl; + return -1; + } + sendbuf_.push(n); + } else { + auto n = ngtcp2_conn_write_application_close( + conn_, &path.path, sendbuf_.wpos(), max_pktlen_, last_error_.code, + util::timestamp(loop_)); + if (n < 0) { + std::cerr << "ngtcp2_conn_write_application_close: " << ngtcp2_strerror(n) + << std::endl; + return -1; + } + sendbuf_.push(n); } - sendbuf_.push(n); - update_remote_addr(&path.path.remote); return send_packet(); @@ -2083,20 +2089,6 @@ void Client::remove_tx_crypto_data(uint64_t offset, size_t datalen) { offset + datalen); } -int Client::remove_tx_stream_data(int64_t stream_id, uint64_t offset, - size_t datalen) { - auto it = streams_.find(stream_id); - if (it == std::end(streams_)) { - std::cerr << "Stream " << stream_id << "not found" << std::endl; - return 0; - } - auto &stream = (*it).second; - ::remove_tx_stream_data(stream->streambuf, stream->streambuf_idx, - stream->tx_stream_offset, offset + datalen); - - return 0; -} - void Client::on_stream_close(int64_t stream_id) { auto it = streams_.find(stream_id); @@ -2104,13 +2096,19 @@ void Client::on_stream_close(int64_t stream_id) { return; } - if (config.interactive) { - ev_io_stop(loop_, &stdinrev_); + if (httpconn_) { + nghttp3_conn_close_stream(httpconn_, stream_id); } streams_.erase(it); } +void Client::on_stream_reset(int64_t stream_id) { + if (httpconn_) { + nghttp3_conn_reset_stream(httpconn_, stream_id); + } +} + namespace { int write_transport_params(const char *path, const ngtcp2_transport_params *params) { @@ -2178,10 +2176,6 @@ int read_transport_params(const char *path, ngtcp2_transport_params *params) { void Client::make_stream_early() { int rv; - if (config.interactive || datafd_ == -1) { - return; - } - if (nstreams_done_ >= config.nstreams) { return; } @@ -2196,43 +2190,114 @@ void Client::make_stream_early() { return; } + // TODO Handle error + if (setup_httpconn() != 0) { + return; + } + + if (submit_http_request(stream_id) != 0) { + return; + } + auto stream = std::make_unique<Stream>(stream_id); - stream->buffer_file(); streams_.emplace(stream_id, std::move(stream)); } int Client::on_extend_max_streams() { int rv; + int64_t stream_id; - if (config.interactive) { - if (last_stream_id_ != -1) { - return 0; + for (; nstreams_done_ < config.nstreams; ++nstreams_done_) { + rv = ngtcp2_conn_open_bidi_stream(conn_, &stream_id, nullptr); + if (rv != 0) { + assert(NGTCP2_ERR_STREAM_ID_BLOCKED == rv); + break; } - if (start_interactive_input() != 0) { - return -1; + + if (submit_http_request(stream_id) != 0) { + break; } - return 0; + auto stream = std::make_unique<Stream>(stream_id); + streams_.emplace(stream_id, std::move(stream)); } + return 0; +} - if (datafd_ != -1) { - for (; nstreams_done_ < config.nstreams; ++nstreams_done_) { - int64_t stream_id; +namespace { +int read_data(nghttp3_conn *conn, int64_t stream_id, const uint8_t **pdata, + size_t *pdatalen, uint32_t *pflags, void *user_data, + void *stream_user_data) { + *pdata = config.data; + *pdatalen = config.datalen; + *pflags |= NGHTTP3_DATA_FLAG_EOF; - rv = ngtcp2_conn_open_bidi_stream(conn_, &stream_id, nullptr); - if (rv != 0) { - assert(NGTCP2_ERR_STREAM_ID_BLOCKED == rv); - break; - } + return 0; +} +} // namespace - last_stream_id_ = stream_id; +int Client::submit_http_request(int64_t stream_id) { + int rv; - auto stream = std::make_unique<Stream>(stream_id); - stream->buffer_file(); + std::string content_length_str; - streams_.emplace(stream_id, std::move(stream)); - } - return 0; + std::array<nghttp3_nv, 6> nva{ + util::make_nv(":method", config.http_method), + util::make_nv(":scheme", config.scheme), + util::make_nv(":authority", config.authority), + util::make_nv(":path", config.path), + util::make_nv("user-agent", "nghttp3/ngtcp2"), + }; + size_t nvlen = 5; + if (config.fd != -1) { + content_length_str = std::to_string(config.datalen); + nva[nvlen++] = util::make_nv("content-length", content_length_str); + } + + nghttp3_data_reader dr{}; + dr.read_data = read_data; + + nghttp3_priority pri, *ppri = NULL; + if (nghttp3_conn_get_remote_num_placeholders(httpconn_) > 0) { + nghttp3_priority_init(&pri, NGHTTP3_ELEM_DEP_TYPE_PLACEHOLDER, 0, 32); + ppri = &pri; + } + + rv = nghttp3_conn_submit_request(httpconn_, stream_id, ppri, nva.data(), + nvlen, config.fd == -1 ? NULL : &dr, NULL); + if (rv != 0) { + std::cerr << "nghttp3_conn_submit_request: " << nghttp3_strerror(rv) + << std::endl; + return -1; + } + + nghttp3_conn_end_stream(httpconn_, stream_id); + + return 0; +} + +int Client::recv_stream_data(int64_t stream_id, int fin, const uint8_t *data, + size_t datalen) { + auto nconsumed = + nghttp3_conn_read_stream(httpconn_, stream_id, data, datalen, fin); + if (nconsumed < 0) { + std::cerr << "nghttp3_conn_read_stream: " << nghttp3_strerror(nconsumed) + << std::endl; + return -1; + } + + ngtcp2_conn_extend_max_stream_offset(conn_, stream_id, nconsumed); + ngtcp2_conn_extend_max_offset(conn_, nconsumed); + + return 0; +} + +int Client::acked_stream_data_offset(int64_t stream_id, size_t datalen) { + auto rv = nghttp3_conn_add_ack_offset(httpconn_, stream_id, datalen); + if (rv != 0) { + std::cerr << "nghttp3_conn_add_ack_offset: " << nghttp3_strerror(rv) + << std::endl; + return -1; } return 0; @@ -2296,7 +2361,178 @@ int Client::select_preferred_address(Address &selected_addr, void Client::start_wev() { ev_io_start(loop_, &wev_); } -void Client::set_tls_alert(uint8_t alert) { tls_alert_ = alert; } +void Client::set_tls_alert(uint8_t alert) { + last_error_ = quicErrorTransportTLS(alert); +} + +namespace { +int http_acked_stream_data(nghttp3_conn *conn, int64_t stream_id, + size_t datalen, void *user_data, + void *stream_user_data) { + auto c = static_cast<Client *>(user_data); + if (c->http_acked_stream_data(stream_id, datalen) != 0) { + return NGHTTP3_ERR_CALLBACK_FAILURE; + } + return 0; +} +} // namespace + +int Client::http_acked_stream_data(int64_t stream_id, size_t datalen) { + return 0; +} + +namespace { +int http_recv_data(nghttp3_conn *conn, int64_t stream_id, const uint8_t *data, + size_t datalen, void *user_data, void *stream_user_data) { + if (!config.quiet) { + debug::print_http_data(stream_id, data, datalen); + } + auto c = static_cast<Client *>(user_data); + c->http_consume(stream_id, datalen); + return 0; +} +} // namespace + +namespace { +int http_deferred_consume(nghttp3_conn *conn, int64_t stream_id, + size_t nconsumed, void *user_data, + void *stream_user_data) { + auto c = static_cast<Client *>(user_data); + c->http_consume(stream_id, nconsumed); + return 0; +} +} // namespace + +void Client::http_consume(int64_t stream_id, size_t nconsumed) { + ngtcp2_conn_extend_max_stream_offset(conn_, stream_id, nconsumed); + ngtcp2_conn_extend_max_offset(conn_, nconsumed); +} + +namespace { +int http_begin_headers(nghttp3_conn *conn, int64_t stream_id, void *user_data, + void *stream_user_data) { + if (!config.quiet) { + debug::print_http_begin_response_headers(stream_id); + } + return 0; +} +} // namespace + +namespace { +int http_recv_header(nghttp3_conn *conn, int64_t stream_id, int32_t token, + nghttp3_rcbuf *name, nghttp3_rcbuf *value, uint8_t flags, + void *user_data, void *stream_user_data) { + if (!config.quiet) { + debug::print_http_header(stream_id, name, value, flags); + } + return 0; +} +} // namespace + +namespace { +int http_end_headers(nghttp3_conn *conn, int64_t stream_id, void *user_data, + void *stream_user_data) { + if (!config.quiet) { + debug::print_http_end_headers(stream_id); + } + return 0; +} +} // namespace + +int Client::setup_httpconn() { + int rv; + + if (httpconn_) { + return 0; + } + + if (ngtcp2_conn_get_max_local_streams_uni(conn_) < 3) { + std::cerr << "peer does not allow at least 3 unidirectional streams." + << std::endl; + return -1; + } + + nghttp3_conn_callbacks callbacks{ + ::http_acked_stream_data, + nullptr, // stream_close + ::http_recv_data, + ::http_deferred_consume, + ::http_begin_headers, + ::http_recv_header, + ::http_end_headers, + nullptr, // begin_trailers + nullptr, // recv_trailer + nullptr, // end_trailers + nullptr, // begin_push_promise + nullptr, // recv_push_promise + nullptr, // end_push_promise + }; + nghttp3_conn_settings settings; + nghttp3_conn_settings_default(&settings); + settings.qpack_max_table_capacity = 4096; + settings.qpack_blocked_streams = 100; + + auto mem = nghttp3_mem_default(); + + rv = nghttp3_conn_client_new(&httpconn_, &callbacks, &settings, mem, this); + if (rv != 0) { + std::cerr << "nghttp3_conn_client_new: " << nghttp3_strerror(rv) + << std::endl; + return -1; + } + + int64_t ctrl_stream_id; + + rv = ngtcp2_conn_open_uni_stream(conn_, &ctrl_stream_id, NULL); + if (rv != 0) { + std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv) + << std::endl; + return -1; + } + + rv = nghttp3_conn_bind_control_stream(httpconn_, ctrl_stream_id); + if (rv != 0) { + std::cerr << "nghttp3_conn_bind_control_stream: " << nghttp3_strerror(rv) + << std::endl; + return -1; + } + + if (!config.quiet) { + fprintf(stderr, "http: control stream=%" PRIx64 "\n", ctrl_stream_id); + } + + int64_t qpack_enc_stream_id, qpack_dec_stream_id; + + rv = ngtcp2_conn_open_uni_stream(conn_, &qpack_enc_stream_id, NULL); + if (rv != 0) { + std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv) + << std::endl; + return -1; + } + + rv = ngtcp2_conn_open_uni_stream(conn_, &qpack_dec_stream_id, NULL); + if (rv != 0) { + std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv) + << std::endl; + return -1; + } + + rv = nghttp3_conn_bind_qpack_streams(httpconn_, qpack_enc_stream_id, + qpack_dec_stream_id); + if (rv != 0) { + std::cerr << "nghttp3_conn_bind_qpack_streams: " << nghttp3_strerror(rv) + << std::endl; + return -1; + } + + if (!config.quiet) { + fprintf(stderr, + "http: QPACK streams encoder=%" PRIx64 " decoder=%" PRIx64 "\n", + qpack_enc_stream_id, qpack_dec_stream_id); + } + + return 0; +} namespace { int transport_params_add_cb(SSL *ssl, unsigned int ext_type, @@ -2464,8 +2700,7 @@ int run(Client &c, const char *addr, const char *port) { return -1; } - if (c.init(fd, local_addr, remote_addr, addr, port, config.fd, - config.version) != 0) { + if (c.init(fd, local_addr, remote_addr, addr, port, config.version) != 0) { return -1; } @@ -2499,6 +2734,53 @@ int run(Client &c, const char *addr, const char *port) { } } // namespace +namespace { +std::string get_string(const char *uri, const http_parser_url &u, + http_parser_url_fields f) { + auto p = &u.field_data[f]; + return {uri + p->off, uri + p->off + p->len}; +} +} // namespace + +namespace { +int parse_uri(const char *uri) { + http_parser_url u; + + http_parser_url_init(&u); + if (http_parser_parse_url(uri, strlen(uri), /* is_connect = */ 0, &u) != 0) { + return -1; + } + + if (!(u.field_set & (1 << UF_SCHEMA)) || !(u.field_set & (1 << UF_HOST))) { + return -1; + } + + config.scheme = get_string(uri, u, UF_SCHEMA); + + config.authority = get_string(uri, u, UF_HOST); + if (util::numeric_host(config.authority.c_str())) { + config.authority = '[' + config.authority + ']'; + } + if (u.field_set & (1 << UF_PORT)) { + config.authority += ':'; + config.authority += get_string(uri, u, UF_PORT); + } + + if (u.field_set & (1 << UF_PATH)) { + config.path = get_string(uri, u, UF_PATH); + } else { + config.path = "/"; + } + + if (u.field_set & (1 << UF_QUERY)) { + config.path += '?'; + config.path += get_string(uri, u, UF_QUERY); + } + + return 0; +} +} // namespace + namespace { std::ofstream keylog_file; void keylog_callback(const SSL *ssl, const char *line) { @@ -2510,7 +2792,7 @@ void keylog_callback(const SSL *ssl, const char *line) { namespace { void print_usage() { - std::cerr << "Usage: client [OPTIONS] <ADDR> <PORT>" << std::endl; + std::cerr << "Usage: client [OPTIONS] <ADDR> <PORT> <URI>" << std::endl; } } // namespace @@ -2528,6 +2810,7 @@ void config_set_default(Config &config) { config.datalen = 0; config.version = NGTCP2_PROTO_VER_D19; config.timeout = 30; + config.http_method = "GET"; } } // namespace @@ -2549,8 +2832,6 @@ Options: The probability of losing incoming packets. <P> must be [0.0, 1.0], inclusive. 0.0 means no packet loss. 1.0 means 100% packet loss. - -i, --interactive - Read input from stdin, and send them as STREAM data. -d, --data=<PATH> Read data from <PATH>, and send them as STREAM data. -n, --nstreams=<N> @@ -2597,6 +2878,9 @@ Options: --key-update=<T> Client initiates key update when <T> seconds elapse after handshake completes. + -m, --http-method=<METHOD> + Specify HTTP method. Default: )" + << config.http_method << R"( -h, --help Display this help and exit. )"; } @@ -2612,8 +2896,8 @@ int main(int argc, char **argv) { {"help", no_argument, nullptr, 'h'}, {"tx-loss", required_argument, nullptr, 't'}, {"rx-loss", required_argument, nullptr, 'r'}, - {"interactive", no_argument, nullptr, 'i'}, {"data", required_argument, nullptr, 'd'}, + {"http-method", required_argument, nullptr, 'm'}, {"nstreams", required_argument, nullptr, 'n'}, {"version", required_argument, nullptr, 'v'}, {"quiet", no_argument, nullptr, 'q'}, @@ -2631,7 +2915,7 @@ int main(int argc, char **argv) { }; auto optidx = 0; - auto c = getopt_long(argc, argv, "d:hin:qr:st:v:", long_opts, &optidx); + auto c = getopt_long(argc, argv, "d:him:n:qr:st:v:", long_opts, &optidx); if (c == -1) { break; } @@ -2644,6 +2928,10 @@ int main(int argc, char **argv) { // --help print_help(); exit(EXIT_SUCCESS); + case 'm': + // --http-method + config.http_method = optarg; + break; case 'n': // --streams config.nstreams = strtol(optarg, nullptr, 10); @@ -2664,11 +2952,6 @@ int main(int argc, char **argv) { // --tx-loss config.tx_loss_prob = strtod(optarg, nullptr); break; - case 'i': - // --interactive - config.fd = fileno(stdin); - config.interactive = true; - break; case 'v': // --version config.version = strtol(optarg, nullptr, 16); @@ -2730,19 +3013,12 @@ int main(int argc, char **argv) { }; } - if (argc - optind < 2) { + if (argc - optind < 3) { std::cerr << "Too few arguments" << std::endl; print_usage(); exit(EXIT_FAILURE); } - if (data_path && config.interactive) { - std::cerr - << "interactive, data: Exclusive options are specified at the same time" - << std::endl; - exit(EXIT_FAILURE); - } - if (data_path) { auto fd = open(data_path, O_RDONLY); if (fd == -1) { @@ -2764,6 +3040,12 @@ int main(int argc, char **argv) { auto addr = argv[optind++]; auto port = argv[optind++]; + auto uri = argv[optind++]; + + if (parse_uri(uri) != 0) { + std::cerr << "Could not parse URI " << uri << std::endl; + exit(EXIT_FAILURE); + } auto ssl_ctx = create_ssl_ctx(); auto ssl_ctx_d = defer(SSL_CTX_free, ssl_ctx); diff --git a/examples/client.h b/examples/client.h index 643b9908e5574f4aea2dc4593f4a2852b21c37eb..a968372b4231fd7ae17a722ca229e3ee1d933426 100644 --- a/examples/client.h +++ b/examples/client.h @@ -34,6 +34,7 @@ #include <map> #include <ngtcp2/ngtcp2.h> +#include <nghttp3/nghttp3.h> #include <openssl/ssl.h> @@ -42,6 +43,7 @@ #include "network.h" #include "crypto.h" #include "template.h" +#include "shared.h" using namespace ngtcp2; @@ -57,8 +59,6 @@ struct Config { const char *ciphers; // groups is the list of supported groups. const char *groups; - // interactive is true if interactive input mode is on. - bool interactive; // nstreams is the number of streams to open. size_t nstreams; // data is the pointer to memory region which maps file denoted by @@ -88,6 +88,10 @@ struct Config { uint32_t key_update; // nat_rebinding is true if simulated NAT rebinding is enabled. bool nat_rebinding; + std::string http_method; + std::string scheme; + std::string authority; + std::string path; }; struct Buffer { @@ -125,17 +129,7 @@ struct Stream { Stream(int64_t stream_id); ~Stream(); - void buffer_file(); - int64_t stream_id; - std::deque<Buffer> streambuf; - // streambuf_idx is the index in streambuf, which points to the - // buffer to send next. - size_t streambuf_idx; - // tx_stream_offset is the offset where all data before offset is - // acked by the remote endpoint. - uint64_t tx_stream_offset; - bool should_send_fin; }; class Client { @@ -144,10 +138,9 @@ public: ~Client(); int init(int fd, const Address &local_addr, const Address &remote_addr, - const char *addr, const char *port, int datafd, uint32_t version); + const char *addr, const char *port, uint32_t version); int init_ssl(); void disconnect(); - void disconnect(int liberr); void close(); void start_wev(); @@ -157,9 +150,7 @@ public: int on_read(); int on_write(bool retransmit = false); int write_streams(); - int on_write_stream(int64_t stream_id, uint8_t fin, Buffer &data); int write_0rtt_streams(); - int on_write_0rtt_stream(int64_t stream_id, uint8_t fin, Buffer &data); int feed_data(const sockaddr *sa, socklen_t salen, uint8_t *data, size_t datalen); int do_handshake(const ngtcp2_path *path, const uint8_t *data, @@ -203,14 +194,10 @@ public: ngtcp2_conn *conn() const; void update_remote_addr(const ngtcp2_addr *addr); int send_packet(); - int start_interactive_input(); - int send_interactive_input(); - int stop_interactive_input(); void remove_tx_crypto_data(uint64_t offset, size_t datalen); - int remove_tx_stream_data(int64_t stream_id, uint64_t offset, size_t datalen); void on_stream_close(int64_t stream_id); int on_extend_max_streams(); - int handle_error(int liberr); + int handle_error(); void make_stream_early(); void on_recv_retry(); int change_local_addr(); @@ -226,13 +213,21 @@ public: int select_preferred_address(Address &selected_addr, const ngtcp2_preferred_addr *paddr); + int setup_httpconn(); + int submit_http_request(int64_t stream_id); + int recv_stream_data(int64_t stream_id, int fin, const uint8_t *data, + size_t datalen); + int acked_stream_data_offset(int64_t stream_id, size_t datalen); + int http_acked_stream_data(int64_t stream_id, size_t datalen); + void http_consume(int64_t stream_id, size_t nconsumed); + void on_stream_reset(int64_t stream_id); + private: Address local_addr_; Address remote_addr_; size_t max_pktlen_; ev_io wev_; ev_io rev_; - ev_io stdinrev_; ev_timer timer_; ev_timer rttimer_; ev_timer change_local_addr_timer_; @@ -242,7 +237,6 @@ private: SSL_CTX *ssl_ctx_; SSL *ssl_; int fd_; - int datafd_; std::map<int64_t, std::unique_ptr<Stream>> streams_; std::deque<Buffer> chandshake_; // chandshake_idx_ is the index in *chandshake_, which points to the @@ -254,23 +248,21 @@ private: std::vector<uint8_t> rx_secret_; size_t nsread_; ngtcp2_conn *conn_; + nghttp3_conn *httpconn_; // addr_ is the server host address. const char *addr_; // port_ is the server port. const char *port_; crypto::Context hs_crypto_ctx_; crypto::Context crypto_ctx_; + QUICError last_error_; // common buffer used to store packet data before sending Buffer sendbuf_; - int64_t last_stream_id_; // nstreams_done_ is the number of streams opened. uint64_t nstreams_done_; // nkey_update_ is the number of key update occurred. size_t nkey_update_; uint32_t version_; - // tls_alert_ is the last TLS alert description generated by the - // local endpoint. - uint8_t tls_alert_; // resumption_ is true if client attempts to resume session. bool resumption_; }; diff --git a/examples/debug.cc b/examples/debug.cc index b235c7f7ba3d70a6208626964caa53afb9b7c377..9981176b0038435e462bed6627497250fa223dbe 100644 --- a/examples/debug.cc +++ b/examples/debug.cc @@ -197,6 +197,33 @@ void path_validation(const ngtcp2_path *path, << std::endl; } +void print_http_begin_request_headers(int64_t stream_id) { + fprintf(outfile, "http: stream 0x%" PRIx64 " request headers started\n", + stream_id); +} + +void print_http_begin_response_headers(int64_t stream_id) { + fprintf(outfile, "http: stream 0x%" PRIx64 " response headers started\n", + stream_id); +} + +void print_http_header(int64_t stream_id, const nghttp3_rcbuf *name, + const nghttp3_rcbuf *value, uint8_t flags) { + fprintf(outfile, "http: stream 0x%" PRIx64 " [%s: %s]%s\n", stream_id, + nghttp3_rcbuf_get_buf(name).base, nghttp3_rcbuf_get_buf(value).base, + (flags & NGHTTP3_NV_FLAG_NEVER_INDEX) ? "(sensitive)" : ""); +} + +void print_http_end_headers(int64_t stream_id) { + fprintf(outfile, "http: stream 0x%" PRIx64 " ended\n", stream_id); +} + +void print_http_data(int64_t stream_id, const uint8_t *data, size_t datalen) { + fprintf(outfile, "http: stream 0x%" PRIx64 " body %zu bytes\n", stream_id, + datalen); + util::hexdump(outfile, data, datalen); +} + } // namespace debug } // namespace ngtcp2 diff --git a/examples/debug.h b/examples/debug.h index b77c2866b94736052555777d7b7603a754c1974f..471fa88eeabad1b0d03ba774aca198af350afc94 100644 --- a/examples/debug.h +++ b/examples/debug.h @@ -36,6 +36,7 @@ #include <chrono> #include <ngtcp2/ngtcp2.h> +#include <nghttp3/nghttp3.h> namespace ngtcp2 { @@ -91,6 +92,17 @@ void log_printf(void *user_data, const char *fmt, ...); void path_validation(const ngtcp2_path *path, ngtcp2_path_validation_result res); +void print_http_begin_request_headers(int64_t stream_id); + +void print_http_begin_response_headers(int64_t stream_id); + +void print_http_header(int64_t stream_id, const nghttp3_rcbuf *name, + const nghttp3_rcbuf *value, uint8_t flags); + +void print_http_end_headers(int64_t stream_id); + +void print_http_data(int64_t stream_id, const uint8_t *data, size_t datalen); + } // namespace debug } // namespace ngtcp2 diff --git a/examples/server.cc b/examples/server.cc index 9009fd0794ebfb74888049dec7e2f2e00c1042f0..19940f495fa1193b9f42fe5d5d3272d2bff548f8 100644 --- a/examples/server.cc +++ b/examples/server.cc @@ -41,6 +41,8 @@ #include <openssl/bio.h> #include <openssl/err.h> +#include <http-parser/http_parser.h> + #include "server.h" #include "network.h" #include "debug.h" @@ -145,13 +147,17 @@ int Handler::on_key(int name, const uint8_t *secret, size_t secretlen) { ngtcp2_conn_set_aead_overhead(conn_, crypto::aead_max_overhead(crypto_ctx_)); switch (name) { - case SSL_KEY_CLIENT_EARLY_TRAFFIC: + case SSL_KEY_CLIENT_EARLY_TRAFFIC: { if (!config.quiet) { std::cerr << "client_early_traffic" << std::endl; } ngtcp2_conn_install_early_keys(conn_, key.data(), keylen, iv.data(), ivlen, hp.data(), hplen); + if (setup_httpconn() != 0) { + return -1; + } break; + } case SSL_KEY_CLIENT_HANDSHAKE_TRAFFIC: if (!config.quiet) { std::cerr << "client_handshake_traffic" << std::endl; @@ -301,84 +307,8 @@ BIO_METHOD *create_bio_method() { } } // namespace -namespace { -int on_msg_begin(http_parser *htp) { - auto s = static_cast<Stream *>(htp->data); - if (s->resp_state != RESP_IDLE) { - return -1; - } - return 0; -} -} // namespace - -namespace { -int on_url_cb(http_parser *htp, const char *data, size_t datalen) { - auto s = static_cast<Stream *>(htp->data); - s->uri.append(data, datalen); - return 0; -} -} // namespace - -namespace { -int on_header_field(http_parser *htp, const char *data, size_t datalen) { - auto s = static_cast<Stream *>(htp->data); - if (s->prev_hdr_key) { - s->hdrs.back().first.append(data, datalen); - } else { - s->prev_hdr_key = true; - s->hdrs.emplace_back(std::string(data, datalen), ""); - } - return 0; -} -} // namespace - -namespace { -int on_header_value(http_parser *htp, const char *data, size_t datalen) { - auto s = static_cast<Stream *>(htp->data); - s->prev_hdr_key = false; - s->hdrs.back().second.append(data, datalen); - return 0; -} -} // namespace - -namespace { -int on_headers_complete(http_parser *htp) { - auto s = static_cast<Stream *>(htp->data); - if (s->start_response() != 0) { - return -1; - } - return 0; -} -} // namespace - -auto htp_settings = http_parser_settings{ - on_msg_begin, // on_message_begin - on_url_cb, // on_url - nullptr, // on_status - on_header_field, // on_header_field - on_header_value, // on_header_value - on_headers_complete, // on_headers_complete - nullptr, // on_body - nullptr, // on_message_complete - nullptr, // on_chunk_header, - nullptr, // on_chunk_complete -}; - Stream::Stream(int64_t stream_id) - : stream_id(stream_id), - streambuf_idx(0), - tx_stream_offset(0), - should_send_fin(false), - resp_state(RESP_IDLE), - http_major(0), - http_minor(0), - prev_hdr_key(false), - fd(-1), - data(nullptr), - datalen(0) { - http_parser_init(&htp, HTTP_REQUEST); - htp.data = this; -} + : stream_id(stream_id), fd(-1), data(nullptr), datalen(0) {} Stream::~Stream() { munmap(data, datalen); @@ -388,12 +318,6 @@ Stream::~Stream() { } int Stream::recv_data(uint8_t fin, const uint8_t *data, size_t datalen) { - auto nread = http_parser_execute( - &htp, &htp_settings, reinterpret_cast<const char *>(data), datalen); - if (nread != datalen) { - return -1; - } - return 0; } @@ -492,64 +416,70 @@ int Stream::map_file(size_t len) { return 0; } -void Stream::buffer_file() { - streambuf.emplace_back(data, data + datalen); - should_send_fin = true; -} - -void Stream::send_status_response(unsigned int status_code, - const std::string &extra_headers) { - auto body = make_status_body(status_code); - std::string hdr; - if (http_major >= 1) { - hdr += "HTTP/"; - hdr += std::to_string(http_major); - hdr += '.'; - hdr += std::to_string(http_minor); - hdr += ' '; - hdr += std::to_string(status_code); - hdr += " "; - hdr += http::get_reason_phrase(status_code); - hdr += "\r\n"; - hdr += "Server: "; - hdr += NGTCP2_SERVER; - hdr += "\r\n"; - hdr += "Content-Type: text/html; charset=UTF-8\r\n"; - hdr += "Content-Length: "; - hdr += std::to_string(body.size()); - hdr += "\r\n"; - hdr += extra_headers; - hdr += "\r\n"; - } - - auto v = Buffer{hdr.size() + ((htp.method == HTTP_HEAD) ? 0 : body.size())}; - auto p = std::begin(v.buf); - p = std::copy(std::begin(hdr), std::end(hdr), p); - if (htp.method != HTTP_HEAD) { - p = std::copy(std::begin(body), std::end(body), p); - } - v.push(std::distance(std::begin(v.buf), p)); - streambuf.emplace_back(std::move(v)); - should_send_fin = true; - resp_state = RESP_COMPLETED; -} - -void Stream::send_redirect_response(unsigned int status_code, +namespace { +int read_data(nghttp3_conn *conn, int64_t stream_id, const uint8_t **pdata, + size_t *pdatalen, uint32_t *pflags, void *user_data, + void *stream_user_data) { + auto stream = static_cast<Stream *>(stream_user_data); + + *pdata = stream->data; + *pdatalen = stream->datalen; + *pflags |= NGHTTP3_DATA_FLAG_EOF; + + return 0; +} +} // namespace + +void Stream::send_status_response( + nghttp3_conn *httpconn, unsigned int status_code, + const std::vector<HTTPHeader> &extra_headers) { + status_resp_body = make_status_body(status_code); + + auto status_code_str = std::to_string(status_code); + auto content_length_str = std::to_string(status_resp_body.size()); + + std::vector<nghttp3_nv> nva(4 + extra_headers.size()); + nva[0] = util::make_nv(":status", status_code_str); + nva[1] = util::make_nv("server", NGTCP2_SERVER); + nva[2] = util::make_nv("content-type", "text/html; charset=UTF-8"); + nva[3] = util::make_nv("content-length", content_length_str); + for (auto i = 0; i < extra_headers.size(); ++i) { + auto &hdr = extra_headers[i]; + auto &nv = nva[4 + i]; + nv = util::make_nv(hdr.name, hdr.value); + } + + data = (uint8_t *)status_resp_body.data(); + datalen = status_resp_body.size(); + + nghttp3_data_reader dr{}; + dr.read_data = read_data; + + auto rv = nghttp3_conn_submit_response(httpconn, stream_id, nva.data(), + nva.size(), &dr); + if (rv != 0) { + std::cerr << "nghttp3_conn_submit_response: " << nghttp3_strerror(rv) + << std::endl; + } +} + +void Stream::send_redirect_response(nghttp3_conn *httpconn, + unsigned int status_code, const std::string &path) { - std::string hdrs = "Location: "; - hdrs += path; - hdrs += "\r\n"; - send_status_response(status_code, hdrs); + send_status_response(httpconn, status_code, {{"location", path}}); } -int Stream::start_response() { - http_major = htp.http_major; - http_minor = htp.http_minor; +int Stream::start_response(nghttp3_conn *httpconn) { + // TODO This should be handled by nghttp3 + if (uri.empty() || method.empty()) { + send_status_response(httpconn, 400); + return 0; + } - auto req_path = request_path(uri, htp.method == HTTP_CONNECT); + auto req_path = request_path(uri, method == "CONNECT"); auto path = resolve_path(req_path); if (path.empty() || open_file(path) != 0) { - send_status_response(404); + send_status_response(httpconn, 404); return 0; } @@ -559,56 +489,47 @@ int Stream::start_response() { if (fstat(fd, &st) == 0) { if (st.st_mode & S_IFDIR) { - send_redirect_response(308, path.substr(config.htdocs.size() - 1) + '/'); + send_redirect_response(httpconn, 308, + path.substr(config.htdocs.size() - 1) + '/'); return 0; } content_length = st.st_size; } else { - send_status_response(404); + send_status_response(httpconn, 404); return 0; } if (map_file(content_length) != 0) { - send_status_response(500); + send_status_response(httpconn, 500); return 0; } - if (http_major >= 1) { - std::string hdr; - hdr += "HTTP/"; - hdr += std::to_string(http_major); - hdr += '.'; - hdr += std::to_string(http_minor); - hdr += " 200 OK\r\n"; - hdr += "Server: "; - hdr += NGTCP2_SERVER; - hdr += "\r\n"; - if (content_length != -1) { - hdr += "Content-Length: "; - hdr += std::to_string(content_length); - hdr += "\r\n"; - } - hdr += "\r\n"; - - auto v = Buffer{hdr.size()}; - auto p = std::begin(v.buf); - p = std::copy(std::begin(hdr), std::end(hdr), p); - v.push(std::distance(std::begin(v.buf), p)); - streambuf.emplace_back(std::move(v)); + if (method == "HEAD") { + close(fd); + fd = -1; } - resp_state = RESP_COMPLETED; + auto content_length_str = std::to_string(content_length); - switch (htp.method) { - case HTTP_HEAD: - should_send_fin = true; - close(fd); - fd = -1; - break; - default: - buffer_file(); + std::array<nghttp3_nv, 4> nva{ + util::make_nv(":status", "200"), + util::make_nv("server", NGTCP2_SERVER), + util::make_nv("content-type", "text/html; charset=UTF-8"), + util::make_nv("content-length", content_length_str), + }; + + nghttp3_data_reader dr{}; + dr.read_data = read_data; + + auto rv = nghttp3_conn_submit_response(httpconn, stream_id, nva.data(), + nva.size(), &dr); + if (rv != 0) { + std::cerr << "nghttp3_conn_submit_response: " << nghttp3_strerror(rv) + << std::endl; } + nghttp3_conn_end_stream(httpconn, stream_id); + return 0; } @@ -714,10 +635,11 @@ Handler::Handler(struct ev_loop *loop, SSL_CTX *ssl_ctx, Server *server, rcid_(*rcid), hs_crypto_ctx_{}, crypto_ctx_{}, + httpconn_{nullptr}, sendbuf_{NGTCP2_MAX_PKTLEN_IPV4}, tx_crypto_offset_(0), + last_error_{QUICErrorType::Transport, 0}, nkey_update_(0), - tls_alert_(0), initial_(true), draining_(false) { ev_timer_init(&timer_, timeoutcb, 0., config.timeout); @@ -734,6 +656,10 @@ Handler::~Handler() { ev_timer_stop(loop_, &rttimer_); ev_timer_stop(loop_, &timer_); + if (httpconn_) { + nghttp3_conn_del(httpconn_); + } + if (conn_) { ngtcp2_conn_del(conn_); } @@ -764,7 +690,9 @@ int handshake_completed(ngtcp2_conn *conn, void *user_data) { debug::handshake_completed(conn, user_data); } - h->send_greeting(); + if (h->setup_httpconn() != 0) { + return NGHTTP3_ERR_CALLBACK_FAILURE; + } return 0; } @@ -936,13 +864,42 @@ int acked_stream_data_offset(ngtcp2_conn *conn, int64_t stream_id, uint64_t offset, size_t datalen, void *user_data, void *stream_user_data) { auto h = static_cast<Handler *>(user_data); - if (h->remove_tx_stream_data(stream_id, offset, datalen) != 0) { + if (h->acked_stream_data_offset(stream_id, datalen) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } } // namespace +int Handler::acked_stream_data_offset(int64_t stream_id, size_t datalen) { + if (!httpconn_) { + return 0; + } + + auto rv = nghttp3_conn_add_ack_offset(httpconn_, stream_id, datalen); + if (rv != 0) { + std::cerr << "nghttp3_conn_add_ack_offset: " << nghttp3_error(rv) + << std::endl; + return -1; + } + + return 0; +} + +namespace { +int stream_open(ngtcp2_conn *conn, int64_t stream_id, void *user_data) { + auto h = static_cast<Handler *>(user_data); + h->on_stream_open(stream_id); + return 0; +} +} // namespace + +void Handler::on_stream_open(int64_t stream_id) { + auto it = streams_.find(stream_id); + assert(it == std::end(streams_)); + streams_.emplace(stream_id, std::make_unique<Stream>(stream_id)); +} + namespace { int stream_close(ngtcp2_conn *conn, int64_t stream_id, uint16_t app_error_code, void *user_data, void *stream_user_data) { @@ -952,6 +909,22 @@ int stream_close(ngtcp2_conn *conn, int64_t stream_id, uint16_t app_error_code, } } // namespace +namespace { +int stream_reset(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size, + uint16_t app_error_code, void *user_data, + void *stream_user_data) { + auto h = static_cast<Handler *>(user_data); + h->on_stream_reset(stream_id); + return 0; +} +} // namespace + +void Handler::on_stream_reset(int64_t stream_id) { + if (httpconn_) { + nghttp3_conn_reset_stream(httpconn_, stream_id); + } +} + namespace { int rand(ngtcp2_conn *conn, uint8_t *dest, size_t destlen, ngtcp2_rand_ctx ctx, void *user_data) { @@ -1007,6 +980,221 @@ int path_validation(ngtcp2_conn *conn, const ngtcp2_path *path, } } // namespace +namespace { +int extend_max_remote_streams_bidi(ngtcp2_conn *conn, uint64_t max_streams, + void *user_data) { + auto h = static_cast<Handler *>(user_data); + h->extend_max_remote_streams_bidi(max_streams); + return 0; +} +} // namespace + +void Handler::extend_max_remote_streams_bidi(uint64_t max_streams) { + if (!httpconn_) { + return; + } + + nghttp3_conn_set_max_client_streams_bidi(httpconn_, max_streams); +} + +namespace { +int http_recv_data(nghttp3_conn *conn, int64_t stream_id, const uint8_t *data, + size_t datalen, void *user_data, void *stream_user_data) { + if (!config.quiet) { + debug::print_http_data(stream_id, data, datalen); + } + auto h = static_cast<Handler *>(user_data); + h->http_consume(stream_id, datalen); + return 0; +} +} // namespace + +namespace { +int http_deferred_consume(nghttp3_conn *conn, int64_t stream_id, + size_t nconsumed, void *user_data, + void *stream_user_data) { + auto h = static_cast<Handler *>(user_data); + h->http_consume(stream_id, nconsumed); + return 0; +} +} // namespace + +void Handler::http_consume(int64_t stream_id, size_t nconsumed) { + ngtcp2_conn_extend_max_stream_offset(conn_, stream_id, nconsumed); + ngtcp2_conn_extend_max_offset(conn_, nconsumed); +} + +namespace { +int http_begin_request_headers(nghttp3_conn *conn, int64_t stream_id, + void *user_data, void *stream_user_data) { + if (!config.quiet) { + debug::print_http_begin_request_headers(stream_id); + } + + auto h = static_cast<Handler *>(user_data); + h->http_begin_request_headers(stream_id); + return 0; +} +} // namespace + +void Handler::http_begin_request_headers(int64_t stream_id) { + auto it = streams_.find(stream_id); + assert(it != std::end(streams_)); + auto &stream = (*it).second; + + nghttp3_conn_set_stream_user_data(httpconn_, stream_id, stream.get()); +} + +namespace { +int http_recv_request_header(nghttp3_conn *conn, int64_t stream_id, + int32_t token, nghttp3_rcbuf *name, + nghttp3_rcbuf *value, uint8_t flags, + void *user_data, void *stream_user_data) { + if (!config.quiet) { + debug::print_http_header(stream_id, name, value, flags); + } + + auto h = static_cast<Handler *>(user_data); + h->http_recv_request_header(stream_id, token, name, value); + return 0; +} +} // namespace + +void Handler::http_recv_request_header(int64_t stream_id, int32_t token, + nghttp3_rcbuf *name, + nghttp3_rcbuf *value) { + auto it = streams_.find(stream_id); + assert(it != std::end(streams_)); + auto &stream = (*it).second; + auto v = nghttp3_rcbuf_get_buf(value); + + switch (token) { + case NGHTTP3_QPACK_TOKEN__PATH: + stream->uri = std::string{v.base, v.base + v.len}; + break; + case NGHTTP3_QPACK_TOKEN__METHOD: + stream->method = std::string{v.base, v.base + v.len}; + break; + } +} + +namespace { +int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, + void *user_data, void *stream_user_data) { + if (!config.quiet) { + debug::print_http_end_headers(stream_id); + } + + auto h = static_cast<Handler *>(user_data); + h->http_end_request_headers(stream_id); + return 0; +} +} // namespace + +void Handler::http_end_request_headers(int64_t stream_id) { + auto it = streams_.find(stream_id); + assert(it != std::end(streams_)); + auto &stream = (*it).second; + + stream->start_response(httpconn_); +} + +int Handler::setup_httpconn() { + int rv; + + if (httpconn_) { + return 0; + } + + if (ngtcp2_conn_get_max_local_streams_uni(conn_) < 3) { + std::cerr << "peer does not allow at least 3 unidirectional streams." + << std::endl; + return -1; + } + + nghttp3_conn_callbacks callbacks{ + nullptr, // acked_stream_data + nullptr, // stream_close + ::http_recv_data, + ::http_deferred_consume, + ::http_begin_request_headers, + ::http_recv_request_header, + ::http_end_request_headers, + nullptr, // begin_trailers + nullptr, // recv_trailer + nullptr, // end_trailers + nullptr, // begin_push_promise + nullptr, // recv_push_promise + nullptr, // end_push_promise + }; + nghttp3_conn_settings settings; + nghttp3_conn_settings_default(&settings); + settings.qpack_max_table_capacity = 4096; + settings.qpack_blocked_streams = 100; + settings.num_placeholders = 10; + + auto mem = nghttp3_mem_default(); + + rv = nghttp3_conn_server_new(&httpconn_, &callbacks, &settings, mem, this); + if (rv != 0) { + std::cerr << "nghttp3_conn_server_new: " << nghttp3_strerror(rv) + << std::endl; + return -1; + } + + int64_t ctrl_stream_id; + + rv = ngtcp2_conn_open_uni_stream(conn_, &ctrl_stream_id, NULL); + if (rv != 0) { + std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv) + << std::endl; + return -1; + } + + rv = nghttp3_conn_bind_control_stream(httpconn_, ctrl_stream_id); + if (rv != 0) { + std::cerr << "nghttp3_conn_bind_control_stream: " << nghttp3_strerror(rv) + << std::endl; + return -1; + } + + if (!config.quiet) { + fprintf(stderr, "http: control stream=%" PRIx64 "\n", ctrl_stream_id); + } + + int64_t qpack_enc_stream_id, qpack_dec_stream_id; + + rv = ngtcp2_conn_open_uni_stream(conn_, &qpack_enc_stream_id, NULL); + if (rv != 0) { + std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv) + << std::endl; + return -1; + } + + rv = ngtcp2_conn_open_uni_stream(conn_, &qpack_dec_stream_id, NULL); + if (rv != 0) { + std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv) + << std::endl; + return -1; + } + + rv = nghttp3_conn_bind_qpack_streams(httpconn_, qpack_enc_stream_id, + qpack_dec_stream_id); + if (rv != 0) { + std::cerr << "nghttp3_conn_bind_qpack_streams: " << nghttp3_strerror(rv) + << std::endl; + return -1; + } + + if (!config.quiet) { + fprintf(stderr, + "http: QPACK streams encoder=%" PRIx64 " decoder=%" PRIx64 "\n", + qpack_enc_stream_id, qpack_dec_stream_id); + } + + return 0; +} + int Handler::init(const Endpoint &ep, const sockaddr *sa, socklen_t salen, const ngtcp2_cid *dcid, const ngtcp2_cid *ocid, uint32_t version) { @@ -1052,8 +1240,8 @@ int Handler::init(const Endpoint &ep, const sockaddr *sa, socklen_t salen, do_hp_mask, ::recv_stream_data, acked_crypto_offset, - acked_stream_data_offset, - nullptr, // stream_open + ::acked_stream_data_offset, + stream_open, stream_close, nullptr, // recv_stateless_reset nullptr, // recv_retry @@ -1065,9 +1253,9 @@ int Handler::init(const Endpoint &ep, const sockaddr *sa, socklen_t salen, ::update_key, path_validation, nullptr, // select_preferred_addr - nullptr, // stream_reset - nullptr, // extend_max_remote_streams_bidi, - nullptr, // extend_max_remote_streams_uni, + ::stream_reset, + ::extend_max_remote_streams_bidi, // extend_max_remote_streams_bidi, + nullptr, // extend_max_remote_streams_uni, }; ngtcp2_settings settings; @@ -1079,7 +1267,7 @@ int Handler::init(const Endpoint &ep, const sockaddr *sa, socklen_t salen, settings.max_stream_data_uni = 256_k; settings.max_data = 1_m; settings.max_streams_bidi = 100; - settings.max_streams_uni = 0; + settings.max_streams_uni = 3; settings.idle_timeout = config.timeout; settings.stateless_reset_token_present = 1; @@ -1450,6 +1638,7 @@ int Handler::do_handshake_read_once(const ngtcp2_path *path, if (rv != 0) { std::cerr << "ngtcp2_conn_read_handshake: " << ngtcp2_strerror(rv) << std::endl; + last_error_ = quicErrorTransport(rv); return -1; } return 0; @@ -1461,6 +1650,7 @@ ssize_t Handler::do_handshake_write_once() { if (nwrite < 0) { std::cerr << "ngtcp2_conn_write_handshake: " << ngtcp2_strerror(nwrite) << std::endl; + last_error_ = quicErrorTransport(nwrite); return -1; } @@ -1538,7 +1728,8 @@ int Handler::feed_data(const Endpoint &ep, const sockaddr *sa, socklen_t salen, start_draining_period(); return NETWORK_ERR_CLOSE_WAIT; } - return handle_error(rv); + last_error_ = quicErrorTransport(rv); + return handle_error(); } return 0; @@ -1546,7 +1737,7 @@ int Handler::feed_data(const Endpoint &ep, const sockaddr *sa, socklen_t salen, rv = do_handshake(&path, data, datalen); if (rv != 0) { - return handle_error(rv); + return handle_error(); } return 0; @@ -1592,16 +1783,12 @@ int Handler::on_write() { } } - for (auto &p : streams_) { - auto &stream = p.second; - rv = on_write_stream(*stream); - if (rv != 0) { - if (rv == NETWORK_ERR_SEND_NON_FATAL) { - schedule_retransmit(); - return rv; - } - return rv; + rv = write_streams(); + if (rv != 0) { + if (rv == NETWORK_ERR_SEND_NON_FATAL) { + schedule_retransmit(); } + return rv; } if (!ngtcp2_conn_get_handshake_completed(conn_)) { @@ -1616,7 +1803,8 @@ int Handler::on_write() { max_pktlen_, util::timestamp(loop_)); if (n < 0) { std::cerr << "ngtcp2_conn_write_pkt: " << ngtcp2_strerror(n) << std::endl; - return handle_error(n); + last_error_ = quicErrorTransport(n); + return handle_error(); } if (n == 0) { break; @@ -1641,79 +1829,104 @@ int Handler::on_write() { return 0; } -int Handler::on_write_stream(Stream &stream) { - if (stream.streambuf_idx == stream.streambuf.size()) { - if (stream.should_send_fin) { - auto v = Buffer{}; - if (write_stream_data(stream, 1, v) != 0) { - return -1; - } - } +int Handler::write_streams() { + if (!httpconn_) { return 0; } - for (auto it = std::begin(stream.streambuf) + stream.streambuf_idx; - it != std::end(stream.streambuf); ++it) { - auto &v = *it; - auto fin = stream.should_send_fin && - stream.streambuf_idx == stream.streambuf.size() - 1; - auto rv = write_stream_data(stream, fin, v); - if (rv != 0) { - return rv; - } - if (v.size() > 0) { - break; - } - ++stream.streambuf_idx; - } - - return 0; -} + std::array<nghttp3_vec, 16> vec; + int rv; -int Handler::write_stream_data(Stream &stream, int fin, Buffer &data) { - ssize_t ndatalen; - PathStorage path; + if (ngtcp2_conn_get_max_data_left(conn_) == 0) { + return 0; + } for (;;) { - auto n = ngtcp2_conn_write_stream(conn_, &path.path, sendbuf_.wpos(), - max_pktlen_, &ndatalen, stream.stream_id, - fin, data.rpos(), data.size(), - util::timestamp(loop_)); - if (n < 0) { - switch (n) { - case NGTCP2_ERR_STREAM_DATA_BLOCKED: - case NGTCP2_ERR_STREAM_SHUT_WR: - return 0; - } - std::cerr << "ngtcp2_conn_write_stream: " << ngtcp2_strerror(n) + int64_t stream_id; + int fin; + + auto sveccnt = nghttp3_conn_writev_stream(httpconn_, &stream_id, &fin, + vec.data(), vec.size()); + if (sveccnt < 0) { + std::cerr << "nghttp3_conn_writev_stream: " << nghttp3_strerror(sveccnt) << std::endl; - return handle_error(n); + last_error_ = quicErrorApplication(sveccnt); + return handle_error(); } - if (n == 0) { - return 0; + if (sveccnt == 0) { + break; } - if (ndatalen >= 0) { - if (fin && static_cast<size_t>(ndatalen) == data.size()) { - stream.should_send_fin = false; + ssize_t ndatalen; + PathStorage path; + auto v = vec.data(); + auto vcnt = static_cast<size_t>(sveccnt); + for (;;) { + auto nwrite = ngtcp2_conn_writev_stream( + conn_, &path.path, sendbuf_.wpos(), max_pktlen_, &ndatalen, stream_id, + fin, reinterpret_cast<const ngtcp2_vec *>(v), vcnt, + util::timestamp(loop_)); + if (nwrite < 0) { + auto should_break = false; + switch (nwrite) { + case NGTCP2_ERR_STREAM_DATA_BLOCKED: + if (ngtcp2_conn_get_max_data_left(conn_) == 0) { + return 0; + } + + rv = nghttp3_conn_block_stream(httpconn_, stream_id); + if (rv != 0) { + std::cerr << "nghttp3_conn_block_stream: " << nghttp3_strerror(rv) + << std::endl; + last_error_ = quicErrorApplication(rv); + return handle_error(); + } + should_break = true; + break; + case NGTCP2_ERR_STREAM_SHUT_WR: + should_break = true; + break; + } + + if (should_break) { + break; + } + + std::cerr << "ngtcp2_conn_write_stream: " << ngtcp2_strerror(nwrite) + << std::endl; + last_error_ = quicErrorTransport(nwrite); + return handle_error(); } - data.seek(ndatalen); - } + if (nwrite == 0) { + // We are congestion limited. + return 0; + } - sendbuf_.push(n); + sendbuf_.push(nwrite); - update_endpoint(&path.path.local); - update_remote_addr(&path.path.remote); + if (ndatalen > 0) { + rv = nghttp3_conn_add_write_offset(httpconn_, stream_id, ndatalen); + if (rv != 0) { + std::cerr << "nghttp3_conn_add_write_offset: " << nghttp3_strerror(rv) + << std::endl; + last_error_ = quicErrorApplication(rv); + return handle_error(); + } + } - auto rv = server_->send_packet(*endpoint_, remote_addr_, sendbuf_); - if (rv != NETWORK_ERR_OK) { - return rv; - } + update_endpoint(&path.path.local); + update_remote_addr(&path.path.remote); - if (ndatalen >= 0 && data.size() == 0) { - break; + auto rv = server_->send_packet(*endpoint_, remote_addr_, sendbuf_); + if (rv != NETWORK_ERR_OK) { + return rv; + } + + if (ndatalen > 0) { + break; + } } } @@ -1735,7 +1948,7 @@ void Handler::start_draining_period() { } } -int Handler::start_closing_period(int liberr) { +int Handler::start_closing_period() { if (!conn_ || ngtcp2_conn_is_in_closing_period(conn_)) { return 0; } @@ -1754,35 +1967,39 @@ int Handler::start_closing_period(int liberr) { conn_closebuf_ = std::make_unique<Buffer>(NGTCP2_MAX_PKTLEN_IPV4); - uint16_t err_code; - if (tls_alert_) { - err_code = NGTCP2_CRYPTO_ERROR | tls_alert_; - } else { - err_code = ngtcp2_err_infer_quic_transport_error_code(liberr); - } - PathStorage path; - auto n = ngtcp2_conn_write_connection_close( - conn_, &path.path, conn_closebuf_->wpos(), max_pktlen_, err_code, - util::timestamp(loop_)); - if (n < 0) { - std::cerr << "ngtcp2_conn_write_connection_close: " << ngtcp2_strerror(n) - << std::endl; - return -1; + if (last_error_.type == QUICErrorType::Transport) { + auto n = ngtcp2_conn_write_connection_close( + conn_, &path.path, conn_closebuf_->wpos(), max_pktlen_, + last_error_.code, util::timestamp(loop_)); + if (n < 0) { + std::cerr << "ngtcp2_conn_write_connection_close: " << ngtcp2_strerror(n) + << std::endl; + return -1; + } + conn_closebuf_->push(n); + } else { + auto n = ngtcp2_conn_write_application_close( + conn_, &path.path, conn_closebuf_->wpos(), max_pktlen_, + last_error_.code, util::timestamp(loop_)); + if (n < 0) { + std::cerr << "ngtcp2_conn_write_application_close: " << ngtcp2_strerror(n) + << std::endl; + return -1; + } + conn_closebuf_->push(n); } - conn_closebuf_->push(n); - update_endpoint(&path.path.local); update_remote_addr(&path.path.remote); return 0; } -int Handler::handle_error(int liberr) { +int Handler::handle_error() { int rv; - rv = start_closing_period(liberr); + rv = start_closing_period(); if (rv != 0) { return -1; } @@ -1825,41 +2042,25 @@ void Handler::schedule_retransmit() { int Handler::recv_stream_data(int64_t stream_id, uint8_t fin, const uint8_t *data, size_t datalen) { - int rv; - if (!config.quiet) { debug::print_stream_data(stream_id, data, datalen); } - auto it = streams_.find(stream_id); - if (it == std::end(streams_)) { - it = streams_.emplace(stream_id, std::make_unique<Stream>(stream_id)).first; + if (!httpconn_) { + return 0; } - auto &stream = (*it).second; - - ngtcp2_conn_extend_max_stream_offset(conn_, stream_id, datalen); - ngtcp2_conn_extend_max_offset(conn_, datalen); - - if (stream->recv_data(fin, data, datalen) != 0) { - if (stream->resp_state == RESP_IDLE) { - stream->send_status_response(400); - rv = ngtcp2_conn_shutdown_stream_read(conn_, stream_id, NGTCP2_APP_PROTO); - if (rv != 0) { - std::cerr << "ngtcp2_conn_shutdown_stream_read: " << ngtcp2_strerror(rv) - << std::endl; - return -1; - } - } else { - rv = ngtcp2_conn_shutdown_stream(conn_, stream_id, NGTCP2_APP_PROTO); - if (rv != 0) { - std::cerr << "ngtcp2_conn_shutdown_stream: " << ngtcp2_strerror(rv) - << std::endl; - return -1; - } - } + auto nconsumed = + nghttp3_conn_read_stream(httpconn_, stream_id, data, datalen, fin); + if (nconsumed < 0) { + std::cerr << "nghttp3_conn_read_stream: " << nghttp3_strerror(nconsumed) + << std::endl; + return -1; } + ngtcp2_conn_extend_max_stream_offset(conn_, stream_id, nconsumed); + ngtcp2_conn_extend_max_offset(conn_, nconsumed); + return 0; } @@ -1972,56 +2173,24 @@ void Handler::remove_tx_crypto_data(uint64_t offset, size_t datalen) { offset + datalen); } -int Handler::remove_tx_stream_data(int64_t stream_id, uint64_t offset, - size_t datalen) { - int rv; +void Handler::on_stream_close(int64_t stream_id) { + if (!config.quiet) { + std::cerr << "QUIC stream " << stream_id << " closed" << std::endl; + } auto it = streams_.find(stream_id); assert(it != std::end(streams_)); - auto &stream = (*it).second; - ::remove_tx_stream_data(stream->streambuf, stream->streambuf_idx, - stream->tx_stream_offset, offset + datalen); - - if (stream->streambuf.empty() && stream->resp_state == RESP_COMPLETED) { - rv = ngtcp2_conn_shutdown_stream_read(conn_, stream_id, NGTCP2_APP_NOERROR); - if (rv != 0 && rv != NGTCP2_ERR_STREAM_NOT_FOUND) { - std::cerr << "ngtcp2_conn_shutdown_stream_read: " << ngtcp2_strerror(rv) - << std::endl; - return -1; - } - } - return 0; -} - -int Handler::send_greeting() { - int rv; - int64_t stream_id; - - rv = ngtcp2_conn_open_uni_stream(conn_, &stream_id, nullptr); - if (rv != 0) { - return 0; + if (httpconn_) { + nghttp3_conn_close_stream(httpconn_, stream_id); } - auto stream = std::make_unique<Stream>(stream_id); - - static constexpr uint8_t hw[] = "Hello World!"; - stream->streambuf.emplace_back(hw, str_size(hw)); - stream->should_send_fin = true; - stream->resp_state = RESP_COMPLETED; - - streams_.emplace(stream_id, std::move(stream)); - - return 0; -} - -void Handler::on_stream_close(int64_t stream_id) { - auto it = streams_.find(stream_id); - assert(it != std::end(streams_)); streams_.erase(it); } -void Handler::set_tls_alert(uint8_t alert) { tls_alert_ = alert; } +void Handler::set_tls_alert(uint8_t alert) { + last_error_ = quicErrorTransportTLS(alert); +} namespace { void swritecb(struct ev_loop *loop, ev_io *w, int revents) { @@ -2065,9 +2234,7 @@ Server::~Server() { close(); } -void Server::disconnect() { disconnect(0); } - -void Server::disconnect(int liberr) { +void Server::disconnect() { config.tx_loss_prob = 0; for (auto &ep : endpoints_) { @@ -2080,7 +2247,7 @@ void Server::disconnect(int liberr) { auto it = std::begin(handlers_); auto &h = (*it).second; - h->handle_error(0); + h->handle_error(); remove(it); } diff --git a/examples/server.h b/examples/server.h index 3af829628d5012f5141e705701d6fb2f4e8aaeab..d606f29a6be3145d46c3d6cc4a9895dfdcf6bd6d 100644 --- a/examples/server.h +++ b/examples/server.h @@ -35,14 +35,15 @@ #include <string> #include <ngtcp2/ngtcp2.h> +#include <nghttp3/nghttp3.h> #include <openssl/ssl.h> #include <ev.h> -#include <http-parser/http_parser.h> #include "network.h" #include "crypto.h" #include "template.h" +#include "shared.h" using namespace ngtcp2; @@ -101,10 +102,12 @@ struct Buffer { uint8_t *tail; }; -enum { - RESP_IDLE, - RESP_STARTED, - RESP_COMPLETED, +struct HTTPHeader { + template <typename T1, typename T2> + HTTPHeader(const T1 &name, const T2 &value) : name(name), value(value) {} + + std::string name; + std::string value; }; struct Stream { @@ -112,41 +115,22 @@ struct Stream { ~Stream(); int recv_data(uint8_t fin, const uint8_t *data, size_t datalen); - int start_response(); + int start_response(nghttp3_conn *conn); int open_file(const std::string &path); int map_file(size_t len); - void buffer_file(); - void send_status_response(unsigned int status_code, - const std::string &extra_headers = ""); - void send_redirect_response(unsigned int status_code, + void send_status_response(nghttp3_conn *conn, unsigned int status_code, + const std::vector<HTTPHeader> &extra_headers = {}); + void send_redirect_response(nghttp3_conn *conn, unsigned int status_code, const std::string &path); int64_t stream_id; - std::deque<Buffer> streambuf; - // streambuf_idx is the index in streambuf, which points to the - // buffer to send next. - size_t streambuf_idx; - // tx_stream_offset is the offset where all data before offset is - // acked by the remote endpoint. - uint64_t tx_stream_offset; - // should_send_fin tells that fin should be sent after currently - // buffered data is sent. After sending fin, it is set to false. - bool should_send_fin; - // resp_state is the state of response. - int resp_state; - http_parser htp; - unsigned int http_major; - unsigned int http_minor; // uri is request uri/path. std::string uri; - // hdrs contains request HTTP header fields. - std::vector<std::pair<std::string, std::string>> hdrs; - // prev_hdr_key is true if the previous modification to hdrs is - // adding key (header field name). - bool prev_hdr_key; + std::string method; // fd is a file descriptor to read file to send its content to a // client. int fd; + std::string status_resp_body; // data is a pointer to the memory which maps file denoted by fd. uint8_t *data; // datalen is the length of mapped file by data. @@ -178,8 +162,7 @@ public: int on_read(const Endpoint &ep, const sockaddr *sa, socklen_t salen, uint8_t *data, size_t datalen); int on_write(); - int on_write_stream(Stream &stream); - int write_stream_data(Stream &stream, int fin, Buffer &data); + int write_streams(); int feed_data(const Endpoint &ep, const sockaddr *sa, socklen_t salen, uint8_t *data, size_t datalen); int do_handshake_read_once(const ngtcp2_path *path, const uint8_t *data, @@ -226,29 +209,38 @@ public: ngtcp2_conn *conn() const; int recv_stream_data(int64_t stream_id, uint8_t fin, const uint8_t *data, size_t datalen); + int acked_stream_data_offset(int64_t stream_id, size_t datalen); const ngtcp2_cid *scid() const; const ngtcp2_cid *pscid() const; const ngtcp2_cid *rcid() const; uint32_t version() const; void remove_tx_crypto_data(uint64_t offset, size_t datalen); - int remove_tx_stream_data(int64_t stream_id, uint64_t offset, size_t datalen); + void on_stream_open(int64_t stream_id); void on_stream_close(int64_t stream_id); void start_draining_period(); - int start_closing_period(int liberror); + int start_closing_period(); bool draining() const; - int handle_error(int liberror); + int handle_error(); int send_conn_close(); void update_endpoint(const ngtcp2_addr *addr); void update_remote_addr(const ngtcp2_addr *addr); - int send_greeting(); - int on_key(int name, const uint8_t *secret, size_t secretlen); void set_tls_alert(uint8_t alert); int update_key(); + int setup_httpconn(); + void http_consume(int64_t stream_id, size_t nconsumed); + void extend_max_remote_streams_bidi(uint64_t max_streams); + Stream *find_stream(int64_t stream_id); + void http_begin_request_headers(int64_t stream_id); + void http_recv_request_header(int64_t stream_id, int32_t token, + nghttp3_rcbuf *name, nghttp3_rcbuf *value); + void http_end_request_headers(int64_t stream_id); + void on_stream_reset(int64_t stream_id); + private: Endpoint *endpoint_; Address remote_addr_; @@ -271,6 +263,7 @@ private: ngtcp2_cid rcid_; crypto::Context hs_crypto_ctx_; crypto::Context crypto_ctx_; + nghttp3_conn *httpconn_; std::map<int64_t, std::unique_ptr<Stream>> streams_; // common buffer used to store packet data before sending Buffer sendbuf_; @@ -283,11 +276,9 @@ private: // tx_crypto_offset_ is the offset where all data before offset is // acked by the remote endpoint. uint64_t tx_crypto_offset_; + QUICError last_error_; // nkey_update_ is the number of key update occurred. size_t nkey_update_; - // tls_alert_ is the last TLS alert description generated by the - // local endpoint. - uint8_t tls_alert_; // initial_ is initially true, and used to process first packet from // client specially. After first packet, it becomes false. bool initial_; @@ -304,7 +295,6 @@ public: int init(const char *addr, const char *port); void disconnect(); - void disconnect(int liberr); void close(); int on_write(); diff --git a/examples/shared.cc b/examples/shared.cc new file mode 100644 index 0000000000000000000000000000000000000000..578894646dfbb116f693716b7c3e5b84f3212a50 --- /dev/null +++ b/examples/shared.cc @@ -0,0 +1,49 @@ +/* + * ngtcp2 + * + * Copyright (c) 2019 ngtcp2 contributors + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shared.h" + +#include <nghttp3/nghttp3.h> + +namespace ngtcp2 { + +QUICError quicErrorTransport(int liberr) { + if (liberr == NGTCP2_ERR_VERSION_NEGOTIATION) { + return {QUICErrorType::TransportVersionNegotiation, 0}; + } + return {QUICErrorType::Transport, + ngtcp2_err_infer_quic_transport_error_code(liberr)}; +} + +QUICError quicErrorTransportTLS(int alert) { + return {QUICErrorType::Transport, ngtcp2_err_infer_quic_transport_error_code( + NGTCP2_CRYPTO_ERROR | alert)}; +} + +QUICError quicErrorApplication(int liberr) { + return {QUICErrorType::Application, + nghttp3_err_infer_quic_app_error_code(liberr)}; +} + +} // namespace ngtcp2 diff --git a/examples/shared.h b/examples/shared.h index e9fa47b58f636ef7888c126065e0654798f1c4e0..3e1dcaa31afe1d0c1c1ce189c3a91a6ce9d05d7d 100644 --- a/examples/shared.h +++ b/examples/shared.h @@ -36,6 +36,25 @@ namespace ngtcp2 { constexpr uint16_t NGTCP2_APP_NOERROR = 0xff00; constexpr uint16_t NGTCP2_APP_PROTO = 0xff01; +enum class QUICErrorType { + Application, + Transport, + TransportVersionNegotiation, +}; + +struct QUICError { + QUICError(QUICErrorType type, uint16_t code) : type(type), code(code) {} + + QUICErrorType type; + uint16_t code; +}; + +QUICError quicErrorTransport(int liberr); + +QUICError quicErrorTransportTLS(int alert); + +QUICError quicErrorApplication(int liberr); + } // namespace ngtcp2 #endif // SHARED_H diff --git a/examples/util.h b/examples/util.h index 0c37c27f33925ed0fcdbbcd36c32de1fa0a7837b..d0d4b1a9e249ad786d95b240b05bddaf3d2a62db 100644 --- a/examples/util.h +++ b/examples/util.h @@ -36,6 +36,7 @@ #include <random> #include <ngtcp2/ngtcp2.h> +#include <nghttp3/nghttp3.h> #include <ev.h> @@ -43,6 +44,24 @@ namespace ngtcp2 { namespace util { +template <typename T, size_t N1, size_t N2> +constexpr nghttp3_nv make_nv(const T (&name)[N1], const T (&value)[N2]) { + return nghttp3_nv{(uint8_t *)name, (uint8_t *)value, N1 - 1, N2 - 1, + NGHTTP3_NV_FLAG_NONE}; +} + +template <typename T, size_t N, typename S> +constexpr nghttp3_nv make_nv(const T (&name)[N], const S &value) { + return nghttp3_nv{(uint8_t *)name, (uint8_t *)value.data(), N - 1, + value.size(), NGHTTP3_NV_FLAG_NONE}; +} + +template <typename S1, typename S2> +constexpr nghttp3_nv make_nv(const S1 &name, const S2 &value) { + return nghttp3_nv{(uint8_t *)name.data(), (uint8_t *)value.data(), + name.size(), value.size(), NGHTTP3_NV_FLAG_NONE}; +} + std::string format_hex(uint8_t c); std::string format_hex(const uint8_t *s, size_t len); @@ -127,6 +146,11 @@ std::string make_cid_key(const ngtcp2_cid *cid); // straddr stringifies |sa| of length |salen| in a format "[IP]:PORT". std::string straddr(const sockaddr *sa, socklen_t salen); +template <typename T, size_t N> +bool streq_l(const T (&a)[N], const nghttp3_vec &b) { + return N - 1 == b.len && memcmp(a, b.base, N - 1) == 0; +} + } // namespace util } // namespace ngtcp2 diff --git a/lib/includes/ngtcp2/ngtcp2.h b/lib/includes/ngtcp2/ngtcp2.h index e04fad321adf55399319deb39b5fe8e95f5b4767..58973c0b5029042cb74599030dd66d245ba292ea 100644 --- a/lib/includes/ngtcp2/ngtcp2.h +++ b/lib/includes/ngtcp2/ngtcp2.h @@ -164,7 +164,7 @@ typedef struct { /* NGTCP2_ALPN_* is a serialized form of ALPN protocol identifier this library supports. Notice that the first byte is the length of the following protocol identifier. */ -#define NGTCP2_ALPN_D19 "\x5hq-19" +#define NGTCP2_ALPN_D19 "\x5h3-19" #define NGTCP2_MAX_PKTLEN_IPV4 1252 #define NGTCP2_MAX_PKTLEN_IPV6 1232 @@ -2587,6 +2587,22 @@ NGTCP2_EXTERN int ngtcp2_conn_initiate_migration(ngtcp2_conn *conn, const ngtcp2_path *path, ngtcp2_tstamp ts); +/** + * @function + * + * `ngtcp2_conn_get_max_local_streams_uni` returns the cumulative + * number of streams which local endpoint can open. + */ +NGTCP2_EXTERN uint64_t ngtcp2_conn_get_max_local_streams_uni(ngtcp2_conn *conn); + +/** + * @function + * + * `ngtcp2_conn_get_max_data_left` returns the number of bytes that + * this local endpoint can send in this connection. + */ +NGTCP2_EXTERN uint64_t ngtcp2_conn_get_max_data_left(ngtcp2_conn *conn); + /** * @function * diff --git a/lib/ngtcp2_conn.c b/lib/ngtcp2_conn.c index 10269f712c0f5b34deb334ae66485a902eadafe9..630f730c44b515341a189c540af0b1390aacec66 100644 --- a/lib/ngtcp2_conn.c +++ b/lib/ngtcp2_conn.c @@ -7641,10 +7641,6 @@ int ngtcp2_conn_set_remote_transport_params( int ngtcp2_conn_set_early_remote_transport_params( ngtcp2_conn *conn, const ngtcp2_transport_params *params) { - if (conn->server) { - return NGTCP2_ERR_INVALID_STATE; - } - settings_copy_from_transport_params(&conn->remote.settings, params); conn_sync_stream_id_limit(conn); @@ -8552,6 +8548,14 @@ int ngtcp2_conn_initiate_migration(ngtcp2_conn *conn, const ngtcp2_path *path, return 0; } +uint64_t ngtcp2_conn_get_max_local_streams_uni(ngtcp2_conn *conn) { + return conn->local.uni.max_streams; +} + +uint64_t ngtcp2_conn_get_max_data_left(ngtcp2_conn *conn) { + return conn->tx.max_offset - conn->tx.offset; +} + void ngtcp2_path_challenge_entry_init(ngtcp2_path_challenge_entry *pcent, const ngtcp2_path *path, const uint8_t *data) {