From 22b2efacfe1fdca8834ebf89b44135c2d01e482f Mon Sep 17 00:00:00 2001 From: Alessandro Ghedini Date: Fri, 14 Jul 2023 09:57:31 +0100 Subject: [PATCH] Initial QUIC and HTTP/3 implementation using quiche --- auto/lib/conf | 4 + auto/lib/make | 4 + auto/lib/openssl/make | 12 +- auto/lib/quiche/conf | 23 + auto/lib/quiche/make | 23 + auto/make | 3 +- auto/modules | 44 + auto/options | 9 + auto/unix | 46 + src/core/ngx_connection.c | 82 + src/core/ngx_connection.h | 11 + src/core/ngx_core.h | 3 + src/event/ngx_event.c | 44 +- src/event/ngx_event.h | 4 - src/event/ngx_event_posted.c | 1 + src/event/ngx_event_posted.h | 3 + src/event/ngx_event_quic.c | 902 +++++++++ src/event/ngx_event_quic.h | 63 + src/event/ngx_event_udp.c | 39 +- src/http/modules/ngx_http_ssl_module.c | 13 +- src/http/ngx_http.c | 33 +- src/http/ngx_http.h | 4 + src/http/ngx_http_core_module.c | 7 + src/http/ngx_http_core_module.h | 3 + src/http/ngx_http_request.c | 140 +- src/http/ngx_http_request.h | 3 + src/http/ngx_http_request_body.c | 33 + src/http/ngx_http_upstream.c | 13 + src/http/v3/ngx_http_v3.c | 2273 +++++++++++++++++++++++ src/http/v3/ngx_http_v3.h | 80 + src/http/v3/ngx_http_v3_filter_module.c | 74 + src/http/v3/ngx_http_v3_module.c | 321 ++++ src/http/v3/ngx_http_v3_module.h | 36 + src/os/unix/ngx_linux_config.h | 10 + src/os/unix/ngx_udp_sendmsg_chain.c | 117 +- 35 files changed, 4451 insertions(+), 29 deletions(-) create mode 100644 auto/lib/quiche/conf create mode 100644 auto/lib/quiche/make create mode 100644 src/event/ngx_event_quic.c create mode 100644 src/event/ngx_event_quic.h create mode 100644 src/http/v3/ngx_http_v3.c create mode 100644 src/http/v3/ngx_http_v3.h create mode 100644 src/http/v3/ngx_http_v3_filter_module.c create mode 100644 src/http/v3/ngx_http_v3_module.c create mode 100644 src/http/v3/ngx_http_v3_module.h diff --git a/auto/lib/conf b/auto/lib/conf index 2c7af1040..abf920bae 100644 --- a/auto/lib/conf +++ b/auto/lib/conf @@ -25,6 +25,10 @@ if [ $USE_OPENSSL = YES ]; then . auto/lib/openssl/conf fi +if [ $USE_QUICHE = YES ]; then + . auto/lib/quiche/conf +fi + if [ $USE_ZLIB = YES ]; then . auto/lib/zlib/conf fi diff --git a/auto/lib/make b/auto/lib/make index b64e32908..c8f34ae2e 100644 --- a/auto/lib/make +++ b/auto/lib/make @@ -11,6 +11,10 @@ if [ $OPENSSL != NONE -a $OPENSSL != NO -a $OPENSSL != YES ]; then . auto/lib/openssl/make fi +if [ $QUICHE != NONE -a $QUICHE != NO -a $QUICHE != YES ]; then + . auto/lib/quiche/make +fi + if [ $ZLIB != NONE -a $ZLIB != NO -a $ZLIB != YES ]; then . auto/lib/zlib/make fi diff --git a/auto/lib/openssl/make b/auto/lib/openssl/make index 126a23875..139008207 100644 --- a/auto/lib/openssl/make +++ b/auto/lib/openssl/make @@ -49,11 +49,13 @@ END cat << END >> $NGX_MAKEFILE $OPENSSL/.openssl/include/openssl/ssl.h: $NGX_MAKEFILE - cd $OPENSSL \\ - && if [ -f Makefile ]; then \$(MAKE) clean; fi \\ - && ./config --prefix=$ngx_prefix no-shared no-threads $OPENSSL_OPT \\ - && \$(MAKE) \\ - && \$(MAKE) install_sw LIBDIR=lib + mkdir -p $OPENSSL/build $OPENSSL/.openssl/lib $OPENSSL/.openssl/include/openssl \\ + && cd $OPENSSL/build \\ + && cmake -DCMAKE_C_FLAGS="$OPENSSL_OPT" -DCMAKE_CXX_FLAGS="$OPENSSL_OPT" .. \\ + && \$(MAKE) VERBOSE=1 \\ + && cd .. \\ + && cp -r src/include/openssl/*.h .openssl/include/openssl \\ + && cp build/libssl.a build/libcrypto.a .openssl/lib END diff --git a/auto/lib/quiche/conf b/auto/lib/quiche/conf new file mode 100644 index 000000000..b853c1f18 --- /dev/null +++ b/auto/lib/quiche/conf @@ -0,0 +1,23 @@ + +# Copyright (C) Cloudflare, Inc. + + +if [ $QUICHE != NONE ]; then + + have=NGX_QUIC . auto/have + + QUICHE_BUILD_TARGET="release" + + if [ $NGX_DEBUG = YES ]; then + QUICHE_BUILD_TARGET="debug" + fi + + CORE_INCS="$CORE_INCS $QUICHE/quiche/include" + CORE_DEPS="$CORE_DEPS $QUICHE/target/$QUICHE_BUILD_TARGET/libquiche.a" + CORE_LIBS="$CORE_LIBS $QUICHE/target/$QUICHE_BUILD_TARGET/libquiche.a $NGX_LIBPTHREAD -lm" + + if [ "$NGX_SYSTEM" = "Darwin" ]; then + CORE_LIBS+=" -framework Security" + fi + +fi diff --git a/auto/lib/quiche/make b/auto/lib/quiche/make new file mode 100644 index 000000000..65b50e1d1 --- /dev/null +++ b/auto/lib/quiche/make @@ -0,0 +1,23 @@ + +# Copyright (C) Cloudflare, Inc. + +QUICHE_COMMON_FLAGS="--package quiche --verbose --no-default-features --features ffi" + +# Default is release build +QUICHE_BUILD_FLAGS="$QUICHE_COMMON_FLAGS --release" +QUICHE_BUILD_TARGET="release" + +if [ $NGX_DEBUG = YES ]; then + QUICHE_BUILD_FLAGS="$QUICHE_COMMON_FLAGS" + QUICHE_BUILD_TARGET="debug" +fi + + +cat << END >> $NGX_MAKEFILE + +$QUICHE/target/$QUICHE_BUILD_TARGET/libquiche.a: \\ + $OPENSSL/.openssl/include/openssl/ssl.h \\ + $NGX_MAKEFILE + cd $QUICHE && cargo build $QUICHE_BUILD_FLAGS $QUICHE_OPT + +END diff --git a/auto/make b/auto/make index 34c40cdd5..136c0a64e 100644 --- a/auto/make +++ b/auto/make @@ -7,7 +7,8 @@ echo "creating $NGX_MAKEFILE" mkdir -p $NGX_OBJS/src/core $NGX_OBJS/src/event $NGX_OBJS/src/event/modules \ $NGX_OBJS/src/os/unix $NGX_OBJS/src/os/win32 \ - $NGX_OBJS/src/http $NGX_OBJS/src/http/v2 $NGX_OBJS/src/http/modules \ + $NGX_OBJS/src/http $NGX_OBJS/src/http/v2 $NGX_OBJS/src/http/v3 \ + $NGX_OBJS/src/http/modules \ $NGX_OBJS/src/http/modules/perl \ $NGX_OBJS/src/mail \ $NGX_OBJS/src/stream \ diff --git a/auto/modules b/auto/modules index 09bfcb08d..2b2e6a889 100644 --- a/auto/modules +++ b/auto/modules @@ -134,6 +134,7 @@ if [ $HTTP = YES ]; then # ngx_http_header_filter # ngx_http_chunked_filter # ngx_http_v2_filter + # ngx_http_v3_filter # ngx_http_range_header_filter # ngx_http_gzip_filter # ngx_http_postpone_filter @@ -166,6 +167,7 @@ if [ $HTTP = YES ]; then ngx_http_header_filter_module \ ngx_http_chunked_filter_module \ ngx_http_v2_filter_module \ + ngx_http_v3_filter_module \ ngx_http_range_header_filter_module \ ngx_http_gzip_filter_module \ ngx_http_postpone_filter_module \ @@ -227,6 +229,17 @@ if [ $HTTP = YES ]; then . auto/module fi + if [ $HTTP_V3 = YES ]; then + ngx_module_name=ngx_http_v3_filter_module + ngx_module_incs= + ngx_module_deps= + ngx_module_srcs=src/http/v3/ngx_http_v3_filter_module.c + ngx_module_libs= + ngx_module_link=$HTTP_V3 + + . auto/module + fi + if :; then ngx_module_name=ngx_http_range_header_filter_module ngx_module_incs= @@ -438,6 +451,24 @@ if [ $HTTP = YES ]; then . auto/module fi + if [ $HTTP_V3 = YES ]; then + USE_QUICHE=YES + USE_OPENSSL=YES + have=NGX_HTTP_V3 . auto/have + have=NGX_HTTP_HEADERS . auto/have + + ngx_module_name=ngx_http_v3_module + ngx_module_incs=src/http/v3 + ngx_module_deps="src/http/v3/ngx_http_v3.h \ + src/http/v3/ngx_http_v3_module.h" + ngx_module_srcs="src/http/v3/ngx_http_v3.c \ + src/http/v3/ngx_http_v3_module.c" + ngx_module_libs= + ngx_module_link=$HTTP_V3 + + . auto/module + fi + if :; then ngx_module_name=ngx_http_static_module ngx_module_incs= @@ -1268,6 +1299,19 @@ if [ $USE_OPENSSL = YES ]; then fi +if [ $USE_QUICHE = YES ]; then + ngx_module_type=CORE + ngx_module_name=ngx_quic_module + ngx_module_incs= + ngx_module_deps=src/event/ngx_event_quic.h + ngx_module_srcs=src/event/ngx_event_quic.c + ngx_module_libs= + ngx_module_link=YES + + . auto/module +fi + + if [ $USE_PCRE = YES ]; then ngx_module_type=CORE ngx_module_name=ngx_regex_module diff --git a/auto/options b/auto/options index d8b421b0f..6b443f048 100644 --- a/auto/options +++ b/auto/options @@ -59,6 +59,7 @@ HTTP_CHARSET=YES HTTP_GZIP=YES HTTP_SSL=NO HTTP_V2=NO +HTTP_V3=NO HTTP_SSI=YES HTTP_POSTPONE=NO HTTP_REALIP=NO @@ -148,6 +149,9 @@ PCRE_JIT=NO USE_OPENSSL=NO OPENSSL=NONE +USE_QUICHE=NO +QUICHE=NONE + USE_ZLIB=NO ZLIB=NONE ZLIB_OPT= @@ -225,6 +229,7 @@ $0: warning: the \"--with-ipv6\" option is deprecated" --with-http_ssl_module) HTTP_SSL=YES ;; --with-http_v2_module) HTTP_V2=YES ;; + --with-http_v3_module) HTTP_V3=YES ;; --with-http_realip_module) HTTP_REALIP=YES ;; --with-http_addition_module) HTTP_ADDITION=YES ;; --with-http_xslt_module) HTTP_XSLT=YES ;; @@ -358,6 +363,9 @@ use the \"--with-mail_ssl_module\" option instead" --with-openssl=*) OPENSSL="$value" ;; --with-openssl-opt=*) OPENSSL_OPT="$value" ;; + --with-quiche=*) QUICHE="$value" ;; + --with-quiche-opt=*) QUICHE_OPT="$value" ;; + --with-md5=*) NGX_POST_CONF_MSG="$NGX_POST_CONF_MSG $0: warning: the \"--with-md5\" option is deprecated" @@ -440,6 +448,7 @@ cat << END --with-http_ssl_module enable ngx_http_ssl_module --with-http_v2_module enable ngx_http_v2_module + --with-http_v3_module enable ngx_http_v3_module --with-http_realip_module enable ngx_http_realip_module --with-http_addition_module enable ngx_http_addition_module --with-http_xslt_module enable ngx_http_xslt_module diff --git a/auto/unix b/auto/unix index 43d3b25a5..35e0d6bcf 100644 --- a/auto/unix +++ b/auto/unix @@ -434,6 +434,30 @@ ngx_feature_test="struct in_pktinfo pkt; setsockopt(0, IPPROTO_IP, IP_PKTINFO, NULL, 0)" . auto/feature +# Linux UDP segmentation offloading + +ngx_feature="UDP_SEGMENT" +ngx_feature_name="NGX_HAVE_UDP_SEGMENT" +ngx_feature_run=no +ngx_feature_incs="#include + /* Can use only with newer glibc */ + #include " +ngx_feature_path= +ngx_feature_libs= +ngx_feature_test="setsockopt(0, /* SOL_UDP */ 17, UDP_SEGMENT, NULL, 0)" +. auto/feature + +# Linux time based packet transmission + +ngx_feature="SO_TXTIME" +ngx_feature_name="NGX_HAVE_SO_TXTIME" +ngx_feature_run=no +ngx_feature_incs="#include + #include " +ngx_feature_path= +ngx_feature_libs= +ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_TXTIME, NULL, 0)" +. auto/feature # RFC 3542 way to get IPv6 datagram destination address @@ -448,6 +472,28 @@ ngx_feature_test="setsockopt(0, IPPROTO_IPV6, IPV6_RECVPKTINFO, NULL, 0)" . auto/feature +ngx_feature="IP_MTU_DISCOVER" +ngx_feature_name="NGX_HAVE_IP_MTU_DISCOVER" +ngx_feature_run=no +ngx_feature_incs="#include + #include " +ngx_feature_path= +ngx_feature_libs= +ngx_feature_test="setsockopt(0, IPPROTO_IP, IP_MTU_DISCOVER, NULL, 0)" +. auto/feature + + +ngx_feature="IPV6_MTU_DISCOVER" +ngx_feature_name="NGX_HAVE_IPV6_MTU_DISCOVER" +ngx_feature_run=no +ngx_feature_incs="#include + #include " +ngx_feature_path= +ngx_feature_libs= +ngx_feature_test="setsockopt(0, IPPROTO_IPV6, IPV6_MTU_DISCOVER, NULL, 0)" +. auto/feature + + ngx_feature="TCP_DEFER_ACCEPT" ngx_feature_name="NGX_HAVE_DEFERRED_ACCEPT" ngx_feature_run=no diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 33682532a..87e73783a 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -1010,6 +1010,72 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle) } } +#endif + +#if (NGX_QUIC) + +#if (NGX_HAVE_SO_TXTIME) + + /* Set SO_TXTIME socket option for pacing the QUIC outgoing + * packets using FQ qdisc. */ + if (ls[i].quic) { + struct sock_txtime sk_txtime; + + sk_txtime.clockid = CLOCK_MONOTONIC; + sk_txtime.flags = 0; + + if (setsockopt(ls[i].fd, SOL_SOCKET, SO_TXTIME, &sk_txtime, + sizeof(sk_txtime)) == -1 ) + { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, + "setsockopt(SO_TXTIME) " + "for %V failed, ignored.", + &ls[i].addr_text); + } else { + ls[i].quic_so_txtime = 1; + } + } + +#endif + +#if (NGX_HAVE_IP_MTU_DISCOVER) + + /* Disable path MTU discovery and enable DF flag for QUIC. */ + if (ls[i].quic && ls[i].sockaddr->sa_family == AF_INET) { + int pmtud = IP_PMTUDISC_PROBE; + + if (setsockopt(ls[i].fd, IPPROTO_IP, IP_MTU_DISCOVER, &pmtud, + sizeof(pmtud)) == -1 ) + + { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, + "setsockopt(IP_MTU_DISCOVER) " + "for %V failed, ignored.", + &ls[i].addr_text); + } + } + +#endif + +#if (NGX_HAVE_IPV6_MTU_DISCOVER) + + /* Disable path MTU discovery and enable DF flag for QUIC. */ + if (ls[i].quic && ls[i].sockaddr->sa_family == AF_INET6) { + int pmtud = IPV6_PMTUDISC_PROBE; + + if (setsockopt(ls[i].fd, IPPROTO_IPV6, IPV6_MTU_DISCOVER, &pmtud, + sizeof(pmtud)) == -1 ) + + { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, + "setsockopt(IPV6_MTU_DISCOVER) " + "for %V failed, ignored.", + &ls[i].addr_text); + } + } + +#endif + #endif } @@ -1053,6 +1119,22 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle) } } + if (c->write->active) { + if (ngx_event_flags & NGX_USE_EPOLL_EVENT) { + + /* + * it seems that Linux-2.6.x OpenVZ sends events + * for closed shared listening sockets unless + * the events was explicitly deleted + */ + + ngx_del_event(c->write, NGX_WRITE_EVENT, 0); + + } else { + ngx_del_event(c->write, NGX_WRITE_EVENT, NGX_CLOSE_EVENT); + } + } + ngx_free_connection(c); c->fd = (ngx_socket_t) -1; diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h index 54059629e..db39965c0 100644 --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -79,6 +79,13 @@ struct ngx_listening_s { unsigned deferred_accept:1; unsigned delete_deferred:1; unsigned add_deferred:1; +#if (NGX_QUIC) + unsigned quic:1; + ngx_queue_t quic_blocked_events; +#if (NGX_HAVE_SO_TXTIME) + unsigned quic_so_txtime:1; +#endif +#endif #if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER) char *accept_filter; #endif @@ -156,6 +163,10 @@ struct ngx_connection_s { ngx_udp_connection_t *udp; +#if (NGX_QUIC) + ngx_quic_connection_t *quic; +#endif + struct sockaddr *local_sockaddr; socklen_t local_socklen; diff --git a/src/core/ngx_core.h b/src/core/ngx_core.h index 93ca9174d..d0441f034 100644 --- a/src/core/ngx_core.h +++ b/src/core/ngx_core.h @@ -82,6 +82,9 @@ typedef void (*ngx_connection_handler_pt)(ngx_connection_t *c); #if (NGX_OPENSSL) #include #endif +#if (NGX_QUIC) +#include +#endif #include #include #include diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c index 69c55d7a0..7c6bae343 100644 --- a/src/event/ngx_event.c +++ b/src/event/ngx_event.c @@ -196,6 +196,9 @@ ngx_process_events_and_timers(ngx_cycle_t *cycle) ngx_uint_t flags; ngx_msec_t timer, delta; + ngx_queue_t *q; + ngx_event_t *ev; + if (ngx_timer_resolution) { timer = NGX_TIMER_INFINITE; flags = 0; @@ -215,6 +218,13 @@ ngx_process_events_and_timers(ngx_cycle_t *cycle) #endif } + if (!ngx_queue_empty(&ngx_posted_delayed_events)) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "posted delayed event queue not empty" + " making poll timeout 0"); + timer = 0; + } + if (ngx_use_accept_mutex) { if (ngx_accept_disabled > 0) { ngx_accept_disabled--; @@ -252,11 +262,38 @@ ngx_process_events_and_timers(ngx_cycle_t *cycle) ngx_shmtx_unlock(&ngx_accept_mutex); } - if (delta) { - ngx_event_expire_timers(); - } + ngx_event_expire_timers(); ngx_event_process_posted(cycle, &ngx_posted_events); + + while (!ngx_queue_empty(&ngx_posted_delayed_events)) { + q = ngx_queue_head(&ngx_posted_delayed_events); + + ev = ngx_queue_data(q, ngx_event_t, queue); + if (ev->delayed) { + /* start of newly inserted nodes */ + for (/* void */; + q != ngx_queue_sentinel(&ngx_posted_delayed_events); + q = ngx_queue_next(q)) + { + ev = ngx_queue_data(q, ngx_event_t, queue); + ev->delayed = 0; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "skipping delayed posted event %p," + " till next iteration", ev); + } + + break; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "delayed posted event %p", ev); + + ngx_delete_posted_event(ev); + + ev->handler(ev); + } } @@ -640,6 +677,7 @@ ngx_event_process_init(ngx_cycle_t *cycle) ngx_queue_init(&ngx_posted_accept_events); ngx_queue_init(&ngx_posted_events); + ngx_queue_init(&ngx_posted_delayed_events); if (ngx_event_timer_init(cycle->log) == NGX_ERROR) { return NGX_ERROR; diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h index bb77c4ae6..8101a7061 100644 --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h @@ -101,11 +101,7 @@ struct ngx_event_s { * accept: 1 if accept many, 0 otherwise */ -#if (NGX_HAVE_KQUEUE) || (NGX_HAVE_IOCP) int available; -#else - unsigned available:1; -#endif ngx_event_handler_pt handler; diff --git a/src/event/ngx_event_posted.c b/src/event/ngx_event_posted.c index d851f3d14..b6cea009e 100644 --- a/src/event/ngx_event_posted.c +++ b/src/event/ngx_event_posted.c @@ -12,6 +12,7 @@ ngx_queue_t ngx_posted_accept_events; ngx_queue_t ngx_posted_events; +ngx_queue_t ngx_posted_delayed_events; void diff --git a/src/event/ngx_event_posted.h b/src/event/ngx_event_posted.h index 145d30fea..6c3885537 100644 --- a/src/event/ngx_event_posted.h +++ b/src/event/ngx_event_posted.h @@ -43,6 +43,9 @@ void ngx_event_process_posted(ngx_cycle_t *cycle, ngx_queue_t *posted); extern ngx_queue_t ngx_posted_accept_events; extern ngx_queue_t ngx_posted_events; +extern ngx_queue_t ngx_posted_delayed_events; + +#define HAVE_POSTED_DELAYED_EVENTS_PATCH #endif /* _NGX_EVENT_POSTED_H_INCLUDED_ */ diff --git a/src/event/ngx_event_quic.c b/src/event/ngx_event_quic.c new file mode 100644 index 000000000..bc73cffc9 --- /dev/null +++ b/src/event/ngx_event_quic.c @@ -0,0 +1,902 @@ + +/* + * Copyright (C) Cloudflare, Inc. + */ + +#include +#include +#include + + +/* Max send burst limit */ +#define MAX_SEND_BURST_LIMIT 65507 + +/* errors */ +#define NGX_QUIC_NO_ERROR 0x0 +#define NGX_QUIC_INTERNAL_ERROR 0x1 + +/* From https://github.com/torvalds/linux/blob/24be4d0b46bb0c3c1dc7bacd30957d6144a70dfc/include/linux/udp.h#L98 */ +#define UDP_MAX_SEGMENTS (1 << 6UL) + + +static void ngx_quic_read_handler(ngx_event_t *ev); +static void ngx_quic_write_handler(ngx_event_t *ev); +static void ngx_quic_listener_write_event_handler(ngx_event_t *ev); + +static void ngx_quic_set_timer(ngx_connection_t *c); + +static void ngx_quic_handshake_completed(ngx_connection_t *c); + +static void ngx_quic_shutdown_handler(ngx_event_t *ev); + +static void ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t status); +static void ngx_quic_close_connection(ngx_connection_t *c); + +static void ngx_quic_clear_sending_state(ngx_quic_connection_t *c); +static ngx_int_t ngx_quic_try_send(ngx_connection_t *c); +static ngx_int_t ngx_quic_send_udp_packet(ngx_connection_t *c, uint8_t *buf, + size_t len, uint16_t segment_size, bool retry); + + +static ngx_command_t ngx_quic_commands[] = { + + ngx_null_command +}; + + +static ngx_core_module_t ngx_quic_module_ctx = { + ngx_string("quic"), + NULL, + NULL +}; + + +ngx_module_t ngx_quic_module = { + NGX_MODULE_V1, + &ngx_quic_module_ctx, /* module context */ + ngx_quic_commands, /* module directives */ + NGX_CORE_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +ngx_int_t +ngx_quic_create_conf(ngx_quic_t *quic) +{ + quic->config = quiche_config_new(QUICHE_PROTOCOL_VERSION); + if (quic->config == NULL) { + ngx_log_error(NGX_LOG_EMERG, quic->log, 0, "failed to create quic config"); + return NGX_ERROR; + } + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_validate_initial(ngx_event_t *ev, u_char *buf, ssize_t buf_len) +{ + /* Check incoming packet type, if it's not Initial we shouldn't be here. */ + if (((buf[0] & 0x30) >> 4) != 0) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "packet is not quic client initial"); + return NGX_ERROR; + } + + /* Client Initial packets must be at least 1200 bytes. */ + if (buf_len < QUICHE_MIN_CLIENT_INITIAL_LEN) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "quic initial packet is too short"); + return NGX_ERROR; + } + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_create_connection(ngx_quic_t *quic, ngx_connection_t *c) +{ + int rc; + u_char *buf; + size_t buf_len; + quiche_conn *conn; + static uint8_t out[MAX_DATAGRAM_SIZE]; + + uint8_t pkt_type; + uint32_t pkt_version; + + uint8_t scid[QUICHE_MAX_CONN_ID_LEN]; + size_t scid_len = sizeof(scid); + + uint8_t dcid[QUICHE_MAX_CONN_ID_LEN]; + size_t dcid_len = sizeof(dcid); + + uint8_t token[1]; + size_t token_len = sizeof(token); + + ngx_quic_connection_t *qc; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic init connection"); + + /* Extract some fields from the client's Initial packet, which was saved + * into c->buffer by ngx_event_recvmsg(). */ + buf = c->buffer->pos; + buf_len = ngx_buf_size(c->buffer); + + rc = quiche_header_info(buf, buf_len, QUICHE_MAX_CONN_ID_LEN, + &pkt_version, &pkt_type, + scid, &scid_len, dcid, &dcid_len, + token, &token_len); + if (rc < 0) { + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "failed to parse quic header: %d", rc); + return NGX_ERROR; + } + + /* Version mismatch, do version negotiation. */ + if (!quiche_version_is_supported(pkt_version)) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic version negotiation"); + + ssize_t written = quiche_negotiate_version(scid, scid_len, + dcid, dcid_len, + out, sizeof(out)); + + if (written < 0) { + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "failed to create quic vneg packet: %d", written); + return NGX_ERROR; + } + + if (ngx_quic_send_udp_packet(c, out, written, 0, false) == NGX_ERROR) { + return NGX_ERROR; + } + + return NGX_DONE; + } + + /* Initialize source connection ID with some random bytes. */ + RAND_bytes(scid, sizeof(scid)); + +#if (NGX_DEBUG) + { + uint8_t dcid_hex[QUICHE_MAX_CONN_ID_LEN * 2], + scid_hex[QUICHE_MAX_CONN_ID_LEN * 2]; + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, + "new quic connection dcid:%*.s new_scid:%*.s", + ngx_hex_dump(dcid_hex, dcid, dcid_len) - dcid_hex, dcid_hex, + ngx_hex_dump(scid_hex, scid, sizeof(scid)) - scid_hex, scid_hex); + } +#endif + + conn = quiche_conn_new_with_tls(dcid, sizeof(dcid), NULL, 0, + c->local_sockaddr, c->local_socklen, + c->sockaddr, c->socklen, quic->config, + c->ssl->connection, true); + if (conn == NULL) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "failed to create quic connection"); + return NGX_ERROR; + } + + qc = ngx_pcalloc(c->pool, sizeof(ngx_quic_connection_t)); + if (qc == NULL) { + quiche_conn_free(conn); + return NGX_ERROR; + } + + qc->handler = NULL; + + qc->conn = conn; + + qc->pacing = quic->pacing; + + qc->send_buf = ngx_palloc(c->pool, MAX_SEND_BURST_LIMIT); + if (qc->send_buf == NULL) { + quiche_conn_free(conn); + return NGX_ERROR; + } + + ngx_quic_clear_sending_state(qc); + + c->quic = qc; + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_handshake(ngx_connection_t *c) +{ + u_char *buf; + size_t buf_len; + ssize_t done; + + quiche_recv_info recv_info = { + c->sockaddr, + c->socklen, + c->local_sockaddr, + c->local_socklen, + }; + + /* Process the client's Initial packet, which was saved into c->buffer by + * ngx_event_recvmsg(). */ + buf = c->buffer->pos; + buf_len = ngx_buf_size(c->buffer); + + done = quiche_conn_recv(c->quic->conn, buf, buf_len, &recv_info); + + if ((done < 0) && (done != QUICHE_ERR_DONE)) { + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "failed to process quic packet: %d", done); + return NGX_ERROR; + } + + c->read->handler = ngx_quic_read_handler; + c->write->handler = ngx_quic_write_handler; + + ngx_post_event(c->write, &ngx_posted_events); + + return NGX_AGAIN; +} + + +static void +ngx_quic_read_handler(ngx_event_t *rev) +{ + int n; + static uint8_t buf[65535]; + ngx_connection_t *c; + uint8_t *b; + + c = rev->data; + b = buf; + + c->log->action = "reading QUIC packets"; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic read handler"); + + if (rev->timedout) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic connection timed out"); + + if (c->quic->handler != NULL) { + c->quic->handler(c); + } + + return; + } + + for (;;) { + n = c->recv(c, buf, sizeof(buf)); + if (n == NGX_AGAIN) { + break; + } + + if (n == NGX_ERROR) { + ngx_quic_finalize_connection(c, NGX_QUIC_INTERNAL_ERROR); + return; + } + + quiche_recv_info recv_info = { + c->sockaddr, + c->socklen, + c->local_sockaddr, + c->local_socklen, + }; + + ssize_t done = quiche_conn_recv(c->quic->conn, b, n, &recv_info); + + if (done == QUICHE_ERR_DONE) { + break; + } + + if (done < 0) { + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "failed to process quic packet: %d", done); + + ngx_quic_finalize_connection(c, NGX_QUIC_INTERNAL_ERROR); + return; + } + } + + if (quiche_conn_is_in_early_data(c->quic->conn) || + quiche_conn_is_established(c->quic->conn)) { + if (!c->ssl->handshaked) { + ngx_quic_handshake_completed(c); + } + + if ((c->quic == NULL) || (c->quic->handler == NULL)) { + return; + } + + /* Notify application layer that there might be stream data to read. */ + c->quic->handler(c); + } + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic done reading"); + + ngx_post_event(c->write, &ngx_posted_events); +} + +static void +ngx_quic_listener_write_event_handler(ngx_event_t *ev) +{ + ngx_queue_t *q; + ngx_event_t *wev; + ngx_listening_t *ls; + ngx_connection_t *c; + + c = ev->data; + ls = c->listening; + + /* Unblock the quic events waiting on socket buffer. */ + while (!ngx_queue_empty(&ls->quic_blocked_events)) { + + q = ngx_queue_head(&ls->quic_blocked_events); + wev = ngx_queue_data(q, ngx_event_t, queue); + + ngx_delete_posted_event(wev); + + ngx_post_event(wev, &ngx_posted_events); + } + + /* Unregister from spurious wake ups */ + if (ls->connection->write->active) { + ngx_del_event(ls->connection->write, NGX_WRITE_EVENT, 0); + } +} + +static void +ngx_quic_clear_sending_state(ngx_quic_connection_t *qc) +{ + qc->segment_size = 0; + qc->last_segment_size = 0; + qc->send_buf_size = 0; + qc->send_buf_offset = 0; + qc->blocked = 0; +} + +static void +ngx_quic_write_handler(ngx_event_t *wev) +{ + bool done; + size_t total_written; + size_t total_segments; + ssize_t prev_written; + uint16_t last_segment_size; + uint16_t segment_size; + ngx_connection_t *c; + quiche_send_info send_info; + ngx_connection_t *ls_c; + ngx_listening_t *ls; + ngx_event_t *ls_wev; + ngx_quic_connection_t *qc; + uint8_t *out; + + c = wev->data; + ls = c->listening; + ls_c = ls->connection; + ls_wev = ls_c->write; + qc = c->quic; + + c->log->action = "writing QUIC packets"; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic write handler"); + + if (wev->timedout) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic alarm fired"); + + quiche_conn_on_timeout(c->quic->conn); + } + + if (quiche_conn_is_closed(c->quic->conn)) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic connection is closed"); + + ngx_quic_finalize_connection(c, NGX_QUIC_NO_ERROR); + return; + } + + /* Socket is not ready, add to blocked queue. */ + if (!ls_wev->ready) { + ngx_post_event(wev, &ls->quic_blocked_events); + return; + } + + /* Try sending blocked packets first. */ + if (qc->blocked) { + + if (ngx_quic_try_send(c) == NGX_OK) { + ngx_post_event(wev, &ngx_posted_events); + } + + return; + } + + /* Round down by MAX_DATAGRAM_SIZE. */ + const size_t max_send_burst = + ngx_min(quiche_conn_send_quantum(c->quic->conn), + MAX_SEND_BURST_LIMIT) + / MAX_DATAGRAM_SIZE * MAX_DATAGRAM_SIZE; + + prev_written = 0; + total_written = 0; + total_segments = 0; + segment_size = 0; + last_segment_size = 0; + done = 0; + + out = qc->send_buf; + + for (;;) { + /* Make sure we don't exceed the max send burst limit. */ + size_t max_len = ngx_min(max_send_burst - total_written, + MAX_DATAGRAM_SIZE); + + ssize_t written = quiche_conn_send(c->quic->conn, out + total_written, + max_len, &send_info); + + /* Flush and exit if there are no more packets to send. */ + if (written == QUICHE_ERR_DONE) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic done writing"); + + /* Nothing more to send. */ + done = 1; + + break; + } + + if (written < 0) { + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "failed to create quic packet: %d", written); + + ngx_quic_finalize_connection(c, NGX_QUIC_INTERNAL_ERROR); + return; + } + + total_written += written; + + total_segments += 1; + + /* + * segment_size is the first packet size in the loop, if + * there are 2 or more packets. + */ + if (!segment_size) { + segment_size = written; + + /* 1st packet timestamp is used when sending in a batch */ + ngx_memcpy(&c->quic->send_info, &send_info, sizeof(quiche_send_info)); + } + + last_segment_size = written; + + /* Flush but keep sending if the max burst limit is reached. */ + if (total_written >= max_send_burst) { + break; + } + + /* Flush but keep sending if the number of segments reached the GSO + * limit. */ + if (total_segments >= UDP_MAX_SEGMENTS) { + break; + } + + /* Flush but keep sending if the packet size changes but the max burst + * limit isn't reached yet. */ + if (prev_written && written != prev_written) { + break; + } + + prev_written = written; + } + + /* Send packets to the network. */ + if (total_written > 0) { + if (last_segment_size > segment_size) { + qc->last_segment_size = last_segment_size; + } + + qc->send_buf_size = total_written; + qc->segment_size = segment_size; + + if (ngx_quic_try_send(c) == NGX_ERROR) { + return; + } + + /* Still need to send more. Schedule to next worker cycle. */ + if (!done) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic pause writing"); + + wev->delayed = 1; + ngx_post_event(wev, &ngx_posted_delayed_events); + } + } + + ngx_quic_set_timer(c); +} + +static ngx_int_t +ngx_quic_try_send(ngx_connection_t *c) +{ + int rc; + size_t send_size; + size_t off; + uint16_t segment_size; + uint16_t last_segment_size; + uint8_t *out; + ngx_quic_connection_t *qc; + + qc = c->quic; + + out = qc->send_buf; + off = qc->send_buf_offset; + + segment_size = qc->segment_size; + last_segment_size = qc->last_segment_size; + + send_size = qc->send_buf_size - last_segment_size; + + rc = NGX_OK; + + /* + * Resume sending only the last packet which is bigger than + * the segment size of the rest of the packets in the batch. + */ + if (send_size == off) { + goto last_segment; + } + +#if (NGX_HAVE_UDP_SEGMENT) + rc = ngx_quic_send_udp_packet(c, out, send_size, segment_size, true); +#else + while (off < send_size) { + size_t len = (send_size - off) > segment_size + ? segment_size : (send_size - off); + + rc = ngx_quic_send_udp_packet(c, out + off, len, 0, true); + + if (rc != NGX_OK) { + break; + } + + off += len; + } +#endif + + if (rc == NGX_AGAIN) { + /* Update where to resume sending once unblocked. */ + qc->send_buf_offset = off; + qc->blocked = 1; + + return NGX_OK; + } + + if (rc == NGX_ERROR) { + ngx_quic_finalize_connection(c, NGX_QUIC_INTERNAL_ERROR); + return rc; + } + +last_segment: + /* + * If the last segment size is bigger than the segment size, + * send packets with the same size using GSO (if available), + * and then send the last one separately at the end. + */ + if (rc == NGX_OK && last_segment_size > 0) { + + c->quic->segment_size = 0; + rc = ngx_quic_send_udp_packet(c, + out + send_size, last_segment_size, 0, true); + + if (rc == NGX_AGAIN) { + /* Update where to resume sending once unblocked. */ + qc->send_buf_offset = send_size; + qc->blocked = 1; + + return NGX_OK; + } + + if (rc == NGX_ERROR) { + ngx_quic_finalize_connection(c, NGX_QUIC_INTERNAL_ERROR); + return rc; + } + } + + /* Packets sent, clear sending state for next cycle. */ + ngx_quic_clear_sending_state(c->quic); + + return NGX_OK; +} + +static void +ngx_quic_set_timer(ngx_connection_t *c) +{ + uint64_t expiry; + ngx_event_t *wev; + + wev = c->write; + + expiry = quiche_conn_timeout_as_millis(c->quic->conn); + expiry = ngx_max(expiry, 1); + + if (wev->timer_set) { + ngx_del_timer(wev); + } + + /* quiche_conn_timeout_as_millis() will return UINT64_MAX when the timer + * should be unset (this would be equvalent to returning Option::None in + * Rust). To avoid overflow we need to explicitly check for this value. */ + if (expiry != UINT64_MAX) { + ngx_add_timer(wev, (ngx_msec_t)expiry); + } +} + + +static void +ngx_quic_handshake_completed(ngx_connection_t *c) +{ +#if (NGX_DEBUG) + { + char buf[129], *s, *d; +#if OPENSSL_VERSION_NUMBER >= 0x10000000L + const +#endif + SSL_CIPHER *cipher; + + cipher = SSL_get_current_cipher(c->ssl->connection); + + if (cipher) { + SSL_CIPHER_description(cipher, &buf[1], 128); + + for (s = &buf[1], d = buf; *s; s++) { + if (*s == ' ' && *d == ' ') { + continue; + } + + if (*s == LF || *s == CR) { + continue; + } + + *++d = *s; + } + + if (*d != ' ') { + d++; + } + + *d = '\0'; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "QUIC: %s, cipher: \"%s\"", + SSL_get_version(c->ssl->connection), &buf[1]); + + if (SSL_session_reused(c->ssl->connection)) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic reused session"); + } + + } else { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quic no shared ciphers"); + } + } +#endif + + ngx_del_timer(c->read); + + c->ssl->handshaked = 1; + + /* Notify application layer that the handshake is complete. */ + c->ssl->handler(c); +} + + +ngx_int_t +ngx_quic_shutdown(ngx_connection_t *c) +{ + ssize_t written; + static uint8_t out[MAX_DATAGRAM_SIZE]; + + /* Connection is closed, free memory. */ + if (quiche_conn_is_closed(c->quic->conn)) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "free quic connection"); + + quiche_conn_free(c->quic->conn); + + c->quic = NULL; + c->ssl = NULL; + + return NGX_OK; + } + + /* We can't free the connection state yet, as we need to wait for the + * draining timeout to expire. + * + * Setup event handlers such that we will try again when that happens (or + * when another event is triggered). */ + c->read->handler = ngx_quic_shutdown_handler; + c->write->handler = ngx_quic_shutdown_handler; + + /* Try sending a packet in order to flush pending frames (CONNECTION_CLOSE + * for example), but ignore errors as we are already closing the connection + * anyway. */ + written = quiche_conn_send(c->quic->conn, out, sizeof(out), &c->quic->send_info); + + if (written > 0) { + ngx_quic_send_udp_packet(c, out, written, 0, false); + } + + ngx_quic_set_timer(c); + + return NGX_AGAIN; +} + + +static void +ngx_quic_shutdown_handler(ngx_event_t *ev) +{ + ngx_connection_t *c; + ngx_connection_handler_pt handler; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic shutdown handler"); + + c = ev->data; + handler = c->quic->handler; + + if (ev->timedout) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic alarm fired"); + + quiche_conn_on_timeout(c->quic->conn); + } + + if (ngx_quic_shutdown(c) == NGX_AGAIN) { + return; + } + + handler(c); +} + + +static void +ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t status) +{ + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "finalize quic connection: %d", c->fd); + + c->error = 1; + + if (quiche_conn_is_closed(c->quic->conn)) { + c->close = 1; + } + + ngx_quic_clear_sending_state(c->quic); + + quiche_conn_close(c->quic->conn, false, status, NULL, 0); + + /* Notify the application layer that the connection is in an error + * state and will be closed. */ + if (c->quic->handler != NULL) { + c->quic->handler(c); + return; + } + + ngx_quic_close_connection(c); +} + + +static void +ngx_quic_close_connection(ngx_connection_t *c) +{ + ngx_pool_t *pool; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "close quic connection: %d", c->fd); + + if (c->quic) { + if (ngx_quic_shutdown(c) == NGX_AGAIN) { + c->quic->handler = ngx_quic_close_connection; + return; + } + } + +#if (NGX_STAT_STUB) + (void) ngx_atomic_fetch_add(ngx_stat_active, -1); +#endif + + c->destroyed = 1; + + pool = c->pool; + + ngx_close_connection(c); + + ngx_destroy_pool(pool); +} + + +void +ngx_quic_cleanup_ctx(void *data) +{ + ngx_quic_t *quic = data; + + quiche_config_free(quic->config); +} + + +static ngx_int_t +ngx_quic_send_udp_packet(ngx_connection_t *c, uint8_t *buf, size_t len, + uint16_t segment_size, bool retry) +{ + ngx_event_t *ls_wev; + ngx_event_t *wev; + ngx_connection_t *ls_c; + ngx_listening_t *ls; + ngx_buf_t out_buf = {0}; + ngx_chain_t out_chain = {0}; + ngx_chain_t *cl; + + wev = c->write; + ls = c->listening; + ls_c = ls->connection; + ls_wev = ls_c->write; + + /* The send_chain() API takes an ngx_chain_t parameter instead of a simple + * buffer, so we need to initialize the chain such that it contains only a + * single buffer. + * + * The c->send_chain() call is required (instead of just c->send()) because + * it uses the sendmsg(2) syscall (instead of sendto(2)), which allows us to + * specify the correct source IP address for the connection. */ + + out_buf.start = out_buf.pos = buf; + out_buf.end = out_buf.last = buf + len; + out_buf.memory = 1; + out_buf.flush = 1; + + out_chain.buf = &out_buf; + out_chain.next = NULL; + + c->write->ready = 1; + + cl = c->send_chain(c, &out_chain, 0); + + if (cl != NULL) { + if (retry) { + ls_wev->ready = 0; + + if (!ls_wev->active) { + + ls_wev->handler = ngx_quic_listener_write_event_handler; + + if (ngx_handle_write_event(ls_wev, 0) != NGX_OK) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "failed to add quic write event"); + + return NGX_ERROR; + } + } + + ngx_post_event(wev, &ls->quic_blocked_events); + } + + return NGX_AGAIN; + } + + if (cl == NGX_CHAIN_ERROR) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "failed to send quic packet"); + + return NGX_ERROR; + } + + return NGX_OK; +} diff --git a/src/event/ngx_event_quic.h b/src/event/ngx_event_quic.h new file mode 100644 index 000000000..1459f3b79 --- /dev/null +++ b/src/event/ngx_event_quic.h @@ -0,0 +1,63 @@ + +/* + * Copyright (C) Cloudflare, Inc. + */ + + +#ifndef _NGX_EVENT_QUIC_H_INCLUDED_ +#define _NGX_EVENT_QUIC_H_INCLUDED_ + + +#include + +#include +#include + +#include + +/* Limit outgoing packets to 1200 bytes. This is the minimum value allowed. */ +#define MAX_DATAGRAM_SIZE 1200 + +typedef struct ngx_quic_s ngx_quic_t; +typedef struct ngx_quic_connection_s ngx_quic_connection_t; + +struct ngx_quic_s { + quiche_config *config; + ngx_log_t *log; + bool pacing; +}; + +struct ngx_quic_connection_s { + quiche_conn *conn; + + ngx_connection_handler_pt handler; + + uint8_t *send_buf; + size_t send_buf_offset; + uint16_t send_buf_size; + uint16_t segment_size; + uint16_t last_segment_size; + + bool blocked; + bool pacing; + quiche_send_info send_info; +}; + + +ngx_int_t ngx_quic_create_conf(ngx_quic_t *quic); + +ngx_int_t ngx_quic_validate_initial(ngx_event_t *ev, u_char *buf, + ssize_t buf_len); + +ngx_int_t ngx_quic_create_connection(ngx_quic_t *quic, ngx_connection_t *c); + +ngx_int_t ngx_quic_create_ssl_connection(ngx_ssl_t *ssl, ngx_connection_t *c, + ngx_uint_t flags); + +ngx_int_t ngx_quic_handshake(ngx_connection_t *c); + +ngx_int_t ngx_quic_shutdown(ngx_connection_t *c); + +void ngx_quic_cleanup_ctx(void *data); + +#endif /* _NGX_EVENT_QUIC_H_INCLUDED_ */ diff --git a/src/event/ngx_event_udp.c b/src/event/ngx_event_udp.c index 557283050..daa504bb0 100644 --- a/src/event/ngx_event_udp.c +++ b/src/event/ngx_event_udp.c @@ -9,6 +9,10 @@ #include #include +/* + * Max number of recvmsg() tried in ngx_event_recvmsg() + */ +#define NGX_RECVMSG_MAX 64 #if !(NGX_WIN32) @@ -70,8 +74,9 @@ ngx_event_recvmsg(ngx_event_t *ev) ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module); + /* For non-kqueue, ev->available is a packet counter */ if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) { - ev->available = ecf->multi_accept; + ev->available = NGX_RECVMSG_MAX; } lc = ev->data; @@ -276,6 +281,14 @@ ngx_event_recvmsg(ngx_event_t *ev) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif +#if (NGX_QUIC) + if (ls->quic) { + if (ngx_quic_validate_initial(ev, buffer, n) != NGX_OK) { + goto next; + } + } +#endif + ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; @@ -413,13 +426,37 @@ ngx_event_recvmsg(ngx_event_t *ev) ls->handler(c); + /* When multi_accept is off, exit the loop now */ + if (!ecf->multi_accept) { + break; + } + next: if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available -= n; + } else { + ev->available --; } } while (ev->available); + + /* Reschedule recvmsg if there is more packets to read */ + n = recvmsg(lc->fd, &msg, MSG_PEEK); + + if (n == -1) { + err = ngx_socket_errno; + + if (err != EAGAIN) { + ngx_log_error(NGX_LOG_ALERT, ev->log, err, "recvmsg() peek error"); + } + } else { + ev->delayed = 1; + ngx_post_event(ev, &ngx_posted_delayed_events); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "recvmsg on %V, reschedule: %d", &ls->addr_text, ev->available); + } } diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index b3f8f4795..00dd9c61a 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -371,7 +371,7 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, #if (NGX_DEBUG) unsigned int i; #endif -#if (NGX_HTTP_V2) +#if (NGX_HTTP_V2 || NGX_HTTP_V3) ngx_http_connection_t *hc; #endif #if (NGX_HTTP_V2 || NGX_DEBUG) @@ -388,9 +388,11 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, } #endif -#if (NGX_HTTP_V2) +#if (NGX_HTTP_V2 || NGX_HTTP_V3) hc = c->data; +#endif +#if (NGX_HTTP_V2) if (hc->addr_conf->http2) { srv = (unsigned char *) NGX_HTTP_V2_ALPN_ADVERTISE NGX_HTTP_NPN_ADVERTISE; @@ -398,6 +400,13 @@ ngx_http_ssl_alpn_select(ngx_ssl_conn_t *ssl_conn, const unsigned char **out, } else #endif +#if (NGX_HTTP_V3) + if (hc->addr_conf->quic) { + srv = (unsigned char *) QUICHE_H3_APPLICATION_PROTOCOL; + srvlen = sizeof(QUICHE_H3_APPLICATION_PROTOCOL) - 1; + + } else +#endif { srv = (unsigned char *) NGX_HTTP_NPN_ADVERTISE; srvlen = sizeof(NGX_HTTP_NPN_ADVERTISE) - 1; diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c index 79ef9c644..a29bff836 100644 --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -1141,6 +1141,7 @@ ngx_int_t ngx_http_add_listen(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, ngx_http_listen_opt_t *lsopt) { + int t; in_port_t p; ngx_uint_t i; struct sockaddr *sa; @@ -1159,11 +1160,13 @@ ngx_http_add_listen(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, sa = lsopt->sockaddr; p = ngx_inet_get_port(sa); + t = lsopt->quic ? SOCK_DGRAM : SOCK_STREAM; port = cmcf->ports->elts; for (i = 0; i < cmcf->ports->nelts; i++) { - if (p != port[i].port || sa->sa_family != port[i].family) { + if (p != port[i].port || sa->sa_family != port[i].family + || t != port[i].type) { continue; } @@ -1182,6 +1185,7 @@ ngx_http_add_listen(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, port->family = sa->sa_family; port->port = p; port->addrs.elts = NULL; + port->type = t; return ngx_http_add_address(cf, cscf, port, lsopt); } @@ -1199,6 +1203,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, #if (NGX_HTTP_V2) ngx_uint_t http2; #endif +#if (NGX_HTTP_V3) + ngx_uint_t quic; +#endif /* * we cannot compare whole sockaddr struct's as kernel @@ -1234,6 +1241,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, #if (NGX_HTTP_V2) http2 = lsopt->http2 || addr[i].opt.http2; #endif +#if (NGX_HTTP_V3) + quic = lsopt->quic || addr[i].opt.quic; +#endif if (lsopt->set) { @@ -1270,6 +1280,9 @@ ngx_http_add_addresses(ngx_conf_t *cf, ngx_http_core_srv_conf_t *cscf, #if (NGX_HTTP_V2) addr[i].opt.http2 = http2; #endif +#if (NGX_HTTP_V3) + addr[i].opt.quic = quic; +#endif return NGX_OK; } @@ -1688,6 +1701,12 @@ ngx_http_init_listening(ngx_conf_t *cf, ngx_http_conf_port_t *port) break; } +#if (NGX_HTTP_V3) + if (addr[i].opt.quic) { + ls->type = SOCK_DGRAM; + } +#endif + addr++; last--; } @@ -1770,6 +1789,12 @@ ngx_http_add_listening(ngx_conf_t *cf, ngx_http_conf_addr_t *addr) ls->reuseport = addr->opt.reuseport; #endif +#if (NGX_HTTP_V3) + ls->quic = addr->opt.quic; + + ls->wildcard = addr->opt.wildcard; +#endif + return ls; } @@ -1803,6 +1828,9 @@ ngx_http_add_addrs(ngx_conf_t *cf, ngx_http_port_t *hport, addrs[i].conf.http2 = addr[i].opt.http2; #endif addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; +#if (NGX_HTTP_V3) + addrs[i].conf.quic = addr[i].opt.quic; +#endif if (addr[i].hash.buckets == NULL && (addr[i].wc_head == NULL @@ -1868,6 +1896,9 @@ ngx_http_add_addrs6(ngx_conf_t *cf, ngx_http_port_t *hport, addrs6[i].conf.http2 = addr[i].opt.http2; #endif addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol; +#if (NGX_HTTP_V3) + addrs6[i].conf.quic = addr[i].opt.quic; +#endif if (addr[i].hash.buckets == NULL && (addr[i].wc_head == NULL diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h index 8b43857ee..444f93536 100644 --- a/src/http/ngx_http.h +++ b/src/http/ngx_http.h @@ -20,6 +20,7 @@ typedef struct ngx_http_file_cache_s ngx_http_file_cache_t; typedef struct ngx_http_log_ctx_s ngx_http_log_ctx_t; typedef struct ngx_http_chunked_s ngx_http_chunked_t; typedef struct ngx_http_v2_stream_s ngx_http_v2_stream_t; +typedef struct ngx_http_v3_stream_s ngx_http_v3_stream_t; typedef ngx_int_t (*ngx_http_header_handler_pt)(ngx_http_request_t *r, ngx_table_elt_t *h, ngx_uint_t offset); @@ -38,6 +39,9 @@ typedef u_char *(*ngx_http_log_handler_pt)(ngx_http_request_t *r, #if (NGX_HTTP_V2) #include #endif +#if (NGX_HTTP_V3) +#include +#endif #if (NGX_HTTP_CACHE) #include #endif diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index cb49ef74a..1e991c0b3 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -4099,6 +4099,13 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } +#if (NGX_HTTP_V3) + if (ngx_strcmp(value[n].data, "quic") == 0) { + lsopt.quic = 1; + continue; + } +#endif + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid parameter \"%V\"", &value[n]); return NGX_CONF_ERROR; diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h index 85f6d66dc..fa08e3631 100644 --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -82,6 +82,7 @@ typedef struct { unsigned reuseport:1; unsigned so_keepalive:2; unsigned proxy_protocol:1; + unsigned quic:1; int backlog; int rcvbuf; @@ -238,6 +239,7 @@ struct ngx_http_addr_conf_s { unsigned ssl:1; unsigned http2:1; unsigned proxy_protocol:1; + unsigned quic:1; }; @@ -268,6 +270,7 @@ typedef struct { ngx_int_t family; in_port_t port; ngx_array_t addrs; /* array of ngx_http_conf_addr_t */ + ngx_int_t type; } ngx_http_conf_port_t; diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index 80c19656f..a4f396753 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -64,6 +64,10 @@ static void ngx_http_ssl_handshake(ngx_event_t *rev); static void ngx_http_ssl_handshake_handler(ngx_connection_t *c); #endif +#if (NGX_HTTP_V3) +static void ngx_http_quic_handshake(ngx_event_t *rev); +#endif + static char *ngx_http_client_errors[] = { @@ -349,6 +353,19 @@ ngx_http_init_connection(ngx_connection_t *c) c->log->action = "reading PROXY protocol"; } +#if (NGX_HTTP_V3) + if (hc->addr_conf->quic) { + hc->quic = 1; + c->log->action = "QUIC handshaking"; + + /* We already have a UDP packet in the connection buffer, so we don't + * need to wait for another read event to kick-off the handshake. */ + ngx_add_timer(rev, c->listening->post_accept_timeout); + ngx_http_quic_handshake(rev); + return; + } +#endif + if (rev->ready) { /* the deferred accept(), iocp */ @@ -797,7 +814,7 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c) c->ssl->no_wait_shutdown = 1; -#if (NGX_HTTP_V2 \ +#if ((NGX_HTTP_V2 || NGX_HTTP_V3) \ && (defined TLSEXT_TYPE_application_layer_protocol_negotiation \ || defined TLSEXT_TYPE_next_proto_neg)) { @@ -807,7 +824,7 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c) hc = c->data; - if (hc->addr_conf->http2) { + if (hc->addr_conf->http2 || hc->addr_conf->quic) { #ifdef TLSEXT_TYPE_application_layer_protocol_negotiation SSL_get0_alpn_selected(c->ssl->connection, &data, &len); @@ -822,11 +839,29 @@ ngx_http_ssl_handshake_handler(ngx_connection_t *c) SSL_get0_next_proto_negotiated(c->ssl->connection, &data, &len); #endif + } + +#if (NGX_HTTP_V2) + if (hc->addr_conf->http2) { if (len == 2 && data[0] == 'h' && data[1] == '2') { ngx_http_v2_init(c->read); return; } } +#endif + +#if (NGX_HTTP_V3) + if (hc->addr_conf->quic) { + if (len >= 2 && data[0] == 'h' && data[1] == '3') { + ngx_http_v3_init(c->read); + return; + } + + ngx_http_close_connection(c); + return; + } +#endif + } #endif @@ -1033,6 +1068,68 @@ failed: #endif +#if (NGX_HTTP_V3) + +static void +ngx_http_quic_handshake(ngx_event_t *rev) +{ + ngx_int_t rc; + ngx_connection_t *c; + ngx_http_connection_t *hc; + ngx_http_v3_srv_conf_t *qscf; + ngx_http_ssl_srv_conf_t *sscf; + + c = rev->data; + hc = c->data; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, rev->log, 0, + "http check quic handshake"); + + if (rev->timedout) { + ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); + ngx_http_close_connection(c); + return; + } + + if (c->close) { + ngx_http_close_connection(c); + return; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, rev->log, 0, "https quic handshake"); + + sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, + ngx_http_ssl_module); + + if (ngx_ssl_create_connection(&sscf->ssl, c, 0) != NGX_OK) { + ngx_http_close_connection(c); + return; + } + + qscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); + + if (ngx_quic_create_connection(&qscf->quic, c) != NGX_OK) { + ngx_http_close_connection(c); + return; + } + + rc = ngx_quic_handshake(c); + + if (rc == NGX_AGAIN) { + + if (!rev->timer_set) { + ngx_add_timer(rev, c->listening->post_accept_timeout); + } + + c->ssl->handler = ngx_http_ssl_handshake_handler; + return; + } + + ngx_http_ssl_handshake_handler(c); +} + +#endif + static void ngx_http_process_request_line(ngx_event_t *rev) @@ -2687,6 +2784,13 @@ ngx_http_finalize_connection(ngx_http_request_t *r) } #endif +#if (NGX_HTTP_V3) + if (r->qstream) { + ngx_http_close_request(r, 0); + return; + } +#endif + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); if (r->main->count != 1) { @@ -2896,6 +3000,19 @@ ngx_http_test_reading(ngx_http_request_t *r) #endif +#if (NGX_HTTP_V3) + + if (r->qstream) { + if (c->error) { + err = 0; + goto closed; + } + + return; + } + +#endif + #if (NGX_HAVE_KQUEUE) if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { @@ -3563,7 +3680,15 @@ ngx_http_close_request(ngx_http_request_t *r, ngx_int_t rc) } #endif +#if (NGX_HTTP_V3) + if (r->qstream) { + ngx_http_v3_close_stream(r->qstream, rc); + return; + } +#endif + ngx_http_free_request(r, rc); + ngx_http_close_connection(c); } @@ -3684,6 +3809,17 @@ ngx_http_close_connection(ngx_connection_t *c) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "close http connection: %d", c->fd); +#if (NGX_HTTP_V3) + + if (c->quic) { + if (ngx_quic_shutdown(c) == NGX_AGAIN) { + c->quic->handler = ngx_http_close_connection; + return; + } + } + +#endif + #if (NGX_HTTP_SSL) if (c->ssl) { diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index fce70efe6..8ac19658c 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -24,6 +24,7 @@ #define NGX_HTTP_VERSION_10 1000 #define NGX_HTTP_VERSION_11 1001 #define NGX_HTTP_VERSION_20 2000 +#define NGX_HTTP_VERSION_3 3000 #define NGX_HTTP_UNKNOWN 0x0001 #define NGX_HTTP_GET 0x0002 @@ -323,6 +324,7 @@ typedef struct { ngx_chain_t *free; unsigned ssl:1; + unsigned quic:1; unsigned proxy_protocol:1; } ngx_http_connection_t; @@ -445,6 +447,7 @@ struct ngx_http_request_s { ngx_http_connection_t *http_connection; ngx_http_v2_stream_t *stream; + ngx_http_v3_stream_t *qstream; ngx_http_log_handler_pt log_handler; diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c index c4f092e59..220cd142f 100644 --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -312,6 +312,12 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) ngx_del_timer(c->read); } +#if (NGX_HTTP_V3) + if (r->qstream) { + return NGX_AGAIN; + } +#endif + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } @@ -404,6 +410,12 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ngx_add_timer(c->read, clcf->client_body_timeout); +#if (NGX_HTTP_V3) + if (r->qstream) { + return NGX_AGAIN; + } +#endif + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } @@ -525,6 +537,17 @@ ngx_http_discard_request_body(ngx_http_request_t *r) } #endif +#if (NGX_HTTP_V3) + if (r->qstream) { + r->qstream->skip_data = 1; + + /* disable stream read to avoid pointless data events */ + ngx_http_v3_stop_stream_read(r->qstream, 0); + + return NGX_OK; + } +#endif + if (ngx_http_test_expect(r) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } @@ -808,6 +831,9 @@ ngx_http_test_expect(ngx_http_request_t *r) || r->http_version < NGX_HTTP_VERSION_11 #if (NGX_HTTP_V2) || r->stream != NULL +#endif +#if (NGX_HTTP_V3) + || r->qstream != NULL #endif ) { @@ -848,6 +874,13 @@ ngx_http_test_expect(ngx_http_request_t *r) static ngx_int_t ngx_http_request_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { + +#if (NGX_HTTP_V3) + if (r->qstream) { + return ngx_http_v3_request_body_filter(r, in); + } +#endif + if (r->headers_in.chunked) { return ngx_http_request_body_chunked_filter(r, in); diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index a7391d09a..398af2797 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -526,6 +526,13 @@ ngx_http_upstream_init(ngx_http_request_t *r) } #endif +#if (NGX_HTTP_V3) + if (r->qstream) { + ngx_http_upstream_init_request(r); + return; + } +#endif + if (c->read->timer_set) { ngx_del_timer(c->read); } @@ -1351,6 +1358,12 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, } #endif +#if (NGX_HTTP_V3) + if (r->qstream) { + return; + } +#endif + #if (NGX_HAVE_KQUEUE) if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { diff --git a/src/http/v3/ngx_http_v3.c b/src/http/v3/ngx_http_v3.c new file mode 100644 index 000000000..d7c4fbf1d --- /dev/null +++ b/src/http/v3/ngx_http_v3.c @@ -0,0 +1,2270 @@ + +/* + * Copyright (C) Cloudflare, Inc. + */ + + +#include +#include +#include +#include + + +typedef struct { + ngx_str_t name; + ngx_uint_t offset; + ngx_uint_t hash; + ngx_http_header_t *hh; +} ngx_http_v3_parse_header_t; + + +/* errors */ +#define NGX_HTTP_V3_NO_ERROR 0x0100 +#define NGX_HTTP_V3_PROTOCOL_ERROR 0x0101 +#define NGX_HTTP_V3_INTERNAL_ERROR 0x0102 +#define NGX_HTTP_V3_TRANSPORT_STREAM_INVALID -1007 +#define NGX_HTTP_V3_TRANSPORT_STREAM_STOPPED -1015 + + +static void ngx_http_v3_handler(ngx_connection_t *c); + +static void ngx_http_v3_idle_handler(ngx_connection_t *c); + +static void ngx_http_v3_handle_connection(ngx_http_v3_connection_t *h3c); + +static ngx_http_v3_stream_t *ngx_http_v3_stream_lookup( + ngx_http_v3_connection_t *h3c, ngx_uint_t stream_id); +static ngx_http_v3_stream_t *ngx_http_v3_create_stream( + ngx_http_v3_connection_t *h3c); +static void ngx_http_v3_close_stream_handler(ngx_event_t *ev); + +static ngx_int_t ngx_http_v3_validate_header(ngx_http_request_t *r, + ngx_http_v3_header_t *header); +static ngx_int_t ngx_http_v3_pseudo_header(ngx_http_request_t *r, + ngx_http_v3_header_t *header); +static ngx_int_t ngx_http_v3_parse_path(ngx_http_request_t *r, + ngx_str_t *value); +static ngx_int_t ngx_http_v3_parse_method(ngx_http_request_t *r, + ngx_str_t *value); +static ngx_int_t ngx_http_v3_parse_scheme(ngx_http_request_t *r, + ngx_str_t *value); +static ngx_int_t ngx_http_v3_parse_authority(ngx_http_request_t *r, + ngx_str_t *value); +static ngx_int_t ngx_http_v3_parse_header(ngx_http_request_t *r, + ngx_http_v3_parse_header_t *header, ngx_str_t *value); +static ngx_int_t ngx_http_v3_cookie(ngx_http_request_t *r, + ngx_http_v3_header_t *header); +static ngx_int_t ngx_http_v3_construct_cookie_header(ngx_http_request_t *r); +static ngx_int_t ngx_http_v3_construct_request_line(ngx_http_request_t *r); + +static void ngx_http_v3_run_request(ngx_http_request_t *r); + +static ssize_t ngx_http_v3_recv_body(ngx_connection_t *c, u_char *buf, + size_t size); +static ngx_chain_t *ngx_http_v3_send_chain(ngx_connection_t *fc, + ngx_chain_t *in, off_t limit); + +static void ngx_http_v3_finalize_connection(ngx_http_v3_connection_t *h3c, + ngx_uint_t status); + +static void ngx_http_v3_pool_cleanup(void *data); + + +static ngx_http_v3_parse_header_t ngx_http_v3_parse_headers[] = { + { ngx_string("host"), + offsetof(ngx_http_headers_in_t, host), 0, NULL }, + + { ngx_string("accept-encoding"), + offsetof(ngx_http_headers_in_t, accept_encoding), 0, NULL }, + + { ngx_string("accept-language"), + offsetof(ngx_http_headers_in_t, accept_language), 0, NULL }, + + { ngx_string("user-agent"), + offsetof(ngx_http_headers_in_t, user_agent), 0, NULL }, + + { ngx_null_string, 0, 0, NULL } +}; + + +void +ngx_http_v3_init(ngx_event_t *rev) +{ + ngx_connection_t *c; + ngx_pool_cleanup_t *cln; + ngx_http_connection_t *hc; + ngx_http_v3_srv_conf_t *h3scf; + ngx_http_v3_connection_t *h3c; + + c = rev->data; + hc = c->data; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "init http3 connection"); + + c->log->action = "processing HTTP/3 connection"; + + h3c = ngx_pcalloc(c->pool, sizeof(ngx_http_v3_connection_t)); + if (h3c == NULL) { + ngx_http_close_connection(c); + return; + } + + h3scf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_v3_module); + + h3c->h3 = quiche_h3_conn_new_with_transport(c->quic->conn, h3scf->http3); + if (h3c->h3 == NULL) { + ngx_http_close_connection(c); + return; + } + + h3c->http_connection = hc; + + h3c->connection = c; + + h3c->pool = c->pool; + + c->data = h3c; + + c->quic->handler = ngx_http_v3_handler; + + cln = ngx_pool_cleanup_add(c->pool, 0); + if (cln == NULL) { + ngx_http_close_connection(c); + return; + } + + cln->handler = ngx_http_v3_pool_cleanup; + cln->data = h3c; + + ngx_rbtree_init(&h3c->streams, &h3c->streams_sentinel, + ngx_rbtree_insert_value); +} + + +static int +ngx_http_v3_for_each_header(uint8_t *name, size_t name_len, + uint8_t *value, size_t value_len, void *argp) +{ + ngx_int_t rc; + ngx_table_elt_t *h; + ngx_http_header_t *hh; + ngx_http_request_t *r; + ngx_http_v3_header_t header; + ngx_http_core_srv_conf_t *cscf; + ngx_http_core_main_conf_t *cmcf; + + static ngx_str_t cookie = ngx_string("cookie"); + + r = argp; + + /* Duplicate the header name because we don't own it. */ + header.name.data = ngx_pnalloc(r->pool, name_len); + if (header.name.data == NULL) { + return NGX_ERROR; + } + header.name.len = name_len; + + ngx_memcpy(header.name.data, name, name_len); + + /* Duplicate the header value because we don't own it. Some of the + * functions that process headers require a NULL-terminated string, + * so allocate enough memory for that. */ + header.value.data = ngx_pcalloc(r->pool, value_len + 1); + if (header.value.data == NULL) { + return NGX_ERROR; + } + header.value.len = value_len; + + ngx_memcpy(header.value.data, value, value_len); + + if (ngx_http_v3_validate_header(r, &header) != NGX_OK) { + return NGX_ERROR; + } + + /* Check for pseudo-header. */ + if (header.name.data[0] == ':') { + rc = ngx_http_v3_pseudo_header(r, &header); + + if (rc == NGX_OK) { + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http3 header: \":%V: %V\"", + &header.name, &header.value); + + return NGX_OK; + } + + return NGX_ERROR; + } + + if (r->invalid_header) { + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + + if (cscf->ignore_invalid_headers) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent invalid header: \"%V\"", &header.name); + + return NGX_ERROR; + } + } + + /* Handle Cookie header separately. Not sure why, but the HTTP/2 code does + * the same. */ + if (header.name.len == cookie.len + && ngx_memcmp(header.name.data, cookie.data, cookie.len) == 0) + { + if (ngx_http_v3_cookie(r, &header) != NGX_OK) { + return NGX_ERROR; + } + + } else { + h = ngx_list_push(&r->headers_in.headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->key.len = header.name.len; + h->key.data = header.name.data; + + /* + * TODO Optimization: precalculate hash + * and handler for indexed headers. + */ + h->hash = ngx_hash_key(h->key.data, h->key.len); + + h->value.len = header.value.len; + h->value.data = header.value.data; + + h->lowcase_key = h->key.data; + + cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); + + hh = ngx_hash_find(&cmcf->headers_in_hash, h->hash, + h->lowcase_key, h->key.len); + + if (hh && hh->handler(r, h, hh->offset) != NGX_OK) { + return NGX_ERROR; + } + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http3 header: \"%V: %V\"", + &header.name, &header.value); + + return NGX_OK; +} + + +static void +ngx_http_v3_process_headers(ngx_connection_t *c, quiche_h3_event *ev, + int64_t stream_id) +{ + int rc; + ngx_http_v3_stream_t *stream; + ngx_http_v3_srv_conf_t *h3scf; + ngx_http_v3_connection_t *h3c; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 process headers"); + + h3c = c->data; + + h3scf = ngx_http_get_module_srv_conf(h3c->http_connection->conf_ctx, + ngx_http_v3_module); + + if (h3c->connection->requests >= h3scf->max_requests) { + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_NO_ERROR); + return; + } + + /* Create a new stream to handle the incoming request. */ + stream = ngx_http_v3_create_stream(h3c); + if (stream == NULL) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "failed to create HTTP/3 stream"); + + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_INTERNAL_ERROR); + return; + } + + stream->id = stream_id; + + stream->node.key = stream_id; + + ngx_rbtree_insert(&h3c->streams, &stream->node); + + /* Populate ngx_http_request_t from raw HTTP/3 headers. */ + rc = quiche_h3_event_for_each_header(ev, + ngx_http_v3_for_each_header, stream->request); + + if (rc != NGX_OK) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "received invalid HTTP/3 headers"); + + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_INTERNAL_ERROR); + return; + } + + stream->in_closed = !quiche_h3_event_headers_has_body(ev); + + ngx_http_v3_run_request(stream->request); +} + + +static void +ngx_http_v3_process_blocked_streams(ngx_http_v3_connection_t *h3c) +{ + ngx_event_t *wev; + quiche_stream_iter *writable; + ngx_http_v3_stream_t *stream; + uint64_t stream_id; + + writable = quiche_conn_writable(h3c->connection->quic->conn); + + while (quiche_stream_iter_next(writable, &stream_id)) { + stream = ngx_http_v3_stream_lookup(h3c, stream_id); + + if (stream == NULL) { + continue; + } + + if (!stream->blocked) { + continue; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h3c->connection->log, 0, + "http3 stream unblocked %ui", stream->id); + + stream->blocked = 0; + + wev = stream->request->connection->write; + + wev->active = 0; + wev->ready = 1; + + if (!stream->headers_sent) { + ngx_http_v3_send_response(stream->request); + } + + if (!wev->delayed) { + wev->handler(wev); + } + } + + quiche_stream_iter_free(writable); +} + + +static void +ngx_http_v3_handler(ngx_connection_t *c) +{ + ngx_chain_t out; + ngx_connection_t *fc; + ngx_http_request_t *r; + ngx_http_v3_connection_t *h3c; + ngx_http_v3_stream_t *stream; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 handler"); + + h3c = c->data; + + if (c->read->timedout) { + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_PROTOCOL_ERROR); + return; + } + + if (c->error) { + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_INTERNAL_ERROR); + return; + } + + ngx_http_v3_process_blocked_streams(h3c); + + while (!c->error) { + quiche_h3_event *ev; + + int64_t stream_id = quiche_h3_conn_poll(h3c->h3, c->quic->conn, &ev); + if (stream_id == QUICHE_H3_ERR_DONE) { + break; + } + + if (stream_id < 0) { + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_PROTOCOL_ERROR); + return; + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, h3c->connection->log, 0, + "http3 event stream:%ui ev:%ui", stream_id, + quiche_h3_event_type(ev)); + + switch (quiche_h3_event_type(ev)) { + case QUICHE_H3_EVENT_HEADERS: { + /* If there is no stream, these are headers. If there is a + * stream, these are trailers and they are ignored.*/ + stream = ngx_http_v3_stream_lookup(h3c, stream_id); + + if (stream == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http3 headers"); + + ngx_http_v3_process_headers(c, ev, stream_id); + } else { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http3 trailers"); + } + + break; + } + + case QUICHE_H3_EVENT_DATA: { + /* Lookup stream. If there isn't one, it means it has already + * been closed, so ignore the event. */ + stream = ngx_http_v3_stream_lookup(h3c, stream_id); + + if (stream != NULL && !stream->in_closed) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http3 data"); + + ngx_post_event(stream->request->connection->read, + &ngx_posted_events); + } + + break; + } + + case QUICHE_H3_EVENT_FINISHED: { + /* Lookup stream. If there isn't one, it means it has already + * been closed, so ignore the event. */ + stream = ngx_http_v3_stream_lookup(h3c, stream_id); + + if (stream != NULL && !stream->in_closed) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http3 finished"); + + /* Flush request body that was buffered. */ + if (stream->request->request_body) { + out.buf = stream->request->request_body->buf; + out.next = NULL; + + ngx_http_v3_request_body_filter(stream->request, &out); + + ngx_post_event(stream->request->connection->read, + &ngx_posted_events); + } + + stream->in_closed = 1; + } + + break; + } + + case QUICHE_H3_EVENT_RESET: { + /* Lookup stream. If there isn't one, it means it has already + * been closed, so ignore the event. */ + stream = ngx_http_v3_stream_lookup(h3c, stream_id); + + if (stream != NULL && !stream->in_closed) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http3 reset"); + + r = stream->request; + fc = r->connection; + + fc->error = 1; + + ngx_post_event(stream->request->connection->read, + &ngx_posted_events); + + stream->in_closed = 1; + } + + break; + } + + case QUICHE_H3_EVENT_PRIORITY_UPDATE: + break; + + case QUICHE_H3_EVENT_GOAWAY: + break; + } + + quiche_h3_event_free(ev); + } + + ngx_http_v3_handle_connection(h3c); +} + + +static void +ngx_http_v3_idle_handler(ngx_connection_t *c) +{ + ngx_http_v3_connection_t *h3c; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 idle handler"); + + h3c = c->data; + + if (c->read->timedout) { + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_NO_ERROR); + return; + } + + if (c->error) { + ngx_http_v3_finalize_connection(h3c, NGX_HTTP_V3_INTERNAL_ERROR); + return; + } + + if (!quiche_conn_is_readable(c->quic->conn)) { + return; + } + + if (c->read->timer_set) { + ngx_del_timer(c->read); + } + + c->quic->handler = ngx_http_v3_handler; + + ngx_http_v3_handler(c); +} + + +static void +ngx_http_v3_handle_connection(ngx_http_v3_connection_t *h3c) +{ + ngx_connection_t *c; + ngx_http_v3_srv_conf_t *h3scf; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h3c->connection->log, 0, + "http3 handle connection"); + + c = h3c->connection; + + if (h3c->processing || c->error) { + return; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h3c->connection->log, 0, + "http3 connection is idle"); + + h3scf = ngx_http_get_module_srv_conf(h3c->http_connection->conf_ctx, + ngx_http_v3_module); + + c->quic->handler = ngx_http_v3_idle_handler; + + ngx_add_timer(c->read, h3scf->idle_timeout); +} + + +static ngx_http_v3_stream_t * +ngx_http_v3_create_stream(ngx_http_v3_connection_t *h3c) +{ + ngx_log_t *log; + ngx_event_t *rev, *wev; + ngx_connection_t *fc; + ngx_http_log_ctx_t *ctx; + ngx_http_request_t *r; + ngx_http_v3_stream_t *stream; + ngx_http_core_srv_conf_t *cscf; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h3c->connection->log, 0, + "http3 create stream"); + + fc = h3c->free_fake_connections; + + if (fc) { + h3c->free_fake_connections = fc->data; + + rev = fc->read; + wev = fc->write; + log = fc->log; + ctx = log->data; + + } else { + fc = ngx_palloc(h3c->pool, sizeof(ngx_connection_t)); + if (fc == NULL) { + return NULL; + } + + rev = ngx_palloc(h3c->pool, sizeof(ngx_event_t)); + if (rev == NULL) { + return NULL; + } + + wev = ngx_palloc(h3c->pool, sizeof(ngx_event_t)); + if (wev == NULL) { + return NULL; + } + + log = ngx_palloc(h3c->pool, sizeof(ngx_log_t)); + if (log == NULL) { + return NULL; + } + + ctx = ngx_palloc(h3c->pool, sizeof(ngx_http_log_ctx_t)); + if (ctx == NULL) { + return NULL; + } + + ctx->connection = fc; + ctx->request = NULL; + ctx->current_request = NULL; + } + + ngx_memcpy(log, h3c->connection->log, sizeof(ngx_log_t)); + + log->data = ctx; + + ngx_memzero(rev, sizeof(ngx_event_t)); + + rev->data = fc; + rev->ready = 1; + rev->handler = ngx_http_v3_close_stream_handler; + rev->log = log; + + ngx_memcpy(wev, rev, sizeof(ngx_event_t)); + + wev->write = 1; + + ngx_memcpy(fc, h3c->connection, sizeof(ngx_connection_t)); + + fc->data = h3c->http_connection; + fc->quic = h3c->connection->quic; + fc->read = rev; + fc->write = wev; + fc->sent = 0; + fc->buffer = NULL; + fc->log = log; + fc->buffered = 0; + fc->sndlowat = 1; + fc->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; + + fc->recv = ngx_http_v3_recv_body; + + fc->send_chain = ngx_http_v3_send_chain; + fc->need_last_buf = 1; + + r = ngx_http_create_request(fc); + if (r == NULL) { + return NULL; + } + + ngx_str_set(&r->http_protocol, "HTTP/3"); + + r->http_version = NGX_HTTP_VERSION_3; + r->valid_location = 1; + + fc->data = r; + h3c->connection->requests++; + + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + + r->header_in = ngx_create_temp_buf(r->pool, + cscf->client_header_buffer_size); + if (r->header_in == NULL) { + ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NULL; + } + + if (ngx_list_init(&r->headers_in.headers, r->pool, 20, + sizeof(ngx_table_elt_t)) + != NGX_OK) + { + ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NULL; + } + + r->headers_in.connection_type = NGX_HTTP_CONNECTION_CLOSE; + + stream = ngx_pcalloc(h3c->pool, sizeof(ngx_http_v3_stream_t)); + if (stream == NULL) { + ngx_http_free_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NULL; + } + + /* Default value */ + stream->priority.urgency = 3; + stream->priority.incremental = false; + + r->qstream = stream; + + stream->request = r; + stream->connection = h3c; + + h3c->processing++; + + return stream; +} + + +static ngx_http_v3_stream_t * +ngx_http_v3_stream_lookup(ngx_http_v3_connection_t *h3c, ngx_uint_t stream_id) +{ + ngx_rbtree_node_t *node, *sentinel; + + node = h3c->streams.root; + sentinel = h3c->streams.sentinel; + + while (node != sentinel) { + + if (stream_id < node->key) { + node = node->left; + continue; + } + + if (stream_id > node->key) { + node = node->right; + continue; + } + + /* stream_id == node->key */ + + return (ngx_http_v3_stream_t *) node; + } + + /* not found */ + + return NULL; +} + + +/* The following functions are copied from the HTTP/2 module, and adapted to + * work independently. In theory we could refactor the HTTP/2 module to expose + * these functions, but that would be fairly invasive and likely cause more + * merge conflicts in the future. */ + + +static ngx_int_t +ngx_http_v3_validate_header(ngx_http_request_t *r, ngx_http_v3_header_t *header) +{ + u_char ch; + ngx_uint_t i; + ngx_http_core_srv_conf_t *cscf; + + if (header->name.len == 0) { + return NGX_ERROR; + } + + r->invalid_header = 0; + + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + + for (i = (header->name.data[0] == ':'); i != header->name.len; i++) { + ch = header->name.data[i]; + + if ((ch >= 'a' && ch <= 'z') + || (ch == '-') + || (ch >= '0' && ch <= '9') + || (ch == '_' && cscf->underscores_in_headers)) + { + continue; + } + + if (ch == '\0' || ch == LF || ch == CR || ch == ':' + || (ch >= 'A' && ch <= 'Z')) + { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent invalid header name: \"%V\"", + &header->name); + + return NGX_ERROR; + } + + r->invalid_header = 1; + } + + for (i = 0; i != header->value.len; i++) { + ch = header->value.data[i]; + + if (ch == '\0' || ch == LF || ch == CR) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent header \"%V\" with " + "invalid value: \"%V\"", + &header->name, &header->value); + + return NGX_ERROR; + } + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_v3_pseudo_header(ngx_http_request_t *r, ngx_http_v3_header_t *header) +{ + header->name.len--; + header->name.data++; + + switch (header->name.len) { + case 4: + if (ngx_memcmp(header->name.data, "path", sizeof("path") - 1) + == 0) + { + return ngx_http_v3_parse_path(r, &header->value); + } + + break; + + case 6: + if (ngx_memcmp(header->name.data, "method", sizeof("method") - 1) + == 0) + { + return ngx_http_v3_parse_method(r, &header->value); + } + + if (ngx_memcmp(header->name.data, "scheme", sizeof("scheme") - 1) + == 0) + { + return ngx_http_v3_parse_scheme(r, &header->value); + } + + break; + + case 9: + if (ngx_memcmp(header->name.data, "authority", sizeof("authority") - 1) + == 0) + { + return ngx_http_v3_parse_authority(r, &header->value); + } + + break; + } + + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent unknown pseudo-header \":%V\"", + &header->name); + + return NGX_DECLINED; +} + + +static ngx_int_t +ngx_http_v3_parse_path(ngx_http_request_t *r, ngx_str_t *value) +{ + if (r->unparsed_uri.len) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent duplicate :path header"); + + return NGX_DECLINED; + } + + if (value->len == 0) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent empty :path header"); + + return NGX_DECLINED; + } + + r->uri_start = value->data; + r->uri_end = value->data + value->len; + + if (ngx_http_parse_uri(r) != NGX_OK) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent invalid :path header: \"%V\"", value); + + return NGX_DECLINED; + } + + if (ngx_http_process_request_uri(r) != NGX_OK) { + /* + * request has been finalized already + * in ngx_http_process_request_uri() + */ + return NGX_ABORT; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_v3_parse_method(ngx_http_request_t *r, ngx_str_t *value) +{ + size_t k, len; + ngx_uint_t n; + const u_char *p, *m; + + /* + * This array takes less than 256 sequential bytes, + * and if typical CPU cache line size is 64 bytes, + * it is prefetched for 4 load operations. + */ + static const struct { + u_char len; + const u_char method[11]; + uint32_t value; + } tests[] = { + { 3, "GET", NGX_HTTP_GET }, + { 4, "POST", NGX_HTTP_POST }, + { 4, "HEAD", NGX_HTTP_HEAD }, + { 7, "OPTIONS", NGX_HTTP_OPTIONS }, + { 8, "PROPFIND", NGX_HTTP_PROPFIND }, + { 3, "PUT", NGX_HTTP_PUT }, + { 5, "MKCOL", NGX_HTTP_MKCOL }, + { 6, "DELETE", NGX_HTTP_DELETE }, + { 4, "COPY", NGX_HTTP_COPY }, + { 4, "MOVE", NGX_HTTP_MOVE }, + { 9, "PROPPATCH", NGX_HTTP_PROPPATCH }, + { 4, "LOCK", NGX_HTTP_LOCK }, + { 6, "UNLOCK", NGX_HTTP_UNLOCK }, + { 5, "PATCH", NGX_HTTP_PATCH }, + { 5, "TRACE", NGX_HTTP_TRACE } + }, *test; + + if (r->method_name.len) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent duplicate :method header"); + + return NGX_DECLINED; + } + + if (value->len == 0) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent empty :method header"); + + return NGX_DECLINED; + } + + r->method_name.len = value->len; + r->method_name.data = value->data; + + len = r->method_name.len; + n = sizeof(tests) / sizeof(tests[0]); + test = tests; + + do { + if (len == test->len) { + p = r->method_name.data; + m = test->method; + k = len; + + do { + if (*p++ != *m++) { + goto next; + } + } while (--k); + + r->method = test->value; + return NGX_OK; + } + + next: + test++; + + } while (--n); + + p = r->method_name.data; + + do { + if ((*p < 'A' || *p > 'Z') && *p != '_' && *p != '-') { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent invalid method: \"%V\"", + &r->method_name); + + return NGX_DECLINED; + } + + p++; + + } while (--len); + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_v3_parse_scheme(ngx_http_request_t *r, ngx_str_t *value) +{ + u_char c, ch; + ngx_uint_t i; + + if (r->schema.len) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent duplicate :scheme header"); + + return NGX_DECLINED; + } + + if (value->len == 0) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent empty :scheme header"); + + return NGX_DECLINED; + } + + for (i = 0; i < value->len; i++) { + ch = value->data[i]; + + c = (u_char) (ch | 0x20); + if (c >= 'a' && c <= 'z') { + continue; + } + + if (((ch >= '0' && ch <= '9') || ch == '+' || ch == '-' || ch == '.') + && i > 0) + { + continue; + } + + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent invalid :scheme header: \"%V\"", value); + + return NGX_DECLINED; + } + + r->schema = *value; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_v3_parse_authority(ngx_http_request_t *r, ngx_str_t *value) +{ + return ngx_http_v3_parse_header(r, &ngx_http_v3_parse_headers[0], value); +} + + +static ngx_int_t +ngx_http_v3_parse_header(ngx_http_request_t *r, + ngx_http_v3_parse_header_t *header, ngx_str_t *value) +{ + ngx_table_elt_t *h; + ngx_http_core_main_conf_t *cmcf; + + h = ngx_list_push(&r->headers_in.headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->key.len = header->name.len; + h->key.data = header->name.data; + h->lowcase_key = header->name.data; + + if (header->hh == NULL) { + header->hash = ngx_hash_key(header->name.data, header->name.len); + + cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); + + header->hh = ngx_hash_find(&cmcf->headers_in_hash, header->hash, + h->lowcase_key, h->key.len); + if (header->hh == NULL) { + return NGX_ERROR; + } + } + + h->hash = header->hash; + + h->value.len = value->len; + h->value.data = value->data; + + if (header->hh->handler(r, h, header->hh->offset) != NGX_OK) { + /* header handler has already finalized request */ + return NGX_ABORT; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_v3_construct_request_line(ngx_http_request_t *r) +{ + u_char *p; + + static const u_char ending[] = " HTTP/3"; + + if (r->method_name.len == 0 + || r->schema.len == 0 + || r->unparsed_uri.len == 0) + { + if (r->method_name.len == 0) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent no :method header"); + + } else if (r->schema.len == 0) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent no :scheme header"); + + } else { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client sent no :path header"); + } + + ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); + return NGX_ERROR; + } + + r->request_line.len = r->method_name.len + 1 + + r->unparsed_uri.len + + sizeof(ending) - 1; + + p = ngx_pnalloc(r->pool, r->request_line.len + 1); + if (p == NULL) { + ngx_http_v3_close_stream(r->qstream, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NGX_ERROR; + } + + r->request_line.data = p; + + p = ngx_cpymem(p, r->method_name.data, r->method_name.len); + + *p++ = ' '; + + p = ngx_cpymem(p, r->unparsed_uri.data, r->unparsed_uri.len); + + ngx_memcpy(p, ending, sizeof(ending)); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http3 request line: \"%V\"", &r->request_line); + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_v3_cookie(ngx_http_request_t *r, ngx_http_v3_header_t *header) +{ + ngx_str_t *val; + ngx_array_t *cookies; + + cookies = r->qstream->cookies; + + if (cookies == NULL) { + cookies = ngx_array_create(r->pool, 2, sizeof(ngx_str_t)); + if (cookies == NULL) { + return NGX_ERROR; + } + + r->qstream->cookies = cookies; + } + + val = ngx_array_push(cookies); + if (val == NULL) { + return NGX_ERROR; + } + + val->len = header->value.len; + val->data = header->value.data; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_v3_construct_cookie_header(ngx_http_request_t *r) +{ + u_char *buf, *p, *end; + size_t len; + ngx_str_t *vals; + ngx_uint_t i; + ngx_array_t *cookies; + ngx_table_elt_t *h; + ngx_http_header_t *hh; + ngx_http_core_main_conf_t *cmcf; + + static ngx_str_t cookie = ngx_string("cookie"); + + cookies = r->qstream->cookies; + + if (cookies == NULL) { + return NGX_OK; + } + + vals = cookies->elts; + + i = 0; + len = 0; + + do { + len += vals[i].len + 2; + } while (++i != cookies->nelts); + + len -= 2; + + buf = ngx_pnalloc(r->pool, len + 1); + if (buf == NULL) { + ngx_http_v3_close_stream(r->qstream, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NGX_ERROR; + } + + p = buf; + end = buf + len; + + for (i = 0; /* void */ ; i++) { + + p = ngx_cpymem(p, vals[i].data, vals[i].len); + + if (p == end) { + *p = '\0'; + break; + } + + *p++ = ';'; *p++ = ' '; + } + + h = ngx_list_push(&r->headers_in.headers); + if (h == NULL) { + ngx_http_v3_close_stream(r->qstream, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NGX_ERROR; + } + + h->hash = ngx_hash(ngx_hash(ngx_hash(ngx_hash( + ngx_hash('c', 'o'), 'o'), 'k'), 'i'), 'e'); + + h->key.len = cookie.len; + h->key.data = cookie.data; + + h->value.len = len; + h->value.data = buf; + + h->lowcase_key = cookie.data; + + cmcf = ngx_http_get_module_main_conf(r, ngx_http_core_module); + + hh = ngx_hash_find(&cmcf->headers_in_hash, h->hash, + h->lowcase_key, h->key.len); + + if (hh == NULL) { + ngx_http_v3_close_stream(r->qstream, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NGX_ERROR; + } + + if (hh->handler(r, h, hh->offset) != NGX_OK) { + /* + * request has been finalized already + * in ngx_http_process_multi_header_lines() + */ + return NGX_ERROR; + } + + return NGX_OK; +} + + +static void +ngx_http_v3_run_request(ngx_http_request_t *r) +{ + if (ngx_http_v3_construct_request_line(r) != NGX_OK) { + return; + } + + if (ngx_http_v3_construct_cookie_header(r) != NGX_OK) { + return; + } + + r->http_state = NGX_HTTP_PROCESS_REQUEST_STATE; + + if (ngx_http_process_request_header(r) != NGX_OK) { + return; + } + + if (r->headers_in.content_length_n > 0 && r->qstream->in_closed) { + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "client prematurely closed stream"); + + r->qstream->skip_data = 1; + + ngx_http_finalize_request(r, NGX_HTTP_BAD_REQUEST); + return; + } + + if (r->headers_in.content_length_n == -1 && !r->qstream->in_closed) { + r->headers_in.chunked = 1; + } + + ngx_http_process_request(r); +} + + +/* End of functions copied from HTTP/2 module. */ + + +ngx_int_t +ngx_http_v3_request_body_filter(ngx_http_request_t *r, ngx_chain_t *in) +{ + size_t size; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *cl, *tl, *out, **ll; + ngx_connection_t *c; + ngx_http_request_body_t *rb; + ngx_http_core_loc_conf_t *clcf; + bool stream_fin; + + c = r->qstream->connection->connection; + + rb = r->request_body; + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + if (rb->rest == -1) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http3 request body filter"); + + if (r->headers_in.chunked) { + rb->rest = clcf->client_body_buffer_size; + r->headers_in.content_length_n = 0; + } else { + rb->rest = r->headers_in.content_length_n; + } + } + + out = NULL; + ll = &out; + + for (cl = in; cl; cl = cl->next) { + + if (rb->rest == 0) { + break; + } + + stream_fin = quiche_conn_stream_finished(c->quic->conn, r->qstream->id); + + if ((ngx_buf_size(cl->buf) == 0) && !stream_fin) { + continue; + } + + tl = ngx_chain_get_free_buf(r->pool, &rb->free); + if (tl == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b = tl->buf; + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->temporary = 1; + b->tag = (ngx_buf_tag_t) &ngx_http_read_client_request_body; + b->start = cl->buf->pos; + b->pos = cl->buf->pos; + b->last = cl->buf->last; + b->end = cl->buf->end; + b->flush = r->request_body_no_buffering; + + size = cl->buf->last - cl->buf->pos; + + cl->buf->pos = cl->buf->last; + + if (r->headers_in.chunked) { + r->headers_in.content_length_n += size; + } + + if (stream_fin) { + rb->rest = 0; + b->last = cl->buf->pos; + b->last_buf = 1; + } + + *ll = tl; + ll = &tl->next; + } + + rc = ngx_http_top_request_body_filter(r, out); + + ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, + (ngx_buf_tag_t) &ngx_http_read_client_request_body); + + return rc; +} + + +size_t +ngx_http_v3_get_headers_out_count(ngx_http_request_t *r) +{ + size_t headers_count; + ngx_uint_t i; + ngx_list_part_t *part; + ngx_table_elt_t *header; + + headers_count = 1; /* :status */ + + if (r->headers_out.server == NULL) { + headers_count += 1; + } + + if (r->headers_out.date == NULL) { + headers_count += 1; + } + + if (r->headers_out.content_type.len) { + headers_count += 1; + } + + if (r->headers_out.content_length == NULL + && r->headers_out.content_length_n >= 0) + { + headers_count += 1; + } + + if (r->headers_out.last_modified == NULL + && r->headers_out.last_modified_time != -1) + { + headers_count += 1; + } + + if (r->headers_out.location && r->headers_out.location->value.len) { + headers_count += 1; + } + +#if (NGX_HTTP_GZIP) + if (r->gzip_vary) { + headers_count += 1; + } +#endif + + part = &r->headers_out.headers.part; + header = part->elts; + + for (i = 0; /* void */; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + break; + } + + part = part->next; + header = part->elts; + i = 0; + } + + if (header[i].hash == 0) { + continue; + } + + headers_count += 1; + } + + return headers_count; +} + + +ngx_int_t +ngx_http_v3_push_response_headers(ngx_http_request_t *r) +{ + u_char *tmp; + size_t len, headers_count; + ngx_str_t host, location; + ngx_uint_t i, port; + ngx_list_part_t *part; + ngx_table_elt_t *header; + ngx_connection_t *fc; + quiche_h3_header *h; + ngx_http_core_loc_conf_t *clcf; + ngx_http_core_srv_conf_t *cscf; + u_char addr[NGX_SOCKADDR_STRLEN]; + + /* The list of response headers was already generated, so there's nothing + * more to do here. */ + if (r->qstream->headers != NULL) { + return NGX_OK; + } + + fc = r->connection; + + if (r->method == NGX_HTTP_HEAD) { + r->header_only = 1; + } + + switch (r->headers_out.status) { + + case NGX_HTTP_OK: + break; + + case NGX_HTTP_NO_CONTENT: + r->header_only = 1; + + if (!r->headers_out.status_line.len) { + ngx_str_null(&r->headers_out.content_type); + + r->headers_out.content_length = NULL; + r->headers_out.content_length_n = -1; + + r->headers_out.last_modified_time = -1; + r->headers_out.last_modified = NULL; + } + break; + + case NGX_HTTP_PARTIAL_CONTENT: + break; + + case NGX_HTTP_NOT_MODIFIED: + r->header_only = 1; + break; + + default: + r->headers_out.last_modified_time = -1; + r->headers_out.last_modified = NULL; + } + + headers_count = ngx_http_v3_get_headers_out_count(r); + + r->qstream->headers = + ngx_array_create(r->pool, headers_count, sizeof(quiche_h3_header)); + + if (r->qstream->headers == NULL) { + return NGX_ERROR; + } + + /* Generate :status pseudo-header. */ + { + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->name = (u_char *) ":status"; + h->name_len = sizeof(":status") - 1; + + tmp = ngx_pnalloc(r->pool, sizeof("418") - 1); + if (tmp == NULL) { + return NGX_ERROR; + } + + h->value = tmp; + h->value_len = ngx_sprintf(tmp, "%03ui", r->headers_out.status) - tmp; + } + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + /* Generate Server header.*/ + if (r->headers_out.server == NULL) { + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->name = (u_char *) "server"; + h->name_len = sizeof("server") - 1; + + if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { + h->value = (u_char *) NGINX_VER; + h->value_len = sizeof(NGINX_VER) - 1; + + } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { + h->value = (u_char *) NGINX_VER_BUILD; + h->value_len = sizeof(NGINX_VER_BUILD) - 1; + + } else { + h->value = (u_char *) "nginx"; + h->value_len = sizeof("nginx") - 1; + } + } + + /* Generate Date header. */ + if (r->headers_out.date == NULL) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http3 output header: \"date: %V\"", + &ngx_cached_http_time); + + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->name = (u_char *) "date"; + h->name_len = sizeof("date") - 1; + + h->value = ngx_cached_http_time.data; + h->value_len = ngx_cached_http_time.len; + } + + /* Generate Content-Type header. */ + if (r->headers_out.content_type.len) { + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + if (r->headers_out.content_type_len == r->headers_out.content_type.len + && r->headers_out.charset.len) + { + len = r->headers_out.content_type.len + sizeof("; charset=") - 1 + + r->headers_out.charset.len; + + tmp = ngx_pnalloc(r->pool, len); + if (tmp == NULL) { + return NGX_ERROR; + } + + tmp = ngx_cpymem(tmp, r->headers_out.content_type.data, + r->headers_out.content_type.len); + + tmp = ngx_cpymem(tmp, "; charset=", sizeof("; charset=") - 1); + + tmp = ngx_cpymem(tmp, r->headers_out.charset.data, + r->headers_out.charset.len); + + /* updated r->headers_out.content_type is also needed for logging */ + + r->headers_out.content_type.len = len; + r->headers_out.content_type.data = tmp - len; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http3 output header: \"content-type: %V\"", + &r->headers_out.content_type); + + h->name = (u_char *) "content-type"; + h->name_len = sizeof("content-type") - 1; + + h->value = r->headers_out.content_type.data; + h->value_len = r->headers_out.content_type.len; + } + + /* Generate Content-Length header. */ + if (r->headers_out.content_length == NULL + && r->headers_out.content_length_n >= 0) + { + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->name = (u_char *) "content-length"; + h->name_len = sizeof("content-length") - 1; + + tmp = ngx_pnalloc(r->pool, NGX_OFF_T_LEN); + if (tmp == NULL) { + return NGX_ERROR; + } + + h->value = tmp; + h->value_len = + ngx_sprintf(tmp, "%O", r->headers_out.content_length_n) - tmp; + } + + /* Generate Last-Modified header. */ + if (r->headers_out.last_modified == NULL + && r->headers_out.last_modified_time != -1) + { + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->name = (u_char *) "last-modified"; + h->name_len = sizeof("last-modified") - 1; + + tmp = ngx_pnalloc(r->pool, sizeof("Wed, 31 Dec 1986 18:00:00 GMT") - 1); + if (tmp == NULL) { + return NGX_ERROR; + } + + h->value = tmp; + h->value_len = + ngx_http_time(tmp, r->headers_out.last_modified_time) - tmp; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http3 output header: \"last-modified: %*.s\"", + h->value_len, h->value); + } + + /* Generate Location header. */ + if (r->headers_out.location && r->headers_out.location->value.len) { + + if (r->headers_out.location->value.data[0] == '/' + && clcf->absolute_redirect) + { + if (clcf->server_name_in_redirect) { + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + host = cscf->server_name; + + } else if (r->headers_in.server.len) { + host = r->headers_in.server; + + } else { + host.data = addr; + host.len = NGX_SOCKADDR_STRLEN; + + if (ngx_connection_local_sockaddr(fc, &host, 0) != NGX_OK) { + return NGX_ERROR; + } + } + + port = ngx_inet_get_port(fc->local_sockaddr); + + location.len = sizeof("https://") - 1 + host.len + + r->headers_out.location->value.len; + + if (clcf->port_in_redirect) { + +#if (NGX_HTTP_SSL) + if (fc->ssl) + port = (port == 443) ? 0 : port; + else +#endif + port = (port == 80) ? 0 : port; + + } else { + port = 0; + } + + if (port) { + location.len += sizeof(":65535") - 1; + } + + location.data = ngx_pnalloc(r->pool, location.len); + if (location.data == NULL) { + return NGX_ERROR; + } + + tmp = ngx_cpymem(location.data, "http", sizeof("http") - 1); + +#if (NGX_HTTP_SSL) + if (fc->ssl) { + *tmp++ = 's'; + } +#endif + + *tmp++ = ':'; *tmp++ = '/'; *tmp++ = '/'; + tmp = ngx_cpymem(tmp, host.data, host.len); + + if (port) { + tmp = ngx_sprintf(tmp, ":%ui", port); + } + + tmp = ngx_cpymem(tmp, r->headers_out.location->value.data, + r->headers_out.location->value.len); + + /* update r->headers_out.location->value for possible logging */ + + r->headers_out.location->value.len = tmp - location.data; + r->headers_out.location->value.data = location.data; + ngx_str_set(&r->headers_out.location->key, "Location"); + } + + r->headers_out.location->hash = 0; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http3 output header: \"location: %V\"", + &r->headers_out.location->value); + + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->name = (u_char *) "location"; + h->name_len = sizeof("location") - 1; + + h->value = r->headers_out.location->value.data; + h->value_len = r->headers_out.location->value.len; + } + +#if (NGX_HTTP_GZIP) + /* Generate Vary header. */ + if (r->gzip_vary) { + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http3 output header: \"vary: Accept-Encoding\""); + + h->name = (u_char *) "vary"; + h->name_len = sizeof("vary") - 1; + + h->value = (u_char *) "Accept-Encoding"; + h->value_len = sizeof("Accept-Encoding") - 1; + } +#endif + + part = &r->headers_out.headers.part; + header = part->elts; + + /* Generate all other headers. */ + for (i = 0; /* void */; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + break; + } + + part = part->next; + header = part->elts; + i = 0; + } + + if (header[i].hash == 0) { + continue; + } + + h = ngx_array_push(r->qstream->headers); + if (h == NULL) { + return NGX_ERROR; + } + +#if (NGX_DEBUG) + if (fc->log->log_level & NGX_LOG_DEBUG_HTTP) { + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http3 output header: \"%V: %V\"", + &header[i].key, &header[i].value); + } +#endif + + h->name = header[i].key.data; + h->name_len = header[i].key.len; + + h->value = header[i].value.data; + h->value_len = header[i].value.len; + } + + return NGX_OK; +} + + +ngx_int_t +ngx_http_v3_send_response(ngx_http_request_t *r) +{ + int rc; + ngx_uint_t fin; + ngx_connection_t *c, *fc; + ngx_http_v3_connection_t *h3c; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http3 send response stream %ui", r->qstream->id); + + fc = r->connection; + + if (fc->error) { + return NGX_ERROR; + } + + h3c = r->qstream->connection; + c = h3c->connection; + + if (ngx_http_v3_push_response_headers(r) != NGX_OK) { + return NGX_ERROR; + } + + fin = r->header_only + || (r->headers_out.content_length_n == 0 && !r->expect_trailers); + + rc = quiche_h3_send_response_with_priority(h3c->h3, c->quic->conn, r->qstream->id, + r->qstream->headers->elts, + r->qstream->headers->nelts, + &r->qstream->priority, + fin); + + if (rc == QUICHE_H3_ERR_STREAM_BLOCKED) { + r->qstream->blocked = 1; + + fc->write->active = 1; + fc->write->ready = 0; + + return NGX_AGAIN; + } + + if (rc != NGX_OK) { + return NGX_ERROR; + } + + if (fin) { + r->qstream->out_closed = 1; + } + + r->qstream->headers_sent = 1; + + if (r->done && r->main->count == 0) { + fc->write->handler = ngx_http_v3_close_stream_handler; + fc->read->handler = ngx_http_empty_handler; + } + + ngx_post_event(c->write, &ngx_posted_events); + + return NGX_OK; +} + + +static ssize_t +ngx_http_v3_stream_do_send(ngx_connection_t *fc, ngx_buf_t *b, ngx_int_t fin) +{ + ssize_t n; + ngx_connection_t *c; + ngx_http_request_t *r; + ngx_http_v3_connection_t *h3c; + ngx_http_v3_stream_t *stream; + + uint8_t *buf = b ? b->pos : NULL; + size_t buf_len = b ? ngx_buf_size(b) : 0; + + r = fc->data; + stream = r->qstream; + h3c = stream->connection; + c = h3c->connection; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, fc->log, 0, + "http3 stream %uz to write %uz bytes, fin=%d", + stream->id, buf_len, fin); + + if (!stream->headers_sent) { + return NGX_AGAIN; + } + + n = quiche_h3_send_body(h3c->h3, c->quic->conn, r->qstream->id, + buf, buf_len, fin); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, stream->connection->connection->log, 0, + "http3 stream written %z bytes", n); + + if (n == QUICHE_H3_ERR_DONE) { + return NGX_AGAIN; + } + + if (n < 0) { + if (n == NGX_HTTP_V3_TRANSPORT_STREAM_INVALID || + n == NGX_HTTP_V3_TRANSPORT_STREAM_STOPPED ) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, fc->log, 0, "stream write failed: %d", n); + } else { + ngx_log_error(NGX_LOG_ERR, fc->log, 0, "stream write failed: %d", n); + } + + return NGX_ERROR; + } + + return n; +} + + +static ssize_t +ngx_http_v3_recv_body(ngx_connection_t *c, u_char *buf, size_t size) +{ + ssize_t n; + ngx_event_t *rev; + ngx_http_request_t *r; + ngx_http_v3_connection_t *h3c; + + rev = c->read; + + r = c->data; + h3c = r->qstream->connection; + + if (c->error) { + rev->ready = 0; + + return NGX_ERROR; + } + + n = quiche_h3_recv_body(h3c->h3, c->quic->conn, r->qstream->id, buf, size); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "http3 body recv: %z of %uz", n, size); + + if (quiche_conn_stream_finished(c->quic->conn, r->qstream->id)) { + rev->ready = 0; + + /* Re-schedule connection read event to poll for Finished event. */ + ngx_post_event(h3c->connection->read, &ngx_posted_events); + } + + if (n == 0) { + rev->ready = 0; + + return 0; + } + + if (n > 0) { + + if ((size_t) n < size) { + rev->ready = 0; + } + + return n; + } + + if (n == QUICHE_H3_ERR_DONE) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, + "quiche_h3_recv_body() not ready"); + + n = NGX_AGAIN; + + } else { + rev->error = 1; + + n = NGX_ERROR; + } + + rev->ready = 0; + + return n; +} + + +static ngx_chain_t * +ngx_http_v3_send_chain(ngx_connection_t *fc, ngx_chain_t *in, off_t limit) +{ + ssize_t n, sent; + off_t send, prev_send; + ngx_uint_t blocked, fin; + + ngx_http_request_t *r; + ngx_http_v3_stream_t *stream; + + r = fc->data; + stream = r->qstream; + + send = 0; + + blocked = 0; + + while (in) { + off_t size = ngx_buf_size(in->buf); + + if (size || in->buf->last_buf) { + break; + } + + in = in->next; + } + + if (in == NULL || stream->out_closed) { + return NULL; + } + + while (in) { + prev_send = send; + + fin = in->buf->last_buf; + + send += ngx_buf_size(in->buf); + + n = ngx_http_v3_stream_do_send(fc, in->buf, fin); + + if (n == NGX_ERROR) { + return NGX_CHAIN_ERROR; + } + + sent = (n == NGX_AGAIN) ? 0 : n; + + fc->sent += sent; + + in->buf->pos += sent; + + /* Partial (or no) write, end now. */ + if ((n == NGX_AGAIN) || (send - prev_send != sent)) { + blocked = 1; + break; + } + + /* Buffer is fully written, switch to the next. */ + if (in->buf->pos == in->buf->last) { + in = in->next; + } + + if (fin) { + stream->out_closed = 1; + } + } + + if (blocked) { + if (!stream->blocked) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, stream->connection->connection->log, 0, + "http3 stream blocked %ui", stream->id); + + stream->blocked = 1; + + fc->write->active = 1; + fc->write->ready = 0; + } + } + + ngx_post_event(stream->connection->connection->write, &ngx_posted_events); + + return in; +} + + +void +ngx_http_v3_close_stream(ngx_http_v3_stream_t *stream, ngx_int_t rc) +{ + ngx_event_t *ev; + ngx_connection_t *c, *fc; + ngx_http_v3_connection_t *h3c; + + h3c = stream->connection; + c = h3c->connection; + + fc = stream->request->connection; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h3c->connection->log, 0, + "http3 close stream %ui", stream->id); + + if (stream->blocked) { + fc->write->handler = ngx_http_v3_close_stream_handler; + fc->read->handler = ngx_http_empty_handler; + return; + } + + quiche_conn_stream_shutdown(h3c->connection->quic->conn, stream->id, + QUICHE_SHUTDOWN_READ, NGX_HTTP_V3_NO_ERROR); + + /* If the stream has closed, a QUIC stream FIN was sent. In that case don't + send RESET_STREAM or else we risk data loss on the client side.*/ + if (!stream->out_closed) { + quiche_conn_stream_shutdown(h3c->connection->quic->conn, stream->id, + QUICHE_SHUTDOWN_WRITE, NGX_HTTP_V3_NO_ERROR); + } + + /* Post a connection write event to flush QUIC frames */ + ngx_post_event(c->write, &ngx_posted_events); + + ngx_rbtree_delete(&h3c->streams, &stream->node); + + ngx_http_free_request(stream->request, rc); + + ev = fc->read; + + if (ev->timer_set) { + ngx_del_timer(ev); + } + + if (ev->posted) { + ngx_delete_posted_event(ev); + } + + ev = fc->write; + + if (ev->timer_set) { + ngx_del_timer(ev); + } + + if (ev->posted) { + ngx_delete_posted_event(ev); + } + + fc->data = h3c->free_fake_connections; + h3c->free_fake_connections = fc; + + h3c->processing--; + + ngx_http_v3_handle_connection(h3c); +} + + +static void +ngx_http_v3_close_stream_handler(ngx_event_t *ev) +{ + ngx_connection_t *fc; + ngx_http_request_t *r; + + fc = ev->data; + r = fc->data; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, fc->log, 0, + "http3 close stream handler"); + + if (ev->timedout) { + ngx_log_error(NGX_LOG_INFO, fc->log, NGX_ETIMEDOUT, "client timed out"); + + fc->timedout = 1; + + ngx_http_v3_close_stream(r->qstream, NGX_HTTP_REQUEST_TIME_OUT); + return; + } + + ngx_http_v3_close_stream(r->qstream, 0); +} + +void +ngx_http_v3_stop_stream_read(ngx_http_v3_stream_t *stream, ngx_int_t rc) +{ + ngx_http_v3_connection_t *h3c; + + if (!stream) { + return; + } + + h3c = stream->connection; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, h3c->connection->log, 0, + "http3 stream shutdown read %ui", stream->id); + + quiche_conn_stream_shutdown(h3c->connection->quic->conn, + stream->id, + QUICHE_SHUTDOWN_READ, NGX_HTTP_V3_NO_ERROR); +} + + +static void +ngx_http_v3_finalize_connection(ngx_http_v3_connection_t *h3c, + ngx_uint_t status) +{ + ngx_event_t *ev; + ngx_connection_t *c, *fc; + ngx_rbtree_node_t *node, *root, *sentinel; + ngx_http_request_t *r; + ngx_http_v3_stream_t *stream; + + c = h3c->connection; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 finalize connection"); + + quiche_conn_close(c->quic->conn, true, status, NULL, 0); + + c->error = 1; + + if (!h3c->processing) { + ngx_http_close_connection(c); + return; + } + + c->read->handler = ngx_http_empty_handler; + c->write->handler = ngx_http_empty_handler; + + root = h3c->streams.root; + sentinel = h3c->streams.sentinel; + + if (root != sentinel) { + node = ngx_rbtree_min(h3c->streams.root, sentinel); + } else { + node = NULL; + } + + /* Close all pending streams / requests. */ + while (node != NULL) { + stream = (ngx_http_v3_stream_t *) node; + + r = stream->request; + fc = r->connection; + + fc->error = 1; + + if (c->close) { + fc->close = 1; + } + + if (stream->blocked) { + stream->blocked = 0; + + ev = fc->write; + ev->active = 0; + ev->ready = 1; + + } else { + ev = fc->read; + } + + node = ngx_rbtree_next(&h3c->streams, node); + + ev->eof = 1; + ev->handler(ev); + } + + if (h3c->processing) { + return; + } + + ngx_http_close_connection(c); +} + + +static void +ngx_http_v3_pool_cleanup(void *data) +{ + ngx_http_v3_connection_t *h3c = data; + + if (h3c->h3) { + quiche_h3_conn_free(h3c->h3); + + h3c->h3 = NULL; + } +} diff --git a/src/http/v3/ngx_http_v3.h b/src/http/v3/ngx_http_v3.h new file mode 100644 index 000000000..2ca965fd8 --- /dev/null +++ b/src/http/v3/ngx_http_v3.h @@ -0,0 +1,80 @@ + +/* + * Copyright (C) Cloudflare, Inc. + */ + + +#ifndef _NGX_HTTP_V3_H_INCLUDED_ +#define _NGX_HTTP_V3_H_INCLUDED_ + + +#include +#include +#include +#include + + +#define NGX_HTTP_V3_ALPN_ADVERTISE "\x05h3-18" + + +typedef struct ngx_http_v3_connection_s ngx_http_v3_connection_t; + + +struct ngx_http_v3_connection_s { + quiche_h3_conn *h3; + + ngx_connection_t *connection; + ngx_http_connection_t *http_connection; + + ngx_pool_t *pool; + + ngx_uint_t processing; + + ngx_rbtree_t streams; + ngx_rbtree_node_t streams_sentinel; + + ngx_connection_t *free_fake_connections; +}; + +struct ngx_http_v3_stream_s { + ngx_rbtree_node_t node; + + uint64_t id; + + quiche_h3_priority priority; + + ngx_http_request_t *request; + + ngx_http_v3_connection_t *connection; + + ngx_array_t *headers; + ngx_array_t *cookies; + + ngx_http_v3_stream_t *next; + + ngx_uint_t headers_sent:1; + ngx_uint_t in_closed:1; + ngx_uint_t out_closed:1; + ngx_uint_t skip_data:1; + ngx_uint_t blocked:1; +}; + + +typedef struct { + ngx_str_t name; + ngx_str_t value; +} ngx_http_v3_header_t; + + +void ngx_http_v3_init(ngx_event_t *rev); + +ngx_int_t ngx_http_v3_send_response(ngx_http_request_t *r); + +void ngx_http_v3_close_stream(ngx_http_v3_stream_t *stream, ngx_int_t rc); +void ngx_http_v3_stop_stream_read(ngx_http_v3_stream_t *stream, ngx_int_t rc); + +ngx_int_t ngx_http_v3_request_body_filter(ngx_http_request_t *r, + ngx_chain_t *in); + + +#endif /* _NGX_HTTP_V3_H_INCLUDED_ */ diff --git a/src/http/v3/ngx_http_v3_filter_module.c b/src/http/v3/ngx_http_v3_filter_module.c new file mode 100644 index 000000000..7cba70535 --- /dev/null +++ b/src/http/v3/ngx_http_v3_filter_module.c @@ -0,0 +1,74 @@ + +/* + * Copyright (C) Cloudflare, Inc. + */ + + +#include +#include +#include +#include + + +static ngx_int_t ngx_http_v3_filter_init(ngx_conf_t *cf); + + +static ngx_http_module_t ngx_http_v3_filter_module_ctx = { + NULL, /* preconfiguration */ + ngx_http_v3_filter_init, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_v3_filter_module = { + NGX_MODULE_V1, + &ngx_http_v3_filter_module_ctx, /* module context */ + NULL, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_http_output_header_filter_pt ngx_http_next_header_filter; + + +static ngx_int_t +ngx_http_v3_header_filter(ngx_http_request_t *r) +{ + if (!r->qstream) { + return ngx_http_next_header_filter(r); + } + + r->header_sent = 1; + + if (r != r->main) { + return NGX_OK; + } + + return ngx_http_v3_send_response(r); +} + + +static ngx_int_t +ngx_http_v3_filter_init(ngx_conf_t *cf) +{ + ngx_http_next_header_filter = ngx_http_top_header_filter; + ngx_http_top_header_filter = ngx_http_v3_header_filter; + + return NGX_OK; +} diff --git a/src/http/v3/ngx_http_v3_module.c b/src/http/v3/ngx_http_v3_module.c new file mode 100644 index 000000000..d413c887b --- /dev/null +++ b/src/http/v3/ngx_http_v3_module.c @@ -0,0 +1,321 @@ + +/* + * Copyright (C) Cloudflare, Inc. + */ + + +#include +#include +#include +#include + +#include + + +static ngx_int_t ngx_http_v3_add_variables(ngx_conf_t *cf); + +static void *ngx_http_v3_create_srv_conf(ngx_conf_t *cf); +static char *ngx_http_v3_merge_srv_conf(ngx_conf_t *cf, + void *parent, void *child); + +static ngx_int_t ngx_http_v3_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); + +static void ngx_http_v3_cleanup_ctx(void *data); + + +static ngx_command_t ngx_http_v3_commands[] = { + + { ngx_string("http3_max_concurrent_streams"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, concurrent_streams), + NULL }, + + { ngx_string("http3_max_requests"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, max_requests), + NULL }, + + { ngx_string("http3_max_header_size"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, max_header_size), + NULL }, + + { ngx_string("http3_initial_max_data"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, max_data), + NULL }, + + { ngx_string("http3_initial_max_stream_data"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, max_stream_data), + NULL }, + + { ngx_string("http3_idle_timeout"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, idle_timeout), + NULL }, + + { ngx_string("http3_congestion_control"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, congestion_control), + NULL }, + + { ngx_string("http3_pacing"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_HTTP_SRV_CONF_OFFSET, + offsetof(ngx_http_v3_srv_conf_t, pacing), + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_v3_module_ctx = { + ngx_http_v3_add_variables, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + ngx_http_v3_create_srv_conf, /* create server configuration */ + ngx_http_v3_merge_srv_conf, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_v3_module = { + NGX_MODULE_V1, + &ngx_http_v3_module_ctx, /* module context */ + ngx_http_v3_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_http_variable_t ngx_http_v3_variables[] = { + + { ngx_string("http3"), NULL, + ngx_http_v3_variable, 0, + NGX_HTTP_VAR_CHANGEABLE, 0 }, + + ngx_http_null_variable +}; + + +static ngx_int_t +ngx_http_v3_add_variables(ngx_conf_t *cf) +{ + ngx_http_variable_t *var, *v; + + for (v = ngx_http_v3_variables; v->name.len; v++) { + var = ngx_http_add_variable(cf, &v->name, v->flags); + if (var == NULL) { + return NGX_ERROR; + } + + var->get_handler = v->get_handler; + var->data = v->data; + } + + return NGX_OK; +} + + +static void * +ngx_http_v3_create_srv_conf(ngx_conf_t *cf) +{ + ngx_http_v3_srv_conf_t *h3scf; + + h3scf = ngx_pcalloc(cf->pool, sizeof(ngx_http_v3_srv_conf_t)); + if (h3scf == NULL) { + return NULL; + } + + h3scf->idle_timeout = NGX_CONF_UNSET_MSEC; + h3scf->max_data = NGX_CONF_UNSET_SIZE; + h3scf->max_stream_data = NGX_CONF_UNSET_SIZE; + h3scf->max_requests = NGX_CONF_UNSET_UINT; + h3scf->max_header_size = NGX_CONF_UNSET_SIZE; + h3scf->concurrent_streams = NGX_CONF_UNSET_UINT; + /* h3scf->congestion_control = { 0, NULL }; */ + h3scf->pacing = NGX_CONF_UNSET; + + return h3scf; +} + + +#if (NGX_DEBUG) +static void +quiche_log(const char *line, void *argp) +{ + ngx_log_t *log = ngx_cycle->log; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, "%s", line); +} +#endif + + +static char * +ngx_http_v3_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) +{ + ngx_http_v3_srv_conf_t *prev = parent; + ngx_http_v3_srv_conf_t *conf = child; + + ngx_pool_cleanup_t *cln; + + ngx_conf_merge_msec_value(conf->idle_timeout, + prev->idle_timeout, 180000); + + ngx_conf_merge_size_value(conf->max_data, + prev->max_data, 10485760); + + ngx_conf_merge_size_value(conf->max_stream_data, + prev->max_stream_data, 1048576); + + ngx_conf_merge_uint_value(conf->max_requests, + prev->max_requests, 1000); + + ngx_conf_merge_size_value(conf->max_header_size, + prev->max_header_size, 16384); + + ngx_conf_merge_uint_value(conf->concurrent_streams, + prev->concurrent_streams, 128); + + ngx_conf_merge_str_value(conf->congestion_control, + prev->congestion_control, "cubic"); + + ngx_conf_merge_value(conf->pacing, prev->pacing, 0); + + conf->quic.log = cf->log; + +#if (NGX_DEBUG) + /* Enable quiche debug logging. quiche commit ceade4 or later is required */ + quiche_enable_debug_logging(quiche_log, NULL); +#endif + + if (ngx_quic_create_conf(&conf->quic) != NGX_OK) { + return NGX_CONF_ERROR; + } + + quiche_config_set_max_send_udp_payload_size(conf->quic.config, + MAX_DATAGRAM_SIZE); + + quiche_config_set_max_idle_timeout(conf->quic.config, conf->idle_timeout); + + quiche_config_set_initial_max_data(conf->quic.config, conf->max_data); + + quiche_config_set_initial_max_stream_data_bidi_remote(conf->quic.config, + conf->max_stream_data); + + quiche_config_set_initial_max_stream_data_uni(conf->quic.config, + conf->max_stream_data); + + quiche_config_set_initial_max_streams_bidi(conf->quic.config, + conf->concurrent_streams); + + /* For HTTP/3 we only need 3 unidirectional streams. */ + quiche_config_set_initial_max_streams_uni(conf->quic.config, 3); + + if (quiche_config_set_cc_algorithm_name(conf->quic.config, + (char *) conf->congestion_control.data) != 0) + { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to configure congestion control algorithm"); + return NGX_CONF_ERROR; + } + + quiche_config_enable_pacing(conf->quic.config, conf->pacing); + conf->quic.pacing = conf->pacing; + + conf->http3 = quiche_h3_config_new(); + if (conf->http3 == NULL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to create HTTP/3 config"); + return NGX_CONF_ERROR; + } + + quiche_h3_config_set_max_field_section_size(conf->http3, + conf->max_header_size); + + cln = ngx_pool_cleanup_add(cf->pool, 0); + if (cln == NULL) { + return NGX_CONF_ERROR; + } + + cln->handler = ngx_quic_cleanup_ctx; + cln->data = &conf->quic; + + cln = ngx_pool_cleanup_add(cf->pool, 0); + if (cln == NULL) { + return NGX_CONF_ERROR; + } + + cln->handler = ngx_http_v3_cleanup_ctx; + cln->data = conf->http3; + + return NGX_CONF_OK; +} + + +static ngx_int_t +ngx_http_v3_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, + uintptr_t data) +{ + ngx_connection_t *c; + + v->valid = 1; + v->no_cacheable = 1; + v->not_found = 0; + + c = r->connection; + if (c == NULL) { + return NGX_ERROR; + } + + if (c->quic != NULL) { + v->len = sizeof("h3") - 1; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = (u_char *) "h3"; + + return NGX_OK; + } + + *v = ngx_http_variable_null_value; + return NGX_OK; +} + + +static void +ngx_http_v3_cleanup_ctx(void *data) +{ + quiche_h3_config *config = data; + + quiche_h3_config_free(config); +} diff --git a/src/http/v3/ngx_http_v3_module.h b/src/http/v3/ngx_http_v3_module.h new file mode 100644 index 000000000..88f7497e6 --- /dev/null +++ b/src/http/v3/ngx_http_v3_module.h @@ -0,0 +1,36 @@ + +/* + * Copyright (C) Cloudflare, Inc. + */ + + +#ifndef _NGX_HTTP_V3_MODULE_H_INCLUDED_ +#define _NGX_HTTP_V3_MODULE_H_INCLUDED_ + + +#include +#include + +#include + + +typedef struct { + ngx_quic_t quic; + + quiche_h3_config *http3; + + ngx_msec_t idle_timeout; + size_t max_data; + size_t max_stream_data; + ngx_uint_t max_requests; + ngx_uint_t max_header_size; + ngx_uint_t concurrent_streams; + ngx_str_t congestion_control; + ngx_flag_t pacing; +} ngx_http_v3_srv_conf_t; + + +extern ngx_module_t ngx_http_v3_module; + + +#endif /* _NGX_HTTP_V3_MODULE_H_INCLUDED_ */ diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h index 3036caebf..c3a207005 100644 --- a/src/os/unix/ngx_linux_config.h +++ b/src/os/unix/ngx_linux_config.h @@ -104,6 +104,16 @@ typedef struct iocb ngx_aiocb_t; #endif +#if (NGX_HAVE_UDP_SEGMENT) +#include +#endif + + +#if (NGX_HAVE_SO_TXTIME) +#include +#endif + + #define NGX_LISTEN_BACKLOG 511 diff --git a/src/os/unix/ngx_udp_sendmsg_chain.c b/src/os/unix/ngx_udp_sendmsg_chain.c index 5399c7916..f82d088b3 100644 --- a/src/os/unix/ngx_udp_sendmsg_chain.c +++ b/src/os/unix/ngx_udp_sendmsg_chain.c @@ -10,6 +10,11 @@ #include +#ifndef UDP_SEGMENT +#define UDP_SEGMENT 103 +#endif + + static ngx_chain_t *ngx_udp_output_chain_to_iovec(ngx_iovec_t *vec, ngx_chain_t *in, ngx_log_t *log); static ssize_t ngx_sendmsg(ngx_connection_t *c, ngx_iovec_t *vec); @@ -199,20 +204,25 @@ ngx_udp_output_chain_to_iovec(ngx_iovec_t *vec, ngx_chain_t *in, ngx_log_t *log) static ssize_t ngx_sendmsg(ngx_connection_t *c, ngx_iovec_t *vec) { - ssize_t n; - ngx_err_t err; - struct msghdr msg; + ssize_t n; + ngx_err_t err; + struct msghdr msg; + struct cmsghdr *cmsg = NULL; #if (NGX_HAVE_MSGHDR_MSG_CONTROL) #if (NGX_HAVE_IP_SENDSRCADDR) u_char msg_control[CMSG_SPACE(sizeof(struct in_addr))]; #elif (NGX_HAVE_IP_PKTINFO) - u_char msg_control[CMSG_SPACE(sizeof(struct in_pktinfo))]; + u_char msg_control[CMSG_SPACE(sizeof(struct in_pktinfo) + + CMSG_SPACE(sizeof(uint16_t)) + + CMSG_SPACE(sizeof(uint64_t)))]; #endif #if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO) - u_char msg_control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; + u_char msg_control6[CMSG_SPACE(sizeof(struct in6_pktinfo)) + + CMSG_SPACE(sizeof(uint16_t)) + + CMSG_SPACE(sizeof(uint64_t))]; #endif #endif @@ -234,7 +244,6 @@ ngx_sendmsg(ngx_connection_t *c, ngx_iovec_t *vec) #if (NGX_HAVE_IP_SENDSRCADDR) if (c->local_sockaddr->sa_family == AF_INET) { - struct cmsghdr *cmsg; struct in_addr *addr; struct sockaddr_in *sin; @@ -255,12 +264,13 @@ ngx_sendmsg(ngx_connection_t *c, ngx_iovec_t *vec) #elif (NGX_HAVE_IP_PKTINFO) if (c->local_sockaddr->sa_family == AF_INET) { - struct cmsghdr *cmsg; struct in_pktinfo *pkt; struct sockaddr_in *sin; + ngx_memzero(&msg_control, sizeof(msg_control)); + msg.msg_control = &msg_control; - msg.msg_controllen = sizeof(msg_control); + msg.msg_controllen = CMSG_SPACE(sizeof(struct in_pktinfo)); cmsg = CMSG_FIRSTHDR(&msg); cmsg->cmsg_level = IPPROTO_IP; @@ -279,12 +289,13 @@ ngx_sendmsg(ngx_connection_t *c, ngx_iovec_t *vec) #if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO) if (c->local_sockaddr->sa_family == AF_INET6) { - struct cmsghdr *cmsg; struct in6_pktinfo *pkt6; struct sockaddr_in6 *sin6; + ngx_memzero(&msg_control6, sizeof(msg_control6)); + msg.msg_control = &msg_control6; - msg.msg_controllen = sizeof(msg_control6); + msg.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo)); cmsg = CMSG_FIRSTHDR(&msg); cmsg->cmsg_level = IPPROTO_IPV6; @@ -301,6 +312,91 @@ ngx_sendmsg(ngx_connection_t *c, ngx_iovec_t *vec) #endif } +#if (NGX_QUIC && NGX_HAVE_UDP_SEGMENT) + + if (c->quic && c->quic->segment_size && + c->local_sockaddr && + (c->local_sockaddr->sa_family == AF_INET || + c->local_sockaddr->sa_family == AF_INET6)) + { + uint16_t *segment_size; + + if (cmsg == NULL) { + if (c->local_sockaddr->sa_family == AF_INET) { + ngx_memzero(&msg_control, sizeof(msg_control)); + msg.msg_control = &msg_control; + } else { + ngx_memzero(&msg_control6, sizeof(msg_control6)); + msg.msg_control = &msg_control6; + } + + msg.msg_controllen = CMSG_SPACE(sizeof(uint16_t)); + + cmsg = CMSG_FIRSTHDR(&msg); + + } else { + msg.msg_controllen += CMSG_SPACE(sizeof(uint16_t)); + + cmsg = CMSG_NXTHDR(&msg, cmsg); + } + + cmsg->cmsg_level = SOL_UDP; + cmsg->cmsg_type = UDP_SEGMENT; + cmsg->cmsg_len = CMSG_LEN(sizeof(uint16_t)); + + segment_size = (uint16_t *) CMSG_DATA(cmsg); + ngx_memzero(segment_size, sizeof(uint16_t)); + *segment_size = c->quic->segment_size; + } + +#endif + +#if (NGX_QUIC && NGX_HAVE_SO_TXTIME) + + ngx_listening_t *ls = c->listening; + + if (c->quic && c->quic->pacing && + ls && ls->quic_so_txtime && + c->local_sockaddr && + (c->local_sockaddr->sa_family == AF_INET || + c->local_sockaddr->sa_family == AF_INET6)) { + uint64_t timestamp_ns, *tx_time; + + if (cmsg == NULL) { + if (c->local_sockaddr->sa_family == AF_INET) { + ngx_memzero(&msg_control, sizeof(msg_control)); + msg.msg_control = &msg_control; + } else { + ngx_memzero(&msg_control6, sizeof(msg_control6)); + msg.msg_control = &msg_control6; + } + + msg.msg_controllen = CMSG_SPACE(sizeof(uint64_t)); + + cmsg = CMSG_FIRSTHDR(&msg); + + } else { + msg.msg_controllen += CMSG_SPACE(sizeof(uint64_t)); + + cmsg = CMSG_NXTHDR(&msg, cmsg); + } + + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_TXTIME; + cmsg->cmsg_len = CMSG_LEN(sizeof(uint64_t)); + + /* Convert struct timespec to nanoseconds. */ + timestamp_ns = c->quic->send_info.at.tv_sec * + (1000ULL * 1000 * 1000) + + c->quic->send_info.at.tv_nsec; + + tx_time = (uint64_t *) CMSG_DATA(cmsg); + ngx_memzero(tx_time, sizeof(uint64_t)); + *tx_time = timestamp_ns; + } + +#endif + #endif eintr: @@ -315,6 +411,7 @@ eintr: switch (err) { case NGX_EAGAIN: + case ENOBUFS: ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, "sendmsg() not ready"); return NGX_AGAIN; -- 2.40.1