From af68854dfed558104d534c66d0321b69ee220b4d Mon Sep 17 00:00:00 2001 From: Alexander Gerasimov Date: Tue, 8 Sep 2026 11:39:27 +0300 Subject: [PATCH 1/5] v0.0.71 - inherit the HTTP/3 certificate from mod_ssl A host with "h3" in Protocols now serves HTTP/3 with the certificate mod_ssl already resolved for it (SSLCertificateFile and mod_md alike), the way mod_http2 rides mod_ssl. The certificate and key are loaded in the ap_ssl_add_cert_files hook, during startup while httpd still runs privileged, so a key readable only by root loads for QUIC as it does for mod_ssl instead of failing in the unprivileged child. H3CertificatePath and H3CertificateKeyPath are removed; drop them from existing configurations. Author: Alexander Gerasimov --- AUTHORS | 3 +- CHANGES | 13 ++++ CMakeLists.txt | 2 +- INSTALL | 13 ++-- README.md | 3 - container/README.md | 8 +-- container/httpd-linux.conf | 2 - container/httpd-windows.conf | 2 - docs/architecture.md | 2 +- docs/configuration.md | 2 +- docs/configuration_httpd.md | 68 +++++++++++--------- docs/containers.md | 2 +- docs/deploy.md | 3 +- interop/httpd.conf | 2 - mod_http3/include/h3_config.h | 30 ++++++--- mod_http3/include/h3_version.h | 4 +- mod_http3/include/quic/detail/h3q_tls.h | 20 +++--- mod_http3/include/quic/h3q.h | 7 +-- mod_http3/src/h3_config.c | 84 +++++++++++-------------- mod_http3/src/h3_hooks.c | 2 +- mod_http3/src/h3_io.c | 4 +- mod_http3/src/h3_server.c | 2 +- mod_http3/src/mod_http3.c | 6 +- mod_http3/src/quic/detail/h3q_tls.c | 19 +++--- mod_http3/src/quic/h3q.c | 5 +- test/http3/env.py | 7 --- test/http3/test_003_directives.py | 9 ++- 27 files changed, 170 insertions(+), 154 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7addfb8..9bc56c1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -18,5 +18,4 @@ Individuals * Jean-Frédéric Clere * Tarek Ibrahim - - + * Alexander Gerasimov https://codeit.guru/ diff --git a/CHANGES b/CHANGES index fff1d37..0916300 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,19 @@ mod_http3 changes Changes are listed most recent first. Security-related entries always appear at the top of their release block. +v0.0.71 (2026-09-08) +-------------------- + *) SECURITY: Load the HTTP/3 certificate and key in post_config, while httpd + still runs privileged, so a root-only key no longer fails in the + unprivileged child and both mod_ssl and mod_http3 read the same files. + [Alexander Gerasimov ] + + *) A host with "h3" in Protocols now serves HTTP/3 with the certificate + mod_ssl resolved for it (SSLCertificateFile and mod_md alike), the way + mod_http2 rides mod_ssl. H3CertificatePath and H3CertificateKeyPath are + removed; drop them from existing configurations. + [Alexander Gerasimov ] + v0.0.70 (2026-09-06) -------------------- *) SECURITY: Updated the httpd submodule so mpm_event tolerates a connection diff --git a/CMakeLists.txt b/CMakeLists.txt index 08178d7..fc691f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.26) -project(mod_http3 VERSION 0.0.70) +project(mod_http3 VERSION 0.0.71) # -- Compiler and Build Type Checks -- if(NOT CMAKE_C_COMPILER_ID MATCHES "^(GNU|MSVC)$") diff --git a/INSTALL b/INSTALL index b901ae9..9426aae 100644 --- a/INSTALL +++ b/INSTALL @@ -116,15 +116,13 @@ reject a directly-trusted self-signed leaf. Import ca.crt to trust the server. The script refuses to overwrite existing keys. - The key must be readable by the httpd child user: - - $ chgrp daemon /path/to/httpd/conf/certs/server.key - $ chmod 640 /path/to/httpd/conf/certs/server.key + The key is read at startup, before httpd drops privileges, so the + permissions mod_ssl accepts are enough. 3. Configure httpd. - LoadModule must appear before the block. - H3CertificatePath and H3CertificateKeyPath are required. + LoadModule must appear before the block. A host serves + HTTP/3 when h3 is in its Protocols and mod_ssl has its certificate. Minimal httpd.conf: @@ -142,8 +140,7 @@ SSLCertificateFile conf/certs/server.crt SSLCertificateKeyFile conf/certs/server.key - H3CertificatePath conf/certs/server.crt - H3CertificateKeyPath conf/certs/server.key + Protocols h3 h2 http/1.1 DocumentRoot htdocs diff --git a/README.md b/README.md index 69dc368..ba7bd49 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,6 @@ Listen 4433 https Protocols h3 h2 http/1.1 - H3CertificatePath conf/server.crt - H3CertificateKeyPath conf/server.key - DocumentRoot htdocs Require all granted diff --git a/container/README.md b/container/README.md index 2c0b385..fc8225b 100644 --- a/container/README.md +++ b/container/README.md @@ -89,11 +89,11 @@ Listen 8443 https SSLEngine on + SSLCertificateFile conf/certs/server.crt + SSLCertificateKeyFile conf/certs/server.key Protocols h3 - H3CertificatePath conf/certs/server.crt - H3CertificateKeyPath conf/certs/server.key - H3Port 8443 + H3Port 8443 ``` @@ -110,7 +110,7 @@ podman logs mod_http3_dev | Error | Cause | Fix | |---|---|---| | `Cannot load .../mod_http3.so` | Build failed | Check build output | -| `Invalid command 'H3CertificatePath'` | Module not loaded | Verify LoadModule line | +| `Invalid command 'H3Port'` | Module not loaded | Verify LoadModule line | | `Permission denied` | SELinux | Add `:Z` to volume mounts | | HTTP/3 not working but HTTP/2 is | UDP port not mapped | Check `podman port mod_http3_dev` | diff --git a/container/httpd-linux.conf b/container/httpd-linux.conf index 8395725..e68bef2 100755 --- a/container/httpd-linux.conf +++ b/container/httpd-linux.conf @@ -38,8 +38,6 @@ Listen ${H3_PORT} https Protocols h3 - H3CertificatePath /src/dependencies/httpd-dist/conf/certs/server.crt - H3CertificateKeyPath /src/dependencies/httpd-dist/conf/certs/server.key H3Port ${H3_PORT} H3MaxConnections 100 H3MaxConcurrentStreams 128 diff --git a/container/httpd-windows.conf b/container/httpd-windows.conf index 203ddb8..3763730 100644 --- a/container/httpd-windows.conf +++ b/container/httpd-windows.conf @@ -32,8 +32,6 @@ Listen ${H3_PORT} https Protocols h3 - H3CertificatePath "C:/httpd/conf/certs/server.crt" - H3CertificateKeyPath "C:/httpd/conf/certs/server.key" H3Port ${H3_PORT} H3MaxConnections 100 H3MaxConcurrentStreams 128 diff --git a/docs/architecture.md b/docs/architecture.md index d58d98a..e7d7358 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -47,6 +47,6 @@ bytes as acknowledged once `SSL_write_ex` accepts them. HTTP/3 connections are UDP/QUIC connections, but request processing runs through standard Apache machinery. HTTP/3 is advertised over existing TCP responses using `Alt-Svc`; clients then establish QUIC on the advertised UDP port. -The module uses the first VirtualHost with both `H3CertificatePath` and `H3CertificateKeyPath` for its listener. Name-based virtual host selection then uses the request authority. IP-based virtual hosts remain unsupported because the necessary per-connection local address is not currently recovered. +The module's listener presents the certificate of the first VirtualHost that serves HTTP/3 (`h3` in `Protocols` on a host with a mod_ssl certificate); the certificate is loaded in `post_config`, before privileges drop. Name-based virtual host selection then uses the request authority. IP-based virtual hosts remain unsupported because the necessary per-connection local address is not currently recovered. See the [configuration guide](configuration.md) for operational control points. diff --git a/docs/configuration.md b/docs/configuration.md index b161535..04e0e36 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,7 +4,7 @@ Advanced build options, dependency management, and build internals. For quick start and deployment, see [INSTALL](../INSTALL). -For httpd runtime directives (`H3CertificatePath`, VirtualHost), see [httpd Configuration](configuration_httpd.md). +For httpd runtime directives (`Protocols h3`, `H3Port`, VirtualHost), see [httpd Configuration](configuration_httpd.md). ## Build Commands diff --git a/docs/configuration_httpd.md b/docs/configuration_httpd.md index 07b602a..f0d3779 100644 --- a/docs/configuration_httpd.md +++ b/docs/configuration_httpd.md @@ -15,21 +15,26 @@ mod_http3 enables HTTP/3 protocol support in Apache HTTP Server. The module: ## Configuration Directives -### H3CertificatePath +### Enabling HTTP/3 on a host -**Syntax:** `H3CertificatePath /path/to/certificate.pem` -**Context:** server config, virtual host -**Required:** Yes - -Path to the TLS certificate file for HTTP/3 connections. May point to the same file used by `SSLCertificateFile`. +A VirtualHost serves HTTP/3 when `h3` is in its `Protocols` and mod_ssl has a +certificate for it -- the same two things mod_http2 needs for `h2`: -### H3CertificateKeyPath - -**Syntax:** `H3CertificateKeyPath /path/to/private-key.pem` -**Context:** server config, virtual host -**Required:** Yes +```apache + + ServerName www.example.com + SSLEngine on + SSLCertificateFile /etc/httpd/ssl/www.crt + SSLCertificateKeyFile /etc/httpd/ssl/www.key + Protocols h3 h2 http/1.1 + +``` -Path to the TLS private key file for HTTP/3 connections. May point to the same file used by `SSLCertificateKeyFile`. +The certificate and key mod_ssl resolved for the host -- `SSLCertificateFile` +pairs and anything mod_md manages -- are loaded for QUIC as well, during +startup while httpd still runs privileged, so a key readable only by root works +as it does for mod_ssl. A host with `h3` in `Protocols` but no mod_ssl +certificate (`SSLEngine off`, or mod_ssl not loaded) does not serve HTTP/3. ### H3Port @@ -37,7 +42,7 @@ Path to the TLS private key file for HTTP/3 connections. May point to the same f **Context:** server config, virtual host **Default:** the port of the VirtualHost that configured HTTP/3 -UDP port the QUIC listener binds to. When unset, the module reuses the port of the VirtualHost that carries the `H3CertificatePath`/`H3CertificateKeyPath` pair, so TCP (HTTP/1.1, HTTP/2) and UDP (HTTP/3) share the same port number. Set it explicitly to serve HTTP/3 on a different port. +UDP port the QUIC listener binds to. When unset, the module reuses the port of the VirtualHost that serves HTTP/3, so TCP (HTTP/1.1, HTTP/2) and UDP (HTTP/3) share the same port number. Set it explicitly to serve HTTP/3 on a different port. ### H3MaxConcurrentStreams @@ -131,7 +136,7 @@ The idle timeout duration in seconds for QUIC connections. This maps to the stan **Context:** server config, virtual host **Default:** `on` -Whether to issue TLS 1.3 session tickets. A returning client that presents a ticket resumes its session and skips a certificate verification, which is the difference between a two-round-trip and a one-round-trip reconnect. Each worker process holds its own ticket keys, so a client resumes only when it returns to the process that issued its ticket; otherwise the server transparently falls back to a full handshake. Turn this off to force a full handshake on every connection. +Whether to issue TLS 1.3 session tickets. A returning client that presents a ticket resumes its session and skips a certificate verification, which is the difference between a two-round-trip and a one-round-trip reconnect. The ticket keys are created before httpd forks, so every child process resumes tickets issued by any other; a ticket from before a restart falls back to a full handshake. Turn this off to force a full handshake on every connection. ### H3AddressValidation @@ -224,8 +229,10 @@ The module automatically detects the port from the VirtualHost configuration: # HTTP/3 will listen on port 8443 ServerName secure.example.com - H3CertificatePath /etc/httpd/ssl/secure.crt - H3CertificateKeyPath /etc/httpd/ssl/secure.key + SSLEngine on + SSLCertificateFile /etc/httpd/ssl/secure.crt + SSLCertificateKeyFile /etc/httpd/ssl/secure.key + Protocols h3 h2 http/1.1 ``` @@ -233,21 +240,25 @@ Use `H3Port` to bind the QUIC listener to a different UDP port than the VirtualH ### Multiple VirtualHosts -The module uses the **first VirtualHost** that has both `H3CertificatePath` and `H3CertificateKeyPath` configured: +The QUIC listener presents the certificate of the **first VirtualHost** that serves HTTP/3; every other HTTP/3 host still advertises `Alt-Svc` and is selected by request authority: ```apache -# This VirtualHost is used for HTTP/3 +# This VirtualHost's certificate is the one QUIC presents ServerName primary.example.com - H3CertificatePath /etc/httpd/ssl/primary.crt - H3CertificateKeyPath /etc/httpd/ssl/primary.key + SSLEngine on + SSLCertificateFile /etc/httpd/ssl/primary.crt + SSLCertificateKeyFile /etc/httpd/ssl/primary.key + Protocols h3 h2 http/1.1 -# This VirtualHost is ignored for HTTP/3 +# Served over HTTP/3 too, but with primary's certificate ServerName secondary.example.com - H3CertificatePath /etc/httpd/ssl/secondary.crt - H3CertificateKeyPath /etc/httpd/ssl/secondary.key + SSLEngine on + SSLCertificateFile /etc/httpd/ssl/secondary.crt + SSLCertificateKeyFile /etc/httpd/ssl/secondary.key + Protocols h3 h2 http/1.1 ``` @@ -299,8 +310,8 @@ Disable the advertisement entirely with `H3AltSvc off`. The module validates configuration during Apache startup: -1. **Certificate Path Check:** `H3CertificatePath` is configured -2. **Key Path Check:** `H3CertificateKeyPath` is configured +1. At least one host serves HTTP/3: `h3` in `Protocols` on an `SSLEngine on` host +2. That host's certificate and key load If either check fails, Apache refuses to start. @@ -330,15 +341,16 @@ LogLevel http3:trace8 ``` # Successful configuration -h3_post_config: pid=[PID] cert=/path/to/cert key=/path/to/key h3_port=443 mpm=event threaded=1 forked=2 max_threads=25 +mod_http3: serving HTTP/3 with mod_ssl certificate /path/to/cert +h3_post_config: pid=[PID] h3_port=443 mpm=event threaded=1 forked=2 max_threads=25 # Worker thread started h3_child_init worker_thread_main # Errors -mod_http3: H3CertificatePath directive is required but not configured -mod_http3: H3CertificateKeyPath directive is required but not configured +mod_http3: no host serves HTTP/3: add h3 to Protocols on a host with SSLEngine on +mod_http3: loading certificate /path/to/cert with key /path/to/key failed: ... ``` ### Security diff --git a/docs/containers.md b/docs/containers.md index a15278e..ba834d9 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -200,5 +200,5 @@ curl has no HTTP/3 support. `curl -V | grep HTTP3` confirms it either way. **HTTP/1.1 works but HTTP/3 does not.** Almost always certificate permissions — see above. `podman logs mod_http3` shows the error from the child process. -**`Invalid command 'H3CertificatePath'`.** The configuration you mounted does not +**`Invalid command 'H3Port'`.** The configuration you mounted does not load the module. It needs `LoadModule http3_module modules/mod_http3.so`. diff --git a/docs/deploy.md b/docs/deploy.md index fbdd3da..df44966 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -20,8 +20,7 @@ Listen 4433 https SSLEngine on SSLCertificateFile conf/certs/server.crt SSLCertificateKeyFile conf/certs/server.key - H3CertificatePath conf/certs/server.crt - H3CertificateKeyPath conf/certs/server.key + Protocols h3 h2 http/1.1 DocumentRoot htdocs Require all granted diff --git a/interop/httpd.conf b/interop/httpd.conf index 78dbaf1..44f575c 100644 --- a/interop/httpd.conf +++ b/interop/httpd.conf @@ -45,8 +45,6 @@ Listen 443 https Protocols h3 - H3CertificatePath /interop/certs/cert.pem - H3CertificateKeyPath /interop/certs/priv.key H3Port 443 H3MaxConcurrentStreams 1000 diff --git a/mod_http3/include/h3_config.h b/mod_http3/include/h3_config.h index 9ccc12f..ca421f2 100644 --- a/mod_http3/include/h3_config.h +++ b/mod_http3/include/h3_config.h @@ -38,8 +38,9 @@ struct h3_server_conf { apr_port_t host_port; - const char* h3_cert_path; - const char* h3_key_path; + /** QUIC TLS context, built from mod_ssl's certificate when the host lists h3 in Protocols. */ + struct ssl_ctx_st* ssl_ctx; + apr_port_t h3_port; apr_uint32_t h3_max_concurrent_streams; apr_uint32_t h3_max_connections; @@ -83,8 +84,7 @@ void* h3_create_server_config(apr_pool_t* p, server_rec* s); /** * ap_merge_server_config callback: produce a child vhost config that - * inherits each unset field from the parent. cert/key/h3_port use the - * new value if non-NULL/non-zero, else the base. + * inherits each unset field from the parent. * @param p Pool for the merged config. * @param base_conf Parent h3_server_conf. * @param new_conf Child h3_server_conf. @@ -113,16 +113,26 @@ void* h3_create_dir_config(apr_pool_t* p, char* dir); void* h3_merge_dir_config(apr_pool_t* p, void* base, void* add); /** - * ap_post_config hook: resolve cert/key/h3_port for the listening vhost - * and log the resolved values. No-op in AP_SQ_MS_CREATE_PRE_CONFIG - * (pre-config phase). Returns OK if a fully-configured vhost is found, - * HTTP_INTERNAL_SERVER_ERROR otherwise. + * ap_ssl_add_cert_files hook: mod_ssl runs it for every SSLEngine vhost with + * the certificate and key files it is about to load (SSLCertificateFile plus + * anything mod_md added). When the vhost lists h3 in Protocols, builds its + * QUIC TLS context from them, here, before the server drops privileges. + * @param s The vhost being configured. + * @param p Config pool; owns the context. + * @param cert_files Certificate chain files, const char* elements. + * @param key_files Private key files, const char* elements. + * @return DECLINED, or HTTP_INTERNAL_SERVER_ERROR if the files do not load. + */ +int h3_ssl_add_cert_files(server_rec* s, apr_pool_t* p, apr_array_header_t* cert_files, apr_array_header_t* key_files); + +/** + * ap_post_config hook: fill in defaults on every vhost that serves HTTP/3; + * the first one owns the listener. No-op in AP_SQ_MS_CREATE_PRE_CONFIG. * @param p Config pool (unused). * @param plog Log pool (unused). * @param ptemp Temp pool (unused). * @param s The first server_rec in the configuration. - * @return OK, or HTTP_INTERNAL_SERVER_ERROR if no vhost has both - * H3CertificatePath and H3CertificateKeyPath set. + * @return OK, or HTTP_INTERNAL_SERVER_ERROR if no vhost serves HTTP/3. */ int h3_post_config(apr_pool_t* p, apr_pool_t* plog, apr_pool_t* ptemp, server_rec* s); diff --git a/mod_http3/include/h3_version.h b/mod_http3/include/h3_version.h index 4aa3ee9..41161f8 100644 --- a/mod_http3/include/h3_version.h +++ b/mod_http3/include/h3_version.h @@ -22,13 +22,13 @@ #define MOD_HTTP3_VERSION_MAJOR 0 #define MOD_HTTP3_VERSION_MINOR 0 -#define MOD_HTTP3_VERSION_PATCH 70 +#define MOD_HTTP3_VERSION_PATCH 71 // Construct a 24-bit packed version number from major, minor and patch. Version 1.2.3 becomes 0x010203. #define MOD_HTTP3_MAKE_VERSION(major, minor, patch) (((major) << 16) | ((minor) << 8) | (patch)) #define MOD_HTTP3_VERSION MOD_HTTP3_MAKE_VERSION(MOD_HTTP3_VERSION_MAJOR, MOD_HTTP3_VERSION_MINOR, MOD_HTTP3_VERSION_PATCH) -#define MOD_HTTP3_VERSION_STRING "0.0.70" +#define MOD_HTTP3_VERSION_STRING "0.0.71" #endif /* H3_VERSION_H */ diff --git a/mod_http3/include/quic/detail/h3q_tls.h b/mod_http3/include/quic/detail/h3q_tls.h index 58c51fb..cd197d1 100644 --- a/mod_http3/include/quic/detail/h3q_tls.h +++ b/mod_http3/include/quic/detail/h3q_tls.h @@ -26,15 +26,21 @@ #include "quic/h3q.h" /** - * Build the TLS context the listener serves from: certificate and key from - * @p cfg, "h3" as the only ALPN protocol, and a key log when SSLKEYLOGFILE is - * set. - * @param cfg Configuration supplying the certificate and key paths. - * @param err Buffer receiving the reason on failure; may be NULL. - * @param errlen Capacity of @p err. + * Build the TLS context the listener serves from: the certificate chain files + * with their keys (one per key type, as mod_ssl allows; a missing key file + * means the key sits in the chain file), "h3" as the only ALPN protocol, and + * a key log when SSLKEYLOGFILE is set. Safe before the server forks and drops + * privileges; children inherit the loaded keys. + * @param cert_files Certificate chain files; at least one. + * @param ncerts Number of entries in @p cert_files. + * @param key_files Private key files, matched by index to @p cert_files. + * @param nkeys Number of entries in @p key_files; may be fewer. + * @param session_tickets Non-zero to issue TLS 1.3 session tickets. + * @param err Buffer receiving the reason on failure; may be NULL. + * @param errlen Capacity of @p err. * @return New context, or NULL on failure. */ -SSL_CTX* h3q_tls_ctx_create(const h3q_config* cfg, char* err, size_t errlen); +SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const char* const* key_files, size_t nkeys, int session_tickets, char* err, size_t errlen); /** * Record a message in a caller-supplied error buffer, appending the OpenSSL diff --git a/mod_http3/include/quic/h3q.h b/mod_http3/include/quic/h3q.h index c5cac25..9ba8ace 100644 --- a/mod_http3/include/quic/h3q.h +++ b/mod_http3/include/quic/h3q.h @@ -34,16 +34,15 @@ typedef struct h3q_stream h3q_stream; * a per-connection setting, applied by h3q_conn_prepare(). */ typedef struct h3q_config { - const char* cert_path; - const char* key_path; + /** TLS context from h3q_tls_ctx_create(); the engine takes its own reference. */ + struct ssl_ctx_st* ssl_ctx; unsigned address_validation : 1; - unsigned session_tickets : 1; } h3q_config; /** * Build the QUIC listener on @p udp_fd, together with the filter BIO that * recovers peer addresses from OpenSSL's accept queue. - * @param cfg Certificate, key and address validation. + * @param cfg TLS context and address validation. * @param udp_fd Pre-opened non-blocking UDP socket bound to the listen port, * borrowed for the engine's lifetime. * @param err Buffer receiving the reason on failure; may be NULL. diff --git a/mod_http3/src/h3_config.c b/mod_http3/src/h3_config.c index 460477e..389c119 100644 --- a/mod_http3/src/h3_config.c +++ b/mod_http3/src/h3_config.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,7 @@ #include "h3_os.h" #include "h3_request.h" #include "mod_http3.h" +#include "quic/detail/h3q_tls.h" apr_port_t get_server_port(const server_rec* s) { @@ -60,8 +62,6 @@ void* h3_merge_server_config(apr_pool_t* p, void* base_conf, void* new_conf) h3_server_conf* base = (h3_server_conf*)base_conf; h3_server_conf* new = (h3_server_conf*)new_conf; - merged->h3_cert_path = new->h3_cert_path ? new->h3_cert_path : base->h3_cert_path; - merged->h3_key_path = new->h3_key_path ? new->h3_key_path : base->h3_key_path; merged->h3_port = new->h3_port ? new->h3_port : base->h3_port; merged->h3_max_concurrent_streams = new->h3_max_concurrent_streams ? new->h3_max_concurrent_streams : base->h3_max_concurrent_streams; merged->h3_max_connections = new->h3_max_connections ? new->h3_max_connections : base->h3_max_connections; @@ -95,28 +95,6 @@ static int mpm_query(int code) return ap_mpm_query(code, &value) == APR_SUCCESS ? value : -1; } -static const char* set_string(cmd_parms* cmd, const char* arg, const char* field) -{ - h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module); - if (!conf) - { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, cmd->server, "mod_http3: server config missing in directive"); - return "mod_http3: internal error: no server config"; - } - *(const char**)((char*)conf + (apr_size_t)field) = apr_pstrdup(cmd->pool, arg); - return NULL; -} - -static const char* set_h3_cert_path(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg) -{ - return set_string(cmd, arg, (const char*)offsetof(h3_server_conf, h3_cert_path)); -} - -static const char* set_h3_key_path(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg) -{ - return set_string(cmd, arg, (const char*)offsetof(h3_server_conf, h3_key_path)); -} - static const char* set_h3_port(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg) { if (!arg || !*arg) @@ -585,6 +563,33 @@ static const char* set_h3_alt_svc_max_age(cmd_parms* cmd, void* dummy H3_UNUSED, return NULL; } +static apr_status_t ssl_ctx_cleanup(void* data) +{ + SSL_CTX_free(data); + return APR_SUCCESS; +} + +int h3_ssl_add_cert_files(server_rec* s, apr_pool_t* p, apr_array_header_t* cert_files, apr_array_header_t* key_files) +{ + CHECK(s && p && cert_files && key_files, return DECLINED;); + h3_server_conf* conf = ap_get_module_config(s->module_config, &http3_module); + if (!conf || ap_state_query(AP_SQ_MAIN_STATE) == AP_SQ_MS_CREATE_PRE_CONFIG || !ap_is_allowed_protocol(NULL, NULL, s, "h3") || cert_files->nelts == 0) + { + return DECLINED; + } + /* Still privileged here, so a root-only key loads the way it does for mod_ssl. */ + char err[H3Q_ERRLEN] = {0}; + conf->ssl_ctx = h3q_tls_ctx_create((const char* const*)cert_files->elts, (size_t)cert_files->nelts, (const char* const*)key_files->elts, (size_t)key_files->nelts, conf->h3_session_tickets != H3_FLAG_OFF, err, sizeof(err)); + if (!conf->ssl_ctx) + { + ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: %s", err); + return HTTP_INTERNAL_SERVER_ERROR; + } + apr_pool_cleanup_register(p, conf->ssl_ctx, ssl_ctx_cleanup, apr_pool_cleanup_null); + ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "mod_http3: serving HTTP/3 with mod_ssl certificate %s", APR_ARRAY_IDX(cert_files, 0, const char*)); + return DECLINED; +} + int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, server_rec* s) { CHECK(ptemp); @@ -599,7 +604,7 @@ int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool for (server_rec* vs = s; vs; vs = vs->next) { h3_server_conf* vc = ap_get_module_config(vs->module_config, &http3_module); - if (vc->h3_cert_path && vc->h3_key_path) + if (vc->ssl_ctx) { vc->host_port = get_server_port(vs); if (vc->h3_port == 0) @@ -683,34 +688,21 @@ int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool { vc->h3_idle_timeout = H3_IDLE_TIMEOUT_DEFAULT; } - conf = vc; - break; + if (!conf) + { + conf = vc; /* the first host owns the listener; the rest still advertise it */ + ap_log_error(APLOG_MARK, APLOG_INFO, 0, vs, "h3_post_config: pid=%d h3_port=%d mpm=%s threaded=%d forked=%d max_threads=%d", h3_getpid(), (int)vc->h3_port, ap_show_mpm(), mpm_query(AP_MPMQ_IS_THREADED), mpm_query(AP_MPMQ_IS_FORKED), mpm_query(AP_MPMQ_MAX_THREADS)); + } } } - CHECK(conf && conf->h3_cert_path && conf->h3_key_path, return HTTP_INTERNAL_SERVER_ERROR;); - - /* Validate cert and key files are readable */ - apr_file_t* f = NULL; - if (apr_file_open(&f, conf->h3_cert_path, APR_READ, APR_OS_DEFAULT, ptemp) != APR_SUCCESS) - { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: H3CertificatePath not readable: %s", conf->h3_cert_path); - return HTTP_INTERNAL_SERVER_ERROR; - } - apr_file_close(f); - f = NULL; - - if (apr_file_open(&f, conf->h3_key_path, APR_READ, APR_OS_DEFAULT, ptemp) != APR_SUCCESS) + if (!conf) { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: H3CertificateKeyPath not readable: %s", conf->h3_key_path); + ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: no host serves HTTP/3: add h3 to Protocols on a host with SSLEngine on"); return HTTP_INTERNAL_SERVER_ERROR; } - apr_file_close(f); h3_request_init(); - - ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "h3_post_config: pid=%d cert=%s key=%s h3_port=%d mpm=%s threaded=%d forked=%d max_threads=%d", h3_getpid(), conf->h3_cert_path, conf->h3_key_path, (int)conf->h3_port, ap_show_mpm(), mpm_query(AP_MPMQ_IS_THREADED), mpm_query(AP_MPMQ_IS_FORKED), - mpm_query(AP_MPMQ_MAX_THREADS)); return OK; } @@ -727,8 +719,6 @@ void* h3_merge_dir_config(apr_pool_t* p H3_UNUSED, void* base, void* add H3_UNUS } const command_rec h3_cmds[] = { - AP_INIT_TAKE1("H3CertificatePath", set_h3_cert_path, NULL, RSRC_CONF, "Path to the SSL certificate file for HTTP/3"), - AP_INIT_TAKE1("H3CertificateKeyPath", set_h3_key_path, NULL, RSRC_CONF, "Path to the SSL certificate key file for HTTP/3"), AP_INIT_TAKE1("H3Port", set_h3_port, NULL, RSRC_CONF, "UDP port to listen on for QUIC/HTTP-3 (default: same as main server)"), AP_INIT_TAKE1("H3MaxConcurrentStreams", set_h3_max_concurrent_streams, NULL, RSRC_CONF, "Maximum number of concurrent HTTP/3 streams per connection (default: 100)"), AP_INIT_TAKE1("H3MaxConnections", set_h3_max_connections, NULL, RSRC_CONF, "Maximum concurrent QUIC/HTTP/3 connections per child process (default: 256)"), diff --git a/mod_http3/src/h3_hooks.c b/mod_http3/src/h3_hooks.c index 4090372..fdd6f78 100644 --- a/mod_http3/src/h3_hooks.c +++ b/mod_http3/src/h3_hooks.c @@ -84,7 +84,7 @@ int h3_hook_fixups(request_rec* r) h3_server_conf* conf = ap_get_module_config(r->server->module_config, &http3_module); - if (!conf || !conf->h3_cert_path || !conf->h3_key_path || conf->h3_port == 0) + if (!conf || !conf->ssl_ctx || conf->h3_port == 0) { return DECLINED; } diff --git a/mod_http3/src/h3_io.c b/mod_http3/src/h3_io.c index 7e74937..e190205 100644 --- a/mod_http3/src/h3_io.c +++ b/mod_http3/src/h3_io.c @@ -133,10 +133,8 @@ apr_status_t h3_io_listen_start(apr_pool_t* pchild, server_rec* s, h3_server_con char qerr[H3Q_ERRLEN] = {0}; h3q_config qcfg = { - .cert_path = conf->h3_cert_path, - .key_path = conf->h3_key_path, + .ssl_ctx = conf->ssl_ctx, .address_validation = (conf->h3_address_validation != H3_FLAG_OFF), - .session_tickets = (conf->h3_session_tickets != H3_FLAG_OFF), }; io->qengine = h3q_engine_create(&qcfg, udp_fd, qerr, sizeof(qerr)); if (!io->qengine) diff --git a/mod_http3/src/h3_server.c b/mod_http3/src/h3_server.c index ec22a21..10c2cfa 100644 --- a/mod_http3/src/h3_server.c +++ b/mod_http3/src/h3_server.c @@ -86,7 +86,7 @@ static h3_server_conf* find_h3_server(server_rec* s, server_rec** out_server) while (current) { h3_server_conf* tmp = ap_get_module_config(current->module_config, &http3_module); - if (tmp && tmp->h3_cert_path && tmp->h3_key_path && !conf) + if (tmp && tmp->ssl_ctx && !conf) { conf = tmp; conf->host_port = get_server_port(current); diff --git a/mod_http3/src/mod_http3.c b/mod_http3/src/mod_http3.c index fb8db48..da71e89 100644 --- a/mod_http3/src/mod_http3.c +++ b/mod_http3/src/mod_http3.c @@ -40,7 +40,11 @@ static void register_hooks(apr_pool_t* p H3_UNUSED) { ap_hook_handler(h3_status_handler, NULL, NULL, APR_HOOK_MIDDLE); - ap_hook_post_config(h3_post_config, NULL, NULL, APR_HOOK_MIDDLE); + /* mod_ssl hands out the certificate files it resolved before it loads them; + * post_config runs after it so those files are known by then. */ + static const char* const after_ssl[] = {"mod_ssl.c", NULL}; + ap_hook_ssl_add_cert_files(h3_ssl_add_cert_files, NULL, NULL, APR_HOOK_LAST); + ap_hook_post_config(h3_post_config, after_ssl, NULL, APR_HOOK_MIDDLE); ap_hook_create_request(h3_hook_http_create_request, NULL, NULL, APR_HOOK_REALLY_FIRST); ap_hook_pre_read_request(h3_hook_pre_read_request, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_post_read_request(h3_hook_post_read_request, NULL, NULL, APR_HOOK_REALLY_FIRST); diff --git a/mod_http3/src/quic/detail/h3q_tls.c b/mod_http3/src/quic/detail/h3q_tls.c index 62a476c..6e2e72d 100644 --- a/mod_http3/src/quic/detail/h3q_tls.c +++ b/mod_http3/src/quic/detail/h3q_tls.c @@ -74,9 +74,10 @@ static void h3q_tls_keylog_cb(const SSL* ssl, const char* line) } } -SSL_CTX* h3q_tls_ctx_create(const h3q_config* cfg, char* err, size_t errlen) +SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const char* const* key_files, size_t nkeys, int session_tickets, char* err, size_t errlen) { - CHECK(cfg); + CHECK(cert_files && ncerts > 0); + CHECK(key_files || nkeys == 0); SSL_CTX* ssl_ctx = SSL_CTX_new(OSSL_QUIC_server_method()); if (!ssl_ctx) @@ -88,17 +89,21 @@ SSL_CTX* h3q_tls_ctx_create(const h3q_config* cfg, char* err, size_t errlen) SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION); - if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cfg->cert_path) <= 0 || SSL_CTX_use_PrivateKey_file(ssl_ctx, cfg->key_path, SSL_FILETYPE_PEM) <= 0) + for (size_t i = 0; i < ncerts; i++) { - h3q_tls_error(err, errlen, "loading the certificate or private key failed"); - SSL_CTX_free(ssl_ctx); - return NULL; + const char* key_file = i < nkeys ? key_files[i] : cert_files[i]; + if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_files[i]) <= 0 || SSL_CTX_use_PrivateKey_file(ssl_ctx, key_file, SSL_FILETYPE_PEM) <= 0) + { + h3q_tls_error(err, errlen, "loading certificate %s with key %s failed", cert_files[i], key_file); + SSL_CTX_free(ssl_ctx); + return NULL; + } } static const unsigned char sid_ctx[] = "mod_http3"; SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1); - if (!cfg->session_tickets) + if (!session_tickets) { /* TLS 1.3 resumption travels in tickets, so issuing none turns it off. */ SSL_CTX_set_num_tickets(ssl_ctx, 0); diff --git a/mod_http3/src/quic/h3q.c b/mod_http3/src/quic/h3q.c index db86952..ff71440 100644 --- a/mod_http3/src/quic/h3q.c +++ b/mod_http3/src/quic/h3q.c @@ -39,12 +39,13 @@ h3q_engine* h3q_engine_create(const h3q_config* cfg, int udp_fd, char* err, size } engine->peer_addr_ex_index = -1; - engine->ssl_ctx = h3q_tls_ctx_create(cfg, err, errlen); - if (!engine->ssl_ctx) + if (!cfg->ssl_ctx || !SSL_CTX_up_ref(cfg->ssl_ctx)) { + h3q_tls_error(err, errlen, "no TLS context to serve from"); h3q_engine_destroy(engine); return NULL; } + engine->ssl_ctx = cfg->ssl_ctx; BIO_METHOD* bm = BIO_meth_new(BIO_TYPE_FILTER | BIO_get_new_index(), "h3q_peer_addr"); if (!bm) diff --git a/test/http3/env.py b/test/http3/env.py index 2b821aa..8d4fdde 100644 --- a/test/http3/env.py +++ b/test/http3/env.py @@ -77,8 +77,6 @@ def add_vhost_test1( proxy_self=False, h2proxy_self=False, h3_port=True, - h3_cert_path=None, - h3_key_path=None, h3_max_concurrent_streams=None, h3_stream_buffer_size=None, h3_max_request_body_size=None, @@ -107,11 +105,6 @@ def add_vhost_test1( if h3_port: port = h3_port if not isinstance(h3_port, bool) else self.env.https_port self.add(f"H3Port {port}") - - cert = h3_cert_path if h3_cert_path else self.env.test_cert_file - key = h3_key_path if h3_key_path else self.env.test_key_file - self.add(f"H3CertificatePath {cert}") - self.add(f"H3CertificateKeyPath {key}") if h3_max_concurrent_streams is not None: self.add(f"H3MaxConcurrentStreams {h3_max_concurrent_streams}") diff --git a/test/http3/test_003_directives.py b/test/http3/test_003_directives.py index 29a35c9..0d1bcb7 100644 --- a/test/http3/test_003_directives.py +++ b/test/http3/test_003_directives.py @@ -30,12 +30,11 @@ def test_002_h3port_in_vhost(self, env): assert "H3Port" in conf assert str(env.https_port) in conf - def test_003_h3_cert_directives_in_vhost(self, env): + def test_003_h3_cert_inherited_from_mod_ssl(self, env): + # No mod_http3 certificate directive exists; mod_ssl's pair is what QUIC serves. conf = _read_test_conf(env) - assert "H3CertificatePath" in conf - assert "H3CertificateKeyPath" in conf - assert env.test_cert_file in conf - assert env.test_key_file in conf + assert "SSLCertificateFile" in conf + assert env.apache_restart() == 0 def test_004_protocols_h3_in_vhost(self, env): conf = _read_test_conf(env) From ebaef8b3fc59437d8d1646c3f9dbfd7666ab860a Mon Sep 17 00:00:00 2001 From: Alexander Gerasimov Date: Tue, 8 Sep 2026 13:50:19 +0300 Subject: [PATCH 2/5] v0.0.71 - select the HTTP/3 certificate by SNI Each virtual host on a shared port now presents its own certificate over HTTP/3, chosen by the client SNI (ServerName and exact ServerAlias names); an unmatched name gets the listener default. A cert_cb on the listener context applies the matched host certificate, key and chain to the connection: SSL_set_SSL_CTX does not switch the certificate of a QUIC connection. Landing it exposed that the module ran with AP_MODULE_FLAG_NONE, so a host without H3 directives shared the main server configuration; per-host H3AltSvc and H3AltSvcMaxAge were silently ignored. Set ALWAYS_MERGE. Test suite: stop.conf never named a pid file, so on builds whose default is run/httpd.pid "apachectl -k stop" stopped nothing and every restart talked to the previous server. With that fixed the suite runs in under two minutes: 122 passed, 2 failed (test_028, the 2.4.68 ErrorDocument crash), 5 skipped. Author: Alexander Gerasimov --- CHANGES | 16 +++++++++ docs/configuration_httpd.md | 5 ++- mod_http3/include/h3_config.h | 5 +-- mod_http3/src/h3_config.c | 49 +++++++++++++++++++++++++++- mod_http3/src/mod_http3.c | 2 +- test/http3/test_029_sni.py | 46 ++++++++++++++++++++++++++ test/pyhttpd/conf/stop.conf.template | 5 +++ 7 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 test/http3/test_029_sni.py diff --git a/CHANGES b/CHANGES index 0916300..98c9916 100644 --- a/CHANGES +++ b/CHANGES @@ -16,6 +16,22 @@ v0.0.71 (2026-09-08) removed; drop them from existing configurations. [Alexander Gerasimov ] + *) Select the HTTP/3 certificate by SNI, so each virtual host on a shared + port serves its own certificate instead of the first host's. + [Alexander Gerasimov ] + + *) Give every virtual host its own mod_http3 configuration + (AP_MODULE_FLAG_ALWAYS_MERGE). A host without H3 directives used to share + the main server's, so per-host settings such as H3AltSvc and H3AltSvcMaxAge + were silently ignored and hosts could not carry their own certificate. + [Alexander Gerasimov ] + + *) Test suite: stop.conf now names the same pid file as httpd.conf, so + "apachectl -k stop" actually stops the server on httpd builds whose + default pid file lives in run/; before, every restart in the suite kept + talking to the previous server. + [Alexander Gerasimov ] + v0.0.70 (2026-09-06) -------------------- *) SECURITY: Updated the httpd submodule so mpm_event tolerates a connection diff --git a/docs/configuration_httpd.md b/docs/configuration_httpd.md index f0d3779..6ab59d6 100644 --- a/docs/configuration_httpd.md +++ b/docs/configuration_httpd.md @@ -240,10 +240,9 @@ Use `H3Port` to bind the QUIC listener to a different UDP port than the VirtualH ### Multiple VirtualHosts -The QUIC listener presents the certificate of the **first VirtualHost** that serves HTTP/3; every other HTTP/3 host still advertises `Alt-Svc` and is selected by request authority: +All HTTP/3 hosts on a port share one QUIC listener. The certificate is chosen by the client's SNI: a host is matched on its `ServerName` and exact `ServerAlias` names (wildcard aliases are not matched) and presents its own certificate; a name that matches no host gets the listener's default, the certificate of the first HTTP/3 host in the configuration chain. Requests are then routed by authority as for TCP: ```apache -# This VirtualHost's certificate is the one QUIC presents ServerName primary.example.com SSLEngine on @@ -252,7 +251,7 @@ The QUIC listener presents the certificate of the **first VirtualHost** that ser Protocols h3 h2 http/1.1 -# Served over HTTP/3 too, but with primary's certificate +# Presents secondary.crt to clients that ask for secondary.example.com ServerName secondary.example.com SSLEngine on diff --git a/mod_http3/include/h3_config.h b/mod_http3/include/h3_config.h index ca421f2..0d9d4a9 100644 --- a/mod_http3/include/h3_config.h +++ b/mod_http3/include/h3_config.h @@ -127,8 +127,9 @@ int h3_ssl_add_cert_files(server_rec* s, apr_pool_t* p, apr_array_header_t* cert /** * ap_post_config hook: fill in defaults on every vhost that serves HTTP/3; - * the first one owns the listener. No-op in AP_SQ_MS_CREATE_PRE_CONFIG. - * @param p Config pool (unused). + * the first one owns the listener, and an SNI callback swaps in each other + * host's certificate by name. No-op in AP_SQ_MS_CREATE_PRE_CONFIG. + * @param p Config pool; owns the SNI host table. * @param plog Log pool (unused). * @param ptemp Temp pool (unused). * @param s The first server_rec in the configuration. diff --git a/mod_http3/src/h3_config.c b/mod_http3/src/h3_config.c index 389c119..be97c88 100644 --- a/mod_http3/src/h3_config.c +++ b/mod_http3/src/h3_config.c @@ -590,8 +590,36 @@ int h3_ssl_add_cert_files(server_rec* s, apr_pool_t* p, apr_array_header_t* cert return DECLINED; } -int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, server_rec* s) +/** One HTTP/3 host name and the TLS context that carries its certificate. */ +typedef struct { + const char* name; + SSL_CTX* ctx; +} h3_sni_host; + +/// cert_cb: serve each host its own certificate over one listener, selected by SNI. +/// SSL_set_SSL_CTX does not switch the certificate of a QUIC connection; applying +/// the matched host's certificate, key and chain to the connection does. +static int h3_sni_select_cert(SSL* ssl, void* arg) +{ + const apr_array_header_t* hosts = arg; + const char* sni = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + for (int i = 0; sni && i < hosts->nelts; i++) + { + const h3_sni_host* h = &APR_ARRAY_IDX(hosts, i, h3_sni_host); + if (apr_cstr_casecmp(h->name, sni) == 0) + { + STACK_OF(X509)* chain = NULL; + SSL_CTX_get0_chain_certs(h->ctx, &chain); + return SSL_use_certificate(ssl, SSL_CTX_get0_certificate(h->ctx)) == 1 && SSL_use_PrivateKey(ssl, SSL_CTX_get0_privatekey(h->ctx)) == 1 && (!chain || SSL_set1_chain(ssl, chain) == 1); + } + } + return 1; /* no match: the listener's own certificate */ +} + +int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, server_rec* s) +{ + CHECK(p); CHECK(ptemp); CHECK(s); h3_server_conf* conf = NULL; @@ -601,11 +629,27 @@ int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool return OK; } + /* Names of every HTTP/3 host, so the listener can pick a certificate by SNI. */ + apr_array_header_t* sni = apr_array_make(p, 4, sizeof(h3_sni_host)); + for (server_rec* vs = s; vs; vs = vs->next) { h3_server_conf* vc = ap_get_module_config(vs->module_config, &http3_module); if (vc->ssl_ctx) { + if (vs->server_hostname) + { + h3_sni_host* e = apr_array_push(sni); + e->name = vs->server_hostname; + e->ctx = vc->ssl_ctx; + } + /* ServerAlias exact names; wildcards are not matched (ponytail: exact only). */ + for (int i = 0; vs->names && i < vs->names->nelts; i++) + { + h3_sni_host* e = apr_array_push(sni); + e->name = APR_ARRAY_IDX(vs->names, i, const char*); + e->ctx = vc->ssl_ctx; + } vc->host_port = get_server_port(vs); if (vc->h3_port == 0) { @@ -702,6 +746,9 @@ int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool return HTTP_INTERNAL_SERVER_ERROR; } + /* The listener serves conf's certificate by default and swaps by SNI. */ + SSL_CTX_set_cert_cb(conf->ssl_ctx, h3_sni_select_cert, sni); + h3_request_init(); return OK; } diff --git a/mod_http3/src/mod_http3.c b/mod_http3/src/mod_http3.c index da71e89..a5d71f7 100644 --- a/mod_http3/src/mod_http3.c +++ b/mod_http3/src/mod_http3.c @@ -76,5 +76,5 @@ HTTP3_PUBLIC module http3_module = { h3_merge_server_config, /* merge per-server config structures */ h3_cmds, /* command apr_table_t */ register_hooks, /* register hooks */ - AP_MODULE_FLAG_NONE /* flags */ + AP_MODULE_FLAG_ALWAYS_MERGE /* every vhost gets its own config, H3 directives or not */ }; diff --git a/test/http3/test_029_sni.py b/test/http3/test_029_sni.py new file mode 100644 index 0000000..30e683f --- /dev/null +++ b/test/http3/test_029_sni.py @@ -0,0 +1,46 @@ +import os + +import pytest + +from pyhttpd.certs import HttpdTestCA + + +class TestSni: + """Each HTTP/3 host presents its own certificate, chosen by the client's SNI.""" + + @pytest.fixture(autouse=True, scope="class") + def _class_scope(self, env): + from .env import H3Conf + + # A second certificate with a different subject than the shared test one. + name = f"test2.{env.http_tld}" + creds = HttpdTestCA.create_root(name=name, store_dir=env.gen_dir) + cert = os.path.join(env.gen_dir, "test2-sni.crt") + key = os.path.join(env.gen_dir, "test2-sni.key") + creds.save_cert_pem(cert) + creds.save_pkey_pem(key) + + conf = H3Conf(env) + conf.add_vhost_test1() + conf.start_vhost([name], doc_root="htdocs/two", with_ssl=True, with_certificates=False) + conf.add(f"SSLCertificateFile {cert}") + conf.add(f"SSLCertificateKeyFile {key}") + conf.add("Protocols h3 http/1.1") + conf.end_vhost() + conf.install() + assert env.apache_restart() == 0 + + def _subject(self, env, host): + url = env.mkurl("https", host, "/index.html") + r = env.curl_get(url, options=["--http3-only", "-k", "-v"]) + assert r.exit_code == 0, r.stderr + r.stdout + assert r.response["protocol"] == "HTTP/3" + lines = [l for l in r.stderr.splitlines() if "subject:" in l] + assert lines, r.stderr + return lines[0] + + def test_001_second_host_gets_its_own_certificate(self, env): + assert f"test2.{env.http_tld}" in self._subject(env, "test2") + + def test_002_first_host_keeps_the_shared_certificate(self, env): + assert f"test2.{env.http_tld}" not in self._subject(env, "test1") diff --git a/test/pyhttpd/conf/stop.conf.template b/test/pyhttpd/conf/stop.conf.template index 21bae84..e19c263 100644 --- a/test/pyhttpd/conf/stop.conf.template +++ b/test/pyhttpd/conf/stop.conf.template @@ -5,6 +5,11 @@ ServerName localhost ServerRoot "${server_dir}" +# Same pid file as httpd.conf, or -k stop finds nothing on builds whose +# compiled-in default is run/httpd.pid and every "restart" talks to the old server. +DefaultRuntimeDir logs +PidFile httpd.pid + Include "conf/modules.conf" DocumentRoot "${server_dir}/htdocs" From 6b3726f3bbf66bf3d33eb2807b5fc9c34837b538 Mon Sep 17 00:00:00 2001 From: Alexander Gerasimov Date: Thu, 8 Oct 2026 22:57:52 +0300 Subject: [PATCH 3/5] v0.0.71 - match wildcard ServerAlias names in SNI httpd keeps wildcard ServerAlias names in wild_names, not in names. The SNI table read only names, so a client whose name matched only a wildcard alias got the listener's default certificate, and the handshake failed on the name check. Add the wild_names entries to the table, and match every entry with ap_strcasecmp_match, the matcher that httpd uses for its own virtual hosts. A plain name still matches exactly, case-insensitive. --- CHANGES | 4 +++- docs/configuration_httpd.md | 2 +- mod_http3/src/h3_config.c | 11 +++++++++-- test/http3/test_029_sni.py | 10 +++++++++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 98c9916..91515c3 100644 --- a/CHANGES +++ b/CHANGES @@ -17,7 +17,9 @@ v0.0.71 (2026-09-08) [Alexander Gerasimov ] *) Select the HTTP/3 certificate by SNI, so each virtual host on a shared - port serves its own certificate instead of the first host's. + port serves its own certificate instead of the first host's. Names + are matched as httpd matches virtual hosts, wildcard ServerAlias + included. [Alexander Gerasimov ] *) Give every virtual host its own mod_http3 configuration diff --git a/docs/configuration_httpd.md b/docs/configuration_httpd.md index 6ab59d6..0eb3115 100644 --- a/docs/configuration_httpd.md +++ b/docs/configuration_httpd.md @@ -240,7 +240,7 @@ Use `H3Port` to bind the QUIC listener to a different UDP port than the VirtualH ### Multiple VirtualHosts -All HTTP/3 hosts on a port share one QUIC listener. The certificate is chosen by the client's SNI: a host is matched on its `ServerName` and exact `ServerAlias` names (wildcard aliases are not matched) and presents its own certificate; a name that matches no host gets the listener's default, the certificate of the first HTTP/3 host in the configuration chain. Requests are then routed by authority as for TCP: +All HTTP/3 hosts on a port share one QUIC listener. The certificate is chosen by the client's SNI: a host is matched on its `ServerName` and `ServerAlias` names, wildcards included, the way httpd matches virtual hosts, and presents its own certificate; a name that matches no host gets the listener's default, the certificate of the first HTTP/3 host in the configuration chain. Requests are then routed by authority as for TCP: ```apache diff --git a/mod_http3/src/h3_config.c b/mod_http3/src/h3_config.c index be97c88..c7dce37 100644 --- a/mod_http3/src/h3_config.c +++ b/mod_http3/src/h3_config.c @@ -607,7 +607,8 @@ static int h3_sni_select_cert(SSL* ssl, void* arg) for (int i = 0; sni && i < hosts->nelts; i++) { const h3_sni_host* h = &APR_ARRAY_IDX(hosts, i, h3_sni_host); - if (apr_cstr_casecmp(h->name, sni) == 0) + /* httpd's ServerAlias matcher: wildcards for wild_names, exact otherwise. */ + if (ap_strcasecmp_match(sni, h->name) == 0) { STACK_OF(X509)* chain = NULL; SSL_CTX_get0_chain_certs(h->ctx, &chain); @@ -643,13 +644,19 @@ int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, e->name = vs->server_hostname; e->ctx = vc->ssl_ctx; } - /* ServerAlias exact names; wildcards are not matched (ponytail: exact only). */ + /* ServerAlias: httpd keeps exact names in names and wildcards in wild_names. */ for (int i = 0; vs->names && i < vs->names->nelts; i++) { h3_sni_host* e = apr_array_push(sni); e->name = APR_ARRAY_IDX(vs->names, i, const char*); e->ctx = vc->ssl_ctx; } + for (int i = 0; vs->wild_names && i < vs->wild_names->nelts; i++) + { + h3_sni_host* e = apr_array_push(sni); + e->name = APR_ARRAY_IDX(vs->wild_names, i, const char*); + e->ctx = vc->ssl_ctx; + } vc->host_port = get_server_port(vs); if (vc->h3_port == 0) { diff --git a/test/http3/test_029_sni.py b/test/http3/test_029_sni.py index 30e683f..5a3f10f 100644 --- a/test/http3/test_029_sni.py +++ b/test/http3/test_029_sni.py @@ -21,7 +21,7 @@ def _class_scope(self, env): creds.save_pkey_pem(key) conf = H3Conf(env) - conf.add_vhost_test1() + conf.add_vhost_test1(extra_lines=[f"ServerAlias *.wild.{env.http_tld}"]) conf.start_vhost([name], doc_root="htdocs/two", with_ssl=True, with_certificates=False) conf.add(f"SSLCertificateFile {cert}") conf.add(f"SSLCertificateKeyFile {key}") @@ -44,3 +44,11 @@ def test_001_second_host_gets_its_own_certificate(self, env): def test_002_first_host_keeps_the_shared_certificate(self, env): assert f"test2.{env.http_tld}" not in self._subject(env, "test1") + + def test_003_wildcard_alias_gets_its_host_certificate(self, env): + """A name matched only by test1's wildcard ServerAlias gets test1's certificate.""" + assert self._subject(env, "a.wild") == self._subject(env, "test1") + + def test_004_wildcard_does_not_match_the_bare_domain(self, env): + """*.wild. does not match wild., as in httpd, so it falls back to the listener default.""" + assert self._subject(env, "wild") != self._subject(env, "test1") From ab354f7374ba1da95f2bdd603d8223a31a2b3a4e Mon Sep 17 00:00:00 2001 From: Alexander Gerasimov Date: Thu, 8 Oct 2026 20:17:10 +0300 Subject: [PATCH 4/5] Move the QUIC transport to ngtcp2 and add H3EarlyData for 0-RTT OpenSSL's own QUIC server drops 0-RTT packets, so 0-RTT cannot work on it. This change runs the QUIC transport on ngtcp2 and keeps OpenSSL only for the TLS 1.3 handshake, through its QUIC TLS API. This is the same split that nginx uses. The h3q_ API does not change. Its implementation now reads the UDP socket, maps connection IDs to connections, answers Retry and Version Negotiation, and runs the ngtcp2 timers on a monotonic clock. The BIO filter that recovered peer addresses from the OpenSSL listener is removed. The engine does not copy stream data: it reports each acknowledgement through stream_acked, and nghttp3 releases the buffer when the peer acknowledges it. Connection close follows RFC 9000 section 10. A local close keeps its CONNECTION_CLOSE packet and resends it, at most once per PTO, for three PTOs. An idle timeout closes silently. The transport idle timer runs 2 seconds past H3IdleTimeout, so the module's clean close comes first. flush_nghttp3 now blocks a stream again each time a write blocks. nghttp3_conn_resume_stream reschedules a blocked stream, and the old code then offered the same stream forever. OpenSSL's large write buffer hid this; ngtcp2 reports the real peer flow control. A graceful restart did not advance pending handshakes while it drained. Each pending connection still counted as an MPM connection, so the old child never exited and kept the UDP port; new requests then got no answer. The event loop now advances pending handshakes while it drains. Worker threads allocate from stream pools while the event thread creates and destroys its own pools under the same session pool. These pools share one allocator, which had no lock, so two threads could get the same memory block. The interop run crashed on this. The session allocator now has a mutex, as mod_http2 does. H3EarlyData (default off) turns on 0-RTT. With it on, a returning client sends its first request with the handshake and gets the response one round trip sooner. Only safe methods run before the handshake completes; other methods wait for it, because 0-RTT data can be replayed (RFC 8470). The TLS layer sets SSL_OP_NO_ANTI_REPLAY, because OpenSSL's own anti-replay is built for TLS over TCP and refuses QUIC resumption with early data. ngtcp2 >= 1.25.0, built with OpenSSL, is a new build dependency (submodule dependencies/ngtcp2, or WITH_NGTCP2). The Windows build turns off the ngtcp2 test suite, which needs a nested submodule. --- .github/ISSUE_TEMPLATE/bug_report.md | 1 + .github/workflows/build-linux.yml | 1 + .github/workflows/build-windows.yml | 4 +- .github/workflows/interop.yml | 1 + .github/workflows/test.yml | 1 + .gitmodules | 7 + CHANGES | 25 ++ CMakeLists.txt | 4 +- INSTALL | 4 +- NOTICE | 4 + README.md | 2 + cmake/modules/ngtcp2.cmake | 37 ++ cmake/modules/unix/ngtcp2.cmake | 49 +++ cmake/modules/windows/ngtcp2.cmake | 50 +++ container/Containerfile.linux | 1 + dependencies/README.md | 12 +- dependencies/ngtcp2 | 1 + docs/architecture.md | 41 +- docs/build.md | 8 +- docs/configuration.md | 7 +- docs/configuration_httpd.md | 12 +- docs/containers.md | 2 +- docs/site/pages/index.md | 5 +- mod_http3/include/h3_config.h | 1 + mod_http3/include/h3_stream.h | 3 + mod_http3/include/quic/detail/h3q_addr.h | 114 ----- mod_http3/include/quic/detail/h3q_impl.h | 112 +++++ mod_http3/include/quic/h3q.h | 31 +- mod_http3/include/quic/h3q_conn.h | 21 +- mod_http3/include/quic/h3q_stream.h | 12 +- mod_http3/src/h3_config.c | 14 + mod_http3/src/h3_io.c | 42 +- mod_http3/src/h3_session.c | 1 + mod_http3/src/h3_stream.c | 17 +- mod_http3/src/h3_threads.c | 4 + mod_http3/src/quic/detail/h3q_addr.c | 223 ---------- mod_http3/src/quic/detail/h3q_tls.c | 15 +- mod_http3/src/quic/h3q.c | 373 ++++++++++------ mod_http3/src/quic/h3q_conn.c | 533 ++++++++++++++++++++--- mod_http3/src/quic/h3q_stream.c | 221 ++++++++-- test/http3/env.py | 3 + test/http3/test_023_resumption.py | 15 +- test/http3/test_030_early_data.py | 64 +++ 43 files changed, 1450 insertions(+), 648 deletions(-) create mode 100644 cmake/modules/ngtcp2.cmake create mode 100644 cmake/modules/unix/ngtcp2.cmake create mode 100644 cmake/modules/windows/ngtcp2.cmake create mode 160000 dependencies/ngtcp2 delete mode 100644 mod_http3/include/quic/detail/h3q_addr.h create mode 100644 mod_http3/include/quic/detail/h3q_impl.h delete mode 100644 mod_http3/src/quic/detail/h3q_addr.c create mode 100644 test/http3/test_030_early_data.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index ef60d89..c0f82c1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -31,6 +31,7 @@ A clear description of what you expected to happen. - httpd Version: [e.g. 2.5.0-trunk] - OpenSSL Version: [e.g. 3.5.0] - nghttp3 Version: [e.g. 1.17.0] + - ngtcp2 Version: [e.g. 1.25.0] - APR Version: [e.g. 1.7.0] **Crash Logs & Additional Context** diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 904bc0e..602e76e 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -29,6 +29,7 @@ jobs: run: | git submodule update --init git submodule update --init --recursive dependencies/nghttp3 + git submodule update --init dependencies/ngtcp2 - name: Set up buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index f256db6..139af6a 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -29,6 +29,7 @@ jobs: run: | git submodule update --init git submodule update --init --recursive dependencies/nghttp3 + git submodule update --init dependencies/ngtcp2 - name: Set up NASM shell: pwsh @@ -86,12 +87,13 @@ jobs: @( "dependencies/apr-dist/bin" "dependencies/nghttp3-dist/bin" + "dependencies/ngtcp2-dist/bin" "dependencies/openssl-dist/bin" "$env:VCPKG_INSTALLATION_ROOT/installed/x64-windows/bin" $redist.FullName ) | ForEach-Object { Get-ChildItem $_ -Filter *.dll | Copy-Item -Destination stage/bin -Force } - $required = 'libapr-1.dll', 'libaprutil-1.dll', 'nghttp3.dll', + $required = 'libapr-1.dll', 'libaprutil-1.dll', 'nghttp3.dll', 'ngtcp2.dll', 'ngtcp2_crypto_ossl.dll', 'libssl-3-x64.dll', 'libcrypto-3-x64.dll', 'VCRUNTIME140.dll' foreach ($dll in $required) { if (-not (Test-Path "stage/bin/$dll")) { throw "staging is missing $dll" } diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index dd47de3..e5d4a7f 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -31,6 +31,7 @@ jobs: run: | git submodule update --init git submodule update --init --recursive dependencies/nghttp3 + git submodule update --init dependencies/ngtcp2 - name: Set up buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fddb248..1c926d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,7 @@ jobs: run: | git submodule update --init git submodule update --init --recursive dependencies/nghttp3 + git submodule update --init dependencies/ngtcp2 - name: Set up buildx uses: docker/setup-buildx-action@v4 diff --git a/.gitmodules b/.gitmodules index 4631b7a..af2ce86 100644 --- a/.gitmodules +++ b/.gitmodules @@ -33,3 +33,10 @@ ignore = dirty update = checkout branch = openssl-3.5 +[submodule "dependencies/ngtcp2"] + path = dependencies/ngtcp2 + url = https://github.com/ngtcp2/ngtcp2.git + shallow = true + ignore = untracked + update = checkout + #branch = main # tag: v1.25.0 diff --git a/CHANGES b/CHANGES index 91515c3..a73913d 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,31 @@ mod_http3 changes Changes are listed most recent first. Security-related entries always appear at the top of their release block. +unreleased +-------------------- + *) Added H3EarlyData (default off). A resumed client may send its first + request as 0-RTT data and gets the response one round trip sooner. + Only safe methods run before the handshake completes; other methods + wait for it, because 0-RTT data can be replayed (RFC 8470). + [Alexander Gerasimov ] + + *) Moved the QUIC transport from the OpenSSL QUIC server to ngtcp2, with + OpenSSL as the TLS backend through its QUIC TLS API. OpenSSL still + drops server-side 0-RTT packets. ngtcp2 >= 1.25.0 is a new build + dependency (WITH_NGTCP2). Stream data is no longer copied: nghttp3 + releases a buffer when the peer acknowledges it. + [Alexander Gerasimov ] + + *) Fixed an event-thread spin when a client holds its flow-control window + shut: a blocked stream is now blocked again each time nghttp3 offers + it. H3IdleTimeout now always closes with NO_ERROR; the QUIC idle timer + runs 2 seconds longer and closes silently. A graceful restart no + longer waits forever on a connection whose handshake was still + running, so the new child gets the UDP port once the old one drains. + Fixed a crash under load: worker threads and the event thread used + pools that share one allocator, and the allocator had no lock. + [Alexander Gerasimov ] + v0.0.71 (2026-09-08) -------------------- *) SECURITY: Load the HTTP/3 certificate and key in post_config, while httpd diff --git a/CMakeLists.txt b/CMakeLists.txt index fc691f7..e02a7cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,7 @@ option(ENABLE_ASAN "Address Sanitizer" OFF) option(ENABLE_WERROR "Treat warnings as errors" OFF) set(WITH_NGHTTP3 "" CACHE PATH "Path to nghttp3 installation prefix") +set(WITH_NGTCP2 "" CACHE PATH "Path to ngtcp2 installation prefix, built with OpenSSL") set(WITH_SSL "" CACHE PATH "Path to OpenSSL installation prefix") set(WITH_HTTPD "" CACHE PATH "Path to httpd installation prefix (includes APR/APU)") set(WITH_APR "" CACHE PATH "Path to APR installation prefix") @@ -63,8 +64,9 @@ include(flags) add_library(${PROJECT_NAME}-deps INTERFACE) include(nghttp3) include(openssl) +include(ngtcp2) include(httpd) -target_link_libraries(${PROJECT_NAME}-deps INTERFACE nghttp3 openssl httpd) +target_link_libraries(${PROJECT_NAME}-deps INTERFACE nghttp3 openssl ngtcp2 httpd) # -- Core object library -- file(GLOB_RECURSE sources CONFIGURE_DEPENDS mod_http3/src/*.c) diff --git a/INSTALL b/INSTALL index 9426aae..b3871bd 100644 --- a/INSTALL +++ b/INSTALL @@ -13,6 +13,7 @@ To use system-installed dependencies, provide WITH_* paths: $ git submodule update --init dependencies/nghttp3 + $ git submodule update --init dependencies/ngtcp2 $ cmake -B build \ -DWITH_SSL=/opt/openssl \ -DWITH_HTTPD=/opt/httpd \ @@ -35,11 +36,12 @@ Requirements: - OpenSSL >= 3.5.0 (with QUIC support) + OpenSSL >= 3.5.0 (with the QUIC TLS API) httpd MMN >= 20211221 APR >= 1.7.0 APU >= 1.6.0 nghttp3 >= 1.18.0 + ngtcp2 >= 1.25.0 (built with OpenSSL) APR-util needs expat and httpd needs PCRE2. Neither is a submodule: both belong to the server stack rather than to mod_http3, and both are diff --git a/NOTICE b/NOTICE index ba62134..653885a 100644 --- a/NOTICE +++ b/NOTICE @@ -30,6 +30,10 @@ This product makes use of the following third-party libraries: Copyright 2019-2026 nghttp3 contributors Licensed under the MIT License. + ngtcp2 (https://github.com/ngtcp2/ngtcp2) + Copyright 2016-2026 ngtcp2 contributors + Licensed under the MIT License. + OpenSSL (https://www.openssl.org/) Copyright 1998-2026 The OpenSSL Project Authors Licensed under the Apache License 2.0. diff --git a/README.md b/README.md index ba7bd49..3d573e0 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Default build compiles all dependencies (OpenSSL, APR, APR-util, httpd) from sub ```sh git submodule update --init git submodule update --init --recursive dependencies/nghttp3 +git submodule update --init dependencies/ngtcp2 cmake -B build cmake --build build ``` @@ -39,6 +40,7 @@ See [INSTALL](INSTALL) for full build instructions. | `WITH_APR` | (empty) | Path to APR prefix (overrides source build) | | `WITH_APU` | (empty) | Path to APR-util prefix (overrides source build) | | `WITH_NGHTTP3` | (empty) | Path to nghttp3 prefix (overrides source build) | +| `WITH_NGTCP2` | (empty) | Path to ngtcp2 prefix, built with OpenSSL (overrides source build) | | `ENABLE_ASAN` | `OFF` | Address Sanitizer (requires `Debug`) | | `ENABLE_UBSAN` | `OFF` | UB Sanitizer (requires `Debug`) | | `ENABLE_WERROR` | `OFF` | Treat warnings as errors | diff --git a/cmake/modules/ngtcp2.cmake b/cmake/modules/ngtcp2.cmake new file mode 100644 index 0000000..5e2b4c2 --- /dev/null +++ b/cmake/modules/ngtcp2.cmake @@ -0,0 +1,37 @@ +# -- ngtcp2 -- +if(TARGET ngtcp2) + return() +endif() + +set(NGTCP2_VERSION_MIN "1.25.0") + +if(WIN32) + include(windows/ngtcp2) +else() + include(unix/ngtcp2) +endif() + +find_library(NGTCP2_LIBRARY NAMES ngtcp2 + PATHS "${NGTCP2_OUTPUT_DIRECTORY}/lib" "${NGTCP2_OUTPUT_DIRECTORY}/lib64" NO_DEFAULT_PATH) +find_library(NGTCP2_CRYPTO_OSSL_LIBRARY NAMES ngtcp2_crypto_ossl + PATHS "${NGTCP2_OUTPUT_DIRECTORY}/lib" "${NGTCP2_OUTPUT_DIRECTORY}/lib64" NO_DEFAULT_PATH) +if(NOT NGTCP2_LIBRARY OR NOT NGTCP2_CRYPTO_OSSL_LIBRARY) + message(FATAL_ERROR + "[ngtcp2] error: ngtcp2 or ngtcp2_crypto_ossl not found in ${NGTCP2_OUTPUT_DIRECTORY}. " + "ngtcp2 must be built with OpenSSL support.") +endif() + +file(READ "${NGTCP2_OUTPUT_DIRECTORY}/include/ngtcp2/version.h" _NGTCP2_VERSION_H_CONTENT) +string(REGEX MATCH "#define NGTCP2_VERSION \"([0-9]+\\.[0-9]+\\.[0-9]+)" _ "${_NGTCP2_VERSION_H_CONTENT}") +set(NGTCP2_VERSION "${CMAKE_MATCH_1}") + +if(NOT NGTCP2_VERSION OR NGTCP2_VERSION VERSION_LESS NGTCP2_VERSION_MIN) + message(FATAL_ERROR + "[ngtcp2] error: need >= ${NGTCP2_VERSION_MIN}, found ${NGTCP2_VERSION} in ${NGTCP2_OUTPUT_DIRECTORY}") +endif() + +message(STATUS "[ngtcp2] found (${NGTCP2_VERSION}): ${NGTCP2_OUTPUT_DIRECTORY}") + +add_library(ngtcp2 INTERFACE) +target_include_directories(ngtcp2 SYSTEM INTERFACE "${NGTCP2_OUTPUT_DIRECTORY}/include") +target_link_libraries(ngtcp2 INTERFACE "${NGTCP2_CRYPTO_OSSL_LIBRARY}" "${NGTCP2_LIBRARY}" openssl) diff --git a/cmake/modules/unix/ngtcp2.cmake b/cmake/modules/unix/ngtcp2.cmake new file mode 100644 index 0000000..e11b11d --- /dev/null +++ b/cmake/modules/unix/ngtcp2.cmake @@ -0,0 +1,49 @@ +if(WITH_NGTCP2) + set(NGTCP2_OUTPUT_DIRECTORY "${WITH_NGTCP2}") +else() + set(NGTCP2_DIRECTORY "${DEPENDENCIES_DIRECTORY}/ngtcp2") + set(NGTCP2_OUTPUT_DIRECTORY "${DEPENDENCIES_OUTPUT_DIRECTORY}/ngtcp2-dist") + + if(NOT EXISTS "${NGTCP2_OUTPUT_DIRECTORY}/.done") + require_initialized_submodule("${NGTCP2_DIRECTORY}") + file(MAKE_DIRECTORY "${NGTCP2_OUTPUT_DIRECTORY}/logs") + + message(STATUS "[ngtcp2] Configuring -> ${NGTCP2_OUTPUT_DIRECTORY}") + execute_process( + COMMAND autoreconf -i + WORKING_DIRECTORY "${NGTCP2_DIRECTORY}" + RESULT_VARIABLE _NGTCP2_RESULT + OUTPUT_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-autoreconf.log" + ERROR_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-autoreconf.log") + if(NOT _NGTCP2_RESULT EQUAL 0) + message(FATAL_ERROR "[ngtcp2] error: autoreconf failed -- see ${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-autoreconf.log") + endif() + + # ngtcp2 finds OpenSSL through pkg-config; point it at the one we use. + execute_process( + COMMAND ${CMAKE_COMMAND} -E env + "PKG_CONFIG_PATH=${OPENSSL_OUTPUT_DIRECTORY}/lib64/pkgconfig:${OPENSSL_OUTPUT_DIRECTORY}/lib/pkgconfig" + ./configure --prefix=${NGTCP2_OUTPUT_DIRECTORY} --enable-lib-only --with-openssl + WORKING_DIRECTORY "${NGTCP2_DIRECTORY}" + RESULT_VARIABLE _NGTCP2_RESULT + OUTPUT_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-configure.log" + ERROR_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-configure.log") + if(NOT _NGTCP2_RESULT EQUAL 0) + message(FATAL_ERROR "[ngtcp2] error: configure failed -- see ${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-configure.log") + endif() + + message(STATUS "[ngtcp2] Building (${DEPENDENCIES_PARALLEL} jobs)") + execute_process( + COMMAND make -j${DEPENDENCIES_PARALLEL} install + WORKING_DIRECTORY "${NGTCP2_DIRECTORY}" + RESULT_VARIABLE _NGTCP2_RESULT + OUTPUT_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-build.log" + ERROR_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-build.log") + if(NOT _NGTCP2_RESULT EQUAL 0) + message(FATAL_ERROR "[ngtcp2] error: build failed -- see ${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-build.log") + endif() + + string(TIMESTAMP _NGTCP2_DONE_TIME "%Y-%b-%d_%H-%M-%S") + file(WRITE "${NGTCP2_OUTPUT_DIRECTORY}/.done" "${_NGTCP2_DONE_TIME}") + endif() +endif() diff --git a/cmake/modules/windows/ngtcp2.cmake b/cmake/modules/windows/ngtcp2.cmake new file mode 100644 index 0000000..6a13598 --- /dev/null +++ b/cmake/modules/windows/ngtcp2.cmake @@ -0,0 +1,50 @@ +if(WITH_NGTCP2) + set(NGTCP2_OUTPUT_DIRECTORY "${WITH_NGTCP2}") +else() + set(NGTCP2_DIRECTORY "${DEPENDENCIES_DIRECTORY}/ngtcp2") + set(NGTCP2_OUTPUT_DIRECTORY "${DEPENDENCIES_OUTPUT_DIRECTORY}/ngtcp2-dist") + + if(NOT EXISTS "${NGTCP2_OUTPUT_DIRECTORY}/.done") + require_initialized_submodule("${NGTCP2_DIRECTORY}") + file(MAKE_DIRECTORY "${NGTCP2_OUTPUT_DIRECTORY}/logs") + + message(STATUS "[ngtcp2] Configuring -> ${NGTCP2_OUTPUT_DIRECTORY}") + execute_process( + COMMAND "${CMAKE_COMMAND}" -S "${NGTCP2_DIRECTORY}" -B "${CMAKE_BINARY_DIR}/deps/ngtcp2" -G "${CMAKE_GENERATOR}" + "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" + "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" + -DVCPKG_APPLOCAL_DEPS=OFF + "-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" + "-DCMAKE_INSTALL_PREFIX=${NGTCP2_OUTPUT_DIRECTORY}" + "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" + "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" + "-DOPENSSL_ROOT_DIR=${OPENSSL_OUTPUT_DIRECTORY}" + -DENABLE_OPENSSL=ON + -DENABLE_LIB_ONLY=ON + -DBUILD_TESTING=OFF + -DENABLE_SHARED_LIB=ON + -DENABLE_STATIC_LIB=ON + RESULT_VARIABLE _NGTCP2_RESULT + OUTPUT_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-configure.log" + ERROR_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-configure.log") + if(NOT _NGTCP2_RESULT EQUAL 0) + file(READ "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-configure.log" _NGTCP2_LOG) + message(FATAL_ERROR "[ngtcp2] error: configure failed\n${_NGTCP2_LOG}") + endif() + + message(STATUS "[ngtcp2] Building (${DEPENDENCIES_PARALLEL} jobs)") + execute_process( + COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}/deps/ngtcp2" --parallel ${DEPENDENCIES_PARALLEL} --config ${CMAKE_BUILD_TYPE} --target install + RESULT_VARIABLE _NGTCP2_RESULT + OUTPUT_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-build.log" + ERROR_FILE "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-build.log") + if(NOT _NGTCP2_RESULT EQUAL 0) + file(READ "${NGTCP2_OUTPUT_DIRECTORY}/logs/ngtcp2-build.log" _NGTCP2_LOG) + message(FATAL_ERROR "[ngtcp2] error: build failed\n${_NGTCP2_LOG}") + endif() + + string(TIMESTAMP _NGTCP2_DONE_TIME "%Y-%b-%d_%H-%M-%S") + file(WRITE "${NGTCP2_OUTPUT_DIRECTORY}/.done" "${_NGTCP2_DONE_TIME}") + endif() +endif() diff --git a/container/Containerfile.linux b/container/Containerfile.linux index d59c209..4ae9e7c 100644 --- a/container/Containerfile.linux +++ b/container/Containerfile.linux @@ -64,6 +64,7 @@ COPY --from=builder /src/dependencies/apr-dist/ /src/dependencies/apr-dist/ COPY --from=builder /src/dependencies/apr-util-dist/ /src/dependencies/apr-util-dist/ COPY --from=builder /src/dependencies/openssl-dist/ /src/dependencies/openssl-dist/ COPY --from=builder /src/dependencies/nghttp3-dist/ /src/dependencies/nghttp3-dist/ +COPY --from=builder /src/dependencies/ngtcp2-dist/ /src/dependencies/ngtcp2-dist/ COPY --from=builder /src/dependencies/httpd-dist/ /src/dependencies/httpd-dist/ COPY --from=builder /src/build/lib/mod_http3.so /src/dependencies/httpd-dist/modules/mod_http3.so diff --git a/dependencies/README.md b/dependencies/README.md index e925a4e..02d664f 100644 --- a/dependencies/README.md +++ b/dependencies/README.md @@ -13,6 +13,7 @@ mod_http3 uses **git submodules** for all dependencies. By default, all dependen | APR | `dependencies/apr` | `1.7.x` | 1.7.7 | APR v2-dev (trunk) will subsume APR-util 1.x APIs. | | APR-util | `dependencies/apr-util` | `1.6.x` | 1.6.4 | Legacy companion library; kept for APR 1.x compatibility. | | nghttp3 | `dependencies/nghttp3` | `main` | 1.17.0 | HTTP/3 framing and QPACK. | +| ngtcp2 | `dependencies/ngtcp2` | tag `v1.25.0` | 1.25.0 | QUIC transport; needs OpenSSL >= 3.5 (QUIC TLS API). | All submodules are shallow (`shallow = true`). Initialise them once: @@ -20,6 +21,7 @@ All submodules are shallow (`shallow = true`). Initialise them once: git submodule sync git submodule update --init git submodule update --init --recursive dependencies/nghttp3 +git submodule update --init dependencies/ngtcp2 ``` --- @@ -44,14 +46,15 @@ mod_http3 uses APR bucket types (`AP_BUCKET_IS_RESPONSE`, etc.) that were introd ### Default -- Build from source -CMake builds OpenSSL, APR, APR-util, httpd, and nghttp3 from their respective git submodules at **configure time**, installing each into `dependencies/-dist/`. A small marker file (`dependencies/-dist/.done`) is used to skip rebuilding dependencies that are already up to date. +CMake builds OpenSSL, APR, APR-util, httpd, nghttp3 and ngtcp2 from their respective git submodules at **configure time**, installing each into `dependencies/-dist/`. A small marker file (`dependencies/-dist/.done`) is used to skip rebuilding dependencies that are already up to date. **Build order enforced by CMake:** 1. nghttp3 (`dependencies/nghttp3`) -> `dependencies/nghttp3-dist/` 2. OpenSSL (`dependencies/openssl`) -> `dependencies/openssl-dist/` -3. APR (`dependencies/apr`) -> `dependencies/apr-dist/` -4. APR-util (`dependencies/apr-util`) -> `dependencies/apr-util-dist/` -5. httpd (`dependencies/httpd`) -> `dependencies/httpd-dist/` +3. ngtcp2 (`dependencies/ngtcp2`) -> `dependencies/ngtcp2-dist/` +4. APR (`dependencies/apr`) -> `dependencies/apr-dist/` +5. APR-util (`dependencies/apr-util`) -> `dependencies/apr-util-dist/` +6. httpd (`dependencies/httpd`) -> `dependencies/httpd-dist/` ```sh # default: builds all dependencies from source (first configure is slow; subsequent ones are instant from cache) @@ -82,6 +85,7 @@ Provide `WITH_*` paths to use system-installed dependencies instead of building | `WITH_APR=/path` | APR source build | >= 1.7.0 | | `WITH_APU=/path` | APR-util source build | >= 1.6.0 | | `WITH_NGHTTP3=/path` | nghttp3 source build | ≥ 1.16.0 | +| `WITH_NGTCP2=/path` | ngtcp2 source build, with OpenSSL | ≥ 1.25.0 | ```sh cmake -B build -DWITH_SSL=/opt/openssl -DWITH_HTTPD=/opt/httpd diff --git a/dependencies/ngtcp2 b/dependencies/ngtcp2 new file mode 160000 index 0000000..f9e9ff0 --- /dev/null +++ b/dependencies/ngtcp2 @@ -0,0 +1 @@ +Subproject commit f9e9ff01ad2c8116bc09de4f644b0028a61486a6 diff --git a/docs/architecture.md b/docs/architecture.md index e7d7358..0c0e8d3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,7 +4,7 @@ ```mermaid flowchart LR - Client[HTTP/3 client] -->|UDP QUIC + TLS 1.3| Engine[OpenSSL QUIC] + Client[HTTP/3 client] -->|UDP QUIC + TLS 1.3| Engine[ngtcp2 QUIC + OpenSSL TLS] Engine --> nghttp3[nghttp3 HTTP/3] nghttp3 --> Module[mod_http3] Module --> httpd[Apache httpd request pipeline] @@ -15,8 +15,9 @@ flowchart LR ## Layers -- **OpenSSL** owns transport and TLS 1.3: packets, loss recovery, streams and - the handshake. See [The QUIC layer](#the-quic-layer). +- **ngtcp2** owns the transport: packets, loss recovery, flow control and + streams. **OpenSSL** runs the TLS 1.3 handshake through its QUIC TLS API. + See [The QUIC layer](#the-quic-layer). - **nghttp3** handles HTTP/3 frames, streams, and QPACK interactions. - **mod_http3** bridges QUIC streams with Apache request/response processing. - **Apache httpd** supplies routing, virtual-host selection, filters, and handlers. @@ -24,24 +25,36 @@ flowchart LR ## The QUIC layer -Every OpenSSL QUIC call the module makes lives under `quic/`, behind symbols -prefixed `h3q_`. This is a wrapper, not an abstraction. There is one transport, -OpenSSL's, and the layer exists to keep `SSL*`, `BIO*` and the QUIC listener -out of the rest of the module, not to allow a second implementation. +Every ngtcp2 and OpenSSL call the module makes lives under `quic/`, behind +symbols prefixed `h3q_`. This is a wrapper, not an abstraction. There is one +transport, ngtcp2 with OpenSSL as its TLS backend, and the layer exists to keep +`ngtcp2_conn*` and `SSL*` out of the rest of the module, not to allow a second +implementation. The engine reads the UDP socket itself, maps connection IDs to +connections, answers Retry and Version Negotiation, and runs the ngtcp2 timers. -Nothing outside `quic/` includes an OpenSSL header, and the published API -documentation excludes both `detail/` directories. +Nothing outside `quic/` includes an ngtcp2 or OpenSSL header, and the +published API documentation excludes both `detail/` directories. -The module hands over one `h3q_config`, holding the certificate path, the key -path, and whether to validate client addresses with a Retry packet, and gets -back an opaque `h3q_engine`. Connections and streams are opaque too, so +The module hands over one `h3q_config`, holding the TLS context, the idle +timeout, the stream limit, whether to validate client addresses with a Retry +packet and whether to accept 0-RTT, and gets back an opaque `h3q_engine`. Connections and streams are opaque too, so `` stays out of every header above this layer. Failures come back through a caller-supplied error buffer rather than the log, because `quic/` has no `server_rec` to log against. nghttp3 sits above this layer and never sees it. One behaviour is worth -knowing: OpenSSL exposes no per-stream acknowledgements, so the module counts -bytes as acknowledged once `SSL_write_ex` accepts them. +knowing: ngtcp2 does not copy stream data, so the buffers nghttp3 hands out +stay in place until the peer acknowledges them. The engine reports each +acknowledgement through the `stream_acked` callback and the module passes it +to nghttp3, which then releases the buffer. + +0-RTT: with `H3EarlyData on`, a resumed client may send its first request +before the handshake completes. The engine queues such streams, the module +starts the session at once and runs safe methods immediately; other methods +wait until the handshake completes, because 0-RTT data can be replayed +(RFC 8470). The safe-method rule is the replay defence; the TLS layer sets +`SSL_OP_NO_ANTI_REPLAY`, since OpenSSL's own anti-replay is built for TLS +over TCP and refuses QUIC resumption with early data. ## Important Boundaries diff --git a/docs/build.md b/docs/build.md index d970113..9627366 100644 --- a/docs/build.md +++ b/docs/build.md @@ -1,11 +1,12 @@ # Build -The default build compiles OpenSSL, APR, APR-util, httpd and nghttp3 from the +The default build compiles OpenSSL, APR, APR-util, httpd, nghttp3 and ngtcp2 from the submodules. Use it when the system httpd's module magic number is too old. ```sh git submodule update --init git submodule update --init --recursive dependencies/nghttp3 +git submodule update --init dependencies/ngtcp2 cmake -B build cmake --build build ``` @@ -19,11 +20,12 @@ clones OpenSSL's external test submodules, which the build never uses. | Dependency | Minimum | | --- | --- | -| OpenSSL | 3.5.0 with QUIC support | +| OpenSSL | 3.5.0 with the QUIC TLS API | | Apache httpd | 2.5.1+ or 2.4.69+ | | APR | 1.7.0 | | APR-util | 1.6.0 | | nghttp3 | 1.18.0 | +| ngtcp2 | 1.25.0, built with OpenSSL | The submodule build produces these. Supply your own with the `WITH_*` options only if they meet the minimums; a distribution httpd is usually rejected on MMN. @@ -31,7 +33,7 @@ only if they meet the minimums; a distribution httpd is usually rejected on MMN. ## Custom Prefixes ```sh -git submodule update --init dependencies/nghttp3 +git submodule update --init dependencies/nghttp3 dependencies/ngtcp2 cmake -B build \ -DWITH_SSL=/opt/openssl \ -DWITH_HTTPD=/opt/httpd \ diff --git a/docs/configuration.md b/docs/configuration.md index 04e0e36..376c897 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -18,8 +18,9 @@ For httpd runtime directives (`Protocols h3`, `H3Port`, VirtualHost), see [httpd | `cmake -LH -N -B build` | Print all cache variables | ```sh -# Module only (with system deps via WITH_SSL/WITH_HTTPD/WITH_NGHTTP3) +# Module only (with system deps via WITH_SSL/WITH_HTTPD/WITH_NGHTTP3/WITH_NGTCP2) git submodule update --init --recursive dependencies/nghttp3 +git submodule update --init dependencies/ngtcp2 ``` Reset submodules: @@ -37,7 +38,8 @@ Build order: 1. nghttp3 -> `dependencies/nghttp3-dist/` 2. OpenSSL -> `dependencies/openssl-dist/` -3. APR -> `dependencies/apr-dist/` +3. ngtcp2 -> `dependencies/ngtcp2-dist/` +4. APR -> `dependencies/apr-dist/` 4. APR-util -> `dependencies/apr-util-dist/` 5. httpd -> `dependencies/httpd-dist/` @@ -59,6 +61,7 @@ Provide `WITH_*` variables to override individual dependencies with system-insta | APR | `WITH_APR=/path` | >= 1.7.0 | | APU | `WITH_APU=/path` | >= 1.6.0 | | nghttp3 | `WITH_NGHTTP3=/path` | >= 1.18.0 | +| ngtcp2 | `WITH_NGTCP2=/path` | >= 1.25.0, built with OpenSSL | > Distro-packaged httpd (Ubuntu, Fedora, etc.) ships with MMN < 20211221 and will fail configure. Use build-from-source mode instead. diff --git a/docs/configuration_httpd.md b/docs/configuration_httpd.md index 0eb3115..2318a03 100644 --- a/docs/configuration_httpd.md +++ b/docs/configuration_httpd.md @@ -9,7 +9,7 @@ For build and installation, see [INSTALL](../INSTALL). mod_http3 enables HTTP/3 protocol support in Apache HTTP Server. The module: - Creates a separate worker thread for HTTP/3 connections over UDP/QUIC -- Uses OpenSSL for QUIC/TLS 1.3 support +- Uses ngtcp2 for QUIC and OpenSSL for TLS 1.3 - Uses nghttp3 for HTTP/3 protocol handling - Integrates with Apache's standard request processing pipeline @@ -128,7 +128,7 @@ The timeout duration in seconds for QUIC handshakes to complete. If a connection **Context:** server config, virtual host **Default:** `300` -The idle timeout duration in seconds for QUIC connections. This maps to the standard QUIC `max_idle_timeout` transport parameter. A connection will be closed if no traffic is sent or received within this timeframe. Use a higher value for applications that require long-lived idle connections (e.g., long-polling, WebSockets over HTTP/3). +The idle timeout duration in seconds for QUIC connections. A connection with no application progress for this time is closed cleanly with NO_ERROR. The QUIC `max_idle_timeout` transport parameter is set 2 seconds higher, so the clean close comes first and the transport timer only drops a peer that went silent. Use a higher value for applications that require long-lived idle connections (e.g., long-polling, WebSockets over HTTP/3). ### H3SessionTickets @@ -138,6 +138,14 @@ The idle timeout duration in seconds for QUIC connections. This maps to the stan Whether to issue TLS 1.3 session tickets. A returning client that presents a ticket resumes its session and skips a certificate verification, which is the difference between a two-round-trip and a one-round-trip reconnect. The ticket keys are created before httpd forks, so every child process resumes tickets issued by any other; a ticket from before a restart falls back to a full handshake. Turn this off to force a full handshake on every connection. +### H3EarlyData + +**Syntax:** `H3EarlyData on|off` +**Context:** server config, virtual host +**Default:** `off` + +Whether to accept 0-RTT request data on a resumed connection. A returning client sends its first request together with the handshake and gets the response one round trip sooner. 0-RTT data can be replayed (RFC 9001 section 9.2), so only safe methods (GET, HEAD, OPTIONS, TRACE) run before the handshake completes; other methods wait for it (RFC 8470). Requires `H3SessionTickets on`. + ### H3AddressValidation **Syntax:** `H3AddressValidation on|off` diff --git a/docs/containers.md b/docs/containers.md index ba834d9..db121a6 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -120,7 +120,7 @@ podman compose --profile linux down -v Both services sit behind a profile — `linux` builds from your checkout, `windows` pulls the published image — so a bare `compose up` starts nothing. -A cold build takes about ten minutes — OpenSSL, APR, APR-util, nghttp3 and httpd +A cold build takes about ten minutes — OpenSSL, APR, APR-util, nghttp3, ngtcp2 and httpd are all compiled from source. ## Which tag to pull diff --git a/docs/site/pages/index.md b/docs/site/pages/index.md index 3fbeeda..0390447 100644 --- a/docs/site/pages/index.md +++ b/docs/site/pages/index.md @@ -2,7 +2,7 @@ ## HTTP/3 for Apache httpd -`mod_http3` is an Apache httpd module that serves HTTP/3 over QUIC. It integrates with the standard httpd request pipeline while adding a UDP/QUIC listener, TLS 1.3 handling through OpenSSL, and HTTP/3 framing through nghttp3. +`mod_http3` is an Apache httpd module that serves HTTP/3 over QUIC. It integrates with the standard httpd request pipeline while adding a UDP/QUIC listener, QUIC through ngtcp2, TLS 1.3 through OpenSSL, and HTTP/3 framing through nghttp3. The module advertises HTTP/3 with `Alt-Svc` by default, allowing compatible clients to discover the UDP endpoint from a TCP response. @@ -22,6 +22,7 @@ Configuration and C API may change between releases. Read the [versioning policy | Component | Responsibility | | --- | --- | | Apache httpd | Request routing, virtual hosts, filters, and module hosting | -| OpenSSL 3.5+ | QUIC transport and TLS 1.3 | +| ngtcp2 | QUIC transport | +| OpenSSL 3.5+ | TLS 1.3 handshake and packet protection | | nghttp3 | HTTP/3 framing and stream state | | APR / APR-util | Portable threads, pools, and sockets | diff --git a/mod_http3/include/h3_config.h b/mod_http3/include/h3_config.h index 0d9d4a9..4470a05 100644 --- a/mod_http3/include/h3_config.h +++ b/mod_http3/include/h3_config.h @@ -54,6 +54,7 @@ struct h3_server_conf apr_uint32_t h3_idle_timeout; apr_size_t h3_socket_buffer_size; h3_tri_flag h3_session_tickets; + h3_tri_flag h3_early_data; apr_uint32_t h3_stream_timeout; apr_uint32_t h3_max_stream_errors; int h3_qpack_capacity_set; diff --git a/mod_http3/include/h3_stream.h b/mod_http3/include/h3_stream.h index 9ca8b48..24b4010 100644 --- a/mod_http3/include/h3_stream.h +++ b/mod_http3/include/h3_stream.h @@ -64,4 +64,7 @@ apr_array_header_t* drain_ready_streams(h3_session* session, apr_pool_t* loop_po */ h3_stream* h3_stream_find(h3_session* session, int64_t sid); +/** h3q stream_acked callback: hand acknowledged bytes to nghttp3. */ +void h3_stream_acked(void* user, int64_t stream_id, size_t len); + #endif /* H3_STREAM_H */ diff --git a/mod_http3/include/quic/detail/h3q_addr.h b/mod_http3/include/quic/detail/h3q_addr.h deleted file mode 100644 index 1bc3041..0000000 --- a/mod_http3/include/quic/detail/h3q_addr.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2026 The mod_http3 Project Authors. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef H3Q_DETAIL_ADDR_H -#define H3Q_DETAIL_ADDR_H - -#include -#include - -#include -#include - -#include "quic/h3q.h" - -typedef struct h3q_datagram h3q_datagram; - -/** The listener's state. All but the two SSL handles exist to recover a - * connection's peer address, which OpenSSL 3.5 does not otherwise expose; - * that is why the definition lives beside the machinery that fills it. */ -struct h3q_engine -{ - SSL_CTX* ssl_ctx; - SSL* ssl_listener; - - BIO_METHOD* peer_addr_bio_method; - BIO_ADDR* current_peer_addr; - int peer_addr_ex_index; - h3q_datagram* peer_rx_head; - h3q_datagram* peer_rx_tail; -}; - -/** - * Drop every datagram still queued on the engine. - * @param engine Engine whose receive queue is emptied. - */ -void h3q_peer_addr_queue_clear(h3q_engine* engine); - -/** - * BIO_meth_set_ctrl handler for the peer-address filter BIO. - * @param bio Filter BIO receiving the control operation. - * @param cmd Control command, forwarded to the underlying BIO. - * @param num Command-specific numeric argument. - * @param ptr Command-specific pointer argument. - * @return Whatever the underlying BIO returns for @p cmd. - */ -long h3q_peer_addr_bio_ctrl(BIO* bio, int cmd, long num, void* ptr); - -/** - * BIO_meth_set_sendmmsg handler, forwarding to the underlying BIO. - * @param bio Filter BIO the datagrams are written through. - * @param msg Array of messages to send. - * @param stride Size of one entry in @p msg. - * @param num_msg Number of entries in @p msg. - * @param flags Flags passed through to the underlying BIO. - * @param msgs_processed Out: how many messages were sent. - * @return 1 on success, 0 on failure. - */ -int h3q_peer_addr_bio_sendmmsg(BIO* bio, BIO_MSG* msg, size_t stride, size_t num_msg, uint64_t flags, size_t* msgs_processed); - -/** - * BIO_meth_set_recvmmsg handler, recording each datagram's peer address. - * @param bio Filter BIO the datagrams are read through. - * @param msg Array receiving the messages. - * @param stride Size of one entry in @p msg. - * @param num_msg Capacity of @p msg. - * @param flags Flags passed through to the underlying BIO. - * @param msgs_processed Out: how many messages were received. - * @return 1 on success, 0 on failure. - */ -int h3q_peer_addr_bio_recvmmsg(BIO* bio, BIO_MSG* msg, size_t stride, size_t num_msg, uint64_t flags, size_t* msgs_processed); - -/** - * BIO_meth_set_destroy handler, clearing the datagram queue. - * @param bio Filter BIO being destroyed. - * @return 1 on success. - */ -int h3q_peer_addr_bio_destroy(BIO* bio); - -/** - * SSL ex_data free callback for a connection's stored peer address. - * @param parent Object the ex_data belongs to. - * @param ptr The stored BIO_ADDR, freed here. - * @param ad ex_data store being torn down. - * @param idx Index the value was stored at. - * @param argl Long argument registered with the index. - * @param argp Pointer argument registered with the index. - */ -void h3q_peer_addr_ex_free(void* parent, void* ptr, CRYPTO_EX_DATA* ad, int idx, long argl, void* argp); - -/** - * SSL_CTX new-pending-conn callback, attaching the peer address to @p conn. - * @param ctx Context the connection was created on. - * @param conn Newly pending connection. - * @param arg The owning h3q_engine. - * @return 1 to accept the connection, 0 to reject it. - */ -int h3q_new_pending_conn_cb(SSL_CTX* ctx, SSL* conn, void* arg); - -#endif /* H3Q_DETAIL_ADDR_H */ diff --git a/mod_http3/include/quic/detail/h3q_impl.h b/mod_http3/include/quic/detail/h3q_impl.h new file mode 100644 index 0000000..f0090e3 --- /dev/null +++ b/mod_http3/include/quic/detail/h3q_impl.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026 The mod_http3 Project Authors. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef H3Q_DETAIL_IMPL_H +#define H3Q_DETAIL_IMPL_H + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "quic/h3q.h" + +#define H3Q_SCIDLEN 18 +#define H3Q_PKT_BUF 1500 +#define H3Q_RECV_BUDGET 64 +#define H3Q_RETRY_TOKEN_TIMEOUT (10 * NGTCP2_SECONDS) + +struct h3q_engine +{ + SSL_CTX* ssl_ctx; + int fd; + apr_pool_t* pool; + apr_hash_t* conns; /* CID bytes -> h3q_conn */ + h3q_conn* conns_head; + h3q_conn* accept_head; + h3q_conn* accept_tail; + struct sockaddr_storage local; + socklen_t local_len; + void (*stream_acked)(void* user, int64_t stream_id, size_t len); + uint32_t idle_timeout_secs; + uint32_t max_streams_bidi; + uint8_t secret[32]; + unsigned address_validation : 1; + unsigned early_data : 1; +}; + +struct h3q_conn +{ + h3q_engine* engine; + apr_pool_t* pool; /* CID hash keys */ + ngtcp2_conn* qconn; + ngtcp2_crypto_conn_ref conn_ref; + ngtcp2_crypto_ossl_ctx* ossl_ctx; + SSL* ssl; + ngtcp2_path_storage path; + apr_array_header_t* cids; /* every CID put in the engine map, to remove on free */ + uint8_t* close_pkt; /* CONNECTION_CLOSE, resent while closing (RFC 9000 10.2.1) */ + size_t close_len; + ngtcp2_tstamp close_until; + ngtcp2_tstamp close_next; + int liberr; /* first local ngtcp2 error, for the log */ + h3q_stream* streams_head; + h3q_stream* accept_head; + h3q_stream* accept_tail; + void* user; + h3q_conn* next; + h3q_conn* next_accept; + unsigned handshake_done : 1; + unsigned closed : 1; + unsigned queued_accept : 1; +}; + +struct h3q_stream +{ + h3q_conn* conn; + int64_t id; + unsigned char* rx; + size_t rx_len; + size_t rx_cap; + size_t rx_off; + h3q_stream* next; + h3q_stream* next_accept; + unsigned fin : 1; + unsigned early : 1; + unsigned read_reset : 1; + unsigned write_closed : 1; + unsigned queued_accept : 1; + unsigned engine_closed : 1; + unsigned fin_pending : 1; /* FIN refused by ngtcp2 once; retried on flush */ +}; + +ngtcp2_tstamp h3q_now(void); +void h3q_send(h3q_engine* engine, const ngtcp2_path* path, const uint8_t* buf, size_t len); +h3q_conn* h3q_conn_new(h3q_engine* engine, const ngtcp2_pkt_hd* hd, const ngtcp2_cid* odcid, const ngtcp2_cid* retry_scid, const struct sockaddr* peer, socklen_t peerlen); +void h3q_conn_flush(h3q_conn* conn); +void h3q_conn_close(h3q_conn* conn, const ngtcp2_ccerr* ccerr); +void h3q_conn_fail(h3q_conn* conn, int liberr); +void h3q_conn_resend_close(h3q_conn* conn, const struct sockaddr* peer, socklen_t peerlen); +h3q_stream* h3q_stream_get(h3q_conn* conn, int64_t id); + +#endif /* H3Q_DETAIL_IMPL_H */ diff --git a/mod_http3/include/quic/h3q.h b/mod_http3/include/quic/h3q.h index 9ba8ace..c20cc4d 100644 --- a/mod_http3/include/quic/h3q.h +++ b/mod_http3/include/quic/h3q.h @@ -22,6 +22,8 @@ #include #include +#include + #include "h3_os.h" typedef struct h3q_engine h3q_engine; @@ -30,19 +32,27 @@ typedef struct h3q_stream h3q_stream; #define H3Q_ERRLEN 256 -/** Everything the listener needs to exist. The idle timeout is not here: it is - * a per-connection setting, applied by h3q_conn_prepare(). */ +/** Everything the listener needs to exist. */ typedef struct h3q_config { /** TLS context from h3q_tls_ctx_create(); the engine takes its own reference. */ struct ssl_ctx_st* ssl_ctx; + /** Parent of the engine's own pool. */ + apr_pool_t* pool; + /** Called when the peer acknowledges @p len bytes of stream @p stream_id. */ + void (*stream_acked)(void* user, int64_t stream_id, size_t len); + /** QUIC max_idle_timeout, applied to every connection. */ + uint32_t idle_timeout_secs; + /** Bidirectional streams a client may have open at once. */ + uint32_t max_streams_bidi; unsigned address_validation : 1; + /** Accept 0-RTT request data on resumed connections. */ + unsigned early_data : 1; } h3q_config; /** - * Build the QUIC listener on @p udp_fd, together with the filter BIO that - * recovers peer addresses from OpenSSL's accept queue. - * @param cfg TLS context and address validation. + * Build the QUIC listener on @p udp_fd. + * @param cfg TLS context, limits, callbacks and feature flags. * @param udp_fd Pre-opened non-blocking UDP socket bound to the listen port, * borrowed for the engine's lifetime. * @param err Buffer receiving the reason on failure; may be NULL. @@ -52,7 +62,7 @@ typedef struct h3q_config h3q_engine* h3q_engine_create(const h3q_config* cfg, int udp_fd, char* err, size_t errlen); /** - * Tear down the listener, its TLS context and any datagrams still queued. + * Tear down the listener, its TLS context and any connections still open. * @param engine Engine to destroy; NULL is ignored. */ void h3q_engine_destroy(h3q_engine* engine); @@ -61,13 +71,14 @@ void h3q_engine_destroy(h3q_engine* engine); * Drive one round of listener work: read datagrams, run timers, send. * @param engine Engine to pump; NULL reports no work. * @return 1 if work was done and another pass may be useful, 0 if idle, and - * -1 if the listener stopped processing events. + * -1 if the socket stopped delivering datagrams. */ int h3q_engine_pump(h3q_engine* engine); /** * Report what the engine needs from the next event-loop wait. - * @param engine Engine to query; NULL asks for neither, on a one-second wait. + * @param engine Engine to query; NULL keeps the defaults: reads, on a + * one-second wait. * @param want_read Out: non-zero if the socket should be polled for reads. * @param want_write Out: non-zero if the socket should be polled for writes. * @param timeout_ms In/out: lowered to the next timer when one is due sooner, @@ -76,14 +87,14 @@ int h3q_engine_pump(h3q_engine* engine); void h3q_engine_want(h3q_engine* engine, int* want_read, int* want_write, int* timeout_ms); /** - * Take the next handshaken connection off the accept queue. + * Take the next new connection off the accept queue. * @param engine Engine to accept from; NULL yields NULL. * @return Accepted connection, or NULL if none is ready. */ h3q_conn* h3q_engine_accept_conn(h3q_engine* engine); /** - * Recover the peer address the filter BIO recorded for @p conn. + * Copy the peer address from the connection's network path. * @param engine Engine owning @p conn; NULL reports failure. * @param conn Connection to inspect. * @param addr Out: peer socket address. diff --git a/mod_http3/include/quic/h3q_conn.h b/mod_http3/include/quic/h3q_conn.h index ccf7069..fa32faf 100644 --- a/mod_http3/include/quic/h3q_conn.h +++ b/mod_http3/include/quic/h3q_conn.h @@ -43,13 +43,11 @@ typedef struct h3q_tls_info int h3q_conn_tls_info(h3q_conn* conn, h3q_tls_info* out); /** - * Apply the stream modes, incoming-stream policy and idle timeout a freshly - * accepted connection needs before its handshake is driven. - * @param conn Connection to prepare; NULL reports failure. - * @param idle_timeout_secs Idle timeout to apply, in seconds. - * @return 1 on success, 0 on failure. + * Set the pointer the engine hands to the stream_acked callback. + * @param conn Connection to label; NULL is ignored. + * @param user Caller context; acknowledgements are dropped while it is NULL. */ -int h3q_conn_prepare(h3q_conn* conn, uint32_t idle_timeout_secs); +void h3q_conn_set_user(h3q_conn* conn, void* user); /** * Open a server-initiated unidirectional stream. @@ -73,6 +71,13 @@ h3q_stream* h3q_conn_accept_stream(h3q_conn* conn); */ int h3q_conn_is_handshake_done(h3q_conn* conn); +/** + * Whether 0-RTT request data waits while the handshake is still running. + * @param conn Connection to query; NULL reports 0. + * @return Non-zero when an early stream is ready before the handshake. + */ +int h3q_conn_has_early_data(h3q_conn* conn); + /** * Whether the connection has finished closing. * @param conn Connection to query; NULL counts as closed. @@ -98,8 +103,8 @@ void h3q_conn_free(h3q_conn* conn); /** * Describe why a connection closed, for logging. Reports the peer's error - * code, frame type and reason string where OpenSSL has them, and whatever the - * error queue holds otherwise. + * code, frame type and reason string where ngtcp2 has them, and whatever the + * OpenSSL error queue holds otherwise. * @param conn Connection to inspect; NULL reports the error queue alone. * @param buf Buffer receiving the description, always NUL-terminated. * @param buflen Capacity of @p buf; zero is ignored. diff --git a/mod_http3/include/quic/h3q_stream.h b/mod_http3/include/quic/h3q_stream.h index 7389e48..489938f 100644 --- a/mod_http3/include/quic/h3q_stream.h +++ b/mod_http3/include/quic/h3q_stream.h @@ -53,8 +53,9 @@ int64_t h3q_stream_id(h3q_stream* st); * @param nvec Number of buffers in @p vec. * @param fin Non-zero to close the stream after these bytes. * @return What was accepted, and whether the write blocked or broke. - * @note Bytes count as acknowledged once accepted: OpenSSL reports no - * per-stream acknowledgements. + * @note ngtcp2 does not copy: accepted bytes must stay in place until the + * stream_acked callback reports them, since loss recovery resends + * from the caller's buffers. */ h3q_write_result h3q_stream_write(h3q_stream* st, const h3q_vec* vec, size_t nvec, int fin); @@ -84,6 +85,13 @@ int h3q_stream_read(h3q_stream* st, unsigned char* buf, size_t read_size, size_t */ void h3q_stream_is_read_finished(h3q_stream* st, int* read_finished, int* write_finished); +/** + * Whether any of the stream's data arrived as 0-RTT, so it may be a replay. + * @param st Stream to query; NULL reports 0. + * @return Non-zero for a stream with 0-RTT data. + */ +int h3q_stream_is_early(h3q_stream* st); + /** * Abort the sending half of a stream. * @param st Stream to reset; NULL is ignored. diff --git a/mod_http3/src/h3_config.c b/mod_http3/src/h3_config.c index c7dce37..7c00346 100644 --- a/mod_http3/src/h3_config.c +++ b/mod_http3/src/h3_config.c @@ -75,6 +75,7 @@ void* h3_merge_server_config(apr_pool_t* p, void* base_conf, void* new_conf) merged->h3_idle_timeout = new->h3_idle_timeout ? new->h3_idle_timeout : base->h3_idle_timeout; merged->h3_socket_buffer_size = new->h3_socket_buffer_size ? new->h3_socket_buffer_size : base->h3_socket_buffer_size; merged->h3_session_tickets = new->h3_session_tickets != H3_FLAG_UNSET ? new->h3_session_tickets : base->h3_session_tickets; + merged->h3_early_data = new->h3_early_data != H3_FLAG_UNSET ? new->h3_early_data : base->h3_early_data; merged->h3_stream_timeout = new->h3_stream_timeout ? new->h3_stream_timeout : base->h3_stream_timeout; merged->h3_max_stream_errors = new->h3_max_stream_errors ? new->h3_max_stream_errors : base->h3_max_stream_errors; merged->h3_qpack_capacity_set = new->h3_qpack_capacity_set ? new->h3_qpack_capacity_set : base->h3_qpack_capacity_set; @@ -521,6 +522,14 @@ static const char* set_h3_session_tickets(cmd_parms* cmd, void* dummy H3_UNUSED, return NULL; } +static const char* set_h3_early_data(cmd_parms* cmd, void* dummy H3_UNUSED, int flag) +{ + h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module); + CHECK(conf); + conf->h3_early_data = flag ? H3_FLAG_ON : H3_FLAG_OFF; + return NULL; +} + static const char* set_h3_alt_svc(cmd_parms* cmd, void* dummy H3_UNUSED, int flag) { h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module); @@ -727,6 +736,10 @@ int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, { vc->h3_session_tickets = H3_FLAG_ON; } + if (vc->h3_early_data == H3_FLAG_UNSET) + { + vc->h3_early_data = H3_FLAG_OFF; + } if (vc->h3_alt_svc_max_age == 0) { vc->h3_alt_svc_max_age = H3_ALT_SVC_MAX_AGE_DEFAULT; @@ -786,6 +799,7 @@ const command_rec h3_cmds[] = { AP_INIT_FLAG("H3AddressValidation", set_h3_address_validation, NULL, RSRC_CONF, "Whether to validate client addresses with a QUIC Retry packet before accepting a connection (default: on)"), AP_INIT_TAKE1("H3SocketBufferSize", set_h3_socket_buffer_size, NULL, RSRC_CONF, "Bytes requested for the QUIC socket send and receive buffers; the OS may grant less (default: 2097152)"), AP_INIT_FLAG("H3SessionTickets", set_h3_session_tickets, NULL, RSRC_CONF, "Whether to issue TLS session tickets so returning clients can resume instead of running a full handshake (default: on)"), + AP_INIT_FLAG("H3EarlyData", set_h3_early_data, NULL, RSRC_CONF, "Whether to accept 0-RTT request data on resumed connections; unsafe methods wait for the handshake (default: off)"), AP_INIT_TAKE1("H3StreamTimeout", set_h3_stream_timeout, NULL, RSRC_CONF, "Seconds a response may make no progress before the stream is aborted (default: the server Timeout)"), AP_INIT_TAKE1("H3MaxStreamErrors", set_h3_max_stream_errors, NULL, RSRC_CONF, "Stream errors one connection may cause before it is closed with H3_EXCESSIVE_LOAD (default: 8)"), AP_INIT_TAKE1("H3QpackTableCapacity", set_h3_qpack_table_capacity, NULL, RSRC_CONF, "Bytes of QPACK dynamic table a client may use for request headers; 0 disables it (default: 4096)"), diff --git a/mod_http3/src/h3_io.c b/mod_http3/src/h3_io.c index e190205..e41d575 100644 --- a/mod_http3/src/h3_io.c +++ b/mod_http3/src/h3_io.c @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -134,6 +135,11 @@ apr_status_t h3_io_listen_start(apr_pool_t* pchild, server_rec* s, h3_server_con char qerr[H3Q_ERRLEN] = {0}; h3q_config qcfg = { .ssl_ctx = conf->ssl_ctx, + .pool = pchild, + .stream_acked = h3_stream_acked, + .idle_timeout_secs = conf->h3_idle_timeout, + .max_streams_bidi = conf->h3_max_concurrent_streams, + .early_data = (conf->h3_early_data == H3_FLAG_ON), .address_validation = (conf->h3_address_validation != H3_FLAG_OFF), }; io->qengine = h3q_engine_create(&qcfg, udp_fd, qerr, sizeof(qerr)); @@ -269,6 +275,15 @@ static apr_status_t spawn_serviced_session(h3_io_t* io, h3q_conn* conn) } apr_allocator_owner_set(allocator, session_pool); apr_pool_tag(session_pool, "h3_session"); + /* Workers allocate from stream subpools while this thread makes its own: share the allocator safely. */ + apr_thread_mutex_t* alloc_mutex = NULL; + if (apr_thread_mutex_create(&alloc_mutex, APR_THREAD_MUTEX_DEFAULT, session_pool) != APR_SUCCESS) + { + apr_pool_destroy(session_pool); + h3q_conn_free(conn); + return APR_EGENERAL; + } + apr_allocator_mutex_set(allocator, alloc_mutex); h3_session* session = NULL; if (h3_session_create(&session, io->server, conn, session_pool) != APR_SUCCESS) @@ -332,14 +347,14 @@ void progress_pending_handshakes(h3_io_t* io) rv = -1; why = "peer closed the connection during the handshake"; } - else if (h3q_conn_is_handshake_done(conn)) + else if (h3q_conn_is_handshake_done(conn) || h3q_conn_has_early_data(conn)) { rv = 1; } if (rv == 1) { - ap_log_error(APLOG_MARK, APLOG_INFO, 0, io->server, "QUIC handshake complete"); + ap_log_error(APLOG_MARK, APLOG_INFO, 0, io->server, h3q_conn_is_handshake_done(conn) ? "QUIC handshake complete" : "QUIC 0-RTT request before the handshake"); spawn_serviced_session(io, conn); remove_pending_handshake(io, i, 0); finished = 1; @@ -362,13 +377,6 @@ void progress_pending_handshakes(h3_io_t* io) int prepare_accepted_connection(h3_io_t* io, h3q_conn* conn) { - h3_server_conf* conf = ap_get_module_config(io->server->module_config, &http3_module); - if (!h3q_conn_prepare(conn, conf->h3_idle_timeout)) - { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, io->server, "h3q_conn_prepare failed for accepted connection - dropping it"); - return 0; - } - h3_pending_handshake* pending = (h3_pending_handshake*)apr_array_push(io->pending_handshakes); pending->conn = conn; pending->accepted_at = apr_time_now(); @@ -380,6 +388,13 @@ int prepare_accepted_connection(h3_io_t* io, h3q_conn* conn) return 1; } +/* RFC 9110 section 9.2.1. */ +static int h3_method_is_safe(const char* method) +{ + int m = method ? ap_method_number_of(method) : M_INVALID; + return m == M_GET || m == M_OPTIONS || m == M_TRACE; +} + static void abandon_stalled_responses(h3_session* session, apr_interval_time_t stall_timeout) { if (stall_timeout <= 0) @@ -452,7 +467,9 @@ int service_session_pass(h3_io_t* io, h3_session* session) if (h3q_conn_is_closed(conn)) { - ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "QUIC connection terminated (idle timeout, peer close, or transport error)"); + char detail[H3Q_ERRLEN] = {0}; + h3q_conn_close_reason(conn, detail, sizeof(detail)); + ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "QUIC connection terminated (idle timeout, peer close, or transport error) close=[%s]", detail); session->aborted = 1; return 0; } @@ -500,6 +517,11 @@ int service_session_pass(h3_io_t* io, h3_session* session) for (int i = 0; i < completed->nelts; i++) { h3_stream* h3s = ((h3_stream**)completed->elts)[i]; + /* 0-RTT data can be a replay: unsafe methods wait for the handshake (RFC 8470 section 3). */ + if (h3s->qstream && h3q_stream_is_early(h3s->qstream) && !h3q_conn_is_handshake_done(conn) && !h3_method_is_safe(h3s->method)) + { + continue; + } h3_process_request(session, h3s); } diff --git a/mod_http3/src/h3_session.c b/mod_http3/src/h3_session.c index 319f1b6..a11c6c0 100644 --- a/mod_http3/src/h3_session.c +++ b/mod_http3/src/h3_session.c @@ -68,6 +68,7 @@ apr_status_t h3_session_create(h3_session** psession, server_rec* s, h3q_conn* q session->s = s; session->pool = pool; session->qconn = qconn; + h3q_conn_set_user(qconn, session); session->streams = apr_hash_make(pool); session->pending_free = apr_array_make(pool, 8, sizeof(h3q_stream*)); session->last_activity = apr_time_now(); diff --git a/mod_http3/src/h3_stream.c b/mod_http3/src/h3_stream.c index 894b5bd..eb43ff0 100644 --- a/mod_http3/src/h3_stream.c +++ b/mod_http3/src/h3_stream.c @@ -125,16 +125,16 @@ void flush_nghttp3(h3_session* session) continue; } nghttp3_conn_add_write_offset(session->ngh3, sid, res.accepted); - nghttp3_conn_add_ack_offset(session->ngh3, sid, res.accepted); if (res.blocked) { - /* Send buffer full: skip this stream instead of busy-looping on the same vec. */ + /* Send buffer full: skip this stream instead of busy-looping on the same vec. + * Block it again every time: resume_stream reschedules a blocked stream. */ if (!h3s->write_blocked) { h3s->write_blocked = 1; session->blocked_streams++; - nghttp3_conn_block_stream(session->ngh3, sid); } + nghttp3_conn_block_stream(session->ngh3, sid); continue; } } @@ -430,3 +430,14 @@ apr_array_header_t* drain_ready_streams(h3_session* session, apr_pool_t* loop_po return completed; } + +void h3_stream_acked(void* user, int64_t stream_id, size_t len) +{ + h3_session* session = user; + apr_thread_mutex_lock(session->lock); + if (!session->ngh3_dead) + { + (void)nghttp3_conn_add_ack_offset(session->ngh3, stream_id, len); + } + apr_thread_mutex_unlock(session->lock); +} diff --git a/mod_http3/src/h3_threads.c b/mod_http3/src/h3_threads.c index bfe1d66..68dbc50 100644 --- a/mod_http3/src/h3_threads.c +++ b/mod_http3/src/h3_threads.c @@ -89,6 +89,10 @@ void* APR_THREAD_FUNC h3_event_thread(apr_thread_t* thread, void* data) h3q_conn_free(conn); } } + } + if (io->thread_running) + { + /* Also while draining: a pending handshake counts as an MPM connection, so it must finish or time out. */ progress_pending_handshakes(io); } diff --git a/mod_http3/src/quic/detail/h3q_addr.c b/mod_http3/src/quic/detail/h3q_addr.c deleted file mode 100644 index 126cc85..0000000 --- a/mod_http3/src/quic/detail/h3q_addr.c +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright (c) 2026 The mod_http3 Project Authors. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -#include "h3_os.h" -#include "quic/detail/h3q_addr.h" - -struct h3q_datagram -{ - unsigned char* data; - size_t data_len; - BIO_ADDR* peer; - BIO_ADDR* local; - h3q_datagram* next; -}; - -static void h3q_datagram_free(h3q_datagram* item) -{ - OPENSSL_free(item->data); - BIO_ADDR_free(item->peer); - BIO_ADDR_free(item->local); - OPENSSL_free(item); -} - -static void h3q_queue_drop_head(h3q_engine* engine) -{ - h3q_datagram* item = engine->peer_rx_head; - engine->peer_rx_head = item->next; - if (!engine->peer_rx_head) - { - engine->peer_rx_tail = NULL; - } - h3q_datagram_free(item); -} - -void h3q_peer_addr_queue_clear(h3q_engine* engine) -{ - h3q_datagram* item = engine->peer_rx_head; - while (item) - { - h3q_datagram* next = item->next; - h3q_datagram_free(item); - item = next; - } - engine->peer_rx_head = NULL; - engine->peer_rx_tail = NULL; -} - -static int h3q_queue_fill(h3q_engine* engine, BIO_MSG* msg, size_t stride, size_t count) -{ - for (size_t i = 0; i < count; i++) - { - BIO_MSG* source = (BIO_MSG*)((unsigned char*)msg + i * stride); - h3q_datagram* item = OPENSSL_zalloc(sizeof(*item)); - if (!item || !source->data || source->data_len == 0) - { - OPENSSL_free(item); - h3q_peer_addr_queue_clear(engine); - return 0; - } - item->data = OPENSSL_memdup(source->data, source->data_len); - item->data_len = source->data_len; - item->peer = source->peer ? BIO_ADDR_dup(source->peer) : NULL; - item->local = source->local ? BIO_ADDR_dup(source->local) : NULL; - if (!item->data || (source->peer && !item->peer) || (source->local && !item->local)) - { - OPENSSL_free(item->data); - BIO_ADDR_free(item->peer); - BIO_ADDR_free(item->local); - OPENSSL_free(item); - h3q_peer_addr_queue_clear(engine); - return 0; - } - if (engine->peer_rx_tail) - { - engine->peer_rx_tail->next = item; - } - else - { - engine->peer_rx_head = item; - } - engine->peer_rx_tail = item; - } - return 1; -} - -static int h3q_queue_pop(h3q_engine* engine, BIO_MSG* msg) -{ - h3q_datagram* item = engine->peer_rx_head; - if (!item || !msg || !msg->data) - { - return 0; - } - if (msg->data_len < item->data_len) - { - /* Drop it, or the queue head never clears and the pump spins. */ - h3q_queue_drop_head(engine); - return 0; - } - memcpy(msg->data, item->data, item->data_len); - msg->data_len = item->data_len; - if (msg->peer && item->peer) - { - BIO_ADDR_copy(msg->peer, item->peer); - } - if (msg->local && item->local) - { - BIO_ADDR_copy(msg->local, item->local); - } - h3q_queue_drop_head(engine); - return 1; -} - -long h3q_peer_addr_bio_ctrl(BIO* bio, int cmd, long num, void* ptr) -{ - BIO* next = BIO_next(bio); - return next ? BIO_ctrl(next, cmd, num, ptr) : 0; -} - -int h3q_peer_addr_bio_sendmmsg(BIO* bio, BIO_MSG* msg, size_t stride, size_t num_msg, uint64_t flags, size_t* msgs_processed) -{ - BIO* next = BIO_next(bio); - return next ? BIO_sendmmsg(next, msg, stride, num_msg, flags, msgs_processed) : 0; -} - -int h3q_peer_addr_bio_recvmmsg(BIO* bio, BIO_MSG* msg, size_t stride, size_t num_msg, uint64_t flags, size_t* msgs_processed) -{ - h3q_engine* engine = BIO_get_data(bio); - BIO* next = BIO_next(bio); - if (!engine || !next || !msg || !msgs_processed || num_msg == 0) - { - return 0; - } - - BIO_ADDR_clear(engine->current_peer_addr); - if (engine->peer_rx_head) - { - *msgs_processed = 0; - if (!h3q_queue_pop(engine, msg)) - { - return 0; - } - *msgs_processed = 1; - if (msg->peer) - { - BIO_ADDR_copy(engine->current_peer_addr, msg->peer); - } - return 1; - } - - size_t received = 0; - int rv = BIO_recvmmsg(next, msg, stride, num_msg, flags, &received); - if (rv && received > 0) - { - if (!h3q_queue_fill(engine, msg, stride, received) || !h3q_queue_pop(engine, msg)) - { - *msgs_processed = 0; - return 0; - } - *msgs_processed = 1; - if (msg->peer) - { - BIO_ADDR_copy(engine->current_peer_addr, msg->peer); - } - } - else - { - *msgs_processed = received; - } - return rv; -} - -int h3q_peer_addr_bio_destroy(BIO* bio) -{ - h3q_engine* engine = BIO_get_data(bio); - if (engine) - { - h3q_peer_addr_queue_clear(engine); - } - return 1; -} - -void h3q_peer_addr_ex_free(void* parent H3_UNUSED, void* ptr, CRYPTO_EX_DATA* ad H3_UNUSED, int idx H3_UNUSED, long argl H3_UNUSED, void* argp H3_UNUSED) -{ - BIO_ADDR_free(ptr); -} - -int h3q_new_pending_conn_cb(SSL_CTX* ctx H3_UNUSED, SSL* conn, void* arg) -{ - h3q_engine* engine = arg; - if (!engine || engine->peer_addr_ex_index < 0 || BIO_ADDR_family(engine->current_peer_addr) == AF_UNSPEC) - { - return 1; - } - - BIO_ADDR* peer = BIO_ADDR_dup(engine->current_peer_addr); - if (!peer || !SSL_set_ex_data(conn, engine->peer_addr_ex_index, peer)) - { - BIO_ADDR_free(peer); - return 0; - } - return 1; -} diff --git a/mod_http3/src/quic/detail/h3q_tls.c b/mod_http3/src/quic/detail/h3q_tls.c index 6e2e72d..d467649 100644 --- a/mod_http3/src/quic/detail/h3q_tls.c +++ b/mod_http3/src/quic/detail/h3q_tls.c @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -79,7 +80,12 @@ SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const CHECK(cert_files && ncerts > 0); CHECK(key_files || nkeys == 0); - SSL_CTX* ssl_ctx = SSL_CTX_new(OSSL_QUIC_server_method()); + if (ngtcp2_crypto_ossl_init() != 0) + { + h3q_tls_error(err, errlen, "ngtcp2_crypto_ossl_init failed; OpenSSL lacks the QUIC TLS API"); + return NULL; + } + SSL_CTX* ssl_ctx = SSL_CTX_new(TLS_server_method()); if (!ssl_ctx) { h3q_tls_error(err, errlen, "SSL_CTX_new failed"); @@ -103,6 +109,13 @@ SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const static const unsigned char sid_ctx[] = "mod_http3"; SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1); + /* H3EarlyData is per connection: SSL_set_quic_tls_early_data_enabled makes a + * ticket advertise 0-RTT (0xffffffff, RFC 9001 4.6.1), so a ticket issued + * with it off invites none. OpenSSL's anti-replay is built for TLS over TCP + * and refuses QUIC resumption with early data; QUIC 0-RTT replay safety comes + * from accepting only safe methods before the handshake (RFC 8470), in h3_io. */ + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_ANTI_REPLAY); + if (!session_tickets) { /* TLS 1.3 resumption travels in tickets, so issuing none turns it off. */ diff --git a/mod_http3/src/quic/h3q.c b/mod_http3/src/quic/h3q.c index ff71440..79b0519 100644 --- a/mod_http3/src/quic/h3q.c +++ b/mod_http3/src/quic/h3q.c @@ -16,101 +16,216 @@ * limitations under the License. */ +#include #include #include +#include -#include -#include +#include #include "h3_check.h" #include "h3_os.h" -#include "quic/detail/h3q_addr.h" +#include "quic/detail/h3q_impl.h" #include "quic/detail/h3q_tls.h" #include "quic/h3q.h" +#include "quic/h3q_conn.h" -h3q_engine* h3q_engine_create(const h3q_config* cfg, int udp_fd, char* err, size_t errlen) +#ifdef _WIN32 +typedef int h3q_iolen; /* winsock takes int lengths */ +#else +typedef size_t h3q_iolen; +#endif + +/* ngtcp2 aborts if time goes back, so use a monotonic clock, never the wall clock. */ +ngtcp2_tstamp h3q_now(void) { - CHECK(cfg); - h3q_engine* engine = calloc(1, sizeof(*engine)); - if (!engine) +#ifdef _WIN32 + LARGE_INTEGER f, c; + QueryPerformanceFrequency(&f); + QueryPerformanceCounter(&c); + return (ngtcp2_tstamp)(c.QuadPart / f.QuadPart) * NGTCP2_SECONDS + (ngtcp2_tstamp)(c.QuadPart % f.QuadPart) * NGTCP2_SECONDS / (ngtcp2_tstamp)f.QuadPart; +#else + struct timespec tp; + clock_gettime(CLOCK_MONOTONIC, &tp); + return (ngtcp2_tstamp)tp.tv_sec * NGTCP2_SECONDS + (ngtcp2_tstamp)tp.tv_nsec; +#endif +} + +static void send_raw(h3q_engine* engine, const struct sockaddr* dst, socklen_t dstlen, const uint8_t* buf, size_t len) +{ + /* A failed send is a lost datagram; loss recovery resends it. */ + (void)sendto(engine->fd, (const char*)buf, (h3q_iolen)len, 0, dst, dstlen); +} + +void h3q_send(h3q_engine* engine, const ngtcp2_path* path, const uint8_t* buf, size_t len) +{ + send_raw(engine, (const struct sockaddr*)path->remote.addr, (socklen_t)path->remote.addrlen, buf, len); +} + +/* A peer that offers an unknown version is told which versions we speak. */ +static void send_version_negotiation(h3q_engine* engine, const ngtcp2_version_cid* vc, const struct sockaddr* peer, socklen_t peerlen) +{ + static const uint32_t versions[] = {NGTCP2_PROTO_VER_V1, NGTCP2_PROTO_VER_V2}; + uint8_t rnd = 0; + uint8_t buf[H3Q_PKT_BUF]; + if (RAND_bytes(&rnd, 1) != 1) { - h3q_tls_error(err, errlen, "allocating the engine failed"); - return NULL; + return; + } + ngtcp2_ssize n = ngtcp2_pkt_write_version_negotiation(buf, sizeof(buf), rnd, vc->scid, vc->scidlen, vc->dcid, vc->dcidlen, versions, 2); + if (n > 0) + { + send_raw(engine, peer, peerlen, buf, (size_t)n); } - engine->peer_addr_ex_index = -1; +} - if (!cfg->ssl_ctx || !SSL_CTX_up_ref(cfg->ssl_ctx)) +static void send_retry(h3q_engine* engine, const ngtcp2_pkt_hd* hd, const struct sockaddr* peer, socklen_t peerlen) +{ + ngtcp2_cid scid = {.datalen = H3Q_SCIDLEN}; + uint8_t token[NGTCP2_CRYPTO_MAX_RETRY_TOKENLEN2]; + uint8_t buf[H3Q_PKT_BUF]; + if (RAND_bytes(scid.data, (int)scid.datalen) != 1) { - h3q_tls_error(err, errlen, "no TLS context to serve from"); - h3q_engine_destroy(engine); - return NULL; + return; } - engine->ssl_ctx = cfg->ssl_ctx; + ngtcp2_ssize tokenlen = ngtcp2_crypto_generate_retry_token2(token, engine->secret, sizeof(engine->secret), hd->version, (const ngtcp2_sockaddr*)peer, (ngtcp2_socklen)peerlen, &scid, &hd->dcid, h3q_now()); + if (tokenlen < 0) + { + return; + } + ngtcp2_ssize n = ngtcp2_crypto_write_retry(buf, sizeof(buf), hd->version, &hd->scid, &scid, &hd->dcid, token, (size_t)tokenlen); + if (n > 0) + { + send_raw(engine, peer, peerlen, buf, (size_t)n); + } +} - BIO_METHOD* bm = BIO_meth_new(BIO_TYPE_FILTER | BIO_get_new_index(), "h3q_peer_addr"); - if (!bm) +static h3q_conn* accept_initial(h3q_engine* engine, const uint8_t* pkt, size_t pktlen, const struct sockaddr* peer, socklen_t peerlen) +{ + ngtcp2_pkt_hd hd; + if (ngtcp2_accept(&hd, pkt, pktlen) != 0) { - h3q_tls_error(err, errlen, "BIO_meth_new failed"); - h3q_engine_destroy(engine); return NULL; } - engine->peer_addr_bio_method = bm; - /* An unset handler installs fine and then drops every datagram. */ - if (!BIO_meth_set_ctrl(bm, h3q_peer_addr_bio_ctrl) || !BIO_meth_set_sendmmsg(bm, h3q_peer_addr_bio_sendmmsg) || !BIO_meth_set_recvmmsg(bm, h3q_peer_addr_bio_recvmmsg) || !BIO_meth_set_destroy(bm, h3q_peer_addr_bio_destroy)) + if (!engine->address_validation) { - h3q_tls_error(err, errlen, "installing the peer address BIO handlers failed"); - h3q_engine_destroy(engine); - return NULL; + return h3q_conn_new(engine, &hd, NULL, NULL, peer, peerlen); } - - engine->current_peer_addr = BIO_ADDR_new(); - engine->peer_addr_ex_index = SSL_get_ex_new_index(0, NULL, NULL, NULL, h3q_peer_addr_ex_free); - if (!engine->current_peer_addr || engine->peer_addr_ex_index < 0) + ngtcp2_cid odcid; + if (hd.tokenlen == 0 || hd.token[0] != NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY2 || ngtcp2_crypto_verify_retry_token2(&odcid, hd.token, hd.tokenlen, engine->secret, sizeof(engine->secret), hd.version, (const ngtcp2_sockaddr*)peer, (ngtcp2_socklen)peerlen, &hd.dcid, H3Q_RETRY_TOKEN_TIMEOUT, h3q_now()) != 0) { - h3q_tls_error(err, errlen, "initializing peer address recovery failed"); - h3q_engine_destroy(engine); + send_retry(engine, &hd, peer, peerlen); return NULL; } + return h3q_conn_new(engine, &hd, &odcid, &hd.dcid, peer, peerlen); +} - SSL_CTX_set_new_pending_conn_cb(engine->ssl_ctx, h3q_new_pending_conn_cb, engine); +static void process_dgram(h3q_engine* engine, uint8_t* buf, size_t len, struct sockaddr_storage* peer, socklen_t peerlen) +{ + ngtcp2_version_cid vc; + int rv = ngtcp2_pkt_decode_version_cid(&vc, buf, len, H3Q_SCIDLEN); + if (rv != 0) + { + if (rv == NGTCP2_ERR_VERSION_NEGOTIATION) + { + send_version_negotiation(engine, &vc, (struct sockaddr*)peer, peerlen); + } + return; + } + h3q_conn* conn = apr_hash_get(engine->conns, vc.dcid, (apr_ssize_t)vc.dcidlen); + if (!conn) + { + conn = accept_initial(engine, buf, len, (struct sockaddr*)peer, peerlen); + } + if (conn && conn->close_len) + { + h3q_conn_resend_close(conn, (struct sockaddr*)peer, peerlen); + return; + } + if (!conn || conn->closed) + { + return; + } + ngtcp2_path path = { + .local = {.addr = (ngtcp2_sockaddr*)&engine->local, .addrlen = (ngtcp2_socklen)engine->local_len}, + .remote = {.addr = (ngtcp2_sockaddr*)peer, .addrlen = (ngtcp2_socklen)peerlen}, + }; + ngtcp2_pkt_info pi = {0}; + rv = ngtcp2_conn_read_pkt(conn->qconn, &path, &pi, buf, len, h3q_now()); + switch (rv) + { + case 0: + h3q_conn_flush(conn); + return; + case NGTCP2_ERR_RETRY: + { + ngtcp2_pkt_hd hd; + if (ngtcp2_accept(&hd, buf, len) == 0) + { + send_retry(engine, &hd, (struct sockaddr*)peer, peerlen); + } + return; + } + case NGTCP2_ERR_DROP_CONN: + case NGTCP2_ERR_DRAINING: + case NGTCP2_ERR_CLOSING: + conn->closed = 1; + return; + default: + h3q_conn_fail(conn, rv); + return; + } +} - uint64_t listener_flags = cfg->address_validation ? 0 : (uint64_t)SSL_LISTENER_FLAG_NO_VALIDATE; - engine->ssl_listener = SSL_new_listener(engine->ssl_ctx, listener_flags); - if (!engine->ssl_listener) +/* 0: nothing to read now. 1: that datagram is lost, read on. -1: the socket is broken. */ +static int recv_failed(void) +{ +#ifdef _WIN32 + int e = WSAGetLastError(); + if (e == WSAEWOULDBLOCK) { - h3q_tls_error(err, errlen, "SSL_new_listener failed"); - h3q_engine_destroy(engine); - return NULL; + return 0; + } + return (e == WSAEINTR || e == WSAECONNRESET || e == WSAEMSGSIZE) ? 1 : -1; +#else + if (errno == EAGAIN) + { + return 0; } + return (errno == EINTR || errno == ECONNREFUSED) ? 1 : -1; +#endif +} - BIO* bio = BIO_new_dgram(udp_fd, BIO_NOCLOSE); - if (!bio) +h3q_engine* h3q_engine_create(const h3q_config* cfg, int udp_fd, char* err, size_t errlen) +{ + CHECK(cfg); + if (!cfg->ssl_ctx || !cfg->pool) { - h3q_tls_error(err, errlen, "BIO_new_dgram failed for fd=%d", udp_fd); - h3q_engine_destroy(engine); + h3q_tls_error(err, errlen, "no TLS context to serve from"); return NULL; } - - BIO* filter_bio = BIO_new(bm); - if (!filter_bio) + h3q_engine* engine = calloc(1, sizeof(*engine)); + if (!engine || apr_pool_create(&engine->pool, cfg->pool) != APR_SUCCESS) { - h3q_tls_error(err, errlen, "BIO_new(h3q_peer_addr) failed"); - BIO_free(bio); - h3q_engine_destroy(engine); + free(engine); + h3q_tls_error(err, errlen, "allocating the engine failed"); return NULL; } - - BIO_set_data(filter_bio, engine); - bio = BIO_push(filter_bio, bio); - SSL_set_bio(engine->ssl_listener, bio, bio); - - if (!SSL_listen(engine->ssl_listener) || !SSL_set_blocking_mode(engine->ssl_listener, 0)) + engine->conns = apr_hash_make(engine->pool); + engine->fd = udp_fd; + engine->stream_acked = cfg->stream_acked; + engine->idle_timeout_secs = cfg->idle_timeout_secs; + engine->max_streams_bidi = cfg->max_streams_bidi; + engine->address_validation = cfg->address_validation; + engine->early_data = cfg->early_data; + engine->local_len = (socklen_t)sizeof(engine->local); + if (getsockname(udp_fd, (struct sockaddr*)&engine->local, &engine->local_len) != 0 || RAND_bytes(engine->secret, (int)sizeof(engine->secret)) != 1 || !SSL_CTX_up_ref(cfg->ssl_ctx)) { - h3q_tls_error(err, errlen, "SSL_listen failed"); + h3q_tls_error(err, errlen, "reading the local address of fd=%d failed", udp_fd); h3q_engine_destroy(engine); return NULL; } - + engine->ssl_ctx = cfg->ssl_ctx; return engine; } @@ -120,19 +235,11 @@ void h3q_engine_destroy(h3q_engine* engine) { return; } - h3q_peer_addr_queue_clear(engine); - if (engine->ssl_listener) + while (engine->conns_head) { - SSL_free(engine->ssl_listener); - } - if (engine->current_peer_addr) - { - BIO_ADDR_free(engine->current_peer_addr); - } - if (engine->peer_addr_bio_method) - { - BIO_meth_free(engine->peer_addr_bio_method); + h3q_conn_free(engine->conns_head); } + apr_pool_destroy(engine->pool); if (engine->ssl_ctx) { SSL_CTX_free(engine->ssl_ctx); @@ -142,111 +249,103 @@ void h3q_engine_destroy(h3q_engine* engine) int h3q_engine_pump(h3q_engine* engine) { - if (!engine || !engine->ssl_listener) + if (!engine) { return 0; } + uint8_t buf[65536]; int work = 0; - if (SSL_handle_events(engine->ssl_listener) != 1) + for (int i = 0; i < H3Q_RECV_BUDGET; i++) { - return -1; + struct sockaddr_storage peer; + socklen_t peerlen = (socklen_t)sizeof(peer); + int n = (int)recvfrom(engine->fd, (char*)buf, (h3q_iolen)sizeof(buf), 0, (struct sockaddr*)&peer, &peerlen); + if (n < 0) + { + int why = recv_failed(); + if (why == 0) + { + break; + } + if (why < 0) + { + return -1; + } + continue; + } + work = 1; + if (n > 0) + { + process_dgram(engine, buf, (size_t)n, &peer, peerlen); + } } - while (engine->peer_rx_head) + ngtcp2_tstamp now = h3q_now(); + for (h3q_conn* conn = engine->conns_head; conn; conn = conn->next) { - if (SSL_handle_events(engine->ssl_listener) != 1) + if (conn->closed || ngtcp2_conn_get_expiry(conn->qconn) > now) { - return -1; + continue; } - work = 1; + int rv = ngtcp2_conn_handle_expiry(conn->qconn, now); + if (rv != 0) + { + h3q_conn_fail(conn, rv); + continue; + } + h3q_conn_flush(conn); } return work; } void h3q_engine_want(h3q_engine* engine, int* want_read, int* want_write, int* timeout_ms) { - if (!engine || !engine->ssl_listener) + *want_read = 1; + *want_write = 0; + if (!engine) { - *want_read = 0; - *want_write = 0; - *timeout_ms = 1000; return; } - *want_read = SSL_net_read_desired(engine->ssl_listener); - *want_write = SSL_net_write_desired(engine->ssl_listener); - - struct timeval tv = {0}; - int is_infinite = 0; - if (SSL_get_event_timeout(engine->ssl_listener, &tv, &is_infinite) && !is_infinite) + ngtcp2_tstamp now = h3q_now(); + for (h3q_conn* conn = engine->conns_head; conn; conn = conn->next) { - /* long is 32-bit on Windows, so tv_sec * 1000 overflows it. */ - int64_t ms = (int64_t)tv.tv_sec * 1000 + (int64_t)tv.tv_usec / 1000; - if (ms >= 0 && ms < (int64_t)*timeout_ms) + if (conn->closed) + { + continue; + } + ngtcp2_tstamp expiry = ngtcp2_conn_get_expiry(conn->qconn); + int64_t ms = expiry <= now ? 1 : (int64_t)((expiry - now) / NGTCP2_MILLISECONDS) + 1; + if (ms < *timeout_ms) { *timeout_ms = (int)ms; } } - /* A zero would spin the caller's wait when a timer reports as due. */ - if (*timeout_ms < 1) - { - *timeout_ms = 1; - } } h3q_conn* h3q_engine_accept_conn(h3q_engine* engine) { - if (!engine || !engine->ssl_listener) + if (!engine || !engine->accept_head) { return NULL; } - SSL* conn = SSL_accept_connection(engine->ssl_listener, SSL_ACCEPT_CONNECTION_NO_BLOCK); - return (h3q_conn*)conn; + h3q_conn* conn = engine->accept_head; + engine->accept_head = conn->next_accept; + if (!engine->accept_head) + { + engine->accept_tail = NULL; + } + conn->next_accept = NULL; + conn->queued_accept = 0; + return conn; } int h3q_engine_peer_addr(h3q_engine* engine, h3q_conn* conn, struct sockaddr_storage* addr, socklen_t* addr_len) { - /* Report it: losing a client address is not worth aborting the child. */ - if (!engine || !conn || !addr || !addr_len) - { - return 0; - } - if (engine->peer_addr_ex_index < 0) - { - return 0; - } - - /* Recorded by h3q_new_pending_conn_cb when the connection first appeared. */ - const BIO_ADDR* peer = SSL_get_ex_data((SSL*)conn, engine->peer_addr_ex_index); - if (!peer) + (void)engine; + if (!conn || !addr || !addr_len || conn->path.path.remote.addrlen == 0 || conn->path.path.remote.addrlen > sizeof(*addr)) { return 0; } - - memset(addr, 0, sizeof(*addr)); - size_t rawlen = 0; - int family = BIO_ADDR_family(peer); - if (family == AF_INET) - { - struct sockaddr_in* sin = (struct sockaddr_in*)addr; - if (!BIO_ADDR_rawaddress(peer, &sin->sin_addr, &rawlen) || rawlen != sizeof(sin->sin_addr)) - { - return 0; - } - sin->sin_family = AF_INET; - sin->sin_port = BIO_ADDR_rawport(peer); - *addr_len = sizeof(*sin); - return 1; - } - if (family == AF_INET6) - { - struct sockaddr_in6* sin6 = (struct sockaddr_in6*)addr; - if (!BIO_ADDR_rawaddress(peer, &sin6->sin6_addr, &rawlen) || rawlen != sizeof(sin6->sin6_addr)) - { - return 0; - } - sin6->sin6_family = AF_INET6; - sin6->sin6_port = BIO_ADDR_rawport(peer); - *addr_len = sizeof(*sin6); - return 1; - } - return 0; + memcpy(addr, conn->path.path.remote.addr, conn->path.path.remote.addrlen); + *addr_len = (socklen_t)conn->path.path.remote.addrlen; + return 1; } diff --git a/mod_http3/src/quic/h3q_conn.c b/mod_http3/src/quic/h3q_conn.c index ed43c8c..6eb718c 100644 --- a/mod_http3/src/quic/h3q_conn.c +++ b/mod_http3/src/quic/h3q_conn.c @@ -17,65 +17,389 @@ */ #include +#include #include #include -#include +#include #include "h3_check.h" +#include "quic/detail/h3q_impl.h" #include "quic/h3q_conn.h" -int h3q_conn_prepare(h3q_conn* conn, uint32_t idle_timeout_secs) +/* CID bytes are copied into the connection pool, so the hash key outlives the callback. */ +static void cid_add(h3q_conn* conn, const ngtcp2_cid* cid) { - SSL* ssl_conn = (SSL*)conn; - if (!ssl_conn || !SSL_set_blocking_mode(ssl_conn, 0)) + apr_hash_set(conn->engine->conns, apr_pmemdup(conn->pool, cid->data, cid->datalen), (apr_ssize_t)cid->datalen, conn); + APR_ARRAY_PUSH(conn->cids, ngtcp2_cid) = *cid; +} + +static h3q_stream* stream_lookup(h3q_conn* conn, int64_t id) +{ + return ngtcp2_conn_get_stream_user_data(conn->qconn, id); +} + +static int cb_handshake_completed(ngtcp2_conn* qconn, void* user_data) +{ + (void)qconn; + ((h3q_conn*)user_data)->handshake_done = 1; + return 0; +} + +static int cb_recv_stream_data(ngtcp2_conn* qconn, uint32_t flags, int64_t stream_id, uint64_t offset, const uint8_t* data, size_t datalen, void* user_data, void* stream_user_data) +{ + (void)qconn; + (void)offset; + (void)stream_user_data; + h3q_conn* conn = user_data; + h3q_stream* st = h3q_stream_get(conn, stream_id); + if (!st) + { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + if (datalen > 0) + { + size_t need = st->rx_len + datalen; + if (need > st->rx_cap) + { + size_t cap = st->rx_cap ? st->rx_cap : 4096; + while (cap < need) + { + cap *= 2; + } + unsigned char* grown = realloc(st->rx, cap); + if (!grown) + { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + st->rx = grown; + st->rx_cap = cap; + } + memcpy(st->rx + st->rx_len, data, datalen); + st->rx_len += datalen; + } + st->fin |= (flags & NGTCP2_STREAM_DATA_FLAG_FIN) != 0; + st->early |= (flags & NGTCP2_STREAM_DATA_FLAG_0RTT) != 0; + if (!st->queued_accept) + { + st->queued_accept = 1; + if (conn->accept_tail) + { + conn->accept_tail->next_accept = st; + } + else + { + conn->accept_head = st; + } + conn->accept_tail = st; + } + return 0; +} + +static int cb_acked_stream_data_offset(ngtcp2_conn* qconn, int64_t stream_id, uint64_t offset, uint64_t datalen, void* user_data, void* stream_user_data) +{ + (void)qconn; + (void)offset; + (void)stream_user_data; + h3q_conn* conn = user_data; + if (conn->user && conn->engine->stream_acked) + { + conn->engine->stream_acked(conn->user, stream_id, (size_t)datalen); + } + return 0; +} + +static int cb_stream_close(ngtcp2_conn* qconn, uint32_t flags, int64_t stream_id, uint64_t app_error_code, void* user_data, void* stream_user_data) +{ + (void)qconn; + (void)flags; + (void)app_error_code; + (void)stream_user_data; + h3q_stream* st = stream_lookup(user_data, stream_id); + if (st) + { + st->fin = 1; + st->write_closed = 1; + st->engine_closed = 1; + } + return 0; +} + +static int cb_stream_reset(ngtcp2_conn* qconn, int64_t stream_id, uint64_t final_size, uint64_t app_error_code, void* user_data, void* stream_user_data) +{ + (void)qconn; + (void)final_size; + (void)app_error_code; + (void)stream_user_data; + h3q_stream* st = stream_lookup(user_data, stream_id); + if (st) + { + st->read_reset = 1; + st->fin = 1; + } + return 0; +} + +static void cb_rand(uint8_t* dest, size_t destlen, const ngtcp2_rand_ctx* rand_ctx) +{ + (void)rand_ctx; + RAND_bytes(dest, (int)destlen); +} + +static int cb_get_new_connection_id(ngtcp2_conn* qconn, ngtcp2_cid* cid, uint8_t* token, size_t cidlen, void* user_data) +{ + (void)qconn; + h3q_conn* conn = user_data; + if (RAND_bytes(cid->data, (int)cidlen) != 1) + { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + cid->datalen = cidlen; + if (ngtcp2_crypto_generate_stateless_reset_token(token, conn->engine->secret, sizeof(conn->engine->secret), cid) != 0) + { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + cid_add(conn, cid); + return 0; +} + +static int cb_remove_connection_id(ngtcp2_conn* qconn, const ngtcp2_cid* cid, void* user_data) +{ + (void)qconn; + h3q_conn* conn = user_data; + apr_hash_set(conn->engine->conns, cid->data, (apr_ssize_t)cid->datalen, NULL); + return 0; +} + +static ngtcp2_conn* conn_ref_get_conn(ngtcp2_crypto_conn_ref* conn_ref) +{ + return ((h3q_conn*)conn_ref->user_data)->qconn; +} + +static int tls_init(h3q_conn* conn) +{ + if (ngtcp2_crypto_ossl_ctx_new(&conn->ossl_ctx, NULL) != 0 || !(conn->ssl = SSL_new(conn->engine->ssl_ctx))) { return 0; } - SSL_set_default_stream_mode(ssl_conn, SSL_DEFAULT_STREAM_MODE_NONE); - SSL_set_incoming_stream_policy(ssl_conn, SSL_INCOMING_STREAM_POLICY_ACCEPT, 0); - SSL_set_generic_value_uint(ssl_conn, SSL_VALUE_QUIC_IDLE_TIMEOUT, (uint64_t)idle_timeout_secs * 1000); + ngtcp2_crypto_ossl_ctx_set_ssl(conn->ossl_ctx, conn->ssl); + if (ngtcp2_crypto_ossl_configure_server_session(conn->ssl) != 0) + { + return 0; + } + conn->conn_ref.get_conn = conn_ref_get_conn; + conn->conn_ref.user_data = conn; + SSL_set_app_data(conn->ssl, &conn->conn_ref); + SSL_set_accept_state(conn->ssl); + if (conn->engine->early_data) + { + SSL_set_quic_tls_early_data_enabled(conn->ssl, 1); + } + ngtcp2_conn_set_tls_native_handle(conn->qconn, conn->ossl_ctx); return 1; } +h3q_conn* h3q_conn_new(h3q_engine* engine, const ngtcp2_pkt_hd* hd, const ngtcp2_cid* odcid, const ngtcp2_cid* retry_scid, const struct sockaddr* peer, socklen_t peerlen) +{ + h3q_conn* conn = calloc(1, sizeof(*conn)); + if (!conn) + { + return NULL; + } + conn->engine = engine; + conn->next = engine->conns_head; + engine->conns_head = conn; + ngtcp2_cid scid = {.datalen = H3Q_SCIDLEN}; + if (apr_pool_create(&conn->pool, engine->pool) != APR_SUCCESS || RAND_bytes(scid.data, (int)scid.datalen) != 1) + { + h3q_conn_free(conn); + return NULL; + } + conn->cids = apr_array_make(conn->pool, 4, sizeof(ngtcp2_cid)); + ngtcp2_path_storage_init(&conn->path, (const ngtcp2_sockaddr*)&engine->local, (ngtcp2_socklen)engine->local_len, (const ngtcp2_sockaddr*)peer, (ngtcp2_socklen)peerlen, NULL); + + ngtcp2_settings settings; + ngtcp2_settings_default(&settings); + settings.initial_ts = h3q_now(); + + ngtcp2_transport_params params; + ngtcp2_transport_params_default(¶ms); + /* 2s past H3IdleTimeout: the module's clean close (checked each second) must come first. */ + params.max_idle_timeout = ((ngtcp2_duration)engine->idle_timeout_secs + 2) * NGTCP2_SECONDS; + params.initial_max_data = 1024 * 1024; + params.initial_max_stream_data_bidi_remote = 256 * 1024; + params.initial_max_stream_data_uni = 256 * 1024; + params.initial_max_streams_bidi = engine->max_streams_bidi; + params.initial_max_streams_uni = 100; + params.original_dcid = odcid ? *odcid : hd->dcid; + params.original_dcid_present = 1; + if (retry_scid) + { + params.retry_scid = *retry_scid; + params.retry_scid_present = 1; + } + params.stateless_reset_token_present = ngtcp2_crypto_generate_stateless_reset_token(params.stateless_reset_token, engine->secret, sizeof(engine->secret), &scid) == 0; + + ngtcp2_callbacks cb = { + .recv_client_initial = ngtcp2_crypto_recv_client_initial_cb, + .recv_crypto_data = ngtcp2_crypto_recv_crypto_data_cb, + .encrypt = ngtcp2_crypto_encrypt_cb, + .decrypt = ngtcp2_crypto_decrypt_cb, + .hp_mask = ngtcp2_crypto_hp_mask_cb, + .update_key = ngtcp2_crypto_update_key_cb, + .delete_crypto_aead_ctx = ngtcp2_crypto_delete_crypto_aead_ctx_cb, + .delete_crypto_cipher_ctx = ngtcp2_crypto_delete_crypto_cipher_ctx_cb, + .get_path_challenge_data = ngtcp2_crypto_get_path_challenge_data_cb, + .version_negotiation = ngtcp2_crypto_version_negotiation_cb, + .handshake_completed = cb_handshake_completed, + .recv_stream_data = cb_recv_stream_data, + .acked_stream_data_offset = cb_acked_stream_data_offset, + .stream_close = cb_stream_close, + .stream_reset = cb_stream_reset, + .rand = cb_rand, + .get_new_connection_id = cb_get_new_connection_id, + .remove_connection_id = cb_remove_connection_id, + }; + if (ngtcp2_conn_server_new(&conn->qconn, &hd->scid, &scid, &conn->path.path, hd->version, &cb, &settings, ¶ms, NULL, conn) != 0 || !tls_init(conn)) + { + h3q_conn_free(conn); + return NULL; + } + cid_add(conn, &scid); + /* A retransmitted Initial still carries the original DCID. */ + cid_add(conn, &hd->dcid); + conn->queued_accept = 1; + if (engine->accept_tail) + { + engine->accept_tail->next_accept = conn; + } + else + { + engine->accept_head = conn; + } + engine->accept_tail = conn; + return conn; +} + +/* nghttp3 offers a FIN once; one ngtcp2 refused is retried here. */ +static void retry_pending_fins(h3q_conn* conn) +{ + uint8_t buf[H3Q_PKT_BUF]; + for (h3q_stream* st = conn->streams_head; st; st = st->next) + { + if (!st->fin_pending || st->write_closed) + { + continue; + } + ngtcp2_ssize ndatalen = 0; + ngtcp2_pkt_info pi; + ngtcp2_path_storage ps; + ngtcp2_path_storage_zero(&ps); + ngtcp2_ssize n = ngtcp2_conn_writev_stream(conn->qconn, &ps.path, &pi, buf, sizeof(buf), &ndatalen, NGTCP2_WRITE_STREAM_FLAG_FIN, st->id, NULL, 0, h3q_now()); + if (n < 0) + { + if (n != NGTCP2_ERR_STREAM_DATA_BLOCKED) + { + st->fin_pending = 0; + st->write_closed = 1; + } + continue; + } + if (n == 0) + { + continue; /* congestion limited; the next expiry retries */ + } + h3q_send(conn->engine, &ps.path, buf, (size_t)n); + if (ndatalen >= 0) + { + st->fin_pending = 0; + st->write_closed = 1; + } + } +} + +void h3q_conn_flush(h3q_conn* conn) +{ + if (!conn || conn->closed) + { + return; + } + retry_pending_fins(conn); + uint8_t buf[H3Q_PKT_BUF]; + for (;;) + { + ngtcp2_path_storage ps; + ngtcp2_pkt_info pi; + ngtcp2_path_storage_zero(&ps); + ngtcp2_ssize n = ngtcp2_conn_write_pkt(conn->qconn, &ps.path, &pi, buf, sizeof(buf), h3q_now()); + if (n < 0) + { + h3q_conn_fail(conn, (int)n); + return; + } + if (n == 0) + { + break; + } + h3q_send(conn->engine, &ps.path, buf, (size_t)n); + } + /* Without this ngtcp2 never paces and bursts the whole window. */ + ngtcp2_conn_update_pkt_tx_time(conn->qconn, h3q_now()); +} + +void h3q_conn_set_user(h3q_conn* conn, void* user) +{ + if (conn) + { + conn->user = user; + } +} + h3q_stream* h3q_conn_open_uni_stream(h3q_conn* conn, int64_t* out_id) { - SSL* ssl_conn = (SSL*)conn; - CHECK(ssl_conn); + CHECK(conn); CHECK(out_id); - SSL* stream = SSL_new_stream(ssl_conn, SSL_STREAM_FLAG_UNI); - if (!stream) + if (ngtcp2_conn_open_uni_stream(conn->qconn, out_id, NULL) != 0) { return NULL; } - *out_id = (int64_t)SSL_get_stream_id(stream); - return (h3q_stream*)stream; + return h3q_stream_get(conn, *out_id); } h3q_stream* h3q_conn_accept_stream(h3q_conn* conn) { - SSL* ssl_conn = (SSL*)conn; - if (!ssl_conn) + if (!conn || !conn->accept_head) { return NULL; } - return (h3q_stream*)SSL_accept_stream(ssl_conn, SSL_ACCEPT_STREAM_NO_BLOCK); + h3q_stream* st = conn->accept_head; + conn->accept_head = st->next_accept; + if (!conn->accept_head) + { + conn->accept_tail = NULL; + } + st->next_accept = NULL; + st->queued_accept = 0; + return st; } int h3q_conn_is_handshake_done(h3q_conn* conn) { - SSL* ssl_conn = (SSL*)conn; - return ssl_conn ? SSL_is_init_finished(ssl_conn) : 0; + return conn ? (int)conn->handshake_done : 0; +} + +int h3q_conn_has_early_data(h3q_conn* conn) +{ + return conn && !conn->handshake_done && conn->accept_head != NULL; } int h3q_conn_tls_info(h3q_conn* conn, h3q_tls_info* out) { - SSL* ssl_conn = (SSL*)conn; - if (!ssl_conn || !out) + if (!conn || !out) { return 0; } - const SSL_CIPHER* cipher = SSL_get_current_cipher(ssl_conn); + const SSL_CIPHER* cipher = SSL_get_current_cipher(conn->ssl); if (!cipher) { return 0; @@ -84,63 +408,157 @@ int h3q_conn_tls_info(h3q_conn* conn, h3q_tls_info* out) out->cipher_bits = SSL_CIPHER_get_bits(cipher, &alg_bits); out->cipher_alg_bits = alg_bits; out->cipher = SSL_CIPHER_get_name(cipher); - const char* version = SSL_get_version(ssl_conn); - out->protocol = (version && strncmp(version, "TLS", 3) == 0) ? version : "TLSv1.3"; - out->resumed = SSL_session_reused(ssl_conn) ? 1u : 0u; + out->protocol = SSL_get_version(conn->ssl); + out->resumed = SSL_session_reused(conn->ssl) ? 1u : 0u; return 1; } int h3q_conn_is_closed(h3q_conn* conn) { - SSL* ssl_conn = (SSL*)conn; - return ssl_conn ? (SSL_get_shutdown(ssl_conn) != 0) : 1; + return !conn || conn->closed || ngtcp2_conn_in_closing_period(conn->qconn) || ngtcp2_conn_in_draining_period(conn->qconn); } -int h3q_conn_shutdown(h3q_conn* conn, int is_rapid, uint64_t app_error, const char* reason) +/* Send CONNECTION_CLOSE once and keep it, so the closing period can resend it. */ +void h3q_conn_close(h3q_conn* conn, const ngtcp2_ccerr* ccerr) { - SSL* ssl_conn = (SSL*)conn; - if (!ssl_conn) + if (conn->closed) { - return 1; + return; } - uint64_t flags = is_rapid ? (uint64_t)SSL_SHUTDOWN_FLAG_RAPID : 0; - int ret = 0; - if (reason) + conn->closed = 1; + if (ngtcp2_conn_in_closing_period(conn->qconn) || ngtcp2_conn_in_draining_period(conn->qconn)) { - SSL_SHUTDOWN_EX_ARGS args = {.quic_error_code = app_error, .quic_reason = reason}; - ret = SSL_shutdown_ex(ssl_conn, flags, &args, sizeof(args)); + return; } - else if (flags != 0) + conn->close_pkt = apr_palloc(conn->pool, H3Q_PKT_BUF); + ngtcp2_path_storage ps; + ngtcp2_pkt_info pi; + ngtcp2_path_storage_zero(&ps); + ngtcp2_ssize n = ngtcp2_conn_write_connection_close(conn->qconn, &ps.path, &pi, conn->close_pkt, H3Q_PKT_BUF, ccerr, h3q_now()); + if (n > 0) { - SSL_SHUTDOWN_EX_ARGS args = {0}; - ret = SSL_shutdown_ex(ssl_conn, flags, &args, sizeof(args)); + ngtcp2_duration pto = ngtcp2_conn_get_pto(conn->qconn); + conn->close_len = (size_t)n; + conn->close_until = h3q_now() + 3 * pto; + conn->close_next = h3q_now() + pto; + h3q_send(conn->engine, &ps.path, conn->close_pkt, conn->close_len); + } +} + +/* Close on a local ngtcp2 error and keep the error for the log. */ +void h3q_conn_fail(h3q_conn* conn, int liberr) +{ + if (!conn->liberr) + { + conn->liberr = liberr; + } + /* Idle timeout and drop: close silently, never send CONNECTION_CLOSE (RFC 9000 10.1). */ + if (liberr == NGTCP2_ERR_IDLE_CLOSE || liberr == NGTCP2_ERR_DROP_CONN) + { + conn->closed = 1; + return; + } + ngtcp2_ccerr ccerr; + if (liberr == NGTCP2_ERR_CRYPTO) + { + ngtcp2_ccerr_set_tls_alert(&ccerr, ngtcp2_conn_get_tls_alert(conn->qconn), NULL, 0); } else { - ret = SSL_shutdown(ssl_conn); + ngtcp2_ccerr_set_liberr(&ccerr, liberr, NULL, 0); + } + h3q_conn_close(conn, &ccerr); +} + +/* Closing period: answer the peer with the stored close, at most once per PTO. */ +void h3q_conn_resend_close(h3q_conn* conn, const struct sockaddr* peer, socklen_t peerlen) +{ + ngtcp2_tstamp now = h3q_now(); + if (now < conn->close_next) + { + return; } + conn->close_next = now + ngtcp2_conn_get_pto(conn->qconn); + ngtcp2_path path = {.remote = {.addr = (ngtcp2_sockaddr*)peer, .addrlen = (ngtcp2_socklen)peerlen}}; + h3q_send(conn->engine, &path, conn->close_pkt, conn->close_len); +} - if (ret == 1) +int h3q_conn_shutdown(h3q_conn* conn, int is_rapid, uint64_t app_error, const char* reason) +{ + if (!conn) { return 1; } - if (ret < 0) + ngtcp2_ccerr ccerr; + ngtcp2_ccerr_default(&ccerr); + if (reason) { - int err = SSL_get_error(ssl_conn, ret); - if (err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) - { - return 1; - } + ngtcp2_ccerr_set_application_error(&ccerr, app_error, (const uint8_t*)reason, strlen(reason)); } - return 0; + h3q_conn_close(conn, &ccerr); + return is_rapid || conn->close_len == 0 || h3q_now() >= conn->close_until; } void h3q_conn_free(h3q_conn* conn) { - if (conn) + if (!conn) { - SSL_free((SSL*)conn); + return; } + h3q_engine* engine = conn->engine; + for (h3q_conn** slot = &engine->conns_head; *slot; slot = &(*slot)->next) + { + if (*slot == conn) + { + *slot = conn->next; + break; + } + } + for (h3q_conn** slot = &engine->accept_head; *slot; slot = &(*slot)->next_accept) + { + if (*slot == conn) + { + *slot = conn->next_accept; + if (!*slot) + { + engine->accept_tail = NULL; + } + break; + } + } + for (int i = 0; conn->cids && i < conn->cids->nelts; i++) + { + const ngtcp2_cid* cid = &APR_ARRAY_IDX(conn->cids, i, ngtcp2_cid); + if (apr_hash_get(engine->conns, cid->data, (apr_ssize_t)cid->datalen) == conn) + { + apr_hash_set(engine->conns, cid->data, (apr_ssize_t)cid->datalen, NULL); + } + } + if (conn->qconn) + { + ngtcp2_conn_del(conn->qconn); + } + if (conn->ssl) + { + SSL_set_app_data(conn->ssl, NULL); + SSL_free(conn->ssl); + } + if (conn->ossl_ctx) + { + ngtcp2_crypto_ossl_ctx_del(conn->ossl_ctx); + } + while (conn->streams_head) + { + h3q_stream* next = conn->streams_head->next; + free(conn->streams_head->rx); + free(conn->streams_head); + conn->streams_head = next; + } + if (conn->pool) + { + apr_pool_destroy(conn->pool); + } + free(conn); } void h3q_conn_close_reason(h3q_conn* conn, char* buf, size_t buflen) @@ -149,18 +567,17 @@ void h3q_conn_close_reason(h3q_conn* conn, char* buf, size_t buflen) { return; } - buf[0] = '\0'; - char errbuf[H3Q_ERRLEN] = {0}; ERR_error_string_n(ERR_peek_last_error(), errbuf, sizeof(errbuf)); - - SSL_CONN_CLOSE_INFO cci = {0}; - if (conn && SSL_get_conn_close_info((SSL*)conn, &cci, sizeof(cci))) + if (conn && conn->liberr) + { + snprintf(buf, buflen, "local %s (%s)", ngtcp2_strerror(conn->liberr), errbuf); + return; + } + const ngtcp2_ccerr* cc = conn && conn->qconn ? ngtcp2_conn_get_ccerr(conn->qconn) : NULL; + if (cc) { - const char* origin = (cci.flags & SSL_CONN_CLOSE_FLAG_LOCAL) ? "local" : "remote"; - const char* layer = (cci.flags & SSL_CONN_CLOSE_FLAG_TRANSPORT) ? "transport" : "app"; - const char* reason = cci.reason ? cci.reason : ""; - snprintf(buf, buflen, "%s %s err=0x%llx frame=0x%llx reason=\"%.*s\" (%s)", origin, layer, (unsigned long long)cci.error_code, (unsigned long long)cci.frame_type, (int)cci.reason_len, reason, errbuf); + snprintf(buf, buflen, "type=%d err=0x%llx frame=0x%llx reason=\"%.*s\" (%s)", (int)cc->type, (unsigned long long)cc->error_code, (unsigned long long)cc->frame_type, (int)cc->reasonlen, cc->reason ? (const char*)cc->reason : "", errbuf); return; } snprintf(buf, buflen, "%s", errbuf); diff --git a/mod_http3/src/quic/h3q_stream.c b/mod_http3/src/quic/h3q_stream.c index 09d590b..5f51ff6 100644 --- a/mod_http3/src/quic/h3q_stream.c +++ b/mod_http3/src/quic/h3q_stream.c @@ -16,109 +16,236 @@ * limitations under the License. */ -#include +#include +#include +#include +#include +#include "quic/detail/h3q_impl.h" #include "quic/h3q_stream.h" +static_assert(sizeof(h3q_vec) == sizeof(ngtcp2_vec) && offsetof(h3q_vec, len) == offsetof(ngtcp2_vec, len), "h3q_vec must match ngtcp2_vec"); + +h3q_stream* h3q_stream_get(h3q_conn* conn, int64_t id) +{ + h3q_stream* st = ngtcp2_conn_get_stream_user_data(conn->qconn, id); + if (st) + { + return st; + } + st = calloc(1, sizeof(*st)); + if (!st) + { + return NULL; + } + st->conn = conn; + st->id = id; + st->next = conn->streams_head; + conn->streams_head = st; + ngtcp2_conn_set_stream_user_data(conn->qconn, id, st); + return st; +} + +int64_t h3q_stream_id(h3q_stream* st) +{ + return st ? st->id : -1; +} + h3q_write_result h3q_stream_write(h3q_stream* st, const h3q_vec* vec, size_t nvec, int fin) { - SSL* ssl = (SSL*)st; h3q_write_result res = {0}; - size_t expected = 0; - for (size_t k = 0; k < nvec; k++) + if (!st || st->write_closed || st->conn->closed) { - expected += vec[k].len; + res.broken = 1; + return res; } - for (size_t k = 0; k < nvec; k++) + h3q_conn* conn = st->conn; + const ngtcp2_vec* datav = (const ngtcp2_vec*)vec; + size_t vi = 0; + size_t voff = 0; + uint8_t buf[H3Q_PKT_BUF]; + for (;;) { - size_t w = 0; - int wrv = SSL_write_ex(ssl, vec[k].base, vec[k].len, &w); - if (wrv <= 0) + ngtcp2_vec head = {0}; + size_t cnt = 0; + if (vi < nvec) + { + head.base = datav[vi].base + voff; + head.len = datav[vi].len - voff; + cnt = 1; + } + int last = vi + 1 >= nvec; + uint32_t flags = (fin && last) ? NGTCP2_WRITE_STREAM_FLAG_FIN : NGTCP2_WRITE_STREAM_FLAG_NONE; + ngtcp2_ssize ndatalen = 0; + ngtcp2_pkt_info pi; + ngtcp2_path_storage ps; + ngtcp2_path_storage_zero(&ps); + ngtcp2_ssize n = ngtcp2_conn_writev_stream(conn->qconn, &ps.path, &pi, buf, sizeof(buf), &ndatalen, flags, st->id, cnt ? &head : NULL, cnt, h3q_now()); + if (n < 0) { - if (SSL_get_error(ssl, wrv) == SSL_ERROR_WANT_WRITE) + if (n == NGTCP2_ERR_STREAM_DATA_BLOCKED) { res.blocked = 1; } else { + st->write_closed = 1; res.broken = 1; + if (ngtcp2_err_is_fatal((int)n)) + { + h3q_conn_fail(conn, (int)n); + } } break; } - res.accepted += w; - if (w < vec[k].len) + if (n > 0) { - res.blocked = 1; + h3q_send(conn->engine, &ps.path, buf, (size_t)n); + } + if (ndatalen > 0) + { + res.accepted += (size_t)ndatalen; + for (size_t left = (size_t)ndatalen; left > 0 && vi < nvec;) + { + size_t chunk = datav[vi].len - voff; + if (chunk > left) + { + voff += left; + left = 0; + } + else + { + left -= chunk; + vi++; + voff = 0; + } + } + } + if (n == 0) + { + /* Congestion limited. An owed FIN counts as blocked, or the stream never ends. */ + if (vi < nvec || fin) + { + res.blocked = 1; + st->fin_pending = vi >= nvec && fin; + } + break; + } + if (vi >= nvec) + { + if (flags & NGTCP2_WRITE_STREAM_FLAG_FIN) + { + /* ndatalen stays -1 when other frames crowded the STREAM frame out. */ + st->fin_pending = ndatalen < 0; + st->write_closed = ndatalen >= 0; + res.blocked |= ndatalen < 0; + } break; } } - if (fin && !res.blocked && !res.broken && res.accepted == expected) - { - SSL_stream_conclude(ssl, 0); - } + h3q_conn_flush(conn); return res; } int h3q_stream_is_write_blocked(h3q_stream* st) { - SSL* ssl = (SSL*)st; - uint64_t avail = 0; - if (ssl && SSL_get_generic_value_uint(ssl, SSL_VALUE_STREAM_WRITE_BUF_AVAIL, &avail) == 1 && avail == 0) + if (!st || st->write_closed || st->conn->closed) { return 1; } - return 0; + return ngtcp2_conn_get_max_data_left(st->conn->qconn) == 0 || ngtcp2_conn_get_max_stream_data_left(st->conn->qconn, st->id) == 0; } int h3q_stream_read(h3q_stream* st, unsigned char* buf, size_t read_size, size_t* nread, int* fin) { - SSL* ssl = (SSL*)st; + *nread = 0; *fin = 0; - int rv = SSL_read_ex(ssl, buf, read_size, nread); - if (rv == 1 && *nread > 0) + if (!st) { - return 1; + return 0; + } + size_t avail = st->rx_len - st->rx_off; + if (avail == 0) + { + *fin = st->fin; + return 0; } - if (rv == 1 || SSL_get_error(ssl, rv) == SSL_ERROR_ZERO_RETURN) + size_t n = read_size < avail ? read_size : avail; + memcpy(buf, st->rx + st->rx_off, n); + st->rx_off += n; + *nread = n; + if (st->rx_off == st->rx_len) { - *fin = 1; + st->rx_off = 0; + st->rx_len = 0; + *fin = st->fin; } - return 0; + /* Consumed bytes give the peer window back. */ + ngtcp2_conn_extend_max_stream_offset(st->conn->qconn, st->id, n); + ngtcp2_conn_extend_max_offset(st->conn->qconn, n); + return 1; } void h3q_stream_is_read_finished(h3q_stream* st, int* read_finished, int* write_finished) { - SSL* ssl = (SSL*)st; - int rstate = SSL_get_stream_read_state(ssl); - *read_finished = (rstate == SSL_STREAM_STATE_FINISHED || rstate == SSL_STREAM_STATE_RESET_REMOTE || rstate == SSL_STREAM_STATE_CONN_CLOSED); - int wstate = SSL_STREAM_STATE_FINISHED; - if (rstate != SSL_STREAM_STATE_CONN_CLOSED && rstate != SSL_STREAM_STATE_RESET_REMOTE) + if (!st) { - wstate = SSL_get_stream_write_state(ssl); + *read_finished = 1; + *write_finished = 1; + return; } - *write_finished = (wstate == SSL_STREAM_STATE_FINISHED || wstate == SSL_STREAM_STATE_RESET_LOCAL); + *read_finished = st->read_reset || (st->fin && st->rx_off >= st->rx_len); + /* Not write_closed: ngtcp2 resends from our buffers until it closes the stream. */ + *write_finished = st->engine_closed || st->conn->closed; +} + +int h3q_stream_is_early(h3q_stream* st) +{ + return st ? (int)st->early : 0; } void h3q_stream_reset(h3q_stream* st, uint64_t err) { - if (!st) + if (st && !st->conn->closed) { - return; + ngtcp2_conn_shutdown_stream_write(st->conn->qconn, 0, st->id, err); + st->write_closed = 1; } - SSL_STREAM_RESET_ARGS args = {err}; - SSL_stream_reset((SSL*)st, &args, sizeof(args)); } void h3q_stream_free(h3q_stream* st) { - if (st) + if (!st) { - SSL_free((SSL*)st); + return; } -} - -int64_t h3q_stream_id(h3q_stream* st) -{ - SSL* ssl = (SSL*)st; - return ssl ? (int64_t)SSL_get_stream_id(ssl) : -1; + h3q_conn* conn = st->conn; + if (!st->engine_closed && !conn->closed) + { + /* Drop unsent data now: its owner frees it after this call. */ + ngtcp2_conn_shutdown_stream_write(conn->qconn, 0, st->id, 0); + } + ngtcp2_conn_set_stream_user_data(conn->qconn, st->id, NULL); + for (h3q_stream** slot = &conn->streams_head; *slot; slot = &(*slot)->next) + { + if (*slot == st) + { + *slot = st->next; + break; + } + } + for (h3q_stream** slot = &conn->accept_head; *slot; slot = &(*slot)->next_accept) + { + if (*slot == st) + { + *slot = st->next_accept; + if (!*slot) + { + conn->accept_tail = NULL; + } + break; + } + } + free(st->rx); + free(st); } diff --git a/test/http3/env.py b/test/http3/env.py index 8d4fdde..9768c17 100644 --- a/test/http3/env.py +++ b/test/http3/env.py @@ -88,6 +88,7 @@ def add_vhost_test1( h3_address_validation=None, h3_socket_buffer_size=None, h3_session_tickets=None, + h3_early_data=None, h3_stream_timeout=None, h3_max_stream_errors=None, h3_qpack_table_capacity=None, @@ -131,6 +132,8 @@ def add_vhost_test1( if h3_session_tickets is not None: val = "on" if h3_session_tickets is True else ("off" if h3_session_tickets is False else h3_session_tickets) self.add(f"H3SessionTickets {val}") + if h3_early_data is not None: + self.add(f"H3EarlyData {'on' if h3_early_data else 'off'}") if h3_stream_timeout is not None: self.add(f"H3StreamTimeout {h3_stream_timeout}") if h3_max_stream_errors is not None: diff --git a/test/http3/test_023_resumption.py b/test/http3/test_023_resumption.py index 64b98b8..0dbecb9 100644 --- a/test/http3/test_023_resumption.py +++ b/test/http3/test_023_resumption.py @@ -112,12 +112,19 @@ def test_006_tickets_off_issues_none(self, env): assert resumed is False assert ticket is None, "H3SessionTickets off must issue no ticket" - def test_007_ticket_never_advertises_early_data(self, env): - """OpenSSL's QUIC server drops 0-RTT packets, so our tickets must never invite them.""" + def test_007_no_early_data_advert_by_default(self, env): + """H3EarlyData is off by default, so a ticket must not invite 0-RTT.""" H3Conf(env).add_vhost_test1(h3_session_tickets=True).install() assert env.apache_restart() == 0 _, ticket = _handshake(env) assert ticket is not None assert not getattr(ticket, "max_early_data_size", 0), ( - "the ticket advertises 0-RTT the server cannot honour; clients will send " - "early data it discards, and an unclamped value fails RFC 9001 4.6.1") + "the ticket advertises 0-RTT although H3EarlyData is off") + + def test_008_early_data_advertised_when_on(self, env): + """With H3EarlyData on, the ticket must advertise 0xffffffff (RFC 9001 4.6.1).""" + H3Conf(env).add_vhost_test1(h3_session_tickets=True, h3_early_data=True).install() + assert env.apache_restart() == 0 + _, ticket = _handshake(env) + assert ticket is not None + assert getattr(ticket, "max_early_data_size", 0) == 0xffffffff diff --git a/test/http3/test_030_early_data.py b/test/http3/test_030_early_data.py new file mode 100644 index 0000000..3f384a8 --- /dev/null +++ b/test/http3/test_030_early_data.py @@ -0,0 +1,64 @@ +import json +import os +import re +import shutil +import subprocess + +import pytest + +from .env import H3Conf + +# ngtcp2's osslclient is the one client here that sends QUIC 0-RTT application +# data. aioquic negotiates TLS early data but keeps the request for 1-RTT, so it +# cannot drive this path. curl needs an HTTP/3 build with 0-RTT support. +OSSLCLIENT = shutil.which("osslclient") + +pytestmark = pytest.mark.skipif(OSSLCLIENT is None, reason="ngtcp2 osslclient not on PATH") + + +def _run(env, session, tp, extra=None): + """Run osslclient once against test1; return (exit_code, stdout, stderr).""" + authority = f"test1.{env.http_tld}" + args = [OSSLCLIENT, f"--session-file={session}", f"--tp-file={tp}", + "--timeout=2s", "127.0.0.1", str(env.https_port), f"https://{authority}/"] + e = dict(os.environ, SSL_CERT_FILE=os.path.join(env.server_dir, "certs", "ca.crt")) + # The harness mints certs under test/certs; fall back to that CA. + if not os.path.exists(e["SSL_CERT_FILE"]): + e["SSL_CERT_FILE"] = env.test_cert_file.replace("server.crt", "ca.crt") + p = subprocess.run(args + (extra or []), capture_output=True, text=True, env=e, timeout=15) + return p.returncode, p.stdout, p.stderr + + +def _h3_200s(env): + """Count successful HTTP/3 responses in the access log.""" + log = os.path.join(env.server_dir, "logs", "access_log") + if not os.path.exists(log): + return 0 + n = 0 + for line in open(log): + m = re.search(r'"request":\s*"[^"]*HTTP/3[^"]*".*?"status":\s*200', line) + if m: + n += 1 + return n + + +class TestEarlyData: + + @pytest.fixture(autouse=True, scope="class") + def _scope(self, env): + H3Conf(env).add_vhost_test1(h3_session_tickets=True, h3_early_data=True).install() + assert env.apache_restart() == 0 + + def test_001_zero_rtt_get_is_answered(self, env, tmp_path): + session = str(tmp_path / "sess") + tp = str(tmp_path / "tp") + # First connection: full handshake, saves the ticket and transport params. + rc, _, _ = _run(env, session, tp) + assert rc == 0, "first osslclient run failed" + assert os.path.getsize(session) > 0, "no session ticket was saved" + before = _h3_200s(env) + # Second connection: resumes and sends the GET as 0-RTT. + rc, _, err = _run(env, session, tp) + assert rc == 0, "0-RTT osslclient run failed" + assert "QUIC handshake has completed" in err + assert _h3_200s(env) > before, "the 0-RTT GET was not answered with 200" From 875b4befbaa1dd22677e26feba46452403220555 Mon Sep 17 00:00:00 2001 From: Alexander Gerasimov Date: Fri, 9 Oct 2026 02:27:30 +0300 Subject: [PATCH 5/5] Try each interop client up to 3 times The runner's tshark can crash on a cut capture. Then the runner writes no results.json and the job fails without a server fault. Each failed try shows as a warning, so flaky results stay visible. --- .github/workflows/interop.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index e5d4a7f..25a9738 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -142,12 +142,19 @@ jobs: mv impls.json implementations_quic.json - name: Run matrix - continue-on-error: true - run: python run.py -s mod_http3 -c "$CLIENT" -t http3 -l logs -j results.json + # The runner's tshark can crash on a cut capture: allow 2 failed tries. + run: | + for try in 1 2 3; do + rm -f results.json + python run.py -s mod_http3 -c "$CLIENT" -t http3 -l logs/$try -j results.json || true + result=$(jq -r '.results[0][0].result' results.json 2>/dev/null || true) + case $result in succeeded|unsupported) break ;; esac + echo "::warning title=$CLIENT::try $try: ${result:-no result}" + done - name: Report verdict run: | - result=$(jq -r '.results[0][0].result' results.json) + result=$(jq -r '.results[0][0].result' results.json 2>/dev/null || echo 'no result') echo "### $CLIENT — \`$result\`" >>"$GITHUB_STEP_SUMMARY" case $result in succeeded) ;;