Issue with getting to HTTP/3 or QUIC on first connect

E
  • 9 Mar '23
I’m having an issue where I (think I) have enabled HTTP3 correctly on my nginx server.  When I connect to the server the first time, it indicates HTTP/2 in the logs.  If I hit refresh it indicates HTTP/3 from then on.  So something is wrong with the configuration of the server to offer headers to negotiate it.  I’ve even added HTTPS DNS records to indicate the preferred connection schemes.  Bonus points if you can help me get QUIC working too.

The result of http3check.net <http://http3check.net/> is "HTTP/3 Check could not get the server's advertised QUIC versions due to the error given below.
Bad status code from server.
Thanks in advance for any pointers

Eric

Source code was pulled 2023-02-27

Build information is:

nginx version: nginx/1.23.4
built by gcc 7.3.1 20180712 (Red Hat 7.3.1-15) (GCC)
built with OpenSSL 3.0.8+quic 7 Feb 2023
TLS SNI support enabled
configure arguments: --with-threads --with-cc-opt='-static -static-libgcc' --with-ld-opt=-static --with-debug --with-compat --with-file-aio --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-stream_realip_module --with-http_realip_module --with-http_secure_link_module --with-http_random_index_module --with-http_geoip_module --with-http_ssl_module --with-http_v2_module --with-http_v3_module --with-stream_quic_module --with-http_sub_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-cc-opt=-I/usr/local/include --with-ld-opt=-L/usr/local/lib --with-openssl=../quictls --with-openssl-opt=enable-tls1_3 --add-module=/source/ngx_brotli

Pertinent config for the server block is

    upstream httpd_backend {
        server 172.28.10.91:443;

        keepalive 120;
    }

    log_format quic '$remote_addr - $remote_user [$time_local] '
                        '"$request" $status $body_bytes_sent '
                        '"$http_referer" "$http_user_agent" "$http3"';

    access_log logs/access.log quic;

    # NON-SSL
    server {
        listen                  80;
        listen                  [::]:80;
        server_name             www.example.com;

        location / {
          rewrite               ^ https://www.example.com$request_uri?/ permanent;
        }
    }

    # SSL
    server {
        listen                  443             ssl http2;
        listen                  [::]:443        ssl http2;
        listen                  443             http3 reuseport;
        listen                  [::]:443        http3 reuseport;

        quic_retry              on;

        server_name             noc2.semperen.com;

        # Set up your cert paths
        ssl_certificate_key             /etc/letsencrypt/live/www.example.com/privkey.pem;
        ssl_certificate                 /etc/letsencrypt/live/www.example.com/fullchain.pem;
        ssl_trusted_certificate         /etc/letsencrypt/live/www.example.com/chain.pem;
        ssl_dhparam                     SSLKeys/dhparam.pem;
        ssl_protocols                   TLSv1.3;
        ssl_prefer_server_ciphers       On;
        ssl_ciphers                     TLS13-CHACHA20-POLY1305-SHA256:TLS13-AES-256-GCM-SHA384:AES256+EECDH:AES256+EDH:!aNULL:!CBC;
        ssl_ecdh_curve                  secp384r1;
        ssl_early_data                  on;

        # enable ocsp stapling
        ssl_stapling on;
        ssl_stapling_verify on;

        resolver 8.8.4.4 8.8.8.8 valid=86400s;
        resolver_timeout 10s;

        # upgrade to HTTP3 and HTTP2
        add_header alt-svc              'h3=":443"; ma=86400, h2=":443"; ma=86400';
        add_header alt-svc              'h2=":443"; ma=86400; persist=1';
        add_header alt-svc              'h2=":443"; ma=86400;';

        add_header X-Frame-Options      "SAMEORIGIN";

        # tell users to go to SSL version next time
        add_header Strict-Transport-Security "max-age=31104000; includeSubdomains" always;

        # handle brotli compression
        brotli                          on;
        brotli_static                   on;

    # Note this is one line, even if it wraps and renders as two
        brotli_types                    text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;

        brotli_buffers                  16 8k;
        brotli_comp_level               9;

        client_max_body_size            32M;

        error_page 502 /custom_502.html;
        location = /custom_502.html {
            root /usr/local/nginx/html;
            internal;
        }

        location / {
            proxy_pass                  https://httpd_backend;
            proxy_http_version          1.1;
            proxy_buffering             on;
            proxy_set_header            X-Forwarded-For $remote_addr;
            proxy_set_header            X-Real-IP       $remote_addr;
            proxy_set_header            Host            $host;
            proxy_pass_header           Authorization;
#           proxy_set_header            X-Scheme        $scheme;
#           proxy_set_header            Upgrade         $http_upgrade;
#           proxy_set_header            Connection      "upgrade";
        }

        location /roundcube {
            proxy_pass                  https://172.28.10.100;
            proxy_http_version          1.1;
            proxy_buffering             on;
            proxy_set_header            X-Forwarded-For $remote_addr;
            proxy_set_header            X-Real-IP       $remote_addr;
            proxy_set_header            Host            $host;
            proxy_pass_header           Authorization;
#           proxy_set_header            X-Scheme        $scheme;
#           proxy_set_header            Upgrade         $http_upgrade;
#           proxy_set_header            Connection      "upgrade";
        }
    }

--
Eric Germann
ekgermann {at} semperen {dot} com || ekgermann {at} gmail {dot} com
LinkedIn: https://www.linkedin.com/in/ericgermann
Medium: https://ekgermann.medium.com <https://ekgermann.medium.com/>
Twitter: @ekgermann
Telegram || Signal || Skype || WhatsApp || Phone +1 {dash} 419 {dash} 513 {dash} 0712

GPG Fingerprint: 89ED 36B3 515A 211B 6390  60A9 E30D 9B9B 3EBF F1A1

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20230309/e05ad9ec/attachment-0001.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: Message signed with OpenPGP
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20230309/e05ad9ec/attachment-0001.bin>
E
  • 9 Mar '23
Forgot to add, here are the HTTPS [1] records in DNS

                                IN      HTTPS   100     .       no-default-alpn alpn="h3"
                                IN      HTTPS   200     .       no-default-alpn alpn="h2"

They 100/200 are weights, much like MX records, where the lowest one wins

[1] https://www.ietf.org/archive/id/draft-ietf-dnsop-svcb-https-11.txt

> On Mar 9, 2023, at 01:56, Eric Germann <ekgermann at semperen.com> wrote:
> 
> I’m having an issue where I (think I) have enabled HTTP3 correctly on my nginx server.  When I connect to the server the first time, it indicates HTTP/2 in the logs.  If I hit refresh it indicates HTTP/3 from then on.  So something is wrong with the configuration of the server to offer headers to negotiate it.  I’ve even added HTTPS DNS records to indicate the preferred connection schemes.  Bonus points if you can help me get QUIC working too.
> 
> The result of http3check.net <http://http3check.net/> is "HTTP/3 Check could not get the server's advertised QUIC versions due to the error given below.
> Bad status code from server.
> Thanks in advance for any pointers
> 
> Eric
> 
> Source code was pulled 2023-02-27
> 
> 
> Build information is:
> 
> nginx version: nginx/1.23.4
> built by gcc 7.3.1 20180712 (Red Hat 7.3.1-15) (GCC)
> built with OpenSSL 3.0.8+quic 7 Feb 2023
> TLS SNI support enabled
> configure arguments: --with-threads --with-cc-opt='-static -static-libgcc' --with-ld-opt=-static --with-debug --with-compat --with-file-aio --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-stream_realip_module --with-http_realip_module --with-http_secure_link_module --with-http_random_index_module --with-http_geoip_module --with-http_ssl_module --with-http_v2_module --with-http_v3_module --with-stream_quic_module --with-http_sub_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-cc-opt=-I/usr/local/include --with-ld-opt=-L/usr/local/lib --with-openssl=../quictls --with-openssl-opt=enable-tls1_3 --add-module=/source/ngx_brotli
> 
> 
> Pertinent config for the server block is
> 
>     upstream httpd_backend {
>         server 172.28.10.91:443;
> 
>         keepalive 120;
>     }
> 
>     log_format quic '$remote_addr - $remote_user [$time_local] '
>                         '"$request" $status $body_bytes_sent '
>                         '"$http_referer" "$http_user_agent" "$http3"';
> 
>     access_log logs/access.log quic;
> 
>     # NON-SSL
>     server {
>         listen                  80;
>         listen                  [::]:80;
>         server_name             www.example.com;
> 
>         location / {
>           rewrite               ^ https://www.example.com$request_uri?/ permanent;
>         }
>     }
> 
>     # SSL
>     server {
>         listen                  443             ssl http2;
>         listen                  [::]:443        ssl http2;
>         listen                  443             http3 reuseport;
>         listen                  [::]:443        http3 reuseport;
> 
>         quic_retry              on;
> 
>         server_name             noc2.semperen.com;
> 
>         # Set up your cert paths
>         ssl_certificate_key             /etc/letsencrypt/live/www.example.com/privkey.pem;
>         ssl_certificate                 /etc/letsencrypt/live/www.example.com/fullchain.pem;
>         ssl_trusted_certificate         /etc/letsencrypt/live/www.example.com/chain.pem;
>         ssl_dhparam                     SSLKeys/dhparam.pem;
>         ssl_protocols                   TLSv1.3;
>         ssl_prefer_server_ciphers       On;
>         ssl_ciphers                     TLS13-CHACHA20-POLY1305-SHA256:TLS13-AES-256-GCM-SHA384:AES256+EECDH:AES256+EDH:!aNULL:!CBC;
>         ssl_ecdh_curve                  secp384r1;
>         ssl_early_data                  on;
> 
>         # enable ocsp stapling
>         ssl_stapling on;
>         ssl_stapling_verify on;
> 
>         resolver 8.8.4.4 8.8.8.8 valid=86400s;
>         resolver_timeout 10s;
> 
>         # upgrade to HTTP3 and HTTP2
>         add_header alt-svc              'h3=":443"; ma=86400, h2=":443"; ma=86400';
>         add_header alt-svc              'h2=":443"; ma=86400; persist=1';
>         add_header alt-svc              'h2=":443"; ma=86400;';
> 
>         add_header X-Frame-Options      "SAMEORIGIN";
> 
> 
>         # tell users to go to SSL version next time
>         add_header Strict-Transport-Security "max-age=31104000; includeSubdomains" always;
> 
>         # handle brotli compression
>         brotli                          on;
>         brotli_static                   on;
> 
>   # Note this is one line, even if it wraps and renders as two
>         brotli_types                    text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
> 
>         brotli_buffers                  16 8k;
>         brotli_comp_level               9;
> 
>         client_max_body_size            32M;
> 
>         error_page 502 /custom_502.html;
>         location = /custom_502.html {
>             root /usr/local/nginx/html;
>             internal;
>         }
> 
>         location / {
>             proxy_pass                  https://httpd_backend;
>             proxy_http_version          1.1;
>             proxy_buffering             on;
>             proxy_set_header            X-Forwarded-For $remote_addr;
>             proxy_set_header            X-Real-IP       $remote_addr;
>             proxy_set_header            Host            $host;
>             proxy_pass_header           Authorization;
> #           proxy_set_header            X-Scheme        $scheme;
> #           proxy_set_header            Upgrade         $http_upgrade;
> #           proxy_set_header            Connection      "upgrade";
>         }
> 
>         location /roundcube {
>             proxy_pass                  https://172.28.10.100;
>             proxy_http_version          1.1;
>             proxy_buffering             on;
>             proxy_set_header            X-Forwarded-For $remote_addr;
>             proxy_set_header            X-Real-IP       $remote_addr;
>             proxy_set_header            Host            $host;
>             proxy_pass_header           Authorization;
> #           proxy_set_header            X-Scheme        $scheme;
> #           proxy_set_header            Upgrade         $http_upgrade;
> #           proxy_set_header            Connection      "upgrade";
>         }
>     }
> 
> 
> 
> 
> --
> Eric Germann
> ekgermann {at} semperen {dot} com || ekgermann {at} gmail {dot} com
> LinkedIn: https://www.linkedin.com/in/ericgermann
> Medium: https://ekgermann.medium.com <https://ekgermann.medium.com/>
> Twitter: @ekgermann
> Telegram || Signal || Skype || WhatsApp || Phone +1 {dash} 419 {dash} 513 {dash} 0712
> 
> GPG Fingerprint: 89ED 36B3 515A 211B 6390  60A9 E30D 9B9B 3EBF F1A1
> 
> 
> 
> 
> 
> 
> 

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20230309/dbb0e21e/attachment-0001.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: Message signed with OpenPGP
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20230309/dbb0e21e/attachment-0001.bin>
R
  • 9 Mar '23
Hi Eric,

> On 9 Mar 2023, at 10:56, Eric Germann via nginx <nginx at nginx.org> wrote:
> 
> I’m having an issue where I (think I) have enabled HTTP3 correctly on my nginx server.  When I connect to the server the first time, it indicates HTTP/2 in the logs.  If I hit refresh it indicates HTTP/3 from then on.

Normally you connect to the server using regular https first (http/2 is probably an option as well).
Only after that the client analyses Alt-Svc response header and tries http/3.

> So something is wrong with the configuration of the server to offer headers to negotiate it.  I’ve even added HTTPS DNS records to indicate the preferred connection schemes.  
> Bonus points if you can help me get QUIC working too.
> 
> The result of http3check.net <http://http3check.net/> is "HTTP/3 Check could not get the server's advertised QUIC versions due to the error given below.
> Bad status code from server.
Can you please check nginx error.log for errors? If no errors, can you please enable debugging and send the error.log to me.

> Thanks in advance for any pointers
> 
> Eric
> 
> Source code was pulled 2023-02-27
> 
> 
> Build information is:
> 
> nginx version: nginx/1.23.4
> built by gcc 7.3.1 20180712 (Red Hat 7.3.1-15) (GCC) 
> built with OpenSSL 3.0.8+quic 7 Feb 2023
> TLS SNI support enabled
> configure arguments: --with-threads --with-cc-opt='-static -static-libgcc' --with-ld-opt=-static --with-debug --with-compat --with-file-aio --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-stream_realip_module --with-http_realip_module --with-http_secure_link_module --with-http_random_index_module --with-http_geoip_module --with-http_ssl_module --with-http_v2_module --with-http_v3_module --with-stream_quic_module --with-http_sub_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-cc-opt=-I/usr/local/include --with-ld-opt=-L/usr/local/lib --with-openssl=../quictls --with-openssl-opt=enable-tls1_3 --add-module=/source/ngx_brotli
> 
> 
> Pertinent config for the server block is
> 
>     upstream httpd_backend {
>         server 172.28.10.91:443;
> 
>         keepalive 120;
>     }
> 
>     log_format quic '$remote_addr - $remote_user [$time_local] '
>                         '"$request" $status $body_bytes_sent '
>                         '"$http_referer" "$http_user_agent" "$http3"';
> 
>     access_log logs/access.log quic;
> 
>     # NON-SSL
>     server {
>         listen                  80;
>         listen                  [::]:80;
>         server_name             www.example.com;
> 
>         location / {
>           rewrite               ^ https://www.example.com$request_uri?/ permanent;
>         }
>     }
> 
>     # SSL
>     server {
>         listen                  443             ssl http2;
>         listen                  [::]:443        ssl http2;
>         listen                  443             http3 reuseport;
>         listen                  [::]:443        http3 reuseport;
> 
>         quic_retry              on;
> 
>         server_name             noc2.semperen.com;
> 
>         # Set up your cert paths
>         ssl_certificate_key             /etc/letsencrypt/live/www.example.com/privkey.pem;
>         ssl_certificate                 /etc/letsencrypt/live/www.example.com/fullchain.pem;
>         ssl_trusted_certificate         /etc/letsencrypt/live/www.example.com/chain.pem;
>         ssl_dhparam                     SSLKeys/dhparam.pem;
>         ssl_protocols                   TLSv1.3;
>         ssl_prefer_server_ciphers       On;
>         ssl_ciphers                     TLS13-CHACHA20-POLY1305-SHA256:TLS13-AES-256-GCM-SHA384:AES256+EECDH:AES256+EDH:!aNULL:!CBC;
>         ssl_ecdh_curve                  secp384r1;
>         ssl_early_data                  on;
> 
>         # enable ocsp stapling
>         ssl_stapling on;
>         ssl_stapling_verify on;
> 
>         resolver 8.8.4.4 8.8.8.8 valid=86400s;
>         resolver_timeout 10s;
> 
>         # upgrade to HTTP3 and HTTP2
>         add_header alt-svc              'h3=":443"; ma=86400, h2=":443"; ma=86400';
>         add_header alt-svc              'h2=":443"; ma=86400; persist=1';
>         add_header alt-svc              'h2=":443"; ma=86400;';
> 
>         add_header X-Frame-Options      "SAMEORIGIN";
> 
> 
>         # tell users to go to SSL version next time
>         add_header Strict-Transport-Security "max-age=31104000; includeSubdomains" always;
> 
>         # handle brotli compression
>         brotli                          on;
>         brotli_static                   on;
> 
>   # Note this is one line, even if it wraps and renders as two
>         brotli_types                    text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
> 
>         brotli_buffers                  16 8k;
>         brotli_comp_level               9;
> 
>         client_max_body_size            32M;
> 
>         error_page 502 /custom_502.html;
>         location = /custom_502.html {
>             root /usr/local/nginx/html;
>             internal;
>         }
> 
>         location / {
>             proxy_pass                  https://httpd_backend;
>             proxy_http_version          1.1;
>             proxy_buffering             on;
>             proxy_set_header            X-Forwarded-For $remote_addr;
>             proxy_set_header            X-Real-IP       $remote_addr;
>             proxy_set_header            Host            $host;
>             proxy_pass_header           Authorization;
> #           proxy_set_header            X-Scheme        $scheme;
> #           proxy_set_header            Upgrade         $http_upgrade;
> #           proxy_set_header            Connection      "upgrade";
>         }
> 
>         location /roundcube {
>             proxy_pass                  https://172.28.10.100;
>             proxy_http_version          1.1;
>             proxy_buffering             on;
>             proxy_set_header            X-Forwarded-For $remote_addr;
>             proxy_set_header            X-Real-IP       $remote_addr;
>             proxy_set_header            Host            $host;
>             proxy_pass_header           Authorization;
> #           proxy_set_header            X-Scheme        $scheme;
> #           proxy_set_header            Upgrade         $http_upgrade;
> #           proxy_set_header            Connection      "upgrade";
>         }
>     }
> 
> 
> 
> 
> --
> Eric Germann
> ekgermann {at} semperen {dot} com || ekgermann {at} gmail {dot} com
> LinkedIn: https://www.linkedin.com/in/ericgermann
> Medium: https://ekgermann.medium.com <https://ekgermann.medium.com/> 
> Twitter: @ekgermann
> Telegram || Signal || Skype || WhatsApp || Phone +1 {dash} 419 {dash} 513 {dash} 0712
> 
> GPG Fingerprint: 89ED 36B3 515A 211B 6390  60A9 E30D 9B9B 3EBF F1A1
> 
> 
> 
> 
> 
> 
> 
> _______________________________________________
> nginx mailing list
> nginx at nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx

----
Roman Arutyunyan
arut at nginx.com

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20230309/28782ab6/attachment-0001.htm>
E
  • 9 Mar '23
For the archives:

- I pulled down latest version of nginx-quic and built it.  The error seems to be resolved.  First page on the site now shows http/3
- As for https://http3check.net <https://http3check.net/>, the reason it failed was I had the root page protected behind basic auth.  Since it doesn’t send auth info, it 401’d and caused the page to throw an error.  I’m going to write to them and ask them to toss up a clearer answer, whether it’s a negotiation error or a http error.

Thanks for the assist.  The debug log is where I found the 401

Eric

> On Mar 9, 2023, at 04:41, Roman Arutyunyan <arut at nginx.com> wrote:
> 
> Hi Eric,
> 
>> On 9 Mar 2023, at 10:56, Eric Germann via nginx <nginx at nginx.org <mailto:nginx at nginx.org>> wrote:
>> 
>> I’m having an issue where I (think I) have enabled HTTP3 correctly on my nginx server.  When I connect to the server the first time, it indicates HTTP/2 in the logs.  If I hit refresh it indicates HTTP/3 from then on.
> 
> Normally you connect to the server using regular https first (http/2 is probably an option as well).
> Only after that the client analyses Alt-Svc response header and tries http/3.
> 
>> So something is wrong with the configuration of the server to offer headers to negotiate it.  I’ve even added HTTPS DNS records to indicate the preferred connection schemes.
>> Bonus points if you can help me get QUIC working too.
>> 
>> The result of http3check.net <http://http3check.net/> is "HTTP/3 Check could not get the server's advertised QUIC versions due to the error given below.
>> Bad status code from server.
> Can you please check nginx error.log for errors? If no errors, can you please enable debugging and send the error.log to me.
> 
>> Thanks in advance for any pointers
>> 
>> Eric
>> 
>> Source code was pulled 2023-02-27
>> 
>> 
>> Build information is:
>> 
>> nginx version: nginx/1.23.4
>> built by gcc 7.3.1 20180712 (Red Hat 7.3.1-15) (GCC)
>> built with OpenSSL 3.0.8+quic 7 Feb 2023
>> TLS SNI support enabled
>> configure arguments: --with-threads --with-cc-opt='-static -static-libgcc' --with-ld-opt=-static --with-debug --with-compat --with-file-aio --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-stream_realip_module --with-http_realip_module --with-http_secure_link_module --with-http_random_index_module --with-http_geoip_module --with-http_ssl_module --with-http_v2_module --with-http_v3_module --with-stream_quic_module --with-http_sub_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-cc-opt=-I/usr/local/include --with-ld-opt=-L/usr/local/lib --with-openssl=../quictls --with-openssl-opt=enable-tls1_3 --add-module=/source/ngx_brotli
>> 

<SNIP>

</SNIP>
> 
> ----
> Roman Arutyunyan
> arut at nginx.com

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20230309/12df15f0/attachment-0001.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: Message signed with OpenPGP
URL: <http://mailman.nginx.org/pipermail/nginx/attachments/20230309/12df15f0/attachment-0001.bin>