Drain redirect bodies without buffering, NEWS for #980

NEWS.md

@@ -21,9 +21,17 @@ Unreleased
   another port on the same host, or over plain HTTP after HTTPS. Only a
   redirect to the same scheme, host and port keeps them, unless you set
   `location_trusted` (#979).
+- A buffered response body is now capped at 512 MiB on HTTP/1.1, HTTP/2
+  and HTTP/3, including after `auto_decompress`, so a server sending an
+  endless or highly compressed body can no longer exhaust memory. Past the
+  cap you get `{error, body_too_large}`. Set `max_body_size` (or the 1.x
+  `max_body`) to change it, `infinity` to remove it, and stream larger
+  downloads (#980).
 
 ### Changed
 
+- Buffered downloads over 512 MiB now fail with `{error, body_too_large}`
+  unless you raise `max_body_size` (#980).
 - If you pin `cacerts` that include the bundled public roots, hackney no
   longer falls back to trusting a cross-signed root it finds in the
   server's chain. Add the root you need to your list, or pass your own

include/hackney.hrl

@@ -1,5 +1,10 @@
 -define(RECV_TIMEOUT, 5000).
 
+%% Cap on a response body buffered whole for the caller, after
+%% auto_decompress (GHSA-jq4m, #980). Per request via `max_body_size';
+%% `infinity' disables it. Larger downloads should stream.
+-define(DEFAULT_MAX_BODY_SIZE, 16#20000000).  %% 512 MiB
+
 -record(connection, {transport,
                      host,
                      port,

src/hackney_conn.erl

@@ -153,14 +153,6 @@
 %% stranded prefix is small. Past this, treat the peer as misbehaving (flooding
 %% an idle connection) and drop it rather than buffer unboundedly.
 -define(MAX_IDLE_BUFFER, 65536).
-%% Cap on a response body buffered whole for the caller (hackney:request/5
-%% with a body, hackney:body/1), after decompression when auto_decompress is
-%% on. Without it any server could make the client hold an arbitrarily large
-%% body in memory. Same default and option as the standalone HTTP/3 API
-%% (hackney_h3, GHSA-jq4m): max_body_size, `infinity' disables it, and the
-%% read fails with {error, body_too_large}. Larger downloads should use the
-%% streaming API.
--define(DEFAULT_MAX_BODY_SIZE, 16#20000000).  %% 512 MiB
 
 %% State data record
 -record(conn_data, {
@@ -3160,9 +3152,16 @@ maybe_handle_async_redirect(_, _, _, _) ->
     no_redirect.
 
 %% @private Skip the response body
+%% Chunks are dropped as they arrive, so a large body costs no memory and is
+%% not subject to max_body_size.
+skip_response_body(#conn_data{method = <<"HEAD">>} = Data) ->
+    Data;
+skip_response_body(#conn_data{status = Status} = Data) when Status =:= 204; Status =:= 304 ->
+    Data;
 skip_response_body(Data) ->
-    case read_full_body(Data, <<>>) of
-        {ok, _, NewData} -> NewData;
+    case stream_body_chunk(Data) of
+        {ok, _Chunk, NewData} -> skip_response_body(NewData);
+        {done, NewData} -> NewData;
         {error, _} -> Data
     end.
 

src/hackney_h3.erl

@@ -27,6 +27,7 @@
 
 -behaviour(gen_server).
 
+-include("hackney.hrl").
 -include("hackney_lib.hrl").
 
 -export([
@@ -88,12 +89,9 @@
 
 -define(CONN_TABLE, hackney_h3_conns).
 
-%% GHSA-jq4m: default ceiling on a buffered HTTP/3 response body. The
-%% non-streaming await path holds the whole body in memory, so without a cap
-%% a peer that trickles data forever drives the node to OOM. Configurable via
-%% the `max_body_size' option (`infinity' disables it); very large downloads
-%% should use the streaming API instead.
--define(DEFAULT_MAX_BODY_SIZE, 16#20000000).  %% 512 MiB
+%% GHSA-jq4m: the non-streaming await path holds the whole body in memory,
+%% so it is capped at ?DEFAULT_MAX_BODY_SIZE (hackney.hrl) unless the
+%% `max_body_size' option says otherwise.
 
 -type h3_conn() :: reference().
 -type stream_id() :: non_neg_integer().