diff --git a/NEWS.md b/NEWS.md index c89d32e9..c6cb3e79 100644 --- a/NEWS.md +++ b/NEWS.md @@ -21,6 +21,12 @@ Unreleased another port on the same host, or over plain HTTP after HTTPS. Only a redirect to the same scheme, host and port keeps them, unless you set `location_trusted` (#979). +- A buffered response body (the default, or `body/1`) is now capped at + 512 MiB on HTTP/1.1, HTTP/2 and HTTP/3. A server that sent an endless + body could exhaust the node's memory; only the direct `hackney_h3` API + had a cap. Past it you get `{error, body_too_large}`. Set `max_body_size` + to change the cap, or `infinity` to remove it, and stream larger + downloads. ### Changed diff --git a/guides/http_guide.md b/guides/http_guide.md index e0d830f7..989c5a7e 100644 --- a/guides/http_guide.md +++ b/guides/http_guide.md @@ -146,6 +146,13 @@ with QUERY. {ok, 200, Headers, Body} = hackney:get(URL). ``` +A buffered body is capped at 512 MiB. Past the cap you get `{error, body_too_large}`. On HTTP/2 and HTTP/3 only that stream is cancelled; on HTTP/1.1 the connection is closed. Change the cap per request with `max_body_size`, or turn it off with `infinity`. For larger downloads, stream the body instead. + +```erlang +{ok, 200, _, Body} = hackney:get(URL, [], <<>>, [{max_body_size, 2 * 1024 * 1024 * 1024}]), +{error, body_too_large} = hackney:get(URL, [], <<>>, [{max_body_size, 1024}]). +``` + ### Automatic Decompression Hackney can automatically decompress gzip and deflate encoded responses: diff --git a/include/hackney.hrl b/include/hackney.hrl index 4a49dd36..9c798b7f 100644 --- a/include/hackney.hrl +++ b/include/hackney.hrl @@ -1,5 +1,11 @@ -define(RECV_TIMEOUT, 5000). +%% Default ceiling on a buffered response body (with_body, body/1). A peer +%% that sends a body forever would otherwise drive the node out of memory +%% (GHSA-jq4m, GHSA-pph4). Per request via `max_body_size'; `infinity' +%% disables it. Large downloads should stream instead. +-define(DEFAULT_MAX_BODY_SIZE, 16#20000000). %% 512 MiB + -record(connection, {transport, host, port, diff --git a/src/hackney.erl b/src/hackney.erl index 6e90c4fc..3ab85b7d 100644 --- a/src/hackney.erl +++ b/src/hackney.erl @@ -1420,9 +1420,11 @@ sync_request_with_redirect(ConnPid, Method, Path, Headers, Body, WithBody, Optio %% For streaming body, send headers and return immediately. Forward %% send_timeout so the body chunks use the caller's deadline even on a %% reused pooled connection. + ReqOpts0 = [{max_body_size, proplists:get_value(max_body_size, Options, + ?DEFAULT_MAX_BODY_SIZE)}], ReqOpts = case proplists:get_value(send_timeout, Options) of - undefined -> []; - SendTimeout -> [{send_timeout, SendTimeout}] + undefined -> ReqOpts0; + SendTimeout -> [{send_timeout, SendTimeout} | ReqOpts0] end, case hackney_conn:send_request_headers(ConnPid, Method, Path, HeadersList, ReqOpts) of ok -> {ok, ConnPid}; @@ -1445,10 +1447,14 @@ sync_request_with_redirect_body(ConnPid, Method, Path, HeadersList, FinalBody, false -> ReqOpts0 end, %% Pass recv_timeout through to the connection so it's applied per-request - ReqOpts2 = case proplists:get_value(recv_timeout, Options) of + ReqOpts2a = case proplists:get_value(recv_timeout, Options) of undefined -> ReqOpts1; RecvTimeout -> [{recv_timeout, RecvTimeout} | ReqOpts1] end, + %% Always set: a pooled connection must not keep a previous request's cap. + ReqOpts2 = [{max_body_size, proplists:get_value(max_body_size, Options, + ?DEFAULT_MAX_BODY_SIZE)} + | ReqOpts2a], %% Pass send_timeout through so HTTP/2 body sends blocked on flow control %% use the caller's deadline ReqOpts = case proplists:get_value(send_timeout, Options) of diff --git a/src/hackney_conn.erl b/src/hackney_conn.erl index 3cd04213..024c65ad 100644 --- a/src/hackney_conn.erl +++ b/src/hackney_conn.erl @@ -173,6 +173,8 @@ %% Options connect_timeout = ?CONNECT_TIMEOUT :: timeout(), recv_timeout = ?RECV_TIMEOUT :: timeout(), + %% Cap on a buffered response body, set per request (like recv_timeout). + max_body_size = ?DEFAULT_MAX_BODY_SIZE :: non_neg_integer() | infinity, send_timeout = ?SEND_TIMEOUT :: timeout() | nonblock, %% Effective send_timeout for the streaming-body request in flight, set at %% every stream start (send_headers) so a per-request override never leaks @@ -1187,14 +1189,16 @@ connected({call, From}, {request, Method, Path, Headers, Body, ReqOpts}, #conn_d %% send_timeout is passed along, never stored: a per-request override must %% not leak into later requests on a pooled connection. SendTimeout = proplists:get_value(send_timeout, ReqOpts, Data#conn_data.send_timeout), - NewData = Data#conn_data{recv_timeout = RecvTimeout}, + NewData = Data#conn_data{recv_timeout = RecvTimeout, + max_body_size = req_max_body_size(ReqOpts, Data)}, do_h2_request(From, Method, Path, Headers, Body, SendTimeout, NewData); connected({call, From}, {request, Method, Path, Headers, Body, ReqOpts}, #conn_data{protocol = http3} = Data) -> %% HTTP/3 request - use hackney_h3 (1xx not applicable for HTTP/3) %% Allow recv_timeout to be overridden per-request (fix for issue #832) RecvTimeout = proplists:get_value(recv_timeout, ReqOpts, Data#conn_data.recv_timeout), - NewData = Data#conn_data{recv_timeout = RecvTimeout}, + NewData = Data#conn_data{recv_timeout = RecvTimeout, + max_body_size = req_max_body_size(ReqOpts, Data)}, do_h3_request(From, Method, Path, Headers, Body, NewData); connected({call, From}, {request_async, Method, Path, Headers, Body, AsyncMode, StreamTo}, #conn_data{protocol = http2} = Data) -> @@ -1255,7 +1259,8 @@ connected({call, From}, {request, Method, Path, Headers, Body, ReqOpts}, Data) - stream_to = undefined, inform_fun = InformFun, auto_decompress = AutoDecompress, - recv_timeout = RecvTimeout + recv_timeout = RecvTimeout, + max_body_size = req_max_body_size(ReqOpts, Data) }, {next_state, sending, NewData, [{next_event, internal, {send_request, Method, Path, Headers, Body}}]}; @@ -1287,14 +1292,16 @@ connected({call, From}, {request_async, Method, Path, Headers, Body, AsyncMode, NewData = Data#conn_data{recv_timeout = RecvTimeout}, do_request_async(From, Method, Path, Headers, Body, AsyncMode, StreamTo, FollowRedirect, NewData); -connected({call, From}, {send_headers, Method, Path, Headers, _ReqOpts}, #conn_data{protocol = http3} = Data) -> +connected({call, From}, {send_headers, Method, Path, Headers, ReqOpts}, #conn_data{protocol = http3} = Data) -> %% HTTP/3 streaming body - send headers only via QUIC - do_h3_send_headers(From, Method, Path, Headers, Data); + do_h3_send_headers(From, Method, Path, Headers, + Data#conn_data{max_body_size = req_max_body_size(ReqOpts, Data)}); connected({call, From}, {send_headers, Method, Path, Headers, ReqOpts}, #conn_data{protocol = http2} = Data) -> %% HTTP/2 streaming body - send headers without END_STREAM, then accept body %% chunks via send_body_chunk/finish_send_body. Mirrors do_h3_send_headers/5. - do_h2_send_headers(From, Method, Path, Headers, ReqOpts, Data); + do_h2_send_headers(From, Method, Path, Headers, ReqOpts, + Data#conn_data{max_body_size = req_max_body_size(ReqOpts, Data)}); connected({call, From}, {open_h2_stream, Method, Path, Headers, HandlerPid, Opts}, #conn_data{protocol = http2, h2_conn = H2Conn} = Data) -> @@ -1315,9 +1322,10 @@ connected({call, From}, {open_h2_stream, Method, Path, Headers, HandlerPid, Opts end, {keep_state_and_data, [{reply, From, Reply}]}; -connected({call, From}, {send_headers, Method, Path, Headers, _ReqOpts}, Data) -> +connected({call, From}, {send_headers, Method, Path, Headers, ReqOpts}, Data) -> %% Send only headers for streaming body mode (HTTP/1.1) NewData = Data#conn_data{ + max_body_size = req_max_body_size(ReqOpts, Data), request_from = From, method = Method, path = Path, @@ -2728,7 +2736,13 @@ read_full_body(#conn_data{status = Status} = Data, Acc) when Status =:= 204; Sta read_full_body(Data, Acc) -> case stream_body_chunk(Data) of {ok, Chunk, NewData} -> - read_full_body(NewData, <>); + Acc1 = <>, + case body_within_limit(Acc1, NewData) of + true -> read_full_body(NewData, Acc1); + %% The rest of the body is left on the socket: the caller + %% closes the connection. + false -> {error, body_too_large} + end; {done, NewData} -> %% Body complete - apply decompression if needed maybe_decompress_body(Acc, NewData); @@ -3104,13 +3118,28 @@ maybe_handle_async_redirect(303, <<"POST">>, Headers, true) -> maybe_handle_async_redirect(_, _, _, _) -> no_redirect. -%% @private Skip the response body +%% @private Skip the response body. Chunks are dropped as they arrive, so +%% skipping holds no memory however large the body is. +skip_response_body(#conn_data{method = <<"HEAD">>} = Data) -> + Data; +skip_response_body(#conn_data{status = Status} = Data) when Status =:= 204; Status =:= 304 -> + Data; skip_response_body(Data) -> - case read_full_body(Data, <<>>) of - {ok, _, NewData} -> NewData; + case stream_body_chunk(Data) of + {ok, _Chunk, NewData} -> skip_response_body(NewData); + {done, NewData} -> NewData; {error, _} -> Data end. +%% @private The per-request cap on a buffered response body (GHSA-pph4). +req_max_body_size(ReqOpts, #conn_data{max_body_size = Default}) -> + proplists:get_value(max_body_size, ReqOpts, Default). + +body_within_limit(_Acc, #conn_data{max_body_size = infinity}) -> + true; +body_within_limit(Acc, #conn_data{max_body_size = Max}) -> + byte_size(Acc) =< Max. + %%==================================================================== %% HTTP/2 Support Functions %%==================================================================== @@ -3451,6 +3480,13 @@ handle_h2_recv_timeout(StreamId, TRef, {keep_state, Data} end. +%% @private A buffered HTTP/2 body went over max_body_size: cancel the +%% stream, the connection keeps serving its other streams. +h2_body_too_large(StreamId, From, Data) -> + _ = cancel_h2_stream(Data#conn_data.h2_conn, StreamId), + h2_stream_result(drop_h2_stream(StreamId, Data), + [{reply, From, {error, body_too_large}}]). + %% @private RST_STREAM(CANCEL) a stalled HTTP/2 stream, tolerating a dead conn. cancel_h2_stream(undefined, _StreamId) -> ok; cancel_h2_stream(H2Conn, StreamId) -> @@ -3898,14 +3934,17 @@ h2_on_data(StreamId, Body, EndStream, Data) -> case maps:get(StreamId, Streams, undefined) of {From, {sync, body, Status, Headers, Acc}} -> NewAcc = <>, - case EndStream of - true -> + case body_within_limit(NewAcc, Data) of + false -> + h2_body_too_large(StreamId, From, + Data#conn_data{request_from = undefined}); + true when EndStream -> Data2 = drop_h2_stream( StreamId, Data#conn_data{request_from = undefined}), h2_stream_result(Data2, [{reply, From, {ok, Status, Headers, NewAcc}}]); - false -> + true -> Streams2 = maps:put(StreamId, {From, {sync, body, Status, Headers, NewAcc}}, Streams), @@ -3979,11 +4018,13 @@ h2_on_data(StreamId, Body, EndStream, Data) -> {From, {stream, body_full, Status, Headers, Acc, From}} -> %% Full-body read via body/1 after start_response/1. NewAcc = <>, - case EndStream of - true -> + case body_within_limit(NewAcc, Data) of + false -> + h2_body_too_large(StreamId, From, Data); + true when EndStream -> h2_stream_result(drop_h2_stream(StreamId, Data), [{reply, From, {ok, NewAcc}}]); - false -> + true -> Streams2 = maps:put(StreamId, {From, {stream, body_full, Status, Headers, NewAcc, From}}, Streams), @@ -4565,19 +4606,29 @@ handle_h3_headers(StreamId, Headers, Fin, Streams, Data) -> end end. +%% @private A buffered HTTP/3 body went over max_body_size: reset the +%% stream, the connection keeps serving its other streams. +h3_body_too_large(StreamId, From, Streams, #conn_data{h3_conn = ConnRef} = Data) -> + _ = hackney_h3:reset_stream(ConnRef, StreamId, ?H3_REQUEST_CANCELLED), + {keep_state, Data#conn_data{h3_streams = maps:remove(StreamId, Streams)}, + [{reply, From, {error, body_too_large}}]}. + %% @private Handle HTTP/3 response body data handle_h3_data(StreamId, RecvData, Fin, Streams, Data) -> case maps:get(StreamId, Streams, undefined) of {From, {receiving_body, Status, Headers, AccBody}} -> %% Sync mode - buffer body NewBody = <>, - case Fin of - true -> + case body_within_limit(NewBody, Data) of + false -> + h3_body_too_large(StreamId, From, Streams, + Data#conn_data{request_from = undefined}); + true when Fin -> %% Stream complete - reply with full response UpdatedStreams = maps:remove(StreamId, Streams), {keep_state, Data#conn_data{h3_streams = UpdatedStreams, request_from = undefined}, [{reply, From, {ok, Status, Headers, NewBody}}]}; - false -> + true -> %% More data expected UpdatedStreams = maps:put(StreamId, {From, {receiving_body, Status, Headers, NewBody}}, Streams), @@ -4663,11 +4714,13 @@ handle_h3_data(StreamId, RecvData, Fin, Streams, Data) -> {_, {streaming_body_full, Status, Headers, Acc, From}} -> %% body/1 is waiting for the whole body NewAcc = <>, - case Fin of - true -> + case body_within_limit(NewAcc, Data) of + false -> + h3_body_too_large(StreamId, From, Streams, Data); + true when Fin -> {keep_state, Data#conn_data{h3_streams = maps:remove(StreamId, Streams)}, [{reply, From, {ok, NewAcc}}]}; - false -> + true -> NewStreamState = {streaming_body_full, Status, Headers, NewAcc, From}, {keep_state, Data#conn_data{h3_streams = maps:put(StreamId, {undefined, NewStreamState}, Streams)}} diff --git a/src/hackney_h3.erl b/src/hackney_h3.erl index 394d2d6b..6278561e 100644 --- a/src/hackney_h3.erl +++ b/src/hackney_h3.erl @@ -27,6 +27,7 @@ -behaviour(gen_server). +-include("hackney.hrl"). -include("hackney_lib.hrl"). -export([ @@ -88,12 +89,9 @@ -define(CONN_TABLE, hackney_h3_conns). -%% GHSA-jq4m: default ceiling on a buffered HTTP/3 response body. The -%% non-streaming await path holds the whole body in memory, so without a cap -%% a peer that trickles data forever drives the node to OOM. Configurable via -%% the `max_body_size' option (`infinity' disables it); very large downloads -%% should use the streaming API instead. --define(DEFAULT_MAX_BODY_SIZE, 16#20000000). %% 512 MiB +%% GHSA-jq4m: the non-streaming await path holds the whole body in memory, +%% so it is capped at ?DEFAULT_MAX_BODY_SIZE (hackney.hrl) unless the +%% `max_body_size' option says otherwise. -type h3_conn() :: reference(). -type stream_id() :: non_neg_integer(). diff --git a/test/hackney_max_body_size_tests.erl b/test/hackney_max_body_size_tests.erl new file mode 100644 index 00000000..8b6a7252 --- /dev/null +++ b/test/hackney_max_body_size_tests.erl @@ -0,0 +1,161 @@ +%%% -*- erlang -*- +%%% +%%% This file is part of hackney released under the Apache 2 license. +%%% See the NOTICE for more information. +%%% +%%% A buffered response body (with_body, body/1) is capped by max_body_size +%%% on every protocol, so a server sending an endless body cannot exhaust +%%% memory (GHSA-pph4). Streaming reads are not capped. +-module(hackney_max_body_size_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-define(BODY_SIZE, 32768). + +%%==================================================================== +%% HTTP/1.1 +%%==================================================================== + +h1_test_() -> + {setup, + fun() -> + {ok, _} = application:ensure_all_started(cowboy), + {ok, _} = application:ensure_all_started(hackney), + Dispatch = cowboy_router:compile([{'_', [{"/[...]", test_http_resource, []}]}]), + {ok, _} = cowboy:start_clear(max_body_h1, [{port, 0}], + #{env => #{dispatch => Dispatch}}), + ranch:get_port(max_body_h1) + end, + fun(_) -> cowboy:stop_listener(max_body_h1) end, + fun(Port) -> + Url = iolist_to_binary(["http://127.0.0.1:", integer_to_list(Port), + "/chunked/", integer_to_list(?BODY_SIZE)]), + [{"with_body over the cap is an error", + ?_assertEqual({error, body_too_large}, + h1_get(Url, [{with_body, true}, {max_body_size, 1024}]))}, + {"body/1 after a streamed upload over the cap is an error", + fun() -> + {ok, Conn} = hackney:request(get, Url, [], stream, + [{pool, false}, {max_body_size, 1024}]), + ok = hackney:finish_send_body(Conn), + {ok, 200, _, Conn} = hackney:start_response(Conn), + ?assertEqual({error, body_too_large}, hackney:body(Conn)) + end}, + {"a body under the cap is returned", + fun() -> + {ok, 200, _, Body} = h1_get(Url, [{with_body, true}, + {max_body_size, ?BODY_SIZE}]), + ?assertEqual(?BODY_SIZE, byte_size(Body)) + end}, + {"infinity turns the cap off", + fun() -> + {ok, 200, _, Body} = h1_get(Url, [{with_body, true}, + {max_body_size, infinity}]), + ?assertEqual(?BODY_SIZE, byte_size(Body)) + end}, + {"the default cap leaves ordinary bodies alone", + fun() -> + {ok, 200, _, Body} = h1_get(Url, [{with_body, true}]), + ?assertEqual(?BODY_SIZE, byte_size(Body)) + end}] + end}. + +h1_get(Url, Opts) -> + hackney:request(get, Url, [], <<>>, [{pool, false} | Opts]). + +%%==================================================================== +%% HTTP/2 +%%==================================================================== + +h2_test_() -> + {setup, + fun() -> + _ = application:ensure_all_started(hackney), + _ = application:ensure_all_started(h2), + start_h2_server() + end, + fun(Server) -> _ = (try h2:stop_server(Server) catch _:_ -> ok end) end, + fun(Server) -> + Url = iolist_to_binary(["https://localhost:", + integer_to_list(h2:server_port(Server)), "/"]), + [{"with_body over the cap is an error", + ?_assertEqual({error, body_too_large}, + h2_get(Url, [{with_body, true}, {max_body_size, 1024}]))}, + {"a body under the cap is returned", + fun() -> + {ok, 200, _, Body} = h2_get(Url, [{with_body, true}, + {max_body_size, ?BODY_SIZE}]), + ?assertEqual(?BODY_SIZE, byte_size(Body)) + end}, + {"the connection keeps serving after a capped stream", + fun() -> + Opts = [{pool, max_body_h2_pool}, {with_body, true}], + ?assertEqual({error, body_too_large}, + h2_get(Url, [{max_body_size, 1024} | Opts])), + {ok, 200, _, Body} = h2_get(Url, Opts), + ?assertEqual(?BODY_SIZE, byte_size(Body)), + hackney_pool:stop_pool(max_body_h2_pool) + end}] + end}. + +h2_get(Url, Opts) -> + Base = [{protocols, [http2]}, + {recv_timeout, 5000}, + {ssl_options, [{insecure, true}, {verify, verify_none}]}], + Opts1 = case proplists:is_defined(pool, Opts) of + true -> Opts; + false -> [{pool, false} | Opts] + end, + hackney:request(get, Url, [], <<>>, Opts1 ++ Base). + +%% Sends the body in eight DATA frames so the cap trips mid-stream. +start_h2_server() -> + Chunk = binary:copy(<<"x">>, ?BODY_SIZE div 8), + Handler = fun(Conn, Sid, _Method, _Path, _Headers) -> + ok = h2:send_response(Conn, Sid, 200, + [{<<"content-type">>, <<"application/octet-stream">>}]), + %% The client cancels a capped stream mid-body, so later sends + %% may fail; that is the point. + _ = [h2:send_data(Conn, Sid, Chunk, false) || _ <- lists:seq(1, 7)], + _ = h2:send_data(Conn, Sid, Chunk, true), + ok + end, + Certs = cert_dir(), + {ok, Server} = h2:start_server(0, #{ + cert => filename:join(Certs, "server.pem"), + key => filename:join(Certs, "server.key"), + handler => Handler + }), + Server. + +cert_dir() -> + BeamDir = filename:dirname(code:which(?MODULE)), + Root = filename:join([BeamDir, "..", "..", "..", "..", ".."]), + filename:join([filename:absname(Root), "test", "certs"]). + +%%==================================================================== +%% HTTP/3 +%%==================================================================== + +h3_test_() -> + {setup, + fun hackney_h3_test_server:start/0, + fun hackney_h3_test_server:stop/1, + fun(Server) -> + %% /large is 16 x 4 KiB. + Url = hackney_h3_test_server:url(Server, <<"/large">>), + [{"with_body over the cap is an error", + {timeout, 30, + ?_assertEqual({error, body_too_large}, + h3_get(Url, [{with_body, true}, {max_body_size, 1024}]))}}, + {"a body under the cap is returned", + {timeout, 30, + fun() -> + {ok, 200, _, Body} = h3_get(Url, [{with_body, true}]), + ?assertEqual(65536, byte_size(Body)) + end}}] + end}. + +h3_get(Url, Opts) -> + hackney:request(get, Url, [], <<>>, + [{pool, false} | Opts] ++ hackney_h3_test_server:hackney_opts()).