diff --git a/CHANGELOG.md b/CHANGELOG.md index a3ffa10..153b6e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## 1.5.0 +* Fixed two denial-of-service issues in the decoder. A crafted database could + nest data-section pointers to shared targets so that decoding one record + cost exponential time and memory from a small file, or point many times at + one large string or bytes value so that a record with few values + materialized gigabytes. The decoder now bounds each record it decodes and + the metadata decoded when a database is opened. A database that exceeds a + limit raises `InvalidDatabaseError`. The limits are: + * 65,536 decoded values, as the MaxMind DB specification recommends. + * 512 levels of nesting, as the specification recommends. This also stops + pointer cycles. + * 2 MiB of string, bytes, and integer payload. The specification leaves this + limit to the reader. 2 MiB matches libmaxminddb. +* The decoder limits can be changed with the new `max_values`, + `max_payload_bytes`, and `max_depth` options to `MaxMind::DB.new`. +* Pointers that target other pointers are now rejected as invalid, as required + by the MaxMind DB specification. +* Lookups are faster. The decoder allocates fewer strings and dispatches on the + data type with a jump table. GeoLite City lookups in memory mode on CRuby + 3.4 are about 18% faster than in 1.4.0. * Unnecessary files were removed from the published .gem. ## 1.4.0 (2025-11-20) diff --git a/lib/maxmind/db.rb b/lib/maxmind/db.rb index 98c1442..8e776bd 100644 --- a/lib/maxmind/db.rb +++ b/lib/maxmind/db.rb @@ -72,7 +72,7 @@ class DB # @param database [String] a path to a {MaxMind # DB}[https://maxmind.github.io/MaxMind-DB/]. # - # @param options [Hash] options controlling the behavior of + # @param options [Hash] options controlling the behavior of # the DB. # # @option options [Symbol] :mode Defines how to open the database. It may @@ -80,11 +80,27 @@ class DB # one, DB uses MODE_AUTO. Refer to the definition of those constants for # an explanation of their meaning. # - # @raise [InvalidDatabaseError] if the database is corrupt or invalid. + # @option options [Integer] :max_values The maximum number of values a + # single record, or the metadata, may decode to. The default is 65,536. + # The largest records MaxMind produces decode to a few hundred values. + # + # @option options [Integer] :max_payload_bytes The maximum total size in + # bytes of the strings, bytes, and integers a single record, or the + # metadata, may decode. The default is 2 MiB. The largest records MaxMind + # produces hold about a kilobyte. + # + # @option options [Integer] :max_depth The maximum nesting depth of maps, + # arrays, and pointers in a single record, or the metadata. The default + # is 512. # - # @raise [ArgumentError] if the mode is invalid. + # @raise [InvalidDatabaseError] if the database is corrupt or invalid. A + # database that exceeds any of the limits above raises this error from + # the lookup, or from this constructor if the metadata exceeds them. + # + # @raise [ArgumentError] if the mode or a limit is invalid. def initialize(database, options = {}) options[:mode] = MODE_AUTO unless options.key?(:mode) + limits = decoder_limits(options) case options[:mode] when MODE_AUTO, MODE_FILE @@ -101,11 +117,11 @@ def initialize(database, options = {}) @size = @io.size metadata_start = find_metadata_start - metadata_decoder = Decoder.new(@io, metadata_start) + metadata_decoder = Decoder.new(@io, metadata_start, **limits) metadata_map, = metadata_decoder.decode(metadata_start) @metadata = Metadata.new(metadata_map) @decoder = Decoder.new(@io, @metadata.search_tree_size + - DATA_SECTION_SEPARATOR_SIZE) + DATA_SECTION_SEPARATOR_SIZE, **limits) # Store copies as instance variables to reduce method calls. @ip_version = @metadata.ip_version @@ -271,6 +287,24 @@ def resolve_data_pointer(pointer) data end + LIMIT_OPTIONS = %i[max_values max_payload_bytes max_depth].freeze + private_constant :LIMIT_OPTIONS + + # Return the decoder limits given in +options+ as keyword arguments for + # Decoder.new. An absent option keeps the decoder's default. + def decoder_limits(options) + limits = {} + LIMIT_OPTIONS.each do |name| + next unless options.key?(name) + + value = options[name] + raise ArgumentError, "#{name} must be a positive integer" unless value.is_a?(Integer) && value.positive? + + limits[name] = value + end + limits + end + def find_metadata_start metadata_max_size = [@size, METADATA_MAX_SIZE].min diff --git a/lib/maxmind/db/decoder.rb b/lib/maxmind/db/decoder.rb index 179f665..6a6c0a7 100644 --- a/lib/maxmind/db/decoder.rb +++ b/lib/maxmind/db/decoder.rb @@ -12,12 +12,12 @@ class DB # # @!visibility private class Decoder - # rubocop:disable Style/OptionalBooleanParameter + # rubocop:disable Style/OptionalBooleanParameter, Metrics/ParameterLists # Create a +Decoder+. # - # +io+ is the DB. It must provide a +read+ method. It must be opened in - # binary mode. + # +io+ is the DB. It must provide +read+ and +getbyte+ methods. It must be + # opened in binary mode. # # +pointer_base+ is the base number to use when decoding a pointer. It is # where the data section begins rather than the beginning of the file. @@ -25,21 +25,102 @@ class Decoder # section. # # +pointer_test+ is used for testing pointer code. - def initialize(io, pointer_base = 0, pointer_test = false) + # + # +max_values+, +max_payload_bytes+, and +max_depth+ set the per-decode + # limits described below and default to the constants there. + def initialize(io, pointer_base = 0, pointer_test = false, + max_values: MAX_VALUES, max_payload_bytes: MAX_BYTES, + max_depth: MAX_DEPTH) @io = io @pointer_base = pointer_base @pointer_test = pointer_test + @max_values = max_values + @max_payload_bytes = max_payload_bytes + @max_depth = max_depth end - # rubocop:enable Style/OptionalBooleanParameter + # rubocop:enable Style/OptionalBooleanParameter, Metrics/ParameterLists + + # Per-decode limits. The value and depth limits are the ones the MaxMind DB + # specification recommends. The specification leaves the payload limit to + # the reader, and 2 MiB matches libmaxminddb. +budget+ is a three-element + # array, [values_remaining, depth, bytes_remaining], shared across the + # recursion so every count survives it. It is call-local, which keeps the + # decoder safe for concurrent reads. + # + # The value limit stops a pointer fan-out. It follows the specification's + # flat rule: the root is one value, each array reserves one value per + # element, and each map reserves two values per entry before iterating. A + # pointer is not charged separately from the logical value at the root or + # its position in a container. A re-decoded node drains the budget, and an + # oversized declared size is rejected before the loop reads anything. The + # largest real records decode a few hundred values. + # + # The byte limit stops payload amplification: a crafted database can point + # many times at one large string or bytes value, so a bounded value count + # still materializes gigabytes. Each string and bytes value, and each + # variable-length integer, subtracts its own length before it is read, so a + # re-decoded (fanned-out) target recharges its payload and an oversized + # declared length is rejected before any bytes are copied. Fixed-width + # scalars are not charged. + # + # The depth limit stops a pointer cycle or over-deep data before the stack + # overflows. + MAX_VALUES = 1 << 16 + private_constant :MAX_VALUES + + MAX_BYTES = 1 << 21 + private_constant :MAX_BYTES + + MAX_DEPTH = 512 + private_constant :MAX_DEPTH + + BUDGET_VALUES = 0 + BUDGET_DEPTH = 1 + BUDGET_BYTES = 2 + private_constant :BUDGET_VALUES, :BUDGET_DEPTH, :BUDGET_BYTES + + # JRuby can exhaust the stack before the depth limit is reached and raises + # a Java StackOverflowError, which is not a SystemStackError. Catch both so + # a pointer cycle always becomes an InvalidDatabaseError. + STACK_ERRORS = if defined?(JRUBY_VERSION) + [SystemStackError, Java::JavaLang::StackOverflowError].freeze + else + [SystemStackError].freeze + end + private_constant :STACK_ERRORS private - def decode_array(size, offset) + # The limit checks are inlined at each call site so containers and + # pointers do not add a helper call. Only the raise is factored out. + def raise_depth_exceeded + raise InvalidDatabaseError, + 'The MaxMind DB file\'s data section exceeds the maximum depth' + end + + def raise_values_exceeded + raise InvalidDatabaseError, + 'The MaxMind DB file\'s data section exceeds the maximum number of values' + end + + # Each string, bytes, and variable-length integer decoder charges its size + # against the payload budget inline, before the bytes are read, so an + # oversized declared length is rejected before it is copied. Ruby integers + # are arbitrary precision, so the subtraction cannot overflow. + def raise_bytes_exceeded + raise InvalidDatabaseError, + 'The MaxMind DB file\'s data section exceeds the maximum number of bytes' + end + + def decode_array(size, offset, budget) + raise_values_exceeded if (budget[BUDGET_VALUES] -= size) < 0 + raise_depth_exceeded if (budget[BUDGET_DEPTH] += 1) > @max_depth array = [] size.times do - value, offset = decode(offset) + value, offset = decode_with_budget(offset, budget) array << value end + budget[BUDGET_DEPTH] -= 1 [array, offset] end @@ -47,7 +128,8 @@ def decode_boolean(size, offset) [size != 0, offset] end - def decode_bytes(size, offset) + def decode_bytes(size, offset, budget) + raise_bytes_exceeded if (budget[BUDGET_BYTES] -= size) < 0 [@io.read(offset, size), offset + size] end @@ -66,37 +148,45 @@ def decode_float(size, offset) def verify_size(expected, actual) return if expected == actual + raise_invalid_size + end + + def raise_invalid_size raise InvalidDatabaseError, 'The MaxMind DB file\'s data section contains bad data (unknown data type or corrupt data)' end - def decode_int32(size, offset) - decode_int('l>', 4, size, offset) + def decode_int32(size, offset, budget) + decode_int('l>', 4, size, offset, budget) end - def decode_uint16(size, offset) - decode_int('n', 2, size, offset) + def decode_uint16(size, offset, budget) + decode_int('n', 2, size, offset, budget) end - def decode_uint32(size, offset) - decode_int('N', 4, size, offset) + def decode_uint32(size, offset, budget) + decode_int('N', 4, size, offset, budget) end - def decode_uint64(size, offset) - decode_int('Q>', 8, size, offset) + def decode_uint64(size, offset, budget) + decode_int('Q>', 8, size, offset, budget) end - def decode_int(type_code, type_size, size, offset) + def decode_int(type_code, type_size, size, offset, budget) + raise_invalid_size if size > type_size return 0, offset if size == 0 + raise_bytes_exceeded if (budget[BUDGET_BYTES] -= size) < 0 buf = @io.read(offset, size) buf = buf.rjust(type_size, "\x00") if size != type_size [buf.unpack1(type_code), offset + size] end - def decode_uint128(size, offset) + def decode_uint128(size, offset, budget) + raise_invalid_size if size > 16 return 0, offset if size == 0 + raise_bytes_exceeded if (budget[BUDGET_BYTES] -= size) < 0 buf = @io.read(offset, size) if size <= 8 @@ -112,45 +202,49 @@ def decode_uint128(size, offset) [a | b, offset + size] end - def decode_map(size, offset) + def decode_map(size, offset, budget) + # A map entry decodes a key and a value, so it costs two values. + raise_values_exceeded if (budget[BUDGET_VALUES] -= size * 2) < 0 + raise_depth_exceeded if (budget[BUDGET_DEPTH] += 1) > @max_depth container = {} size.times do - key, offset = decode(offset) - value, offset = decode(offset) + key, offset = decode_with_budget(offset, budget) + value, offset = decode_with_budget(offset, budget) container[key] = value end + budget[BUDGET_DEPTH] -= 1 [container, offset] end def decode_pointer(size, offset) pointer_size = size >> 3 + # Build the pointer with integer arithmetic to avoid temporary strings + # when combining control bits with the payload bytes. case pointer_size when 0 new_offset = offset + 1 - buf = (size & 0x7).chr << @io.read(offset, 1) - pointer = buf.unpack1('n') + @pointer_base + pointer = ((size & 0x7) << 8) | @io.getbyte(offset) when 1 new_offset = offset + 2 - buf = "\x00".b << (size & 0x7).chr << @io.read(offset, 2) - pointer = buf.unpack1('N') + 2048 + @pointer_base + pointer = ((size & 0x7) << 16) | @io.read(offset, 2).unpack1('n') + pointer += 2048 when 2 new_offset = offset + 3 - buf = (size & 0x7).chr << @io.read(offset, 3) - pointer = buf.unpack1('N') + 526_336 + @pointer_base + buf = @io.read(offset, 3) + pointer = ((size & 0x7) << 24) | (buf.getbyte(0) << 16) | + (buf.getbyte(1) << 8) | buf.getbyte(2) + pointer += 526_336 else new_offset = offset + 4 - buf = @io.read(offset, 4) - pointer = buf.unpack1('N') + @pointer_base + pointer = @io.read(offset, 4).unpack1('N') end - - return pointer, new_offset if @pointer_test - - value, = decode(pointer) - [value, new_offset] + pointer += @pointer_base + [pointer, new_offset] end - def decode_utf8_string(size, offset) + def decode_utf8_string(size, offset, budget) + raise_bytes_exceeded if (budget[BUDGET_BYTES] -= size) < 0 new_offset = offset + size buf = @io.read(offset, size) buf.force_encoding(Encoding::UTF_8) @@ -159,23 +253,6 @@ def decode_utf8_string(size, offset) [buf, new_offset] end - TYPE_DECODER = { - 1 => :decode_pointer, - 2 => :decode_utf8_string, - 3 => :decode_double, - 4 => :decode_bytes, - 5 => :decode_uint16, - 6 => :decode_uint32, - 7 => :decode_map, - 8 => :decode_int32, - 9 => :decode_uint64, - 10 => :decode_uint128, - 11 => :decode_array, - 14 => :decode_boolean, - 15 => :decode_float, - }.freeze - private_constant :TYPE_DECODER - public # Decode a section of the data section starting at +offset+. @@ -187,23 +264,82 @@ def decode_utf8_string(size, offset) # # Throws an exception if there is an error. def decode(offset) + # Bound the work per decode so a crafted database cannot exhaust CPU or + # memory. +budget+ carries the remaining value count, the current depth, + # and the remaining payload-byte allowance, and is call-local, which + # keeps the decoder safe for concurrent reads. The root value is charged + # here; containers charge their children. The depth limit catches a + # pointer cycle on MRI. JRuby can exhaust the stack before the limit is + # reached and raises a Java StackOverflowError, so catch that too and + # report the same error. + decode_with_budget(offset, [@max_values - 1, 0, @max_payload_bytes]) + rescue *STACK_ERRORS + raise InvalidDatabaseError, + 'The MaxMind DB file\'s data section exceeds the maximum depth' + end + + private + + # The dispatch below is one branch per data type, so the method's + # cyclomatic complexity is above the cop's default. It is inlined here + # for speed and the branches are uniform. + # rubocop:disable-next Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + def decode_with_budget(offset, budget) + pointer_return_offset = nil new_offset = offset + 1 - buf = @io.read(offset, 1) - ctrl_byte = buf.ord + ctrl_byte = @io.getbyte(offset) type_num = ctrl_byte >> 5 type_num, new_offset = read_extended(new_offset) if type_num == 0 size, new_offset = size_from_ctrl_byte(ctrl_byte, new_offset, type_num) - # We could check an element exists at `type_num', but for performance I - # don't. - send(TYPE_DECODER[type_num], size, new_offset) - end + if type_num == 1 + pointer, pointer_return_offset = decode_pointer(size, new_offset) + return [pointer, pointer_return_offset] if @pointer_test + + # The root or containing collection already charged the logical value + # at the pointer's position. Following it adds depth but no separate + # value. Its target cannot be another pointer, and decoding the target + # still reserves container children and charges payload bytes. + raise_depth_exceeded if (budget[BUDGET_DEPTH] += 1) > @max_depth + new_offset = pointer + 1 + ctrl_byte = @io.getbyte(pointer) + type_num = ctrl_byte >> 5 + if type_num == 1 + raise InvalidDatabaseError, + 'The MaxMind DB file\'s data section contains bad data (pointer points to another pointer)' + end + type_num, new_offset = read_extended(new_offset) if type_num == 0 + size, new_offset = size_from_ctrl_byte(ctrl_byte, new_offset, type_num) + end - private + # Direct case dispatch avoids looking the method up in a Hash and + # calling it with send. + result = case type_num + when 2 then decode_utf8_string(size, new_offset, budget) + when 3 then decode_double(size, new_offset) + when 4 then decode_bytes(size, new_offset, budget) + when 5 then decode_uint16(size, new_offset, budget) + when 6 then decode_uint32(size, new_offset, budget) + when 7 then decode_map(size, new_offset, budget) + when 8 then decode_int32(size, new_offset, budget) + when 9 then decode_uint64(size, new_offset, budget) + when 10 then decode_uint128(size, new_offset, budget) + when 11 then decode_array(size, new_offset, budget) + when 14 then decode_boolean(size, new_offset) + when 15 then decode_float(size, new_offset) + else + raise InvalidDatabaseError, + "The MaxMind DB file's data section contains bad data (unknown data type #{type_num})" + end + return result unless pointer_return_offset + + budget[BUDGET_DEPTH] -= 1 + result[1] = pointer_return_offset + result + end def read_extended(offset) - buf = @io.read(offset, 1) - next_byte = buf.ord + next_byte = @io.getbyte(offset) type_num = next_byte + 7 if type_num < 7 raise InvalidDatabaseError, @@ -218,8 +354,7 @@ def size_from_ctrl_byte(ctrl_byte, offset, type_num) return size, offset if type_num == 1 || size < 29 if size == 29 - size_bytes = @io.read(offset, 1) - size = 29 + size_bytes.ord + size = 29 + @io.getbyte(offset) return size, offset + 1 end diff --git a/lib/maxmind/db/file_reader.rb b/lib/maxmind/db/file_reader.rb index 808b2e3..eb73b53 100644 --- a/lib/maxmind/db/file_reader.rb +++ b/lib/maxmind/db/file_reader.rb @@ -44,6 +44,11 @@ def close @fh.close end + # Return the byte at +offset+ as an Integer. + def getbyte(offset) + read(offset, 1).ord + end + def read(offset, size) return ''.b if size == 0 diff --git a/lib/maxmind/db/memory_reader.rb b/lib/maxmind/db/memory_reader.rb index 061bd54..eab6811 100644 --- a/lib/maxmind/db/memory_reader.rb +++ b/lib/maxmind/db/memory_reader.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'maxmind/db/errors' + module MaxMind class DB # @!visibility private @@ -24,8 +26,23 @@ def inspect def close; end + # Return the byte at +offset+ as an Integer without allocating a String. + def getbyte(offset) + @buf.getbyte(offset) || raise_bad_data + end + def read(offset, size) - @buf[offset, size] + return ''.b if size == 0 + + raise_bad_data if offset + size > @buf.length + + @buf.byteslice(offset, size) + end + + private + + def raise_bad_data + raise InvalidDatabaseError, 'The MaxMind DB file contains bad data' end end end diff --git a/test/data b/test/data index e7b0018..363086b 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit e7b0018644317ad6f33eb408f4479ccc4ab0e6fd +Subproject commit 363086b7d90650100e91f954937794c6a090c2a0 diff --git a/test/test_decoder.rb b/test/test_decoder.rb index 95b211e..471952e 100644 --- a/test/test_decoder.rb +++ b/test/test_decoder.rb @@ -5,6 +5,29 @@ require 'mmdb_util' class DecoderTest < Minitest::Test + class HeaderOnlyReader + def initialize(header) + @header = header + end + + def getbyte(offset) + byte = @header.getbyte(offset) + raise "The decoder read beyond the header at offset #{offset}" unless byte + + byte + end + + def read(offset, size) + bytes = @header.byteslice(offset, size) + if bytes.nil? || bytes.bytesize != size + message = "The decoder read #{size} payload bytes at offset #{offset}" + raise message + end + + bytes + end + end + def test_arrays arrays = { "\x00\x04".b => [], @@ -129,6 +152,221 @@ def test_pointer validate_type_decoding('pointers', pointers) end + def encode_pointer1(target) + # One-byte-payload pointer (type 1, pointer_size 0) with base 0. + [(1 << 5) | ((target >> 8) & 0x7), target & 0xFF].pack('C*').b + end + + def test_pointer_fan_out_is_bounded + # A data section of nested arrays, each holding two pointers to the node + # below, would cost 2**depth decode operations. The decoder bounds the + # number of values it decodes per lookup and rejects the database. + depth = 100 + buf = "\xa0".b # leaf: uint16 with value 0 + prev = 0 + depth.times do + offset = buf.bytesize + buf += "\x02\x04".b + encode_pointer1(prev) + encode_pointer1(prev) + prev = offset + end + + io = MaxMind::DB::MemoryReader.new(buf, is_buffer: true) + assert_raises(MaxMind::DB::InvalidDatabaseError) do + MaxMind::DB::Decoder.new(io, 0).decode(prev) + end + end + + def scalar_pointer_array(pointer_count) + # A uint16 leaf at offset 0 and, at offset 1, an array of pointers to it. + array_header = [0x1e, 4, pointer_count - 285].pack('CCn') + array = array_header + (encode_pointer1(0) * pointer_count) + MaxMind::DB::MemoryReader.new("\xa0".b + array, is_buffer: true) + end + + def test_value_limit_follows_the_flat_rule + # The specification charges the root as one value and each pointer as the + # value it resolves to, not as a separate value. An array of 65,535 + # pointers to a scalar is therefore 65,536 values, exactly the limit, and + # decodes. One more pointer exceeds it. + decoded, = MaxMind::DB::Decoder.new(scalar_pointer_array(65_535), 0).decode(1) + + assert_equal(65_535, decoded.length) + + error = assert_raises(MaxMind::DB::InvalidDatabaseError) do + MaxMind::DB::Decoder.new(scalar_pointer_array(65_536), 0).decode(1) + end + assert_equal( + 'The MaxMind DB file\'s data section exceeds the maximum number of values', + error.message + ) + end + + def test_integer_payload_is_charged + # A variable-length integer charges its declared size against the payload + # budget like a string does. A 4-byte uint32 decodes with a 4-byte budget + # and is rejected with a 3-byte one; a 16-byte uint128 likewise at 16 and + # 15. 0xc4 is uint32 with size 4; 0x10 0x03 is the extended uint128 type + # with size 16. + message = 'The MaxMind DB file\'s data section exceeds the maximum number of bytes' + uint32 = MaxMind::DB::MemoryReader.new("\xc4\x00\x00\x00\x01".b, is_buffer: true) + uint128 = MaxMind::DB::MemoryReader.new( + "\x10\x03".b + ("\x00".b * 15) + "\x01".b, is_buffer: true + ) + + assert_equal(1, MaxMind::DB::Decoder.new(uint32, 0, max_payload_bytes: 4).decode(0)[0]) + assert_equal(1, MaxMind::DB::Decoder.new(uint128, 0, max_payload_bytes: 16).decode(0)[0]) + + [[uint32, 3], [uint128, 15]].each do |io, limit| + error = assert_raises(MaxMind::DB::InvalidDatabaseError, limit.to_s) do + MaxMind::DB::Decoder.new(io, 0, max_payload_bytes: limit).decode(0) + end + assert_equal(message, error.message) + end + end + + def test_oversized_integer_is_rejected_before_read + headers = { + 'uint16' => "\xa3".b, + 'uint32' => "\xc5".b, + 'int32' => "\x05\x01".b, + 'uint64' => "\x09\x02".b, + 'uint128' => "\x11\x03".b, + } + + headers.each do |name, header| + io = HeaderOnlyReader.new(header) + + assert_raises(MaxMind::DB::InvalidDatabaseError, name) do + MaxMind::DB::Decoder.new(io, 0).decode(0) + end + end + end + + def test_unknown_type_raises + # An extended type byte selects type 7 + its value. 0x10 gives type 23, + # which the format does not define; the deprecated end marker is type 13. + # Both must raise InvalidDatabaseError rather than fail inside the dispatch. + ["\x00\x10".b, "\x00\x06".b].each do |buf| + io = MaxMind::DB::MemoryReader.new(buf, is_buffer: true) + error = assert_raises(MaxMind::DB::InvalidDatabaseError, buf.inspect) do + MaxMind::DB::Decoder.new(io, 0).decode(0) + end + assert_match(/unknown data type/, error.message) + end + end + + def test_pointer_to_pointer_raises + # The specification forbids a pointer from targeting another pointer. + io = MaxMind::DB::MemoryReader.new("\x20\x02\x20\x02".b, is_buffer: true) + error = assert_raises(MaxMind::DB::InvalidDatabaseError) do + MaxMind::DB::Decoder.new(io, 0).decode(0) + end + assert_equal( + 'The MaxMind DB file\'s data section contains bad data (pointer points to another pointer)', + error.message + ) + end + + def test_cyclic_pointer_raises + # An array that contains a pointer back to itself is a legal pointer target + # but must still be stopped by the depth limit. + io = MaxMind::DB::MemoryReader.new("\x01\x04\x20\x00".b, is_buffer: true) + assert_raises(MaxMind::DB::InvalidDatabaseError) do + MaxMind::DB::Decoder.new(io, 0).decode(0) + end + end + + def test_depth_limit_boundary + # A shallow explicit limit tests the boundary without depending on the + # native stack available to a particular Ruby implementation. + limit = 32 + io = MaxMind::DB::MemoryReader.new(("\x01\x04".b * limit) + "\xa0".b, is_buffer: true) + decoded, = MaxMind::DB::Decoder.new(io, 0, max_depth: limit).decode(0) + limit.times { decoded = decoded.fetch(0) } + + assert_equal(0, decoded) + + io = MaxMind::DB::MemoryReader.new( + ("\x01\x04".b * (limit + 1)) + "\xa0".b, + is_buffer: true + ) + error = assert_raises(MaxMind::DB::InvalidDatabaseError) do + MaxMind::DB::Decoder.new(io, 0, max_depth: limit).decode(0) + end + assert_equal( + 'The MaxMind DB file\'s data section exceeds the maximum depth', + error.message + ) + end + + def test_container_depth_is_restored_between_siblings + count = 600 + array_header = [0x1e, 4, count - 285].pack('CCn') + containers = { + 'arrays' => "\x00\x04".b, + 'maps' => "\xe0".b, + } + + containers.each do |name, empty_container| + io = MaxMind::DB::MemoryReader.new( + array_header + (empty_container * count), + is_buffer: true + ) + decoded, = MaxMind::DB::Decoder.new(io, 0).decode(0) + + assert_equal(count, decoded.length, name) + assert_empty(decoded.reject(&:empty?), name) + end + end + + def test_oversized_payload_is_rejected_before_read + # Each header declares a two-byte payload, but the reader contains only the + # header and raises if the decoder tries to copy the missing payload. + headers = { + 'UTF-8 string' => "\x42".b, + 'bytes' => "\x82".b, + } + + headers.each do |name, header| + io = HeaderOnlyReader.new(header) + error = assert_raises(MaxMind::DB::InvalidDatabaseError, name) do + MaxMind::DB::Decoder.new(io, 0, max_payload_bytes: 1).decode(0) + end + assert_equal( + 'The MaxMind DB file\'s data section exceeds the maximum number of bytes', + error.message, + name + ) + end + end + + def test_oversized_array_is_bounded + # An array that declares 65,536 children contains 65,537 total values with + # the array itself, so it exceeds the 65,536-value limit. The reader holds + # only the header and raises if the decoder tries to read a child. 0x1e 0x04 + # selects an array with size code 30; 0xfee3 encodes 65,536 - 285. + io = HeaderOnlyReader.new("\x1e\x04\xfe\xe3".b) + error = assert_raises(MaxMind::DB::InvalidDatabaseError) do + MaxMind::DB::Decoder.new(io, 0).decode(0) + end + assert_equal( + 'The MaxMind DB file\'s data section exceeds the maximum number of values', + error.message + ) + end + + def test_oversized_map_is_bounded + # A map entry decodes a key and a value, so a map of N entries costs 2N + # children. A map that declares 32,769 entries has 65,538 children and + # 65,539 total values including the map itself, just past the 65,536 limit, + # and is rejected before any entry is read. 0xfe is a map with size code + # 30, then the two size bytes for 32,769 - 285 = 32,484 (0x7ee4). + io = HeaderOnlyReader.new("\xfe\x7e\xe4".b) + assert_raises(MaxMind::DB::InvalidDatabaseError) do + MaxMind::DB::Decoder.new(io, 0).decode(0) + end + end + # rubocop:disable-next Style/ClassVars @@strings = { "\x40".b => '', diff --git a/test/test_memory_reader.rb b/test/test_memory_reader.rb new file mode 100644 index 0000000..87e5b50 --- /dev/null +++ b/test/test_memory_reader.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'maxmind/db/memory_reader' +require 'minitest/autorun' + +class MemoryReaderTest < Minitest::Test + def setup + @reader = MaxMind::DB::MemoryReader.new('abc'.b, is_buffer: true) + end + + def test_getbyte_requires_an_existing_byte + assert_equal('c'.ord, @reader.getbyte(2)) + assert_raises(MaxMind::DB::InvalidDatabaseError) { @reader.getbyte(3) } + end + + def test_read_requires_the_full_range + assert_equal('bc', @reader.read(1, 2)) + assert_equal(''.b, @reader.read(4, 0)) + assert_raises(MaxMind::DB::InvalidDatabaseError) { @reader.read(2, 2) } + assert_raises(MaxMind::DB::InvalidDatabaseError) { @reader.read(4, 1) } + end + + def test_read_uses_the_current_buffer_length + buffer = 'abc'.b + reader = MaxMind::DB::MemoryReader.new(buffer, is_buffer: true) + buffer.replace('a'.b) + + assert_raises(MaxMind::DB::InvalidDatabaseError) { reader.read(0, 2) } + end +end diff --git a/test/test_reader.rb b/test/test_reader.rb index aafa9a4..76543bf 100644 --- a/test/test_reader.rb +++ b/test/test_reader.rb @@ -241,6 +241,133 @@ def test_broken_database reader.close end + LIMIT_MODES = [MaxMind::DB::MODE_FILE, MaxMind::DB::MODE_MEMORY].freeze + + def fixture(name, **) + MaxMind::DB.new("test/data/test-data/MaxMind-DB-test-#{name}.mmdb", **) + end + + def assert_fixture_rejected(name, message = nil, **options) + LIMIT_MODES.each do |mode| + reader = fixture(name, mode: mode, **options) + error = assert_raises(MaxMind::DB::InvalidDatabaseError, "#{name} (#{mode})") do + reader.get('1.1.1.1') + end + assert_equal(message, error.message, "#{name} (#{mode})") if message + reader.close + end + end + + def assert_fixture_decodes(name, **options) + LIMIT_MODES.each do |mode| + reader = fixture(name, mode: mode, **options) + + refute_nil(reader.get('1.1.1.1'), "#{name} (#{mode})") + reader.close + end + end + + def test_pointer_fan_out_is_bounded + # Each record is a depth-40 pointer fan-out. An unprotected decoder performs + # 2**40 leaf decodes from a few hundred bytes. + assert_fixture_rejected('pointer-decoder-dos') + LIMIT_MODES.each do |mode| + reader = fixture('pointer-decoder-dos-ipv6', mode: mode) + + assert_raises(MaxMind::DB::InvalidDatabaseError, mode.to_s) { reader.get('::1') } + reader.close + end + end + + def test_limit_budget_is_reset_between_lookups + # The at-limit fixtures leave no budget to spare. If the budget lived on + # the shared decoder instead of in each call, a second lookup on the same + # reader would fail. The general thread test covers concurrent reader use. + %w[decoder-value-limit decoder-payload-limit].each do |name| + LIMIT_MODES.each do |mode| + reader = fixture(name, mode: mode) + + 2.times { refute_nil(reader.get('1.1.1.1'), "#{name} (#{mode})") } + reader.close + end + end + end + + def test_value_count_boundary + # The at-limit fixture decodes to exactly 65,536 values under the flat rule + # and must decode. One more value must be rejected. The pointer-heavy + # fixture reaches 65,535 values through pointers, which cost nothing beyond + # the values they resolve to, so it must decode too. + assert_fixture_decodes('decoder-value-limit') + assert_fixture_decodes('decoder-value-limit-pointer-heavy') + assert_fixture_rejected( + 'decoder-value-limit-over', + 'The MaxMind DB file\'s data section exceeds the maximum number of values', + ) + end + + def test_payload_amplification_is_bounded + # Each record points many times at one large string or bytes value. + # Following each pointer would copy the target again, so a reader that + # materializes every occurrence produces far more data than the file holds. + # The -worst-case fixture stays at exactly the value limit, so only the + # payload byte budget stops it. + message = 'The MaxMind DB file\'s data section exceeds the maximum number of bytes' + + assert_fixture_rejected('payload-amplification-dos', message) + assert_fixture_rejected('payload-amplification-dos-string', message) + assert_fixture_rejected('payload-amplification-dos-worst-case', message) + end + + def test_payload_byte_budget_boundary + # The at-limit fixture materializes exactly 2 MiB of payload and must + # decode. The over-limit fixture holds one byte more and must be rejected, + # so an off-by-one in the byte budget is caught. + assert_fixture_decodes('decoder-payload-limit') + assert_fixture_rejected( + 'decoder-payload-limit-over', + 'The MaxMind DB file\'s data section exceeds the maximum number of bytes', + ) + end + + def test_metadata_payload_amplification_is_bounded + # The languages metadata array points many times at one large string. + # Opening the database decodes the metadata, so the same budget must reject + # it there rather than materialize the amplified payload. + LIMIT_MODES.each do |mode| + assert_raises(MaxMind::DB::InvalidDatabaseError, mode.to_s) do + fixture('metadata-payload-limit', mode: mode) + end + end + end + + def test_limits_are_configurable + # Raising a limit accepts a fixture that the default rejects, for records + # and for metadata. Lowering the depth limit rejects an ordinary record. The + # metadata nests three deep (a pointer inside the languages array), so a + # depth of 3 still opens the database. The record nests deeper. + assert_fixture_decodes('decoder-value-limit-over', max_values: 65_537) + assert_fixture_decodes('decoder-payload-limit-over', max_payload_bytes: 1 << 22) + reader = fixture('metadata-payload-limit', max_payload_bytes: 1 << 22) + + refute_nil(reader.metadata.languages) + reader.close + + assert_fixture_rejected( + 'decoder', + 'The MaxMind DB file\'s data section exceeds the maximum depth', + max_depth: 3, + ) + end + + def test_invalid_limit_raises + [0, -1, 1.5, '1', nil].each do |value| + assert_raises(ArgumentError, value.inspect) do + fixture('decoder', max_values: value) + end + end + end + def test_ip_validation reader = MaxMind::DB.new( 'test/data/test-data/MaxMind-DB-test-decoder.mmdb'