Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions src/main/java/com/amazon/ion/BufferConfiguration.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@

package com.amazon.ion;

import com.amazon.ion.impl._Private_IonConstants;

/**
* Provides logic common to all BufferConfiguration implementations.
* @param <Configuration> the type of the concrete subclass of this BufferConfiguration.
Expand Down Expand Up @@ -47,6 +45,12 @@ public static abstract class Builder<
Configuration extends BufferConfiguration<Configuration>,
BuilderType extends BufferConfiguration.Builder<Configuration, BuilderType>
> {
/**
* The default maximum buffer size: 100 MB. This provides protection against memory exhaustion
* from malicious inputs that declare excessively large lengths, while remaining large enough
* for any realistic well-formed Ion stream.
*/
public static final int DEFAULT_MAXIMUM_BUFFER_SIZE = 100 * 1024 * 1024;

/**
* Large enough that most streams will never need to grow the buffer. NOTE: this only needs to be large
Expand All @@ -62,7 +66,13 @@ public static abstract class Builder<
/**
* The maximum number of bytes that will be buffered.
*/
private int maximumBufferSize = _Private_IonConstants.ARRAY_MAXIMUM_SIZE;
private int maximumBufferSize = DEFAULT_MAXIMUM_BUFFER_SIZE;

/**
* Tracks whether the user has explicitly set the maximum buffer size via
* {@link #withMaximumBufferSize(int)}.
*/
private boolean maximumBufferSizeExplicitlySet = false;

/**
* The handler that will be notified when oversized values are encountered.
Expand Down Expand Up @@ -135,13 +145,14 @@ public final DataHandler getDataHandler() {
* Set the maximum size of the buffer. For binary Ion, the minimum value is 5 because all valid binary Ion data
* begins with a 4-byte Ion version marker and the smallest value is 1 byte. For text Ion, the minimum value is
* 2 because the smallest text Ion value is 1 byte and the smallest delimiter is 1 byte.
* Default: Near to the maximum size of an array.
* Default: 100 MB ({@link #DEFAULT_MAXIMUM_BUFFER_SIZE}).
*
* @param maximumBufferSizeInBytes the value.
* @return this builder.
*/
public final BuilderType withMaximumBufferSize(final int maximumBufferSizeInBytes) {
maximumBufferSize = maximumBufferSizeInBytes;
maximumBufferSizeExplicitlySet = true;
return (BuilderType) this;
}

Expand All @@ -152,6 +163,14 @@ public int getMaximumBufferSize() {
return maximumBufferSize;
}

/**
* @return true if the maximum buffer size was explicitly set by the user via
* {@link #withMaximumBufferSize(int)}, false if using the default.
*/
public boolean isMaximumBufferSizeExplicitlySet() {
return maximumBufferSizeExplicitlySet;
}

/**
* Gets the minimum allowed maximum buffer size.
* @return the value.
Expand Down Expand Up @@ -216,7 +235,7 @@ protected BufferConfiguration(Builder<Configuration, ?> builder) {
));
}
if (builder.getOversizedValueHandler() == null) {
requireMaximumBufferSize();
requireMaximumBufferSize(builder);
oversizedValueHandler = builder.getThrowingOversizedValueHandler();
} else {
oversizedValueHandler = builder.getOversizedValueHandler();
Expand All @@ -229,10 +248,11 @@ protected BufferConfiguration(Builder<Configuration, ?> builder) {
}

/**
* Requires that the maximum buffer size not be limited.
* Requires an OversizedValueHandler when the user explicitly sets a custom maximum buffer size.
* When using the default, a throwing handler is applied automatically.
*/
private void requireMaximumBufferSize() {
if (maximumBufferSize < _Private_IonConstants.ARRAY_MAXIMUM_SIZE) {
private void requireMaximumBufferSize(Builder<Configuration, ?> builder) {
if (builder.isMaximumBufferSizeExplicitlySet()) {
throw new IllegalArgumentException(
"Must specify an OversizedValueHandler when a custom maximum buffer size is specified."
);
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/com/amazon/ion/IonBufferConfiguration.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@

package com.amazon.ion;

import com.amazon.ion.impl._Private_IonConstants;

/**
* Configures buffers that hold Ion data.
*/
Expand Down Expand Up @@ -180,18 +178,20 @@ public IonBufferConfiguration build() {
private IonBufferConfiguration(Builder builder) {
super(builder);
if (builder.getOversizedSymbolTableHandler() == null) {
requireMaximumBufferSize();
requireMaximumBufferSize(builder);
oversizedSymbolTableHandler = builder.getThrowingOversizedSymbolTableHandler();
} else {
oversizedSymbolTableHandler = builder.getOversizedSymbolTableHandler();
}
}

/**
* Requires that the maximum buffer size not be limited.
* Requires that the maximum buffer size not be limited unless the user explicitly configured it.
* When the user explicitly sets a custom maximum buffer size, an OversizedSymbolTableHandler is required.
* When using the default maximum buffer size, the throwing handler is used automatically.
*/
private void requireMaximumBufferSize() {
if (getMaximumBufferSize() < _Private_IonConstants.ARRAY_MAXIMUM_SIZE) {
private void requireMaximumBufferSize(Builder builder) {
if (builder.isMaximumBufferSizeExplicitlySet()) {
throw new IllegalArgumentException(
"Must specify both an OversizedValueHandler and OversizedSymbolTableHandler when a custom maximum buffer size is specified."
);
Expand Down
37 changes: 37 additions & 0 deletions src/main/java/com/amazon/ion/impl/UnifiedInputStreamX.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
package com.amazon.ion.impl;

import com.amazon.ion.IonException;
import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint;
import java.io.Closeable;
import java.io.IOException;
Expand Down Expand Up @@ -77,6 +78,16 @@ abstract class UnifiedInputStreamX
UnifiedSavePointManagerX _save_points;


/**
* The maximum total bytes that may be loaded across all pages, or -1 for no limit.
*/
long _maximumTotalBytes = -1;

/**
* Tracks the total bytes loaded across all pages.
*/
long _totalBytesLoaded = 0;

// factories to construct an appropriate input stream
// based on the input source
public static UnifiedInputStreamX makeStream(CharSequence chars) {
Expand All @@ -94,6 +105,9 @@ public static UnifiedInputStreamX makeStream(char[] chars, int offset, int lengt
public static UnifiedInputStreamX makeStream(Reader reader) throws IOException {
return new FromCharStream(reader);
}
public static UnifiedInputStreamX makeStream(Reader reader, long maximumTotalBytes) throws IOException {
return new FromCharStream(reader, maximumTotalBytes);
}
public static UnifiedInputStreamX makeStream(byte[] buffer) {
return new FromByteArray(buffer, 0, buffer.length);
}
Expand All @@ -103,6 +117,9 @@ public static UnifiedInputStreamX makeStream(byte[] buffer, int offset, int leng
public static UnifiedInputStreamX makeStream(InputStream stream) throws IOException {
return new FromByteStream(stream);
}
public static UnifiedInputStreamX makeStream(InputStream stream, long maximumTotalBytes) throws IOException {
return new FromByteStream(stream, maximumTotalBytes);
}
public final InputStream getInputStream() { return _stream; }
public final Reader getReader() { return _reader; }
public final byte[] getByteArray() { return _bytes; }
Expand Down Expand Up @@ -498,6 +515,14 @@ protected final int load(UnifiedDataPageX curr, int start_pos, long file_positio
else {
read = curr.load(_reader, start_pos, file_position);
}
if (read > 0) {
_totalBytesLoaded += read;
if (_maximumTotalBytes > 0 && _totalBytesLoaded > _maximumTotalBytes) {
throw new IonException(
"Text stream exceeded maximum buffer size of " + _maximumTotalBytes + " bytes"
);
}
}
}
return read;
}
Expand Down Expand Up @@ -531,10 +556,16 @@ private static class FromCharArray extends UnifiedInputStreamX
private static class FromCharStream extends UnifiedInputStreamX
{
FromCharStream(Reader reader) throws IOException
{
this(reader, -1);
}

FromCharStream(Reader reader, long maximumTotalBytes) throws IOException
{
_is_byte_data = false;
_is_stream = true;
_reader = reader;
_maximumTotalBytes = maximumTotalBytes;
// If this page size ever becomes configurable watch out for _Private_IonConstants.ARRAY_MAXIMUM_SIZE
_buffer = UnifiedInputBufferX.makePageBuffer(UnifiedInputBufferX.BufferType.CHARS, DEFAULT_PAGE_SIZE);
super.init();
Expand Down Expand Up @@ -569,10 +600,16 @@ static class FromByteArray extends UnifiedInputStreamX
private static class FromByteStream extends UnifiedInputStreamX
{
FromByteStream(InputStream stream) throws IOException
{
this(stream, -1);
}

FromByteStream(InputStream stream, long maximumTotalBytes) throws IOException
{
_is_byte_data = true;
_is_stream = true;
_stream = stream;
_maximumTotalBytes = maximumTotalBytes;
// If this page size ever becomes configurable watch out for _Private_IonConstants.ARRAY_MAXIMUM_SIZE
_buffer = UnifiedInputBufferX.makePageBuffer(UnifiedInputBufferX.BufferType.BYTES, DEFAULT_PAGE_SIZE);
super.init();
Expand Down
31 changes: 27 additions & 4 deletions src/main/java/com/amazon/ion/impl/_Private_IonReaderBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
// SPDX-License-Identifier: Apache-2.0
package com.amazon.ion.impl;

import com.amazon.ion.IonBufferConfiguration;
import com.amazon.ion.IonCatalog;
import com.amazon.ion.IonException;
import com.amazon.ion.IonReader;
import com.amazon.ion.IonTextReader;
import com.amazon.ion.IonValue;
import com.amazon.ion.util.InputStreamInterceptor;
import com.amazon.ion.util.LimitedInflaterInputStream;
import com.amazon.ion.system.IonReaderBuilder;
import com.amazon.ion.util.IonStreamUtils;

Expand Down Expand Up @@ -205,6 +207,24 @@ private static void validateHeaderLength(int maxHeaderLength) {
}
}

/**
* Wraps the given InputStream with a {@link LimitedInflaterInputStream} if the interceptor is for GZIP
* and a finite maximum buffer size is configured.
*/
private static InputStream wrapWithInflationLimitIfNeeded(
_Private_IonReaderBuilder builder,
InputStreamInterceptor streamInterceptor,
InputStream interceptedStream
) {
if ("gzip".equals(streamInterceptor.formatName())) {
long maxDecompressedBytes = builder.getBufferConfiguration().getMaximumBufferSize();
if (maxDecompressedBytes > 0) {
return new LimitedInflaterInputStream(interceptedStream, maxDecompressedBytes);
}
}
return interceptedStream;
}

static IonReader buildReader(
_Private_IonReaderBuilder builder,
byte[] ionData,
Expand All @@ -222,9 +242,11 @@ static IonReader buildReader(
}
if (streamInterceptor.isMatch(ionData, offset, length)) {
try {
InputStream decompressedStream = streamInterceptor.newInputStream(new ByteArrayInputStream(ionData, offset, length));
decompressedStream = wrapWithInflationLimitIfNeeded(builder, streamInterceptor, decompressedStream);
return buildReader(
builder,
streamInterceptor.newInputStream(new ByteArrayInputStream(ionData, offset, length)),
decompressedStream,
_Private_IonReaderFactory::makeReaderBinary,
_Private_IonReaderFactory::makeReaderText,
// The builder provides only one level of detection, e.g. GZIP-compressed binary Ion *or*
Expand Down Expand Up @@ -275,7 +297,7 @@ private static boolean startsWithIvm(byte[] buffer, int length) {

@FunctionalInterface
interface IonReaderFromInputStreamFactoryText {
IonReader makeReader(IonCatalog catalog, InputStream source, _Private_LocalSymbolTableFactory lstFactory);
IonReader makeReader(IonCatalog catalog, InputStream source, _Private_LocalSymbolTableFactory lstFactory, long maximumTotalBytes);
}

@FunctionalInterface
Expand Down Expand Up @@ -357,6 +379,7 @@ static IonReader buildReader(
ionData = streamInterceptor.newInputStream(
new TwoElementInputStream(new ByteArrayInputStream(possibleIVM, 0, bytesRead), ionData)
);
ionData = wrapWithInflationLimitIfNeeded(builder, streamInterceptor, ionData);
} catch (IOException e) {
throw new IonException(e);
}
Expand All @@ -376,7 +399,7 @@ static IonReader buildReader(
} else {
wrapper = ionData;
}
return text.makeReader(builder.validateCatalog(), wrapper, builder.lstFactory);
return text.makeReader(builder.validateCatalog(), wrapper, builder.lstFactory, builder.getBufferConfiguration().getMaximumBufferSize());
}

@Override
Expand All @@ -393,7 +416,7 @@ public IonReader build(InputStream source)

@Override
public IonReader build(Reader ionText) {
return makeReaderText(validateCatalog(), ionText, lstFactory);
return makeReaderText(validateCatalog(), ionText, lstFactory, getBufferConfiguration().getMaximumBufferSize());
}

@Override
Expand Down
39 changes: 33 additions & 6 deletions src/main/java/com/amazon/ion/impl/_Private_IonReaderFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,18 @@ public static final IonReader makeSystemReaderText(CharSequence chars)
public static final IonReader makeReaderText(IonCatalog catalog,
InputStream is,
_Private_LocalSymbolTableFactory lstFactory)
{
return makeReaderText(catalog, is, lstFactory, -1);
}

public static final IonReader makeReaderText(IonCatalog catalog,
InputStream is,
_Private_LocalSymbolTableFactory lstFactory,
long maximumTotalBytes)
{
UnifiedInputStreamX uis;
try {
uis = makeUnifiedStream(is);
uis = makeUnifiedStream(is, maximumTotalBytes);
} catch (IOException e) {
throw new IonException(e);
}
Expand All @@ -104,12 +112,13 @@ public static IonReader makeSystemReaderText(InputStream is)

private static IonReader makeSystemReaderText(IonCatalog catalog,
InputStream is,
_Private_LocalSymbolTableFactory lstFactory)
_Private_LocalSymbolTableFactory lstFactory,
long maximumTotalBytes)
{
UnifiedInputStreamX uis;
try
{
uis = makeUnifiedStream(is);
uis = makeUnifiedStream(is, maximumTotalBytes);
}
catch (IOException e)
{
Expand Down Expand Up @@ -138,9 +147,22 @@ private static IonReader makeSystemReaderText(IonCatalog catalog,
public static final IonTextReader makeReaderText(IonCatalog catalog,
Reader chars,
_Private_LocalSymbolTableFactory lstFactory)
{
return makeReaderText(catalog, chars, lstFactory, -1);
}

public static final IonTextReader makeReaderText(IonCatalog catalog,
Reader chars,
_Private_LocalSymbolTableFactory lstFactory,
long maximumTotalBytes)
{
try {
UnifiedInputStreamX in = makeStream(chars);
UnifiedInputStreamX in;
if (maximumTotalBytes > 0) {
in = makeStream(chars, maximumTotalBytes);
} else {
in = makeStream(chars);
}
return new IonReaderTextUserX(catalog, lstFactory, in);
}
catch (IOException e) {
Expand Down Expand Up @@ -225,14 +247,19 @@ private static UnifiedInputStreamX makeUnifiedStream(byte[] bytes,
return uis;
}

private static UnifiedInputStreamX makeUnifiedStream(InputStream in)
private static UnifiedInputStreamX makeUnifiedStream(InputStream in, long maximumTotalBytes)
throws IOException
{
in.getClass(); // Force NPE

// TODO avoid multiple wrapping streams, use the UIS for the pushback
in = IonStreamUtils.unGzip(in);
UnifiedInputStreamX uis = UnifiedInputStreamX.makeStream(in);
UnifiedInputStreamX uis;
if (maximumTotalBytes > 0) {
uis = UnifiedInputStreamX.makeStream(in, maximumTotalBytes);
} else {
uis = UnifiedInputStreamX.makeStream(in);
}
return uis;
}
}
Loading
Loading