diff --git a/parquet-format-structures/src/main/thrift/parquet-format.version b/parquet-format-structures/src/main/thrift/parquet-format.version index 774d7578dc..762e7a62e8 100644 --- a/parquet-format-structures/src/main/thrift/parquet-format.version +++ b/parquet-format-structures/src/main/thrift/parquet-format.version @@ -1,2 +1,2 @@ # Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh. -parquet-format.commit=c47e2a66e88943fc46fde1b028a9432f14fdf5c0 +parquet-format.commit=04d56f291ff963e98bc37ab8100e2fc133ff583c diff --git a/parquet-format-structures/src/main/thrift/parquet.thrift b/parquet-format-structures/src/main/thrift/parquet.thrift index fe259d61bc..bb582017bf 100644 --- a/parquet-format-structures/src/main/thrift/parquet.thrift +++ b/parquet-format-structures/src/main/thrift/parquet.thrift @@ -468,6 +468,17 @@ struct GeographyType { 2: optional EdgeInterpolationAlgorithm algorithm; } +/** + * File logical type annotation + * + * Annotates a group that represents a reference to a file, or to a range of + * bytes that may be stored inline or in an external file. + * + * See LogicalTypes.md for details. + */ +struct FileType { +} + /** * LogicalType annotations to replace ConvertedType. * @@ -501,6 +512,7 @@ union LogicalType { 16: VariantType VARIANT // no compatible ConvertedType 17: GeometryType GEOMETRY // no compatible ConvertedType 18: GeographyType GEOGRAPHY // no compatible ConvertedType + 19: FileType FILE // no compatible ConvertedType } /** @@ -636,6 +648,14 @@ enum Encoding { Support for INT32, INT64 and FIXED_LEN_BYTE_ARRAY added in 2.11. */ BYTE_STREAM_SPLIT = 9; + + /** Adaptive Lossless floating-Point (ALP) encoding for FLOAT and DOUBLE. + Losslessly converts decimal-like floating-point values to integers via + decimal scaling, then applies Frame of Reference (FOR) encoding and + bit-packing; values that cannot be converted losslessly are stored as + exceptions. See Encodings.md for the detailed specification. + */ + ALP = 10; } /** @@ -1061,6 +1081,9 @@ struct TypeDefinedOrder {} /** Empty struct to signal IEEE 754 total order for floating point types */ struct IEEE754TotalOrder {} +/** Empty struct to signal chronological ordering of physical type INT96 */ +struct Int96TimestampOrder {} + /** * Union to specify the order used for the min_value and max_value fields for a * column. This union takes the role of an enhanced enum that allows rich @@ -1071,6 +1094,8 @@ struct IEEE754TotalOrder {} * physical type (if there is no logical type). * * IEEE754TotalOrder - the floating point column uses IEEE 754 total order. * + * * Int96TimestampOrder - the INT96 column uses chronological timestamp order. + * * If the reader does not support the value of this union, min and max stats * for this column should be ignored. */ @@ -1103,25 +1128,29 @@ union ColumnOrder { * VARIANT - undefined * GEOMETRY - undefined * GEOGRAPHY - undefined + * FILE - undefined * * In the absence of logical types, the sort order is determined by the physical type: * BOOLEAN - false, true * INT32 - signed comparison * INT64 - signed comparison - * INT96 (only used for legacy timestamps) - undefined(+) + * INT96 (only used for legacy timestamps) - depends on sort order (+) * FLOAT - signed comparison of the represented value (*) * DOUBLE - signed comparison of the represented value (*) * BYTE_ARRAY - unsigned byte-wise comparison * FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison * * (+) While the INT96 type has been deprecated, at the time of writing it is - * still used in many legacy systems. If a Parquet implementation chooses - * to write statistics for INT96 columns, it is recommended to order them - * according to the legacy rules: - * - compare the last 4 bytes (days) as a little-endian 32-bit signed integer - * - if equal last 4 bytes, compare the first 8 bytes as a little-endian - * 64-bit signed integer (nanos) - * See https://github.com/apache/parquet-format/issues/502 for more details + * still used in many legacy systems. It is optional for writers to emit + * statistics for INT96 columns. Writers that emit stats for such columns + * should use the INT96_TIMESTAMP_ORDER for this type and order the values + * according to the legacy rules: + * - compare the last 4 bytes (days) as a little-endian 32-bit signed integer + * - if equal last 4 bytes, compare the first 8 bytes as a little-endian + * 64-bit signed integer (nanos) + * If TYPE_ORDER is used for an INT96 column, readers should ignore all statistics + * (`min`/`max` fields in `Statistics` and `min_values`/`max_values` fields in + * `ColumnIndex`) for that column. * * (*) Because TYPE_ORDER is ambiguous for floating point types due to * underspecified handling of NaN and -0/+0, it is recommended that writers @@ -1195,6 +1224,12 @@ union ColumnOrder { * or max_values indicates that all non-null values are NaN. */ 2: IEEE754TotalOrder IEEE_754_TOTAL_ORDER; + + /* + * The INT96 timestamp type is ordered chronologically. Only columns of + * physical type INT96 may use this ordering. + */ + 3: Int96TimestampOrder INT96_TIMESTAMP_ORDER; } struct PageLocation { @@ -1278,6 +1313,13 @@ struct ColumnIndex { * - If the order of this column is IEEE754_TOTAL_ORDER, then min_values[i] * and max_values[i] of that page must be set to the smallest and largest * NaN values as defined by IEEE 754 total order. + * + * For columns of physical type INT96, the writer must do the following: + * - If the order of this column is not INT96_TIMESTAMP_ORDER, then a column + * index must not be written for this column chunk. + * - If the order of this column is INT96_TIMESTAMP_ORDER, the min_values[i] + * and max_values[i] of that page must be set to the smallest and largest + * values as defined by the INT96 chronological timestamp ordering. */ 2: required list min_values 3: required list max_values diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index f6ee73bbc0..cc7805c317 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -1409,6 +1409,10 @@ LogicalTypeAnnotation getLogicalTypeAnnotation(LogicalType type) { case VARIANT: VariantType variant = type.getVARIANT(); return LogicalTypeAnnotation.variantType(variant.getSpecification_version()); + case FILE: + // Present in the format but not mapped to a LogicalTypeAnnotation yet. Ignore it to + // preserve the physical type, as an unrecognised logical type would be. + return null; default: throw new RuntimeException("Unknown logical type " + type); } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index f5222de828..784a837e3e 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -101,6 +101,7 @@ import org.apache.parquet.format.DecimalType; import org.apache.parquet.format.FieldRepetitionType; import org.apache.parquet.format.FileMetaData; +import org.apache.parquet.format.FileType; import org.apache.parquet.format.GeographyType; import org.apache.parquet.format.GeometryType; import org.apache.parquet.format.GeospatialStatistics; @@ -542,6 +543,15 @@ public void testLogicalToConvertedTypeConversion() { .isEqualTo(ConvertedType.MAP_KEY_VALUE); } + @Test + public void testFileLogicalTypeIsIgnoredRatherThanFailing() { + ParquetMetadataConverter converter = new ParquetMetadataConverter(); + // FILE has no LogicalTypeAnnotation yet, so it must degrade to the physical type the way an + // unrecognised logical type does, rather than throwing. + assertThat(converter.getLogicalTypeAnnotation(LogicalType.FILE(new FileType()))) + .isNull(); + } + @Test public void testEnumEquivalence() { ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); @@ -550,6 +560,11 @@ public void testEnumEquivalence() { .isEqualTo(encoding); } for (org.apache.parquet.format.Encoding encoding : org.apache.parquet.format.Encoding.values()) { + // ALP is in the format spec but is not implemented on the Java side yet, so it has no + // org.apache.parquet.column.Encoding to round trip through. Remove this once it does. + if (encoding == org.apache.parquet.format.Encoding.ALP) { + continue; + } assertThat(parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding))) .isEqualTo(encoding); }