fix: accept up to 9 fractional digits for micros timestamps

pyiceberg/utils/datetime.py

@@ -28,11 +28,11 @@
 
 EPOCH_DATE = date.fromisoformat("1970-01-01")
 EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000")
-ISO_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?")
+ISO_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,9})?")
 ISO_TIMESTAMP_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?")
 EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00")
-ISO_TIMESTAMPTZ = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?[-+]\d{2}:\d{2}")
-ISO_TIMESTAMPTZ_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?([-+]\d{2}:\d{2})")
+ISO_TIMESTAMPTZ = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,9})?[-+]\d{2}:\d{2}(:\d{2})?")
+ISO_TIMESTAMPTZ_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?([-+]\d{2}:\d{2}(:\d{2})?)")
 
 
 def micros_to_days(timestamp: int) -> int:
@@ -86,7 +86,8 @@ def datetime_to_micros(dt: datetime) -> int:
 def timestamp_to_micros(timestamp_str: str) -> int:
     """Convert an ISO-9601 formatted timestamp without zone to microseconds from 1970-01-01T00:00:00.000000."""
     if ISO_TIMESTAMP.fullmatch(timestamp_str):
-        return datetime_to_micros(datetime.fromisoformat(timestamp_str))
+        # Parse through the nanosecond path and truncate to microseconds, mirroring Java
+        return nanos_to_micros(timestamp_to_nanos(timestamp_str))
     if ISO_TIMESTAMPTZ.fullmatch(timestamp_str):
         # When we can match a timestamp without a zone, we can give a more specific error
         raise ValueError(f"Zone offset provided, but not expected: {timestamp_str}")
@@ -168,7 +169,8 @@ def millis_to_datetime(millis: int) -> datetime:
 def timestamptz_to_micros(timestamptz_str: str) -> int:
     """Convert an ISO-8601 formatted timestamp with zone to microseconds from 1970-01-01T00:00:00.000000+00:00."""
     if ISO_TIMESTAMPTZ.fullmatch(timestamptz_str):
-        return datetime_to_micros(datetime.fromisoformat(timestamptz_str))
+        # Parse through the nanosecond path and truncate to microseconds, mirroring Java
+        return nanos_to_micros(timestamptz_to_nanos(timestamptz_str))
     if ISO_TIMESTAMP.fullmatch(timestamptz_str):
         # When we can match a timestamp without a zone, we can give a more specific error
         raise ValueError(f"Missing zone offset: {timestamptz_str} (must be ISO-8601)")

tests/test_conversions.py

@@ -609,6 +609,23 @@ def test_json_serialize_roundtrip(primitive_type: PrimitiveType, value: Any) ->
     assert value == conversions.from_json(primitive_type, conversions.to_json(primitive_type, value))
 
 
+@pytest.mark.parametrize(
+    "primitive_type, json_val, expected",
+    [
+        (TimestampType(), "2017-11-16T22:31:08.123456789", datetime(2017, 11, 16, 22, 31, 8, 123456)),
+        (
+            TimestamptzType(),
+            "2017-11-16T22:31:08.123456789+00:00",
+            datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=timezone.utc),
+        ),
+    ],
+)
+def test_json_single_deserialization_with_nano_precision(
+    primitive_type: PrimitiveType, json_val: str, expected: datetime
+) -> None:
+    assert conversions.from_json(primitive_type, json_val) == expected
+
+
 def test_unknown_type_conversions() -> None:
     """Unknown values are always null, so they have no binary or partition representation."""
     unknown = UnknownType()

tests/utils/test_datetime.py

@@ -28,7 +28,9 @@
     nanos_to_micros,
     time_str_to_nanos,
     time_to_nanos,
+    timestamp_to_micros,
     timestamp_to_nanos,
+    timestamptz_to_micros,
     timestamptz_to_nanos,
 )
 
@@ -155,6 +157,55 @@ def test_timestamptz_to_nanos_invalid_timestamp() -> None:
         timestamptz_to_nanos("invalid")
 
 
+@pytest.mark.parametrize(
+    "timestamp, micros",
+    [
+        ("1970-01-01T00:00:00", 0),
+        ("2025-02-23T20:21:44.375612", 1740342104375612),
+        ("2025-02-23T20:21:44.3756127", 1740342104375612),
+        ("2025-02-23T20:21:44.37561278", 1740342104375612),
+        ("2025-02-23T20:21:44.375612001", 1740342104375612),
+    ],
+)
+def test_timestamp_to_micros(timestamp: str, micros: int) -> None:
+    assert micros == timestamp_to_micros(timestamp)
+
+
+def test_timestamp_to_micros_unexpected_zone_offset() -> None:
+    with pytest.raises(ValueError, match="Zone offset provided, but not expected: 2025-02-23T16:21:44.375612001-04:00"):
+        timestamp_to_micros("2025-02-23T16:21:44.375612001-04:00")
+
+
+def test_timestamp_to_micros_invalid_timestamp() -> None:
+    with pytest.raises(ValueError, match=re.escape("Invalid timestamp without zone: invalid (must be ISO-8601)")):
+        timestamp_to_micros("invalid")
+
+
+@pytest.mark.parametrize(
+    "timestamp, micros",
+    [
+        ("1970-01-01T00:00:00+00:00", 0),
+        ("2025-02-23T16:21:44.375612-04:00", 1740342104375612),
+        ("2025-02-23T16:21:44.3756127-04:00", 1740342104375612),
+        ("2025-02-23T16:21:44.37561278-04:00", 1740342104375612),
+        ("2025-02-23T16:21:44.375612001-04:00", 1740342104375612),
+        ("2025-02-23T16:21:44-04:00:00", 1740342104000000),
+    ],
+)
+def test_timestamptz_to_micros(timestamp: str, micros: int) -> None:
+    assert micros == timestamptz_to_micros(timestamp)
+
+
+def test_timestamptz_to_micros_missing_zone_offset() -> None:
+    with pytest.raises(ValueError, match="Missing zone offset: 2025-02-23T20:21:44.375612001"):
+        timestamptz_to_micros("2025-02-23T20:21:44.375612001")
+
+
+def test_timestamptz_to_micros_invalid_timestamp() -> None:
+    with pytest.raises(ValueError, match=re.escape("Invalid timestamp with zone: invalid (must be ISO-8601)")):
+        timestamptz_to_micros("invalid")
+
+
 @pytest.mark.parametrize("nanos, micros", [(1510871468000001001, 1510871468000001), (-1510871468000001001, -1510871468000002)])
 def test_nanos_to_micros(nanos: int, micros: int) -> None:
     assert micros == nanos_to_micros(nanos)