From eb92f242d54c672497fa1bb3d7ab06bfe6d2586b Mon Sep 17 00:00:00 2001 From: auroflow Date: Thu, 10 Sep 2026 17:56:14 +0800 Subject: [PATCH 01/14] [FLINK-40472][python] Support Arrow vectorized DataFrame UDFs Generated-by: Codex (GPT-6) --- .../docs/reference/pyflink.dataframe/udf.rst | 36 ++- .../pyflink/dataframe/tests/test_udf.py | 123 +++++++++- flink-python/pyflink/dataframe/udf.py | 76 ++++-- .../pyflink/fn_execution/coder_impl_fast.pxd | 3 +- .../pyflink/fn_execution/coder_impl_fast.pyx | 18 +- .../pyflink/fn_execution/coder_impl_slow.py | 18 +- flink-python/pyflink/fn_execution/coders.py | 15 +- .../fn_execution/flink_fn_execution_pb2.py | 222 +++++++++--------- .../fn_execution/flink_fn_execution_pb2.pyi | 20 +- .../pyflink/fn_execution/table/operations.py | 17 +- .../fn_execution/tests/test_arrow_udf.py | 119 ++++++++++ .../pyflink/fn_execution/tests/test_coders.py | 112 +++++++++ .../pyflink/fn_execution/utils/arrow_utils.py | 121 ++++++++++ .../fn_execution/utils/operation_utils.py | 12 + .../pyflink/proto/flink-fn-execution.proto | 9 + .../pyflink/table/tests/test_pandas_udf.py | 2 +- flink-python/pyflink/table/udf.py | 36 ++- .../apache/flink/python/util/ProtoUtils.java | 19 ++ .../ArrowPythonScalarFunctionOperator.java | 12 +- ...ArrowPythonScalarFunctionOperatorTest.java | 63 +++++ .../functions/python/PythonFunctionInfo.java | 17 ++ .../functions/python/PythonFunctionKind.java | 4 +- .../exec/common/CommonExecPythonCalc.java | 32 ++- .../nodes/exec/utils/CommonPythonUtil.java | 9 +- .../PythonCalcSplitFunctionKindRule.java | 75 ++++++ .../rules/logical/PythonCalcSplitRule.scala | 51 +--- .../rules/logical/PythonArrowCalcTest.java | 67 ++++++ .../rules/logical/PythonArrowCalcTest.xml | 67 ++++++ 28 files changed, 1154 insertions(+), 221 deletions(-) create mode 100644 flink-python/pyflink/fn_execution/tests/test_arrow_udf.py create mode 100644 flink-python/pyflink/fn_execution/utils/arrow_utils.py create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRule.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.java create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.xml diff --git a/flink-python/docs/reference/pyflink.dataframe/udf.rst b/flink-python/docs/reference/pyflink.dataframe/udf.rst index 73d1ff2abe76d..6f61793ab2142 100644 --- a/flink-python/docs/reference/pyflink.dataframe/udf.rst +++ b/flink-python/docs/reference/pyflink.dataframe/udf.rst @@ -26,10 +26,42 @@ columns. A scalar UDF produces one logical output column and can be used in :meth:`~pyflink.dataframe.DataFrame.with_columns`, and :meth:`~pyflink.dataframe.DataFrame.select`. -DataFrame scalar UDFs support synchronous, asynchronous, and pandas-vectorized -callables. See :func:`pyflink.dataframe.udf` for declaration forms, type +DataFrame scalar UDFs support general synchronous and asynchronous callables, +and synchronous pandas or Arrow vectorized callables. See :func:`pyflink.dataframe.udf` for declaration forms, type inference, execution modes, and examples. +Arrow Vectorized Functions +========================== + +Arrow UDFs receive columns as ``pyarrow.Array`` values and return an ``Array`` +or ``ChunkedArray`` of the same length. They require an explicit logical +``return_dtype``. Container annotations infer Arrow mode; unannotated functions +can select ``func_type="arrow"`` explicitly. + +.. code-block:: python + + import pyarrow as pa + import pyarrow.compute as pc + import pyflink.dataframe as pf + + @pf.udf(return_dtype=pf.DataType.string()) + def normalize_name(names: pa.Array) -> pa.Array: + return pc.utf8_upper(names) + + df = pf.from_records([("Alice",), ("Bob",)], schema=["name"]) + result = df.with_column("normalized_name", normalize_name(pf.col("name"))) + +Multiple column arguments and scalar literals can be combined. Literals remain +Python scalars, and at least one argument must be column-valued. ROW columns use +``StructArray``; nested results and chunked arrays can feed subsequent UDFs. +Declared types, nested nullability, and row counts are validated without implicit +element casts. Python lists or scalars, Arrow tables, and record batches are not +supported scalar results. + +Explicit ``func_type`` overrides annotations. Mixed pandas and Arrow container +annotations require an explicit choice. Arrow UDFs are synchronous; per-UDF +concurrency and batch-size options are not provided. + API Reference ============= diff --git a/flink-python/pyflink/dataframe/tests/test_udf.py b/flink-python/pyflink/dataframe/tests/test_udf.py index 12be3306aebeb..a748725e11ce9 100644 --- a/flink-python/pyflink/dataframe/tests/test_udf.py +++ b/flink-python/pyflink/dataframe/tests/test_udf.py @@ -61,6 +61,73 @@ def _call_module_alias_function(value): class DataFrameUDFDeclarationTests(unittest.TestCase): + def test_arrow_annotation_inference_and_overrides(self): + def arrow_identity(values: pa.Array) -> pa.ChunkedArray: + return pa.chunked_array([values]) + + def mixed(values: pd.Series) -> pa.Array: + return pa.array(values) + + def captured(context: pd.Series, values: pa.Array) -> pa.Array: + return values + + class ArrowCallable: + def __call__(self, values: "pa.Array") -> "pa.Array": + return values + + class ArrowScalar(ScalarFunction): + def eval(self, values: pa.Array) -> pa.Array: + return values + + for func in (arrow_identity, ArrowCallable, ArrowCallable(), ArrowScalar, ArrowScalar(), + functools.partial(captured, pd.Series([1]))): + with self.subTest(func=func): + declaration = pf.udf(func, return_dtype=pf.DataType.int64()) + self.assertEqual(cast(Any, declaration)._func_type, "arrow") + + for mode in ("general", "pandas", "arrow"): + with self.subTest(mode=mode): + declaration = pf.udf(mixed, return_dtype=pf.DataType.int64(), func_type=mode) + self.assertEqual(cast(Any, declaration)._func_type, mode) + + with self.assertRaisesRegex(ValueError, "pandas.*Arrow.*func_type"): + pf.udf(mixed, return_dtype=pf.DataType.int64()) + with self.assertRaisesRegex(TypeError, "return_dtype is required for arrow"): + pf.udf(arrow_identity) + + def test_explicit_arrow_declarations(self): + from pyflink.table.udf import udf as table_udf + + def identity(values): + return values + + declaration = pf.udf(identity, return_dtype=pf.DataType.string(), func_type="arrow") + self.assertEqual(_return_dtype(declaration), pf.DataType.string()) + table_udf(identity, result_type=TableDataTypes.STRING(), func_type="arrow") + + with self.assertRaisesRegex(TypeError, "return_dtype is required for arrow"): + pf.udf(identity, func_type="arrow") + + async def async_identity(values): + return values + + class AsyncCallable: + async def __call__(self, values): + return values + + for declare in ( + lambda: pf.udf(async_identity, return_dtype=pf.DataType.string(), func_type="arrow"), + lambda: table_udf(async_identity, result_type=TableDataTypes.STRING(), + func_type="arrow"), + lambda: table_udf(AsyncCallable(), result_type=TableDataTypes.STRING(), + func_type="arrow"), + lambda: table_udf(functools.partial(AsyncCallable()), + result_type=TableDataTypes.STRING(), func_type="arrow"), + ): + with self.subTest(declare=declare): + with self.assertRaisesRegex(ValueError, "Async.*arrow"): + declare() + def test_function_declarations_return_types_and_metadata(self): class Details(TypedDict): label: str @@ -547,9 +614,9 @@ def __call__(self, context: pd.Series, value: int) -> int: False, ), ( - "pyarrow annotations remain general", + "inferred arrow", lambda: pf.udf(arrow_add_one, return_dtype=pf.DataType.int64()), - "general", + "arrow", False, ), ( @@ -1178,11 +1245,11 @@ def eval(self, value): "name must not be empty", ), ( - "arrow func type", + "unsupported func type", lambda: pf.udf( missing_return, return_dtype=pf.DataType.int64(), - func_type="arrow", + func_type="unsupported", ), ValueError, "func_type must be one of", @@ -1480,6 +1547,21 @@ def close(self): class DataFrameUDFPlannerTests(PyFlinkDataFrameUTTestCase): + def test_arrow_calls_require_a_column_argument(self): + from pyflink.table import ExplainDetail + + @pf.udf(return_dtype=pf.DataType.int64(), func_type="arrow") + def identity(*values): + return values[0] + + dataframe = pf.from_records([(1,)], schema=["id"]) + for args in ((), (1,), (pf.lit(1),), (identity(),)): + with self.subTest(args=args): + with self.assertRaisesRegex(Exception, "at least one column-valued argument"): + result = dataframe.with_columns( + valid=identity(pf.col("id")), invalid=identity(*args)) + result.to_table().explain(ExplainDetail.JSON_EXECUTION_PLAN) + def test_with_columns_binds_expressions_and_resolves_output_schema(self): @pf.udf(name="render_value") def render(value: int, suffix: str) -> str: @@ -1521,6 +1603,25 @@ def describe(value): class DataFrameUDFITCase(PyFlinkStreamDataFrameTestCase): def test_supported_scalar_udfs_in_one_job(self): + import pyarrow.compute as pc + + self.env.set_parallelism(1) + self.t_env.get_config().set("python.fn-execution.bundle.size", "3") + self.t_env.get_config().set("python.fn-execution.arrow.batch.size", "2") + + @pf.udf(return_dtype=pf.DataType.string()) + def normalize_name(names: pa.Array) -> pa.Array: + return pc.utf8_upper(names) + + @pf.udf(return_dtype=pf.DataType.struct({"value": pf.DataType.int64().not_null()})) + def describe(values: pa.Array) -> pa.ChunkedArray: + result = pa.StructArray.from_arrays([pc.multiply(values, 2)], names=["value"]) + return pa.chunked_array([result.slice(0, 1), result.slice(1)]) + + @pf.udf(return_dtype=pf.DataType.int64()) + def struct_value(values: pa.Array) -> pa.Array: + return pc.struct_field(values, "value") + @dataclass class Details: doubled: int @@ -1561,19 +1662,27 @@ def eval(self, *values: int) -> int: opened_scalar_class = pf.udf(OpenedScalarFunction) result = ( - pf.from_records([(1,)], schema=["id"]) + pf.from_records([(1, "alice"), (2, None), (3, "Bob")], schema=["id", "name"]) .with_columns(async_value=add_two(pf.col("id"))) .with_columns( pandas_value=add_three(pf.col("id")), details=details(pf.col("id")), deferred_value=deferred(pf.col("id")), scalar_value=opened_scalar_class(pf.col("id")), + normalized_name=normalize_name(pf.col("name")), + arrow_details=describe(pf.col("id")), + arrow_after_pandas=struct_value(describe(add_three(pf.col("id")))), + pandas_after_arrow=add_three(struct_value(describe(pf.col("id")))), ) ) self.assertEqual( - result.collect(), - [Row(1, 3, 4, Row(2, ["1"]), 5, 6)], + sorted(result.collect(), key=lambda row: row[0]), + [ + Row(1, "alice", 3, 4, Row(2, ["1"]), 5, 6, "ALICE", Row(2), 8, 5), + Row(2, None, 4, 5, Row(4, ["2"]), 6, 7, None, Row(4), 10, 7), + Row(3, "Bob", 5, 6, Row(6, ["3"]), 7, 8, "BOB", Row(6), 12, 9), + ], ) diff --git a/flink-python/pyflink/dataframe/udf.py b/flink-python/pyflink/dataframe/udf.py index b5e55c3c14cfc..c28e5e9f818a3 100644 --- a/flink-python/pyflink/dataframe/udf.py +++ b/flink-python/pyflink/dataframe/udf.py @@ -420,16 +420,41 @@ def udf( ... incremented=increment(pf.col("value"), 2), ... ) + Arrow UDFs operate directly on Arrow columns and require ``return_dtype``. + Select ``func_type="arrow"`` explicitly or annotate an unbound parameter or + the return value with ``pyarrow.Array`` or ``pyarrow.ChunkedArray``:: + + >>> import pyarrow as pa + >>> import pyarrow.compute as pc + >>> @pf.udf(return_dtype=pf.DataType.string()) + ... def normalize_name(names: pa.Array) -> pa.Array: + ... return pc.utf8_upper(names) + >>> result = df.with_column("normalized_name", normalize_name(pf.col("name"))) + + Column arguments arrive as Arrow arrays, including ``StructArray`` for + ``ROW`` columns. Literal arguments remain Python scalars. At least one + column-valued argument is required. Return an ``Array`` or ``ChunkedArray`` + with the declared element type and the same number of rows as the input + batch. Nested types and nullability are validated without implicit element + casts. Intermediate results in composed Arrow calls may remain chunked. + Scalar values, lists, Arrow tables, and record batches are not scalar UDF + results. Arrow mode supports synchronous functions only. + + Explicit ``func_type`` overrides annotations. Without it, a declaration + containing both pandas and Arrow container hints is ambiguous and raises + an error directing the caller to select a mode explicitly. + :param func: Function, callable object, scalar UDF instance, or zero-argument callable/scalar-UDF class. :param return_dtype: DataFrame logical type, Python type, or SQL type string. General UDFs may infer it from a return annotation; - pandas UDFs require it. + pandas and Arrow UDFs require it. :param deterministic: Whether equal inputs always produce equal results. Must agree with scalar-function metadata. :param name: Non-empty function identity used by the Table planner. - :param func_type: ``"general"`` or ``"pandas"``. If omitted, any unbound - pandas container annotation selects pandas mode. + :param func_type: ``"general"``, ``"pandas"``, or ``"arrow"``. If omitted, + unbound container annotations select pandas or Arrow mode; + otherwise general mode is used. :return: A callable that accepts DataFrame expressions or Python literals and returns an :class:`~pyflink.table.expression.Expression`, or a decorator producing such a callable when ``func`` is omitted. @@ -487,18 +512,18 @@ def _validate_scalar_udf_options( return_dtype: Optional[_DataTypeLike], is_async: bool, ) -> None: - if func_type not in ("general", "pandas"): + if func_type not in ("general", "pandas", "arrow"): raise ValueError( - f"The func_type must be one of 'general, pandas', got {func_type}." + f"The func_type must be one of 'general, pandas, arrow', got {func_type}." ) - if return_dtype is None and func_type == "pandas": + if return_dtype is None and func_type in ("pandas", "arrow"): raise TypeError( - "return_dtype is required for pandas UDFs because pandas container " + f"return_dtype is required for {func_type} UDFs because {func_type} container " "annotations do not describe the logical result type." ) - if is_async and func_type == "pandas": + if is_async and func_type in ("pandas", "arrow"): raise ValueError( - "Async scalar functions do not support pandas func_type. " + f"Async scalar functions do not support {func_type} func_type. " "Use func_type='general'." ) @@ -1003,30 +1028,39 @@ def _data_type_from_type_hint(type_hint: Any) -> DataType: def _detect_func_type(declaration_context: _UDFDeclarationContext) -> str: - """Detect pandas mode from an unbound pandas container annotation.""" + """Detect a unique vectorized mode from unbound container annotations.""" hint_func = declaration_context.annotation_target + container_types: Dict[str, Tuple[Type, ...]] = {} + container_globalns: Dict[str, Any] = {} try: import pandas as pd + container_types["pandas"] = (pd.Series, pd.DataFrame) + container_globalns.update(pandas=pd, pd=pd) + except ImportError: + pass + try: + import pyarrow as pa + container_types["arrow"] = (pa.Array, pa.ChunkedArray) + container_globalns.update(pyarrow=pa, pa=pa) except ImportError: - return "general" + pass - pandas_types = (pd.Series, pd.DataFrame) - pandas_globalns = { - "pandas": pd, - "pd": pd, - **declaration_context.globalns, - } + modes: set[str] = set() for name in getattr(hint_func, "__annotations__", {}): if name in declaration_context.ignored_hint_names: continue hint = _resolve_callable_annotation( declaration_context, name, - globalns=pandas_globalns, + globalns={**container_globalns, **declaration_context.globalns}, + ) + modes.update(mode for mode, types in container_types.items() if hint in types) + if len(modes) > 1: + raise ValueError( + "UDF annotations contain both pandas and Arrow containers; " + "specify func_type explicitly." ) - if hint in pandas_types: - return "pandas" - return "general" + return next(iter(modes), "general") # ======================== Worker Adapters ======================== diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pxd b/flink-python/pyflink/fn_execution/coder_impl_fast.pxd index 05d3ba5fd6b9b..aeacced5dc537 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_fast.pxd +++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pxd @@ -90,13 +90,14 @@ cdef class RowCoderImpl(FieldCoderImpl): cdef MaskUtils _mask_utils cdef class ArrowCoderImpl(FieldCoderImpl): + cdef object _batch_format cdef object _schema cdef list _field_types cdef object _timezone cdef object _resettable_io cdef object _batch_reader - cdef list decode_one_batch_from_stream(self, InputStream in_stream, size_t size) + cdef decode_one_batch_from_stream(self, InputStream in_stream, size_t size) cdef class OverWindowArrowCoderImpl(FieldCoderImpl): cdef ArrowCoderImpl _arrow_coder diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx index 92dff893fe922..f4d6fb7b85565 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx +++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx @@ -37,6 +37,7 @@ from pyflink.fn_execution.formats.avro import FlinkAvroDecoder, FlinkAvroDatumRe FlinkAvroBufferWrapper, FlinkAvroEncoder, FlinkAvroDatumWriter from pyflink.fn_execution.ResettableIO import ResettableIO from pyflink.table.utils import pandas_to_arrow, arrow_to_pandas +from pyflink.fn_execution.utils.arrow_utils import validate_arrow_batch ROW_KIND_BIT_SIZE = 2 @@ -431,7 +432,8 @@ cdef class ArrowCoderImpl(FieldCoderImpl): A coder for arrow format data. """ - def __init__(self, schema, row_type, timezone): + def __init__(self, schema, row_type, timezone, batch_format="PANDAS"): + self._batch_format = batch_format self._schema = schema self._field_types = row_type.field_types() self._timezone = timezone @@ -443,16 +445,22 @@ cdef class ArrowCoderImpl(FieldCoderImpl): self._resettable_io.set_output_stream(out_stream) batch_writer = pa.RecordBatchStreamWriter(self._resettable_io, self._schema) - batch_writer.write_batch( - pandas_to_arrow(self._schema, self._timezone, self._field_types, cols)) + if self._batch_format == "ARROW": + batch = validate_arrow_batch(cols, self._schema, self._field_types) + else: + batch = pandas_to_arrow(self._schema, self._timezone, self._field_types, cols) + batch_writer.write_batch(batch) cpdef decode_from_stream(self, InputStream in_stream, size_t size): return self.decode_one_batch_from_stream(in_stream, size) - cdef list decode_one_batch_from_stream(self, InputStream in_stream, size_t size): + cdef decode_one_batch_from_stream(self, InputStream in_stream, size_t size): self._resettable_io.set_input_bytes(in_stream.read(size)) # there is only one arrow batch in the underlying input stream - return arrow_to_pandas(self._timezone, self._field_types, [next(self._batch_reader)]) + batch = next(self._batch_reader) + if self._batch_format == "ARROW": + return batch + return arrow_to_pandas(self._timezone, self._field_types, [batch]) def _load_from_stream(self, stream): import pyarrow as pa diff --git a/flink-python/pyflink/fn_execution/coder_impl_slow.py b/flink-python/pyflink/fn_execution/coder_impl_slow.py index 769720dc27719..7a77cc566167d 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_slow.py +++ b/flink-python/pyflink/fn_execution/coder_impl_slow.py @@ -32,6 +32,7 @@ FlinkAvroBufferWrapper, FlinkAvroEncoder, FlinkAvroDatumWriter from pyflink.fn_execution.stream_slow import InputStream, OutputStream from pyflink.table.utils import pandas_to_arrow, arrow_to_pandas +from pyflink.fn_execution.utils.arrow_utils import validate_arrow_batch ROW_KIND_BIT_SIZE = 2 @@ -278,7 +279,8 @@ class ArrowCoderImpl(FieldCoderImpl): A coder for arrow format data. """ - def __init__(self, schema, row_type, timezone): + def __init__(self, schema, row_type, timezone, batch_format="PANDAS"): + self._batch_format = batch_format self._schema = schema self._field_types = row_type.field_types() self._timezone = timezone @@ -290,16 +292,22 @@ def encode_to_stream(self, cols, out_stream: OutputStream): self._resettable_io.set_output_stream(out_stream) batch_writer = pa.RecordBatchStreamWriter(self._resettable_io, self._schema) - batch_writer.write_batch( - pandas_to_arrow(self._schema, self._timezone, self._field_types, cols)) + if self._batch_format == "ARROW": + batch = validate_arrow_batch(cols, self._schema, self._field_types) + else: + batch = pandas_to_arrow(self._schema, self._timezone, self._field_types, cols) + batch_writer.write_batch(batch) def decode_from_stream(self, in_stream: InputStream, length=0): return self.decode_one_batch_from_stream(in_stream, length) - def decode_one_batch_from_stream(self, in_stream: InputStream, size: int) -> List: + def decode_one_batch_from_stream(self, in_stream: InputStream, size: int): self._resettable_io.set_input_bytes(in_stream.read(size)) # there is only one arrow batch in the underlying input stream - return arrow_to_pandas(self._timezone, self._field_types, [next(self._batch_reader)]) + batch = next(self._batch_reader) + if self._batch_format == "ARROW": + return batch + return arrow_to_pandas(self._timezone, self._field_types, [batch]) @staticmethod def _load_from_stream(stream): diff --git a/flink-python/pyflink/fn_execution/coders.py b/flink-python/pyflink/fn_execution/coders.py index 838528f24d120..6f22c31d8f61b 100644 --- a/flink-python/pyflink/fn_execution/coders.py +++ b/flink-python/pyflink/fn_execution/coders.py @@ -87,7 +87,14 @@ def _to_field_coder(cls, coder_info_descriptor_proto): timezone = pytz.timezone(os.environ['TABLE_LOCAL_TIME_ZONE']) schema_proto = coder_info_descriptor_proto.arrow_type.schema row_type = cls._to_row_type(schema_proto) - return ArrowCoder(cls._to_arrow_schema(row_type), row_type, timezone) + batch_format = coder_info_descriptor_proto.arrow_type.BatchFormat.Name( + coder_info_descriptor_proto.arrow_type.batch_format) + if batch_format == "ARROW": + from pyflink.fn_execution.utils.arrow_utils import to_arrow_schema + schema = to_arrow_schema(row_type) + else: + schema = cls._to_arrow_schema(row_type) + return ArrowCoder(schema, row_type, timezone, batch_format) elif coder_info_descriptor_proto.HasField('over_window_arrow_type'): timezone = pytz.timezone(os.environ['TABLE_LOCAL_TIME_ZONE']) schema_proto = coder_info_descriptor_proto.over_window_arrow_type.schema @@ -242,13 +249,15 @@ class ArrowCoder(FieldCoder): Coder for Arrow. """ - def __init__(self, schema, row_type, timezone): + def __init__(self, schema, row_type, timezone, batch_format="PANDAS"): + self._batch_format = batch_format self._schema = schema self._row_type = row_type self._timezone = timezone def get_impl(self): - return coder_impl.ArrowCoderImpl(self._schema, self._row_type, self._timezone) + return coder_impl.ArrowCoderImpl( + self._schema, self._row_type, self._timezone, self._batch_format) def __repr__(self): return 'ArrowCoder[%s]' % self._schema diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py index 6fabbefcc7cf0..431c21f00ed69 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py @@ -41,7 +41,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x9a\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x12\x12\n\x08refIndex\x18\x04 \x01(\x05H\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xdb\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x16\n\x0eoutput_indices\x18\x08 \x03(\x05\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x9a\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x12\x12\n\x08refIndex\x18\x04 \x01(\x05H\x00\x42\x07\n\x05input\"\x87\x02\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\x12\x14\n\x0cis_arrow_udf\x18\x06 \x01(\x08\x12G\n\x0boutput_type\x18\x07 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xdb\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x16\n\x0eoutput_indices\x18\x08 \x03(\x05\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xfb\x08\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\xce\x01\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x12\x61\n\x0c\x62\x61tch_format\x18\x02 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowType.BatchFormat\"$\n\x0b\x42\x61tchFormat\x12\n\n\x06PANDAS\x10\x00\x12\t\n\x05\x41RROW\x10\x01\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -54,113 +54,115 @@ _globals['_INPUT']._serialized_start=107 _globals['_INPUT']._serialized_end=261 _globals['_USERDEFINEDFUNCTION']._serialized_start=264 - _globals['_USERDEFINEDFUNCTION']._serialized_end=432 - _globals['_ASYNCOPTIONS']._serialized_start=435 - _globals['_ASYNCOPTIONS']._serialized_end=579 - _globals['_USERDEFINEDFUNCTIONS']._serialized_start=582 - _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1057 - _globals['_OVERWINDOW']._serialized_start=1060 - _globals['_OVERWINDOW']._serialized_end=1409 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1201 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1409 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1412 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2191 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1677 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2191 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=1940 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=2024 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=2027 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2178 - _globals['_GROUPWINDOW']._serialized_start=2194 - _globals['_GROUPWINDOW']._serialized_end=2750 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2558 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2649 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2651 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2750 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2753 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3464 - _globals['_SCHEMA']._serialized_start=3467 - _globals['_SCHEMA']._serialized_end=5505 - _globals['_SCHEMA_MAPINFO']._serialized_start=3542 - _globals['_SCHEMA_MAPINFO']._serialized_end=3693 - _globals['_SCHEMA_TIMEINFO']._serialized_start=3695 - _globals['_SCHEMA_TIMEINFO']._serialized_end=3724 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3726 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3760 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3762 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3806 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3808 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3847 - _globals['_SCHEMA_DECIMALINFO']._serialized_start=3849 - _globals['_SCHEMA_DECIMALINFO']._serialized_end=3896 - _globals['_SCHEMA_BINARYINFO']._serialized_start=3898 - _globals['_SCHEMA_BINARYINFO']._serialized_end=3926 - _globals['_SCHEMA_VARBINARYINFO']._serialized_start=3928 - _globals['_SCHEMA_VARBINARYINFO']._serialized_end=3959 - _globals['_SCHEMA_CHARINFO']._serialized_start=3961 - _globals['_SCHEMA_CHARINFO']._serialized_end=3987 - _globals['_SCHEMA_VARCHARINFO']._serialized_start=3989 - _globals['_SCHEMA_VARCHARINFO']._serialized_end=4018 - _globals['_SCHEMA_FIELDTYPE']._serialized_start=4021 - _globals['_SCHEMA_FIELDTYPE']._serialized_end=5093 - _globals['_SCHEMA_FIELD']._serialized_start=5095 - _globals['_SCHEMA_FIELD']._serialized_end=5203 - _globals['_SCHEMA_TYPENAME']._serialized_start=5206 - _globals['_SCHEMA_TYPENAME']._serialized_end=5505 - _globals['_TYPEINFO']._serialized_start=5508 - _globals['_TYPEINFO']._serialized_end=6855 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=6002 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6141 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6144 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6328 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6237 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6328 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6330 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6410 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6412 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6442 - _globals['_TYPEINFO_TYPENAME']._serialized_start=6445 - _globals['_TYPEINFO_TYPENAME']._serialized_end=6842 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6858 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7835 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7353 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7659 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7662 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7835 - _globals['_STATEDESCRIPTOR']._serialized_start=7838 - _globals['_STATEDESCRIPTOR']._serialized_end=9730 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7970 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9730 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8441 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9539 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8619 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8707 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8709 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8784 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8787 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9395 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9397 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9497 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9539 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9541 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9609 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9611 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9685 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9687 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9730 - _globals['_CODERINFODESCRIPTOR']._serialized_start=9733 - _globals['_CODERINFODESCRIPTOR']._serialized_end=10742 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10326 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10400 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10402 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10469 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10471 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10540 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10542 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10621 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10623 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10695 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10697 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10729 + _globals['_USERDEFINEDFUNCTION']._serialized_end=527 + _globals['_ASYNCOPTIONS']._serialized_start=530 + _globals['_ASYNCOPTIONS']._serialized_end=674 + _globals['_USERDEFINEDFUNCTIONS']._serialized_start=677 + _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1152 + _globals['_OVERWINDOW']._serialized_start=1155 + _globals['_OVERWINDOW']._serialized_end=1504 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1296 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1504 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1507 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2286 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1772 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2286 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=2035 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=2119 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=2122 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2273 + _globals['_GROUPWINDOW']._serialized_start=2289 + _globals['_GROUPWINDOW']._serialized_end=2845 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2653 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2744 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2746 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2845 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2848 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3559 + _globals['_SCHEMA']._serialized_start=3562 + _globals['_SCHEMA']._serialized_end=5600 + _globals['_SCHEMA_MAPINFO']._serialized_start=3637 + _globals['_SCHEMA_MAPINFO']._serialized_end=3788 + _globals['_SCHEMA_TIMEINFO']._serialized_start=3790 + _globals['_SCHEMA_TIMEINFO']._serialized_end=3819 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3821 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3855 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3857 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3901 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3903 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3942 + _globals['_SCHEMA_DECIMALINFO']._serialized_start=3944 + _globals['_SCHEMA_DECIMALINFO']._serialized_end=3991 + _globals['_SCHEMA_BINARYINFO']._serialized_start=3993 + _globals['_SCHEMA_BINARYINFO']._serialized_end=4021 + _globals['_SCHEMA_VARBINARYINFO']._serialized_start=4023 + _globals['_SCHEMA_VARBINARYINFO']._serialized_end=4054 + _globals['_SCHEMA_CHARINFO']._serialized_start=4056 + _globals['_SCHEMA_CHARINFO']._serialized_end=4082 + _globals['_SCHEMA_VARCHARINFO']._serialized_start=4084 + _globals['_SCHEMA_VARCHARINFO']._serialized_end=4113 + _globals['_SCHEMA_FIELDTYPE']._serialized_start=4116 + _globals['_SCHEMA_FIELDTYPE']._serialized_end=5188 + _globals['_SCHEMA_FIELD']._serialized_start=5190 + _globals['_SCHEMA_FIELD']._serialized_end=5298 + _globals['_SCHEMA_TYPENAME']._serialized_start=5301 + _globals['_SCHEMA_TYPENAME']._serialized_end=5600 + _globals['_TYPEINFO']._serialized_start=5603 + _globals['_TYPEINFO']._serialized_end=6950 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=6097 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6236 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6239 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6423 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6332 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6423 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6425 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6505 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6507 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6537 + _globals['_TYPEINFO_TYPENAME']._serialized_start=6540 + _globals['_TYPEINFO_TYPENAME']._serialized_end=6937 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6953 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7930 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7448 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7754 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7757 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7930 + _globals['_STATEDESCRIPTOR']._serialized_start=7933 + _globals['_STATEDESCRIPTOR']._serialized_end=9825 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=8065 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9825 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8536 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9634 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8714 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8802 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8804 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8879 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8882 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9490 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9492 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9590 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9592 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9634 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9636 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9704 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9706 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9780 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9782 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9825 + _globals['_CODERINFODESCRIPTOR']._serialized_start=9828 + _globals['_CODERINFODESCRIPTOR']._serialized_end=10975 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10421 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10495 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10497 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10564 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10567 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10773 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE_BATCHFORMAT']._serialized_start=10737 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE_BATCHFORMAT']._serialized_end=10773 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10775 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10854 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10856 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10928 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10930 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10962 # @@protoc_insertion_point(module_scope) diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi index 554c04492d953..7882fa0c96ed7 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi @@ -45,18 +45,22 @@ class Input(_message.Message): def __init__(self, udf: _Optional[_Union[UserDefinedFunction, _Mapping]] = ..., inputOffset: _Optional[int] = ..., inputConstant: _Optional[bytes] = ..., refIndex: _Optional[int] = ...) -> None: ... class UserDefinedFunction(_message.Message): - __slots__ = ("payload", "inputs", "window_index", "takes_row_as_input", "is_pandas_udf") + __slots__ = ("payload", "inputs", "window_index", "takes_row_as_input", "is_pandas_udf", "is_arrow_udf", "output_type") PAYLOAD_FIELD_NUMBER: _ClassVar[int] INPUTS_FIELD_NUMBER: _ClassVar[int] WINDOW_INDEX_FIELD_NUMBER: _ClassVar[int] TAKES_ROW_AS_INPUT_FIELD_NUMBER: _ClassVar[int] IS_PANDAS_UDF_FIELD_NUMBER: _ClassVar[int] + IS_ARROW_UDF_FIELD_NUMBER: _ClassVar[int] + OUTPUT_TYPE_FIELD_NUMBER: _ClassVar[int] payload: bytes inputs: _containers.RepeatedCompositeFieldContainer[Input] window_index: int takes_row_as_input: bool is_pandas_udf: bool - def __init__(self, payload: _Optional[bytes] = ..., inputs: _Optional[_Iterable[_Union[Input, _Mapping]]] = ..., window_index: _Optional[int] = ..., takes_row_as_input: bool = ..., is_pandas_udf: bool = ...) -> None: ... + is_arrow_udf: bool + output_type: Schema.FieldType + def __init__(self, payload: _Optional[bytes] = ..., inputs: _Optional[_Iterable[_Union[Input, _Mapping]]] = ..., window_index: _Optional[int] = ..., takes_row_as_input: bool = ..., is_pandas_udf: bool = ..., is_arrow_udf: bool = ..., output_type: _Optional[_Union[Schema.FieldType, _Mapping]] = ...) -> None: ... class AsyncOptions(_message.Message): __slots__ = ("max_concurrent_operations", "timeout_ms", "retry_enabled", "retry_max_attempts", "retry_delay_ms") @@ -645,10 +649,18 @@ class CoderInfoDescriptor(_message.Message): schema: Schema def __init__(self, schema: _Optional[_Union[Schema, _Mapping]] = ...) -> None: ... class ArrowType(_message.Message): - __slots__ = ("schema",) + __slots__ = ("schema", "batch_format") + class BatchFormat(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + PANDAS: _ClassVar[CoderInfoDescriptor.ArrowType.BatchFormat] + ARROW: _ClassVar[CoderInfoDescriptor.ArrowType.BatchFormat] + PANDAS: CoderInfoDescriptor.ArrowType.BatchFormat + ARROW: CoderInfoDescriptor.ArrowType.BatchFormat SCHEMA_FIELD_NUMBER: _ClassVar[int] + BATCH_FORMAT_FIELD_NUMBER: _ClassVar[int] schema: Schema - def __init__(self, schema: _Optional[_Union[Schema, _Mapping]] = ...) -> None: ... + batch_format: CoderInfoDescriptor.ArrowType.BatchFormat + def __init__(self, schema: _Optional[_Union[Schema, _Mapping]] = ..., batch_format: _Optional[_Union[CoderInfoDescriptor.ArrowType.BatchFormat, str]] = ...) -> None: ... class OverWindowArrowType(_message.Message): __slots__ = ("schema",) SCHEMA_FIELD_NUMBER: _ClassVar[int] diff --git a/flink-python/pyflink/fn_execution/table/operations.py b/flink-python/pyflink/fn_execution/table/operations.py index 35ff6a09d686d..49d71203216ac 100644 --- a/flink-python/pyflink/fn_execution/table/operations.py +++ b/flink-python/pyflink/fn_execution/table/operations.py @@ -141,9 +141,11 @@ def generate_func(self, serialized_fn): Generates a UDF execution function. Uses sequential execution with result references when refIndex is present (CSE mode), otherwise uses lambda-based approach. """ + is_arrow = all(udf.is_arrow_udf for udf in serialized_fn.udfs) + one_arg_optimization = self._one_arg_optimization and not is_arrow udf_infos = [ operation_utils.extract_user_defined_function( - udf, one_arg_optimization=self._one_arg_optimization) + udf, one_arg_optimization=one_arg_optimization) for udf in serialized_fn.udfs] variable_dict = {} @@ -154,6 +156,10 @@ def generate_func(self, serialized_fn): user_defined_funcs.extend(funcs) func_strs.append(func_str) + if is_arrow: + from pyflink.fn_execution.utils.arrow_utils import create_arrow_batch + variable_dict['create_arrow_batch'] = create_arrow_batch + output_indices = list(serialized_fn.output_indices) # Result references require sequential evaluation. A non-empty output_indices does too: # it may repeat or reorder results even when none of the UDFs references another result. @@ -163,7 +169,9 @@ def generate_func(self, serialized_fn): if not requires_sequential_execution: # Keep original lambda-based approach for backward compatibility scalar_functions = ','.join(func_strs) - if self._one_result_optimization: + if is_arrow: + func_str = f'lambda value: create_arrow_batch([{scalar_functions}], value.num_rows)' + elif self._one_result_optimization: func_str = 'lambda value: %s' % scalar_functions else: func_str = 'lambda value: [%s]' % scalar_functions @@ -186,7 +194,10 @@ def generate_func(self, serialized_fn): code_lines.append(' results = [None] * %d' % len(func_strs)) for i, fn in enumerate(func_strs): code_lines.append(' results[%d] = %s' % (i, fn)) - if self._one_result_optimization: + if is_arrow: + outputs = ','.join('results[%d]' % i for i in output_indices) + code_lines.append(f' return create_arrow_batch([{outputs}], value.num_rows)') + elif self._one_result_optimization: code_lines.append(' return results[%d]' % output_indices[0]) else: code_lines.append( diff --git a/flink-python/pyflink/fn_execution/tests/test_arrow_udf.py b/flink-python/pyflink/fn_execution/tests/test_arrow_udf.py new file mode 100644 index 0000000000000..f635cefa0c428 --- /dev/null +++ b/flink-python/pyflink/fn_execution/tests/test_arrow_udf.py @@ -0,0 +1,119 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""Behavior of Arrow scalar operations without a Flink cluster.""" + +import pickle +import unittest + +import cloudpickle +import pyarrow as pa +import pyarrow.compute as pc + +from pyflink.fn_execution import flink_fn_execution_pb2 as proto +from pyflink.fn_execution.table.operations import ScalarFunctionOperation +from pyflink.table.udf import DelegatingScalarFunction, ScalarFunction + + +class Uppercase(ScalarFunction): + def eval(self, values): + return pc.utf8_upper(values) + + +class ArrowScalarOperationTests(unittest.TestCase): + def operation(self, func, inputs): + function = proto.UserDefinedFunction( + payload=cloudpickle.dumps(DelegatingScalarFunction(func)), + is_arrow_udf=True, inputs=inputs) + operation = ScalarFunctionOperation(proto.UserDefinedFunctions(udfs=[function])) + operation.open() + self.addCleanup(operation.close) + return operation + + def test_literals_columns_and_chunked_results(self): + def add(offset, left, right): + if not isinstance(offset, int) or not isinstance(left, pa.Array) \ + or not isinstance(right, pa.Array): + raise TypeError("Expected a scalar literal followed by two Arrow arrays.") + values = pc.add(pc.add(left, right), offset) + return pa.chunked_array([values.slice(0, 1), values.slice(1)]) + + operation = self.operation(add, [ + proto.Input(inputConstant=b"\x00" + pickle.dumps(10)), + proto.Input(inputOffset=0), proto.Input(inputOffset=1)]) + result = operation.process_element(pa.record_batch( + [pa.array([1, None, 3]), pa.array([4, 5, 6])], names=["a", "b"])) + self.assertEqual(result.column(0).to_pylist(), [15, None, 19]) + + def test_invalid_scalar_results(self): + batch = pa.record_batch([pa.array([1, 2, 3])], names=["value"]) + for result, error, message in ( + ([1, 2, 3], TypeError, "Array or pyarrow.ChunkedArray"), + (None, TypeError, "NoneType"), + (pa.scalar(1), TypeError, "Scalar"), + (batch, TypeError, "RecordBatch"), + (pa.Table.from_batches([batch]), TypeError, "Table"), + (pa.array([1]), ValueError, "returned 1 rows, expected 3"), + (pa.chunked_array([[1], [2]]), ValueError, "returned 2 rows, expected 3"), + ): + with self.subTest(result=result): + operation = self.operation(lambda values: result, [proto.Input(inputOffset=0)]) + with self.assertRaisesRegex(error, message): + operation.process_element(batch) + + def test_invalid_intermediate_result_is_not_consumed(self): + inner = proto.UserDefinedFunction( + payload=cloudpickle.dumps(DelegatingScalarFunction(lambda values: values.slice(0, 1))), + is_arrow_udf=True, inputs=[proto.Input(inputOffset=0)]) + # The outer result has the correct batch length, but must not hide the invalid inner result. + operation = self.operation(lambda values: pa.array([1, 2, 3]), [proto.Input(udf=inner)]) + with self.assertRaisesRegex(ValueError, "returned 1 rows, expected 3"): + operation.process_element(pa.record_batch([pa.array([1, 2, 3])], names=["value"])) + + def test_arrow_scalar_operation(self): + function = proto.UserDefinedFunction( + payload=cloudpickle.dumps(Uppercase()), is_arrow_udf=True, + inputs=[proto.Input(inputOffset=0)]) + operation = ScalarFunctionOperation( + proto.UserDefinedFunctions(udfs=[function]), one_arg_optimization=True) + operation.open() + self.addCleanup(operation.close) + result = operation.process_element(pa.record_batch( + [pa.array(["alice", None, "Bob"])], names=["name"])) + self.assertIsInstance(result, pa.RecordBatch) + self.assertEqual(result.column(0).to_pylist(), ["ALICE", None, "BOB"]) + + def test_intermediate_schema_is_enforced(self): + for values, error, message in ( + (pa.array(["wrong", "type"]), TypeError, "expected int64"), + (pa.chunked_array([[1], [None]], type=pa.int64()), ValueError, "not nullable"), + ): + with self.subTest(values=values): + inner = proto.UserDefinedFunction( + payload=cloudpickle.dumps(DelegatingScalarFunction(lambda column: values)), + is_arrow_udf=True, inputs=[proto.Input(inputOffset=0)], + output_type=proto.Schema.FieldType(type_name=proto.Schema.BIGINT, + nullable=False)) + operation = self.operation(lambda column: pa.array([1, 2]), + [proto.Input(udf=inner)]) + with self.assertRaisesRegex(error, message): + operation.process_element(pa.record_batch([pa.array([1, 2])], names=["a"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index f04a33f164999..540efc3559495 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -19,7 +19,12 @@ """Tests common to all coder implementations.""" import decimal import logging +import os import unittest +from unittest import mock + +import pyarrow as pa +import pytz from pyflink.fn_execution.coders import BigIntCoder, TinyIntCoder, BooleanCoder, \ SmallIntCoder, IntCoder, FloatCoder, DoubleCoder, BinaryCoder, CharCoder, DateCoder, \ @@ -30,6 +35,113 @@ from pyflink.testing.test_case_utils import PyFlinkTestCase +class ArrowCodersTests(unittest.TestCase): + from pyflink.fn_execution import coder_impl_slow as implementation + + def arrow_coder(self, schema, row_type): + return self.implementation.ArrowCoderImpl(schema, row_type, pytz.UTC, "ARROW") + + def test_arrow_descriptor_preserves_pandas_default(self): + from pyflink.fn_execution import flink_fn_execution_pb2 as proto + from pyflink.fn_execution.coders import LengthPrefixBaseCoder + import pandas as pd + + arrow_type = proto.CoderInfoDescriptor.ArrowType(schema=proto.Schema(fields=[ + proto.Schema.Field(name="name", type=proto.Schema.FieldType( + type_name=proto.Schema.VARCHAR, nullable=True, + var_char_info=proto.Schema.VarCharInfo(length=2147483647)))])) + descriptor = proto.CoderInfoDescriptor(arrow_type=arrow_type) + with mock.patch.dict(os.environ, {"TABLE_LOCAL_TIME_ZONE": "UTC"}): + pandas_coder = LengthPrefixBaseCoder._to_field_coder(descriptor).get_impl() + result = pandas_coder.decode(pandas_coder.encode([pd.Series(["a", None])])) + self.assertIsInstance(result[0], pd.Series) + self.assertEqual(result[0].tolist(), ["a", None]) + descriptor.arrow_type.batch_format = proto.CoderInfoDescriptor.ArrowType.ARROW + arrow_coder = LengthPrefixBaseCoder._to_field_coder(descriptor).get_impl() + batch = pa.record_batch([pa.array(["a", None])], names=["name"]) + self.assertEqual(arrow_coder.decode(arrow_coder.encode(batch)), batch) + + def test_arrow_nested_nullability(self): + from pyflink.table import DataTypes + from pyflink.table.types import to_arrow_type + + row_type = DataTypes.ROW([ + DataTypes.FIELD("values", DataTypes.ARRAY(DataTypes.INT().not_null()))]) + schema = pa.schema([pa.field("values", to_arrow_type(row_type.field_types()[0]))]) + coder = self.arrow_coder(schema, row_type) + batch = pa.record_batch([pa.array([[1, None]], type=pa.list_(pa.int32()))], schema=schema) + with self.assertRaisesRegex(ValueError, "values.*not nullable"): + coder.encode(batch) + + valid = pa.record_batch([pa.array([[1, 2], None], type=pa.list_(pa.int32()))], + schema=schema) + self.assertEqual(coder.decode(coder.encode(valid)), valid) + + def test_struct_map_and_temporal_results(self): + import datetime + from pyflink.table import DataTypes + from pyflink.fn_execution.utils.arrow_utils import to_arrow_schema + + row_type = DataTypes.ROW([ + DataTypes.FIELD("record", DataTypes.ROW([ + DataTypes.FIELD("inner", DataTypes.ROW([ + DataTypes.FIELD("value", DataTypes.INT().not_null())]))])), + DataTypes.FIELD("lookup", DataTypes.MAP( + DataTypes.STRING().not_null(), DataTypes.INT().not_null())), + DataTypes.FIELD("amount", DataTypes.DECIMAL(6, 2)), + DataTypes.FIELD("time", DataTypes.TIMESTAMP(3))]) + schema = to_arrow_schema(row_type) + coder = self.arrow_coder(schema, row_type) + rows = [ + {"record": {"inner": {"value": 7}}, "lookup": [("a", 1)], + "amount": decimal.Decimal("12.34"), "time": datetime.datetime(2020, 1, 2)}, + {"record": None, "lookup": None, "amount": None, "time": None}, + {"record": {"inner": None}, "lookup": [], + "amount": decimal.Decimal("-0.50"), "time": datetime.datetime(2021, 3, 4)}] + batch = pa.RecordBatch.from_pylist(rows, schema=schema) + self.assertEqual(coder.decode(coder.encode(batch)).to_pylist(), rows) + self.assertEqual(coder.decode(coder.encode(batch.slice(1))).to_pylist(), rows[1:]) + + for field, value, message in ( + ("record", {"inner": {"value": None}}, "record.inner.value.*not nullable"), + ("lookup", [("a", None)], "lookup.value.*not nullable"), + ): + with self.subTest(field=field): + invalid = pa.RecordBatch.from_pylist([{**rows[0], field: value}], schema=schema) + with self.assertRaisesRegex(ValueError, message): + coder.encode(invalid) + + wrong = pa.StructArray.from_arrays([pa.array([1], type=pa.int64())], names=["value"]) + invalid = pa.record_batch([pa.StructArray.from_arrays([wrong], names=["inner"]), + batch.column(1).slice(0, 1), batch.column(2).slice(0, 1), + batch.column(3).slice(0, 1)], names=schema.names) + with self.assertRaisesRegex(TypeError, "record.inner.value.*int64.*int32"): + coder.encode(invalid) + + def test_native_arrow_round_trip(self): + from pyflink.table import DataTypes + + row_type = DataTypes.ROW([DataTypes.FIELD("name", DataTypes.STRING())]) + schema = pa.schema([pa.field("name", pa.string())]) + coder = self.arrow_coder(schema, row_type) + batch = pa.record_batch([pa.array(["ALICE", None, "BOB"])], schema=schema) + self.assertEqual(coder.decode(coder.encode(batch)), batch) + + with self.assertRaisesRegex(TypeError, "name.*string"): + coder.encode(pa.record_batch([pa.array([1, 2])], names=["name"])) + + +try: + from pyflink.fn_execution import coder_impl_fast +except ImportError: + coder_impl_fast = None + + +@unittest.skipIf(coder_impl_fast is None, "Compiled coders are not installed") +class FastArrowCodersTests(ArrowCodersTests): + implementation = coder_impl_fast + + class CodersTest(PyFlinkTestCase): def check_coder(self, coder, *values): diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py new file mode 100644 index 0000000000000..e8ef5c55a2fed --- /dev/null +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -0,0 +1,121 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""Arrow-native scalar UDF result contracts shared by the Python and compiled coders.""" + + +def to_arrow_schema(row_type): + import pyarrow as pa + from pyflink.table.types import ArrayType, MapType, RowType, to_arrow_type + + def field(name, data_type): + if isinstance(data_type, RowType): + arrow_type = pa.struct([field(f.name, f.data_type) for f in data_type.fields]) + elif isinstance(data_type, ArrayType): + arrow_type = pa.list_(field("element", data_type.element_type)) + elif isinstance(data_type, MapType): + arrow_type = pa.map_(field("key", data_type.key_type).with_nullable(False), + field("value", data_type.value_type)) + else: + arrow_type = to_arrow_type(data_type) + return pa.field(name, arrow_type, nullable=data_type._nullable) + + return pa.schema([field(f.name, f.data_type) for f in row_type.fields]) + + +def validate_arrow_batch(batch, schema, field_types): + import pyarrow as pa + + if not isinstance(batch, pa.RecordBatch): + raise TypeError("Arrow transport expects a pyarrow.RecordBatch.") + if batch.num_columns != len(schema): + raise ValueError(f"Arrow result has {batch.num_columns} columns, expected {len(schema)}.") + for column, field, data_type in zip(batch.columns, schema, field_types): + _validate_array(column, field.type, data_type, field.name) + return pa.RecordBatch.from_arrays(batch.columns, schema=schema) + + +def _validate_array(column, expected_type, data_type, path): + import pyarrow as pa + from pyflink.table.types import ArrayType, MapType, RowType + + def wrong_type(): + raise TypeError( + f"Arrow result field '{path}' has type {column.type}, expected {expected_type}.") + + if not data_type._nullable and column.null_count: + raise ValueError(f"Arrow result field '{path}' is not nullable.") + if isinstance(data_type, RowType): + if not pa.types.is_struct(column.type) or column.type.num_fields != len(data_type.fields): + wrong_type() + # Children hidden by a null parent are not logical values and may contain nulls. + visible = column.filter(column.is_valid()) if column.null_count else column + for index, field in enumerate(data_type.fields): + if column.type[index].name != field.name: + wrong_type() + _validate_array(visible.field(index), expected_type[index].type, + field.data_type, f"{path}.{field.name}") + elif isinstance(data_type, ArrayType): + if not pa.types.is_list(column.type): + wrong_type() + # flatten respects the slice offsets and excludes values under null lists. + _validate_array(column.flatten(), expected_type.value_type, + data_type.element_type, f"{path}[]") + elif isinstance(data_type, MapType): + if not pa.types.is_map(column.type): + wrong_type() + visible = column.filter(column.is_valid()) if column.null_count else column + start, end = visible.offsets[0].as_py(), visible.offsets[-1].as_py() + _validate_array(visible.keys.slice(start, end - start), expected_type.key_type, + data_type.key_type.not_null(), f"{path}.key") + _validate_array(visible.items.slice(start, end - start), expected_type.item_type, + data_type.value_type, f"{path}.value") + elif column.type != expected_type: + wrong_type() + + +def check_arrow_udf_result(func, *args, result_type=None, arrow_type=None): + import pyarrow as pa + + result = func(*args) + name = getattr(func, "__qualname__", type(func).__name__) + if not isinstance(result, (pa.Array, pa.ChunkedArray)): + raise TypeError( + f"Arrow UDF '{name}' must return a pyarrow.Array or pyarrow.ChunkedArray, " + f"got {type(result).__name__}.") + for arg in args: + if isinstance(arg, (pa.Array, pa.ChunkedArray)) and len(result) != len(arg): + raise ValueError( + f"Arrow UDF '{name}' returned {len(result)} rows, expected {len(arg)}.") + if result_type is not None: + chunks = result.chunks if isinstance(result, pa.ChunkedArray) else [result] + # An empty ChunkedArray still has an element type that must match the declaration. + for chunk in chunks or [pa.array([], type=result.type)]: + _validate_array(chunk, arrow_type, result_type, name) + return result + + +def create_arrow_batch(results, row_count): + import pyarrow as pa + + columns = [] + for result in results: + if len(result) != row_count: + raise ValueError(f"Arrow UDF returned {len(result)} rows, expected {row_count}.") + columns.append(result.combine_chunks() if isinstance(result, pa.ChunkedArray) else result) + return pa.RecordBatch.from_arrays(columns, names=[f"f{i}" for i in range(len(columns))]) diff --git a/flink-python/pyflink/fn_execution/utils/operation_utils.py b/flink-python/pyflink/fn_execution/utils/operation_utils.py index ff96585e4c251..e47827d78f397 100644 --- a/flink-python/pyflink/fn_execution/utils/operation_utils.py +++ b/flink-python/pyflink/fn_execution/utils/operation_utils.py @@ -160,6 +160,18 @@ def _extract_input(args) -> Tuple[str, Dict, List]: variable_dict[func_name] = user_defined_func.func else: variable_dict[func_name] = user_defined_func.eval + if user_defined_function_proto.is_arrow_udf: + from pyflink.fn_execution.coders import LengthPrefixBaseCoder + from pyflink.fn_execution.utils.arrow_utils import check_arrow_udf_result, to_arrow_schema + from pyflink.table.types import RowField, RowType + result_type = None + arrow_type = None + if user_defined_function_proto.HasField('output_type'): + result_type = LengthPrefixBaseCoder._to_data_type( + user_defined_function_proto.output_type) + arrow_type = to_arrow_schema(RowType([RowField('result', result_type)]))[0].type + variable_dict[func_name] = partial(check_arrow_udf_result, variable_dict[func_name], + result_type=result_type, arrow_type=arrow_type) user_defined_funcs.append(user_defined_func) func_args, input_variable_dict, input_funcs = _extract_input(user_defined_function_proto.inputs) diff --git a/flink-python/pyflink/proto/flink-fn-execution.proto b/flink-python/pyflink/proto/flink-fn-execution.proto index 4644909bad962..e7abbed1eae9d 100644 --- a/flink-python/pyflink/proto/flink-fn-execution.proto +++ b/flink-python/pyflink/proto/flink-fn-execution.proto @@ -64,6 +64,10 @@ message UserDefinedFunction { // Whether it's pandas UDF bool is_pandas_udf = 5; + + // Whether the scalar UDF consumes and returns Arrow arrays directly + bool is_arrow_udf = 6; + Schema.FieldType output_type = 7; } // Async execution configuration for async functions @@ -526,6 +530,11 @@ message CoderInfoDescriptor { message ArrowType { Schema schema = 1; + enum BatchFormat { + PANDAS = 0; + ARROW = 1; + } + BatchFormat batch_format = 2; } // only used in batch over window diff --git a/flink-python/pyflink/table/tests/test_pandas_udf.py b/flink-python/pyflink/table/tests/test_pandas_udf.py index 192ce9c65b474..84d25d4307d9e 100644 --- a/flink-python/pyflink/table/tests/test_pandas_udf.py +++ b/flink-python/pyflink/table/tests/test_pandas_udf.py @@ -32,7 +32,7 @@ class PandasUDFTests(PyFlinkTestCase): def test_non_exist_func_type(self): with self.assertRaisesRegex(ValueError, - 'The func_type must be one of \'general, pandas\''): + "The func_type must be one of 'general, pandas, arrow'"): udf(lambda i: i + 1, result_type=DataTypes.BIGINT(), func_type="non-exist") diff --git a/flink-python/pyflink/table/udf.py b/flink-python/pyflink/table/udf.py index 92bd180d1899e..92dee5ccbf3a5 100644 --- a/flink-python/pyflink/table/udf.py +++ b/flink-python/pyflink/table/udf.py @@ -507,6 +507,8 @@ def get_python_function_kind(): return JPythonFunctionKind.GENERAL elif self._func_type == "pandas": return JPythonFunctionKind.PANDAS + elif self._func_type == "arrow": + return JPythonFunctionKind.ARROW else: raise TypeError("Unsupported func_type: %s." % self._func_type) @@ -760,10 +762,19 @@ def _get_python_env(): def _create_udf(f, input_types, result_type, func_type, deterministic, name): + if func_type == 'arrow': + target = f + while isinstance(target, functools.partial): + target = target.func + if isinstance(target, ScalarFunction): + target = target.eval + if inspect.iscoroutinefunction(target) or inspect.iscoroutinefunction( + getattr(target, '__call__', None)): + raise ValueError("Async scalar functions do not support arrow func_type.") if isinstance(f, AsyncScalarFunction) or inspect.iscoroutinefunction(f): - if func_type == 'pandas': + if func_type in ('pandas', 'arrow'): raise ValueError( - "Async scalar functions do not support pandas func_type. " + f"Async scalar functions do not support {func_type} func_type. " "Please use func_type='general' (default) for async functions.") return UserDefinedAsyncScalarFunctionWrapper( f, input_types, result_type, func_type, deterministic, name) @@ -831,6 +842,20 @@ def udf(f: Union[Callable, ScalarFunction, AsyncScalarFunction, Type] = None, ... return f"value_for_{key}" >>> async_lookup = udf(AsyncLookup(), result_type=DataTypes.STRING()) + Arrow vectorized scalar functions use ``func_type="arrow"`` and operate + directly on Arrow arrays, without converting to pandas:: + + >>> import pyarrow.compute as pc + >>> @udf(result_type=DataTypes.STRING(), func_type="arrow") + ... def uppercase(values): + ... return pc.utf8_upper(values) + + Arrow column arguments are ``pyarrow.Array`` values (``StructArray`` for + ROW columns); literal arguments remain Python scalars. Supply at least one + column-valued argument and return an ``Array`` or ``ChunkedArray`` with the + same row count and declared logical result type. Results are validated + without implicit element-type casts. Async Arrow functions are not supported. + :param f: lambda function, user-defined function, or async function. :param input_types: optional, the input data types. :param result_type: the result data type. @@ -838,16 +863,15 @@ def udf(f: Union[Callable, ScalarFunction, AsyncScalarFunction, Type] = None, this function is guaranteed to always return the same result given the same parameters. (default True) :param name: the function name. - :param func_type: the type of the python function, available value: general, pandas, + :param func_type: the type of the python function, available value: general, pandas, arrow, (default: general) :return: UserDefinedScalarFunctionWrapper, UserDefinedAsyncScalarFunctionWrapper, or function. .. versionadded:: 1.10.0 """ - if func_type not in ('general', 'pandas'): - raise ValueError("The func_type must be one of 'general, pandas', got %s." - % func_type) + if func_type not in ('general', 'pandas', 'arrow'): + raise ValueError(f"The func_type must be one of 'general, pandas, arrow', got {func_type}.") # decorator if f is None: diff --git a/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java b/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java index a4cee4f9ebc68..86159036b55d6 100644 --- a/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java +++ b/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java @@ -109,11 +109,24 @@ public static FlinkFnApi.CoderInfoDescriptor createArrowTypeCoderInfoDescriptorP RowType rowType, FlinkFnApi.CoderInfoDescriptor.Mode mode, boolean separatedWithEndMessage) { + return createArrowTypeCoderInfoDescriptorProto( + rowType, + mode, + separatedWithEndMessage, + FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.PANDAS); + } + + public static FlinkFnApi.CoderInfoDescriptor createArrowTypeCoderInfoDescriptorProto( + RowType rowType, + FlinkFnApi.CoderInfoDescriptor.Mode mode, + boolean separatedWithEndMessage, + FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat batchFormat) { return createCoderInfoDescriptorProto( null, null, FlinkFnApi.CoderInfoDescriptor.ArrowType.newBuilder() .setSchema(toProtoType(rowType).getRowSchema()) + .setBatchFormat(batchFormat) .build(), null, null, @@ -229,6 +242,12 @@ public static FlinkFnApi.UserDefinedFunction createUserDefinedFunctionProto( builder.setIsPandasUdf( pythonFunctionInfo.getPythonFunction().getPythonFunctionKind() == PythonFunctionKind.PANDAS); + builder.setIsArrowUdf( + pythonFunctionInfo.getPythonFunction().getPythonFunctionKind() + == PythonFunctionKind.ARROW); + if (pythonFunctionInfo.getOutputType() != null) { + builder.setOutputType(toProtoType(pythonFunctionInfo.getOutputType())); + } return builder.build(); } diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperator.java index ef5bd046390d3..bee19392571df 100644 --- a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperator.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperator.java @@ -25,6 +25,7 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.functions.python.PythonFunctionKind; import org.apache.flink.table.runtime.arrow.serializers.ArrowSerializer; import org.apache.flink.table.runtime.generated.GeneratedProjection; import org.apache.flink.table.runtime.operators.python.scalar.AbstractPythonScalarFunctionOperator; @@ -49,6 +50,8 @@ public class ArrowPythonScalarFunctionOperator extends AbstractPythonScalarFunct private transient ArrowSerializer arrowSerializer; + private final FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat batchFormat; + public ArrowPythonScalarFunctionOperator( Configuration config, PythonFunctionInfo[] scalarFunctions, @@ -86,6 +89,11 @@ public ArrowPythonScalarFunctionOperator( udfOutputType, udfInputGeneratedProjection, forwardedFieldGeneratedProjection); + batchFormat = + scalarFunctions[0].getPythonFunction().getPythonFunctionKind() + == PythonFunctionKind.ARROW + ? FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.ARROW + : FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.PANDAS; } @Override @@ -100,14 +108,14 @@ public void open() throws Exception { @Override public FlinkFnApi.CoderInfoDescriptor createInputCoderInfoDescriptor(RowType runnerInputType) { return createArrowTypeCoderInfoDescriptorProto( - runnerInputType, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, false); + runnerInputType, FlinkFnApi.CoderInfoDescriptor.Mode.MULTIPLE, false, batchFormat); } @Override public FlinkFnApi.CoderInfoDescriptor createOutputCoderInfoDescriptor( RowType runnerOutputType) { return createArrowTypeCoderInfoDescriptorProto( - runnerOutputType, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false); + runnerOutputType, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false, batchFormat); } @Override diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java index 5aa7dd76e65aa..c73ea4e7b83cc 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java @@ -20,13 +20,18 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.configuration.Configuration; +import org.apache.flink.fnexecution.v1.FlinkFnApi; import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.python.util.ProtoUtils; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.connector.Projection; import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.python.PythonEnv; import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.functions.python.PythonFunctionKind; +import org.apache.flink.table.functions.python.PythonScalarFunction; import org.apache.flink.table.planner.codegen.CodeGeneratorContext; import org.apache.flink.table.planner.codegen.ProjectionCodeGenerator; import org.apache.flink.table.runtime.generated.GeneratedProjection; @@ -39,10 +44,14 @@ import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.RowKind; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + import java.io.IOException; import java.util.Collection; import static org.apache.flink.table.runtime.util.StreamRecordUtils.row; +import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link ArrowPythonScalarFunctionOperator}. */ public class ArrowPythonScalarFunctionOperatorTest @@ -56,6 +65,60 @@ public class ArrowPythonScalarFunctionOperatorTest DataTypes.BIGINT().getLogicalType() }); + @ParameterizedTest + @EnumSource( + value = PythonFunctionKind.class, + names = {"PANDAS", "ARROW"}) + void testScalarBatchFormat(PythonFunctionKind kind) { + final RowType rowType = RowType.of(DataTypes.STRING().getLogicalType()); + final PythonFunctionInfo function = + new PythonFunctionInfo( + new PythonScalarFunction( + "identity", + new byte[0], + kind, + true, + false, + new PythonEnv(PythonEnv.ExecType.PROCESS)), + new Object[] {0}, + kind == PythonFunctionKind.ARROW + ? DataTypes.STRING().notNull().getLogicalType() + : null); + final ArrowPythonScalarFunctionOperator operator = + getTestOperator( + new Configuration(), + new PythonFunctionInfo[] {function}, + rowType, + rowType, + new int[] {0}, + new int[0]); + final FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat expected = + FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.valueOf(kind.name()); + assertThat(operator.createInputCoderInfoDescriptor(rowType).getArrowType().getBatchFormat()) + .isEqualTo(expected); + assertThat( + operator.createOutputCoderInfoDescriptor(rowType) + .getArrowType() + .getBatchFormat()) + .isEqualTo(expected); + assertThat(ProtoUtils.createUserDefinedFunctionProto(function).getIsArrowUdf()) + .isEqualTo(kind == PythonFunctionKind.ARROW); + final FlinkFnApi.UserDefinedFunction functionProto = + ProtoUtils.createUserDefinedFunctionProto(function); + assertThat(functionProto.hasOutputType()).isEqualTo(kind == PythonFunctionKind.ARROW); + if (kind == PythonFunctionKind.ARROW) { + assertThat(functionProto.getOutputType().getTypeName()) + .isEqualTo(FlinkFnApi.Schema.TypeName.VARCHAR); + assertThat(functionProto.getOutputType().getNullable()).isFalse(); + } + assertThat( + ProtoUtils.createArrowTypeCoderInfoDescriptorProto( + rowType, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false) + .getArrowType() + .getBatchFormat()) + .isEqualTo(FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.PANDAS); + } + @Override public ArrowPythonScalarFunctionOperator getTestOperator( Configuration config, diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java index e72e6b403c864..8400a8300aca5 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java @@ -19,8 +19,11 @@ package org.apache.flink.table.functions.python; import org.apache.flink.annotation.Internal; +import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.util.Preconditions; +import javax.annotation.Nullable; + /** * PythonFunctionInfo contains the execution information of a Python function, such as: the actual * Python function, the input arguments, etc. @@ -39,9 +42,23 @@ public class PythonFunctionInfo implements PythonFunctionInput { /** The input arguments of this function. */ private PythonFunctionInput[] inputs; + @Nullable private final LogicalType outputType; + public PythonFunctionInfo(PythonFunction pythonFunction, PythonFunctionInput[] inputs) { + this(pythonFunction, inputs, null); + } + + public PythonFunctionInfo( + PythonFunction pythonFunction, PythonFunctionInput[] inputs, + @Nullable LogicalType outputType) { this.pythonFunction = Preconditions.checkNotNull(pythonFunction); this.inputs = Preconditions.checkNotNull(inputs); + this.outputType = outputType; + } + + @Nullable + public LogicalType getOutputType() { + return outputType; } public PythonFunction getPythonFunction() { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionKind.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionKind.java index cd7a27ef8a830..e8afcdfaaa10e 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionKind.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionKind.java @@ -25,5 +25,7 @@ public enum PythonFunctionKind { GENERAL, - PANDAS + PANDAS, + + ARROW } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java index b0fcd6e463a78..83c8f11a36713 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java @@ -28,7 +28,10 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.connector.Projection; import org.apache.flink.table.data.RowData; +import org.apache.flink.table.functions.python.InputRef; import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.functions.python.PythonFunctionInput; +import org.apache.flink.table.functions.python.ResultRef; import org.apache.flink.table.functions.python.PythonFunctionKind; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.codegen.CodeGeneratorContext; @@ -156,7 +159,10 @@ private OneInputTransformation createPythonOneInputTransformat extractPythonScalarFunctionInfos(cseResult, classLoader); int[] pythonUdfInputOffsets = extractResult.f0; PythonFunctionInfo[] pythonFunctionInfos = extractResult.f1; - + boolean[] hasColumn = new boolean[pythonFunctionInfos.length]; + for (int i = 0; i < pythonFunctionInfos.length; i++) { + hasColumn[i] = validateArrowInputs(pythonFunctionInfos[i], hasColumn); + } LogicalType[] inputLogicalTypes = ((InternalTypeInfo) inputTransform.getOutputType()).toRowFieldTypes(); InternalTypeInfo pythonOperatorInputTypeInfo = @@ -192,7 +198,9 @@ private OneInputTransformation createPythonOneInputTransformat .anyMatch( x -> PythonUtil.containsPythonCall( - x, PythonFunctionKind.PANDAS))); + x, PythonFunctionKind.PANDAS) + || PythonUtil.containsPythonCall( + x, PythonFunctionKind.ARROW))); return ExecNodeUtil.createOneInputTransformation( inputTransform, @@ -203,6 +211,26 @@ private OneInputTransformation createPythonOneInputTransformat false); } + private static boolean validateArrowInputs( + PythonFunctionInfo function, boolean[] resultHasColumn) { + boolean hasColumn = false; + for (PythonFunctionInput input : function.getInputs()) { + if (input instanceof PythonFunctionInfo) { + hasColumn |= validateArrowInputs((PythonFunctionInfo) input, resultHasColumn); + } else if (input instanceof InputRef) { + hasColumn = true; + } else if (input instanceof ResultRef) { + hasColumn |= resultHasColumn[((ResultRef) input).getIndex()]; + } + } + if (function.getPythonFunction().getPythonFunctionKind() == PythonFunctionKind.ARROW + && !hasColumn) { + throw new TableException( + "Arrow scalar UDFs require at least one column-valued argument."); + } + return hasColumn; + } + private Tuple2 extractPythonScalarFunctionInfos( PythonCallCseResult cseResult, ClassLoader classLoader) { List rexCalls = cseResult.getDeduplicatedCalls(); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java index 795e50b152692..371708dd63718 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java @@ -36,6 +36,8 @@ import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.functions.python.PythonFunctionInput; import org.apache.flink.table.functions.python.ResultRef; +import org.apache.flink.table.functions.python.PythonFunctionKind; +import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.functions.aggfunctions.AvgAggFunction; import org.apache.flink.table.planner.functions.aggfunctions.Count1AggFunction; import org.apache.flink.table.planner.functions.aggfunctions.CountAggFunction; @@ -508,8 +510,13 @@ private static PythonFunctionInfo createPythonFunctionInfo( inputNodes.put(operand, inputOffset); } } + final PythonFunction pythonFunction = (PythonFunction) functionDefinition; return new PythonFunctionInfo( - (PythonFunction) functionDefinition, inputs.toArray(new PythonFunctionInput[0])); + pythonFunction, + inputs.toArray(new PythonFunctionInput[0]), + pythonFunction.getPythonFunctionKind() == PythonFunctionKind.ARROW + ? FlinkTypeFactory.toLogicalType(pythonRexCall.getType()) + : null); } private static BuiltInPythonAggregateFunction getBuiltInPythonAggregateFunction( diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRule.java new file mode 100644 index 0000000000000..0e42290ed7081 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRule.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.rules.logical; + +import org.apache.flink.table.functions.python.PythonFunctionKind; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc; +import org.apache.flink.table.planner.plan.utils.PythonUtil; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; + +import java.util.Arrays; + +import scala.Option; + +/** Separates scalar Python functions whose user-facing batch representations differ. */ +public class PythonCalcSplitFunctionKindRule extends RemoteCalcSplitProjectionRuleBase { + + public PythonCalcSplitFunctionKindRule(RemoteCallFinder callFinder) { + super("PythonCalcSplitFunctionKindRule", callFinder); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final FlinkLogicalCalc calc = call.rel(0); + final RexProgram program = calc.getProgram(); + return Arrays.stream(PythonFunctionKind.values()) + .filter( + kind -> + program.getProjectList().stream() + .map(program::expandLocalRef) + .anyMatch( + node -> + PythonUtil.containsPythonCall( + node, kind))) + .count() + > 1; + } + + @Override + public boolean needConvert(RexProgram program, RexNode node, Option matchState) { + // Keep one top-level kind above the split and extract calls of every other kind. + final PythonFunctionKind topLevelKind = + Arrays.stream(PythonFunctionKind.values()) + .filter( + kind -> + program.getProjectList().stream() + .map(program::expandLocalRef) + .anyMatch( + project -> + PythonUtil.isPythonCall( + project, kind))) + .findFirst() + .orElseThrow( + () -> new IllegalStateException("Missing top-level Python call.")); + return PythonUtil.isPythonCall(node) && !PythonUtil.isPythonCall(node, topLevelKind); + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala index 642ca5f5691b8..7983ea96a1a4d 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRule.scala @@ -17,53 +17,10 @@ */ package org.apache.flink.table.planner.plan.rules.logical -import org.apache.flink.table.functions.ScalarFunction -import org.apache.flink.table.functions.python.PythonFunctionKind -import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc -import org.apache.flink.table.planner.plan.utils.{InputRefVisitor, PythonUtil, RexDefaultVisitor} -import org.apache.flink.table.planner.plan.utils.PythonUtil.{containsNonPythonCall, containsPythonCall, isNonPythonCall, isPythonCall} +import org.apache.flink.table.planner.plan.utils.PythonUtil -import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall} -import org.apache.calcite.plan.RelOptRule.{any, operand} -import org.apache.calcite.rex.{RexBuilder, RexCall, RexCorrelVariable, RexFieldAccess, RexInputRef, RexLocalRef, RexNode, RexProgram} -import org.apache.calcite.sql.validate.SqlValidatorUtil - -import java.util.function.Function - -import scala.collection.JavaConversions._ -import scala.collection.JavaConverters._ -import scala.collection.mutable - -/** - * Rule that splits [[FlinkLogicalCalc]]s which contain both general Python functions and pandas - * Python functions in the projection into multiple [[FlinkLogicalCalc]]s. After this rule is - * applied, it will only contain general Python functions or pandas Python functions in the - * projection of each [[FlinkLogicalCalc]]. - */ -class PythonCalcSplitPandasInProjectionRule(callFinder: RemoteCallFinder) - extends RemoteCalcSplitProjectionRuleBase("PythonCalcSplitPandasInProjectionRule", callFinder) { - - override def matches(call: RelOptRuleCall): Boolean = { - val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc] - val projects = calc.getProgram.getProjectList.map(calc.getProgram.expandLocalRef) - - // matches if it contains both general Python functions and - // pandas Python functions in the projection - projects.exists(containsPythonCall(_, PythonFunctionKind.GENERAL)) && - projects.exists(containsPythonCall(_, PythonFunctionKind.PANDAS)) - } - - override def needConvert( - program: RexProgram, - node: RexNode, - matchState: Option[Nothing]): Boolean = { - program.getProjectList - .map(program.expandLocalRef) - .exists(isPythonCall(_, PythonFunctionKind.GENERAL)) == isPythonCall( - node, - PythonFunctionKind.PANDAS) - } -} +import org.apache.calcite.plan.RelOptRule +import org.apache.calcite.rex.RexNode class PythonRemoteCallFinder extends RemoteCallFinder { override def containsRemoteCall(node: RexNode): Boolean = { @@ -105,7 +62,7 @@ object PythonCalcSplitRule { val CONDITION_PROJECTION_CSE: RelOptRule = RemoteCalcConditionProjectionCseRule.Config.DEFAULT.withRemoteCallFinder(callFinder).toRule() val SPLIT_PROJECT: RelOptRule = new RemoteCalcSplitProjectionRule(callFinder) - val SPLIT_PANDAS_IN_PROJECT: RelOptRule = new PythonCalcSplitPandasInProjectionRule(callFinder) + val SPLIT_PANDAS_IN_PROJECT: RelOptRule = new PythonCalcSplitFunctionKindRule(callFinder) val SPLIT_PROJECTION_REX_FIELD: RelOptRule = new RemoteCalcSplitProjectionRexFieldRule(callFinder) val SPLIT_CONDITION_REX_FIELD: RelOptRule = new RemoteCalcSplitConditionRexFieldRule(callFinder) val EXPAND_PROJECT: RelOptRule = new RemoteCalcExpandProjectRule(callFinder) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.java new file mode 100644 index 0000000000000..f2717a2ef3c11 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.rules.logical; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.functions.python.PythonEnv; +import org.apache.flink.table.functions.python.PythonFunctionKind; +import org.apache.flink.table.functions.python.PythonScalarFunction; +import org.apache.flink.table.planner.utils.JavaTableTestUtil; +import org.apache.flink.table.planner.utils.TableTestBase; +import org.apache.flink.table.types.DataType; + +import org.junit.jupiter.api.Test; + +/** Plans compositions of scalar UDFs with different Python argument representations. */ +class PythonArrowCalcTest extends TableTestBase { + + @Test + void testStreamingComposition() { + verifyComposition(javaStreamTestUtil()); + } + + @Test + void testBatchComposition() { + verifyComposition(javaBatchTestUtil()); + } + + private void verifyComposition(JavaTableTestUtil util) { + util.addTableSource("T", Schema.newBuilder().column("a", DataTypes.INT()).build()); + for (PythonFunctionKind kind : PythonFunctionKind.values()) { + util.tableEnv() + .createTemporarySystemFunction( + kind.name().toLowerCase() + "_udf", + new PythonScalarFunction( + kind.name(), + new byte[0], + new DataType[] {DataTypes.INT()}, + DataTypes.INT(), + kind, + true, + false, + new PythonEnv(PythonEnv.ExecType.PROCESS))); + } + util.verifyExecPlan( + "SELECT arrow_udf(a), pandas_udf(a), general_udf(a), " + + "arrow_udf(arrow_udf(a)), arrow_udf(pandas_udf(a)), " + + "pandas_udf(arrow_udf(a)), arrow_udf(general_udf(a)), " + + "general_udf(arrow_udf(a)) FROM T"); + } +} diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.xml new file mode 100644 index 0000000000000..021138e012abf --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + From a65eda57fa36100c1fdf8c7b538d4050974e1b12 Mon Sep 17 00:00:00 2001 From: auroflow Date: Mon, 14 Sep 2026 14:59:26 +0800 Subject: [PATCH 02/14] [FLINK-40472][python] Preserve Arrow collection nullability Generated-by: Codex (GPT-6) --- .../pyflink/fn_execution/tests/test_coders.py | 39 +++++++++++++++++++ flink-python/pyflink/table/types.py | 8 +++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index 540efc3559495..f1bdc96ae1b32 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -35,6 +35,45 @@ from pyflink.testing.test_case_utils import PyFlinkTestCase +class ArrowSchemaTests(unittest.TestCase): + def test_pandas_collection_schema_and_round_trip(self): + import pandas as pd + from pyflink.table import DataTypes + from pyflink.table.types import create_arrow_schema + from pyflink.table.utils import arrow_to_pandas, pandas_to_arrow + + names = ["values", "lookup", "record"] + types = [ + DataTypes.ARRAY(DataTypes.INT().not_null()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT()).not_null()), + DataTypes.ROW([DataTypes.FIELD("values", DataTypes.ARRAY(DataTypes.INT().not_null()))])] + schema = create_arrow_schema(names, types) + self.assertEqual(schema.field("values").type.value_field.name, "item") + self.assertFalse(schema.field("values").type.value_field.nullable) + self.assertFalse(schema.field("lookup").type.key_field.nullable) + self.assertFalse(schema.field("lookup").type.item_field.nullable) + self.assertTrue(schema.field("lookup").type.item_type.value_field.nullable) + self.assertFalse(schema.field("record").type[0].type.value_field.nullable) + + # Correcting schema metadata must not add null checks to the pandas conversion path. + columns = [pd.Series([[1, 2], [], None, [None]]), + pd.Series([[('a', [1])], [], None, [('b', None)]]), + pd.DataFrame({"values": [[3], [], None, [None]]})] + batch = pandas_to_arrow(schema, pytz.UTC, types, columns) + with pa.BufferOutputStream() as output: + with pa.ipc.new_stream(output, schema) as writer: + writer.write_batch(batch) + decoded = pa.ipc.open_stream(output.getvalue()).read_next_batch() + expected = [ + {"values": [1, 2], "lookup": [('a', [1])], "record": {"values": [3]}}, + {"values": [], "lookup": [], "record": {"values": []}}, + {"values": None, "lookup": None, "record": {"values": None}}, + {"values": [None], "lookup": [('b', None)], "record": {"values": [None]}}] + self.assertEqual(decoded.to_pylist(), expected) + restored = arrow_to_pandas(pytz.UTC, types, [decoded]) + self.assertEqual(pandas_to_arrow(schema, pytz.UTC, types, restored).to_pylist(), expected) + + class ArrowCodersTests(unittest.TestCase): from pyflink.fn_execution import coder_impl_slow as implementation diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index 16d92c3c1ff66..ad07937f38618 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -2344,12 +2344,16 @@ def to_arrow_type(data_type: DataType): else: return pa.timestamp('ns') elif isinstance(data_type, MapType): - return pa.map_(to_arrow_type(data_type.key_type), to_arrow_type(data_type.value_type)) + return pa.map_( + pa.field("key", to_arrow_type(data_type.key_type), nullable=False), + pa.field("value", to_arrow_type(data_type.value_type), + nullable=data_type.value_type._nullable)) elif isinstance(data_type, ArrayType): if type(data_type.element_type) in [LocalZonedTimestampType, RowType]: raise ValueError("%s is not supported to be used as the element type of ArrayType." % data_type.element_type) - return pa.list_(to_arrow_type(data_type.element_type)) + return pa.list_(pa.field("item", to_arrow_type(data_type.element_type), + nullable=data_type.element_type._nullable)) elif isinstance(data_type, RowType): for field in data_type: if type(field.data_type) in [LocalZonedTimestampType, RowType]: From f49df7285c4286b4109ed883ec3c4799d50668cd Mon Sep 17 00:00:00 2001 From: auroflow Date: Mon, 14 Sep 2026 14:59:56 +0800 Subject: [PATCH 03/14] [FLINK-40472][python] Share Arrow schema mapping and simplify UDF validation Generated-by: Codex (GPT-6) --- .../docs/reference/pyflink.dataframe/udf.rst | 36 +-- flink-python/pyflink/dataframe/udf.py | 39 ++- flink-python/pyflink/fn_execution/coders.py | 21 +- .../fn_execution/flink_fn_execution_pb2.py | 224 +++++++++--------- .../fn_execution/flink_fn_execution_pb2.pyi | 6 +- .../pyflink/fn_execution/table/operations.py | 9 +- .../fn_execution/tests/test_arrow_udf.py | 119 ---------- .../pyflink/fn_execution/tests/test_coders.py | 39 ++- .../pyflink/fn_execution/utils/arrow_utils.py | 28 +-- .../fn_execution/utils/operation_utils.py | 13 +- .../pyflink/proto/flink-fn-execution.proto | 1 - flink-python/pyflink/table/tests/test_udf.py | 76 +++++- flink-python/pyflink/table/types.py | 29 ++- flink-python/pyflink/table/udf.py | 14 -- .../apache/flink/python/util/ProtoUtils.java | 3 - .../streaming/api/utils/ProtoUtilsTest.java | 51 ++++ ...ArrowPythonScalarFunctionOperatorTest.java | 63 ----- .../functions/python/PythonFunctionInfo.java | 17 -- .../nodes/exec/utils/CommonPythonUtil.java | 9 +- ... PythonCalcSplitFunctionKindRuleTest.java} | 2 +- ...> PythonCalcSplitFunctionKindRuleTest.xml} | 0 21 files changed, 321 insertions(+), 478 deletions(-) delete mode 100644 flink-python/pyflink/fn_execution/tests/test_arrow_udf.py rename flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/{PythonArrowCalcTest.java => PythonCalcSplitFunctionKindRuleTest.java} (97%) rename flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/{PythonArrowCalcTest.xml => PythonCalcSplitFunctionKindRuleTest.xml} (100%) diff --git a/flink-python/docs/reference/pyflink.dataframe/udf.rst b/flink-python/docs/reference/pyflink.dataframe/udf.rst index 6f61793ab2142..c0084bee45d4f 100644 --- a/flink-python/docs/reference/pyflink.dataframe/udf.rst +++ b/flink-python/docs/reference/pyflink.dataframe/udf.rst @@ -27,40 +27,8 @@ columns. A scalar UDF produces one logical output column and can be used in :meth:`~pyflink.dataframe.DataFrame.select`. DataFrame scalar UDFs support general synchronous and asynchronous callables, -and synchronous pandas or Arrow vectorized callables. See :func:`pyflink.dataframe.udf` for declaration forms, type -inference, execution modes, and examples. - -Arrow Vectorized Functions -========================== - -Arrow UDFs receive columns as ``pyarrow.Array`` values and return an ``Array`` -or ``ChunkedArray`` of the same length. They require an explicit logical -``return_dtype``. Container annotations infer Arrow mode; unannotated functions -can select ``func_type="arrow"`` explicitly. - -.. code-block:: python - - import pyarrow as pa - import pyarrow.compute as pc - import pyflink.dataframe as pf - - @pf.udf(return_dtype=pf.DataType.string()) - def normalize_name(names: pa.Array) -> pa.Array: - return pc.utf8_upper(names) - - df = pf.from_records([("Alice",), ("Bob",)], schema=["name"]) - result = df.with_column("normalized_name", normalize_name(pf.col("name"))) - -Multiple column arguments and scalar literals can be combined. Literals remain -Python scalars, and at least one argument must be column-valued. ROW columns use -``StructArray``; nested results and chunked arrays can feed subsequent UDFs. -Declared types, nested nullability, and row counts are validated without implicit -element casts. Python lists or scalars, Arrow tables, and record batches are not -supported scalar results. - -Explicit ``func_type`` overrides annotations. Mixed pandas and Arrow container -annotations require an explicit choice. Arrow UDFs are synchronous; per-UDF -concurrency and batch-size options are not provided. +and synchronous pandas or Arrow vectorized callables. See :func:`pyflink.dataframe.udf` +for declaration forms, type inference, execution modes, and examples. API Reference ============= diff --git a/flink-python/pyflink/dataframe/udf.py b/flink-python/pyflink/dataframe/udf.py index c28e5e9f818a3..f9e14c893b601 100644 --- a/flink-python/pyflink/dataframe/udf.py +++ b/flink-python/pyflink/dataframe/udf.py @@ -410,6 +410,18 @@ def udf( ... def inferred_pandas_add_one(values: pd.Series) -> pd.Series: ... return values + 1 + Synchronous Arrow UDFs operate on ``pyarrow.Array`` columns and return an + ``Array`` or ``ChunkedArray``. They require ``return_dtype``. Select + ``func_type="arrow"`` explicitly, or infer it from an Arrow container + annotation on an unbound parameter or the return value:: + + >>> import pyarrow as pa + >>> import pyarrow.compute as pc + + >>> @pf.udf(return_dtype=pf.DataType.string()) + ... def normalize_name(names: pa.Array) -> pa.Array: + ... return pc.utf8_upper(names) + A declared UDF is called with DataFrame expressions or Python literals to produce a single-column expression:: @@ -420,30 +432,6 @@ def udf( ... incremented=increment(pf.col("value"), 2), ... ) - Arrow UDFs operate directly on Arrow columns and require ``return_dtype``. - Select ``func_type="arrow"`` explicitly or annotate an unbound parameter or - the return value with ``pyarrow.Array`` or ``pyarrow.ChunkedArray``:: - - >>> import pyarrow as pa - >>> import pyarrow.compute as pc - >>> @pf.udf(return_dtype=pf.DataType.string()) - ... def normalize_name(names: pa.Array) -> pa.Array: - ... return pc.utf8_upper(names) - >>> result = df.with_column("normalized_name", normalize_name(pf.col("name"))) - - Column arguments arrive as Arrow arrays, including ``StructArray`` for - ``ROW`` columns. Literal arguments remain Python scalars. At least one - column-valued argument is required. Return an ``Array`` or ``ChunkedArray`` - with the declared element type and the same number of rows as the input - batch. Nested types and nullability are validated without implicit element - casts. Intermediate results in composed Arrow calls may remain chunked. - Scalar values, lists, Arrow tables, and record batches are not scalar UDF - results. Arrow mode supports synchronous functions only. - - Explicit ``func_type`` overrides annotations. Without it, a declaration - containing both pandas and Arrow container hints is ambiguous and raises - an error directing the caller to select a mode explicitly. - :param func: Function, callable object, scalar UDF instance, or zero-argument callable/scalar-UDF class. :param return_dtype: DataFrame logical type, Python type, or SQL type string. @@ -454,7 +442,8 @@ def udf( :param name: Non-empty function identity used by the Table planner. :param func_type: ``"general"``, ``"pandas"``, or ``"arrow"``. If omitted, unbound container annotations select pandas or Arrow mode; - otherwise general mode is used. + otherwise general mode is used. Mixed pandas and Arrow hints + require an explicit mode. :return: A callable that accepts DataFrame expressions or Python literals and returns an :class:`~pyflink.table.expression.Expression`, or a decorator producing such a callable when ``func`` is omitted. diff --git a/flink-python/pyflink/fn_execution/coders.py b/flink-python/pyflink/fn_execution/coders.py index 6f22c31d8f61b..e2e7e2c36d611 100644 --- a/flink-python/pyflink/fn_execution/coders.py +++ b/flink-python/pyflink/fn_execution/coders.py @@ -36,8 +36,8 @@ ExternalTypeInfo from pyflink.table.types import TinyIntType, SmallIntType, IntType, BigIntType, BooleanType, \ FloatType, DoubleType, VarCharType, VarBinaryType, DecimalType, DateType, TimeType, \ - LocalZonedTimestampType, RowType, RowField, to_arrow_type, TimestampType, ArrayType, MapType, \ - BinaryType, NullType, CharType + LocalZonedTimestampType, RowType, RowField, create_arrow_schema, TimestampType, ArrayType, \ + MapType, BinaryType, NullType, CharType __all__ = ['FlattenRowCoder', 'RowCoder', 'BigIntCoder', 'TinyIntCoder', 'BooleanCoder', 'SmallIntCoder', 'IntCoder', 'FloatCoder', 'DoubleCoder', 'BinaryCoder', 'CharCoder', @@ -89,18 +89,16 @@ def _to_field_coder(cls, coder_info_descriptor_proto): row_type = cls._to_row_type(schema_proto) batch_format = coder_info_descriptor_proto.arrow_type.BatchFormat.Name( coder_info_descriptor_proto.arrow_type.batch_format) - if batch_format == "ARROW": - from pyflink.fn_execution.utils.arrow_utils import to_arrow_schema - schema = to_arrow_schema(row_type) - else: - schema = cls._to_arrow_schema(row_type) + schema = create_arrow_schema(row_type.field_names(), row_type.field_types(), + allow_nested=batch_format == "ARROW") return ArrowCoder(schema, row_type, timezone, batch_format) elif coder_info_descriptor_proto.HasField('over_window_arrow_type'): timezone = pytz.timezone(os.environ['TABLE_LOCAL_TIME_ZONE']) schema_proto = coder_info_descriptor_proto.over_window_arrow_type.schema row_type = cls._to_row_type(schema_proto) return OverWindowArrowCoder( - cls._to_arrow_schema(row_type), row_type, timezone) + create_arrow_schema(row_type.field_names(), row_type.field_types()), + row_type, timezone) elif coder_info_descriptor_proto.HasField('raw_type'): type_info_proto = coder_info_descriptor_proto.raw_type.type_info field_coder = from_type_info_proto(type_info_proto) @@ -108,13 +106,6 @@ def _to_field_coder(cls, coder_info_descriptor_proto): else: raise ValueError("Unexpected coder type %s" % coder_info_descriptor_proto) - @classmethod - def _to_arrow_schema(cls, row_type): - import pyarrow as pa - - return pa.schema([pa.field(n, to_arrow_type(t), t._nullable) - for n, t in zip(row_type.field_names(), row_type.field_types())]) - @classmethod def _to_data_type(cls, field_type): from pyflink.fn_execution import flink_fn_execution_pb2 diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py index 431c21f00ed69..54016508148b4 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py @@ -41,7 +41,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x9a\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x12\x12\n\x08refIndex\x18\x04 \x01(\x05H\x00\x42\x07\n\x05input\"\x87\x02\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\x12\x14\n\x0cis_arrow_udf\x18\x06 \x01(\x08\x12G\n\x0boutput_type\x18\x07 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xdb\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x16\n\x0eoutput_indices\x18\x08 \x03(\x05\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xfb\x08\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\xce\x01\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x12\x61\n\x0c\x62\x61tch_format\x18\x02 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowType.BatchFormat\"$\n\x0b\x42\x61tchFormat\x12\n\n\x06PANDAS\x10\x00\x12\t\n\x05\x41RROW\x10\x01\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x9a\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x12\x12\n\x08refIndex\x18\x04 \x01(\x05H\x00\x42\x07\n\x05input\"\xbe\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\x12\x14\n\x0cis_arrow_udf\x18\x06 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xdb\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x16\n\x0eoutput_indices\x18\x08 \x03(\x05\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xfb\x08\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\xce\x01\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x12\x61\n\x0c\x62\x61tch_format\x18\x02 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowType.BatchFormat\"$\n\x0b\x42\x61tchFormat\x12\n\n\x06PANDAS\x10\x00\x12\t\n\x05\x41RROW\x10\x01\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -54,115 +54,115 @@ _globals['_INPUT']._serialized_start=107 _globals['_INPUT']._serialized_end=261 _globals['_USERDEFINEDFUNCTION']._serialized_start=264 - _globals['_USERDEFINEDFUNCTION']._serialized_end=527 - _globals['_ASYNCOPTIONS']._serialized_start=530 - _globals['_ASYNCOPTIONS']._serialized_end=674 - _globals['_USERDEFINEDFUNCTIONS']._serialized_start=677 - _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1152 - _globals['_OVERWINDOW']._serialized_start=1155 - _globals['_OVERWINDOW']._serialized_end=1504 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1296 - _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1504 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1507 - _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2286 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1772 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2286 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=2035 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=2119 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=2122 - _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2273 - _globals['_GROUPWINDOW']._serialized_start=2289 - _globals['_GROUPWINDOW']._serialized_end=2845 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2653 - _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2744 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2746 - _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2845 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2848 - _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3559 - _globals['_SCHEMA']._serialized_start=3562 - _globals['_SCHEMA']._serialized_end=5600 - _globals['_SCHEMA_MAPINFO']._serialized_start=3637 - _globals['_SCHEMA_MAPINFO']._serialized_end=3788 - _globals['_SCHEMA_TIMEINFO']._serialized_start=3790 - _globals['_SCHEMA_TIMEINFO']._serialized_end=3819 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3821 - _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3855 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3857 - _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3901 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3903 - _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3942 - _globals['_SCHEMA_DECIMALINFO']._serialized_start=3944 - _globals['_SCHEMA_DECIMALINFO']._serialized_end=3991 - _globals['_SCHEMA_BINARYINFO']._serialized_start=3993 - _globals['_SCHEMA_BINARYINFO']._serialized_end=4021 - _globals['_SCHEMA_VARBINARYINFO']._serialized_start=4023 - _globals['_SCHEMA_VARBINARYINFO']._serialized_end=4054 - _globals['_SCHEMA_CHARINFO']._serialized_start=4056 - _globals['_SCHEMA_CHARINFO']._serialized_end=4082 - _globals['_SCHEMA_VARCHARINFO']._serialized_start=4084 - _globals['_SCHEMA_VARCHARINFO']._serialized_end=4113 - _globals['_SCHEMA_FIELDTYPE']._serialized_start=4116 - _globals['_SCHEMA_FIELDTYPE']._serialized_end=5188 - _globals['_SCHEMA_FIELD']._serialized_start=5190 - _globals['_SCHEMA_FIELD']._serialized_end=5298 - _globals['_SCHEMA_TYPENAME']._serialized_start=5301 - _globals['_SCHEMA_TYPENAME']._serialized_end=5600 - _globals['_TYPEINFO']._serialized_start=5603 - _globals['_TYPEINFO']._serialized_end=6950 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=6097 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6236 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6239 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6423 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6332 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6423 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6425 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6505 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6507 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6537 - _globals['_TYPEINFO_TYPENAME']._serialized_start=6540 - _globals['_TYPEINFO_TYPENAME']._serialized_end=6937 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6953 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7930 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7448 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7754 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7757 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7930 - _globals['_STATEDESCRIPTOR']._serialized_start=7933 - _globals['_STATEDESCRIPTOR']._serialized_end=9825 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=8065 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9825 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8536 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9634 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8714 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8802 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8804 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8879 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8882 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9490 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9492 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9590 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9592 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9634 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9636 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9704 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9706 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9780 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9782 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9825 - _globals['_CODERINFODESCRIPTOR']._serialized_start=9828 - _globals['_CODERINFODESCRIPTOR']._serialized_end=10975 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10421 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10495 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10497 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10564 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10567 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10773 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE_BATCHFORMAT']._serialized_start=10737 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE_BATCHFORMAT']._serialized_end=10773 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10775 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10854 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10856 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10928 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10930 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10962 + _globals['_USERDEFINEDFUNCTION']._serialized_end=454 + _globals['_ASYNCOPTIONS']._serialized_start=457 + _globals['_ASYNCOPTIONS']._serialized_end=601 + _globals['_USERDEFINEDFUNCTIONS']._serialized_start=604 + _globals['_USERDEFINEDFUNCTIONS']._serialized_end=1079 + _globals['_OVERWINDOW']._serialized_start=1082 + _globals['_OVERWINDOW']._serialized_end=1431 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_start=1223 + _globals['_OVERWINDOW_WINDOWTYPE']._serialized_end=1431 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_start=1434 + _globals['_USERDEFINEDAGGREGATEFUNCTION']._serialized_end=2213 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_start=1699 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC']._serialized_end=2213 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_start=1962 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_LISTVIEW']._serialized_end=2046 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_start=2049 + _globals['_USERDEFINEDAGGREGATEFUNCTION_DATAVIEWSPEC_MAPVIEW']._serialized_end=2200 + _globals['_GROUPWINDOW']._serialized_start=2216 + _globals['_GROUPWINDOW']._serialized_end=2772 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_start=2580 + _globals['_GROUPWINDOW_WINDOWTYPE']._serialized_end=2671 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_start=2673 + _globals['_GROUPWINDOW_WINDOWPROPERTY']._serialized_end=2772 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2775 + _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3486 + _globals['_SCHEMA']._serialized_start=3489 + _globals['_SCHEMA']._serialized_end=5527 + _globals['_SCHEMA_MAPINFO']._serialized_start=3564 + _globals['_SCHEMA_MAPINFO']._serialized_end=3715 + _globals['_SCHEMA_TIMEINFO']._serialized_start=3717 + _globals['_SCHEMA_TIMEINFO']._serialized_end=3746 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_start=3748 + _globals['_SCHEMA_TIMESTAMPINFO']._serialized_end=3782 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_start=3784 + _globals['_SCHEMA_LOCALZONEDTIMESTAMPINFO']._serialized_end=3828 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_start=3830 + _globals['_SCHEMA_ZONEDTIMESTAMPINFO']._serialized_end=3869 + _globals['_SCHEMA_DECIMALINFO']._serialized_start=3871 + _globals['_SCHEMA_DECIMALINFO']._serialized_end=3918 + _globals['_SCHEMA_BINARYINFO']._serialized_start=3920 + _globals['_SCHEMA_BINARYINFO']._serialized_end=3948 + _globals['_SCHEMA_VARBINARYINFO']._serialized_start=3950 + _globals['_SCHEMA_VARBINARYINFO']._serialized_end=3981 + _globals['_SCHEMA_CHARINFO']._serialized_start=3983 + _globals['_SCHEMA_CHARINFO']._serialized_end=4009 + _globals['_SCHEMA_VARCHARINFO']._serialized_start=4011 + _globals['_SCHEMA_VARCHARINFO']._serialized_end=4040 + _globals['_SCHEMA_FIELDTYPE']._serialized_start=4043 + _globals['_SCHEMA_FIELDTYPE']._serialized_end=5115 + _globals['_SCHEMA_FIELD']._serialized_start=5117 + _globals['_SCHEMA_FIELD']._serialized_end=5225 + _globals['_SCHEMA_TYPENAME']._serialized_start=5228 + _globals['_SCHEMA_TYPENAME']._serialized_end=5527 + _globals['_TYPEINFO']._serialized_start=5530 + _globals['_TYPEINFO']._serialized_end=6877 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=6024 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6163 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6166 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6350 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6259 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6350 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6352 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6432 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6434 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6464 + _globals['_TYPEINFO_TYPENAME']._serialized_start=6467 + _globals['_TYPEINFO_TYPENAME']._serialized_end=6864 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6880 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7857 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7375 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7681 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7684 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7857 + _globals['_STATEDESCRIPTOR']._serialized_start=7860 + _globals['_STATEDESCRIPTOR']._serialized_end=9752 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7992 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9752 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8463 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9561 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8641 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8729 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8731 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8806 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8809 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9417 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9419 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9517 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9519 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9561 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9563 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9631 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9633 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9707 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9709 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9752 + _globals['_CODERINFODESCRIPTOR']._serialized_start=9755 + _globals['_CODERINFODESCRIPTOR']._serialized_end=10902 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10348 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10422 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10424 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10491 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10494 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10700 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE_BATCHFORMAT']._serialized_start=10664 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE_BATCHFORMAT']._serialized_end=10700 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10702 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10781 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10783 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10855 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10857 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10889 # @@protoc_insertion_point(module_scope) diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi index 7882fa0c96ed7..520077f395b35 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi @@ -45,22 +45,20 @@ class Input(_message.Message): def __init__(self, udf: _Optional[_Union[UserDefinedFunction, _Mapping]] = ..., inputOffset: _Optional[int] = ..., inputConstant: _Optional[bytes] = ..., refIndex: _Optional[int] = ...) -> None: ... class UserDefinedFunction(_message.Message): - __slots__ = ("payload", "inputs", "window_index", "takes_row_as_input", "is_pandas_udf", "is_arrow_udf", "output_type") + __slots__ = ("payload", "inputs", "window_index", "takes_row_as_input", "is_pandas_udf", "is_arrow_udf") PAYLOAD_FIELD_NUMBER: _ClassVar[int] INPUTS_FIELD_NUMBER: _ClassVar[int] WINDOW_INDEX_FIELD_NUMBER: _ClassVar[int] TAKES_ROW_AS_INPUT_FIELD_NUMBER: _ClassVar[int] IS_PANDAS_UDF_FIELD_NUMBER: _ClassVar[int] IS_ARROW_UDF_FIELD_NUMBER: _ClassVar[int] - OUTPUT_TYPE_FIELD_NUMBER: _ClassVar[int] payload: bytes inputs: _containers.RepeatedCompositeFieldContainer[Input] window_index: int takes_row_as_input: bool is_pandas_udf: bool is_arrow_udf: bool - output_type: Schema.FieldType - def __init__(self, payload: _Optional[bytes] = ..., inputs: _Optional[_Iterable[_Union[Input, _Mapping]]] = ..., window_index: _Optional[int] = ..., takes_row_as_input: bool = ..., is_pandas_udf: bool = ..., is_arrow_udf: bool = ..., output_type: _Optional[_Union[Schema.FieldType, _Mapping]] = ...) -> None: ... + def __init__(self, payload: _Optional[bytes] = ..., inputs: _Optional[_Iterable[_Union[Input, _Mapping]]] = ..., window_index: _Optional[int] = ..., takes_row_as_input: bool = ..., is_pandas_udf: bool = ..., is_arrow_udf: bool = ...) -> None: ... class AsyncOptions(_message.Message): __slots__ = ("max_concurrent_operations", "timeout_ms", "retry_enabled", "retry_max_attempts", "retry_delay_ms") diff --git a/flink-python/pyflink/fn_execution/table/operations.py b/flink-python/pyflink/fn_execution/table/operations.py index 49d71203216ac..e2d5786011abe 100644 --- a/flink-python/pyflink/fn_execution/table/operations.py +++ b/flink-python/pyflink/fn_execution/table/operations.py @@ -157,8 +157,8 @@ def generate_func(self, serialized_fn): func_strs.append(func_str) if is_arrow: - from pyflink.fn_execution.utils.arrow_utils import create_arrow_batch - variable_dict['create_arrow_batch'] = create_arrow_batch + from pyflink.fn_execution.utils.arrow_utils import create_record_batch + variable_dict['create_record_batch'] = create_record_batch output_indices = list(serialized_fn.output_indices) # Result references require sequential evaluation. A non-empty output_indices does too: @@ -170,7 +170,8 @@ def generate_func(self, serialized_fn): # Keep original lambda-based approach for backward compatibility scalar_functions = ','.join(func_strs) if is_arrow: - func_str = f'lambda value: create_arrow_batch([{scalar_functions}], value.num_rows)' + func_str = (f'lambda value: create_record_batch(' + f'[{scalar_functions}], value.num_rows)') elif self._one_result_optimization: func_str = 'lambda value: %s' % scalar_functions else: @@ -196,7 +197,7 @@ def generate_func(self, serialized_fn): code_lines.append(' results[%d] = %s' % (i, fn)) if is_arrow: outputs = ','.join('results[%d]' % i for i in output_indices) - code_lines.append(f' return create_arrow_batch([{outputs}], value.num_rows)') + code_lines.append(f' return create_record_batch([{outputs}], value.num_rows)') elif self._one_result_optimization: code_lines.append(' return results[%d]' % output_indices[0]) else: diff --git a/flink-python/pyflink/fn_execution/tests/test_arrow_udf.py b/flink-python/pyflink/fn_execution/tests/test_arrow_udf.py deleted file mode 100644 index f635cefa0c428..0000000000000 --- a/flink-python/pyflink/fn_execution/tests/test_arrow_udf.py +++ /dev/null @@ -1,119 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -"""Behavior of Arrow scalar operations without a Flink cluster.""" - -import pickle -import unittest - -import cloudpickle -import pyarrow as pa -import pyarrow.compute as pc - -from pyflink.fn_execution import flink_fn_execution_pb2 as proto -from pyflink.fn_execution.table.operations import ScalarFunctionOperation -from pyflink.table.udf import DelegatingScalarFunction, ScalarFunction - - -class Uppercase(ScalarFunction): - def eval(self, values): - return pc.utf8_upper(values) - - -class ArrowScalarOperationTests(unittest.TestCase): - def operation(self, func, inputs): - function = proto.UserDefinedFunction( - payload=cloudpickle.dumps(DelegatingScalarFunction(func)), - is_arrow_udf=True, inputs=inputs) - operation = ScalarFunctionOperation(proto.UserDefinedFunctions(udfs=[function])) - operation.open() - self.addCleanup(operation.close) - return operation - - def test_literals_columns_and_chunked_results(self): - def add(offset, left, right): - if not isinstance(offset, int) or not isinstance(left, pa.Array) \ - or not isinstance(right, pa.Array): - raise TypeError("Expected a scalar literal followed by two Arrow arrays.") - values = pc.add(pc.add(left, right), offset) - return pa.chunked_array([values.slice(0, 1), values.slice(1)]) - - operation = self.operation(add, [ - proto.Input(inputConstant=b"\x00" + pickle.dumps(10)), - proto.Input(inputOffset=0), proto.Input(inputOffset=1)]) - result = operation.process_element(pa.record_batch( - [pa.array([1, None, 3]), pa.array([4, 5, 6])], names=["a", "b"])) - self.assertEqual(result.column(0).to_pylist(), [15, None, 19]) - - def test_invalid_scalar_results(self): - batch = pa.record_batch([pa.array([1, 2, 3])], names=["value"]) - for result, error, message in ( - ([1, 2, 3], TypeError, "Array or pyarrow.ChunkedArray"), - (None, TypeError, "NoneType"), - (pa.scalar(1), TypeError, "Scalar"), - (batch, TypeError, "RecordBatch"), - (pa.Table.from_batches([batch]), TypeError, "Table"), - (pa.array([1]), ValueError, "returned 1 rows, expected 3"), - (pa.chunked_array([[1], [2]]), ValueError, "returned 2 rows, expected 3"), - ): - with self.subTest(result=result): - operation = self.operation(lambda values: result, [proto.Input(inputOffset=0)]) - with self.assertRaisesRegex(error, message): - operation.process_element(batch) - - def test_invalid_intermediate_result_is_not_consumed(self): - inner = proto.UserDefinedFunction( - payload=cloudpickle.dumps(DelegatingScalarFunction(lambda values: values.slice(0, 1))), - is_arrow_udf=True, inputs=[proto.Input(inputOffset=0)]) - # The outer result has the correct batch length, but must not hide the invalid inner result. - operation = self.operation(lambda values: pa.array([1, 2, 3]), [proto.Input(udf=inner)]) - with self.assertRaisesRegex(ValueError, "returned 1 rows, expected 3"): - operation.process_element(pa.record_batch([pa.array([1, 2, 3])], names=["value"])) - - def test_arrow_scalar_operation(self): - function = proto.UserDefinedFunction( - payload=cloudpickle.dumps(Uppercase()), is_arrow_udf=True, - inputs=[proto.Input(inputOffset=0)]) - operation = ScalarFunctionOperation( - proto.UserDefinedFunctions(udfs=[function]), one_arg_optimization=True) - operation.open() - self.addCleanup(operation.close) - result = operation.process_element(pa.record_batch( - [pa.array(["alice", None, "Bob"])], names=["name"])) - self.assertIsInstance(result, pa.RecordBatch) - self.assertEqual(result.column(0).to_pylist(), ["ALICE", None, "BOB"]) - - def test_intermediate_schema_is_enforced(self): - for values, error, message in ( - (pa.array(["wrong", "type"]), TypeError, "expected int64"), - (pa.chunked_array([[1], [None]], type=pa.int64()), ValueError, "not nullable"), - ): - with self.subTest(values=values): - inner = proto.UserDefinedFunction( - payload=cloudpickle.dumps(DelegatingScalarFunction(lambda column: values)), - is_arrow_udf=True, inputs=[proto.Input(inputOffset=0)], - output_type=proto.Schema.FieldType(type_name=proto.Schema.BIGINT, - nullable=False)) - operation = self.operation(lambda column: pa.array([1, 2]), - [proto.Input(udf=inner)]) - with self.assertRaisesRegex(error, message): - operation.process_element(pa.record_batch([pa.array([1, 2])], names=["a"])) - - -if __name__ == "__main__": - unittest.main() diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index f1bdc96ae1b32..312bdc2fc9073 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -36,6 +36,26 @@ class ArrowSchemaTests(unittest.TestCase): + def test_nested_types_require_arrow_mode(self): + from pyflink.table import DataTypes + from pyflink.table.types import create_arrow_schema + + for data_type, error, expected in ( + (DataTypes.ARRAY(DataTypes.ROW([DataTypes.FIELD("v", DataTypes.INT())])), ValueError, + pa.list_(pa.field("item", pa.struct([pa.field("v", pa.int32())])))), + (DataTypes.ARRAY(DataTypes.TIMESTAMP_LTZ(3)), ValueError, pa.list_(pa.timestamp('ms'))), + (DataTypes.ROW([DataTypes.FIELD("r", DataTypes.ROW([ + DataTypes.FIELD("v", DataTypes.INT())]))]), TypeError, + pa.struct([pa.field("r", pa.struct([pa.field("v", pa.int32())]))])), + (DataTypes.ROW([DataTypes.FIELD("t", DataTypes.TIMESTAMP_LTZ(3))]), TypeError, + pa.struct([pa.field("t", pa.timestamp('ms'))])), + ): + with self.subTest(data_type=data_type): + with self.assertRaises(error): + create_arrow_schema(["value"], [data_type]) + schema = create_arrow_schema(["value"], [data_type], allow_nested=True) + self.assertEqual(schema.field("value").type, expected) + def test_pandas_collection_schema_and_round_trip(self): import pandas as pd from pyflink.table import DataTypes @@ -73,13 +93,6 @@ def test_pandas_collection_schema_and_round_trip(self): restored = arrow_to_pandas(pytz.UTC, types, [decoded]) self.assertEqual(pandas_to_arrow(schema, pytz.UTC, types, restored).to_pylist(), expected) - -class ArrowCodersTests(unittest.TestCase): - from pyflink.fn_execution import coder_impl_slow as implementation - - def arrow_coder(self, schema, row_type): - return self.implementation.ArrowCoderImpl(schema, row_type, pytz.UTC, "ARROW") - def test_arrow_descriptor_preserves_pandas_default(self): from pyflink.fn_execution import flink_fn_execution_pb2 as proto from pyflink.fn_execution.coders import LengthPrefixBaseCoder @@ -100,6 +113,13 @@ def test_arrow_descriptor_preserves_pandas_default(self): batch = pa.record_batch([pa.array(["a", None])], names=["name"]) self.assertEqual(arrow_coder.decode(arrow_coder.encode(batch)), batch) + +class ArrowCodersTests(unittest.TestCase): + from pyflink.fn_execution import coder_impl_slow as implementation + + def arrow_coder(self, schema, row_type): + return self.implementation.ArrowCoderImpl(schema, row_type, pytz.UTC, "ARROW") + def test_arrow_nested_nullability(self): from pyflink.table import DataTypes from pyflink.table.types import to_arrow_type @@ -119,7 +139,7 @@ def test_arrow_nested_nullability(self): def test_struct_map_and_temporal_results(self): import datetime from pyflink.table import DataTypes - from pyflink.fn_execution.utils.arrow_utils import to_arrow_schema + from pyflink.table.types import create_arrow_schema row_type = DataTypes.ROW([ DataTypes.FIELD("record", DataTypes.ROW([ @@ -129,7 +149,8 @@ def test_struct_map_and_temporal_results(self): DataTypes.STRING().not_null(), DataTypes.INT().not_null())), DataTypes.FIELD("amount", DataTypes.DECIMAL(6, 2)), DataTypes.FIELD("time", DataTypes.TIMESTAMP(3))]) - schema = to_arrow_schema(row_type) + schema = create_arrow_schema(row_type.field_names(), row_type.field_types(), + allow_nested=True) coder = self.arrow_coder(schema, row_type) rows = [ {"record": {"inner": {"value": 7}}, "lookup": [("a", 1)], diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py index e8ef5c55a2fed..399eeaebeaaed 100644 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -19,25 +19,6 @@ """Arrow-native scalar UDF result contracts shared by the Python and compiled coders.""" -def to_arrow_schema(row_type): - import pyarrow as pa - from pyflink.table.types import ArrayType, MapType, RowType, to_arrow_type - - def field(name, data_type): - if isinstance(data_type, RowType): - arrow_type = pa.struct([field(f.name, f.data_type) for f in data_type.fields]) - elif isinstance(data_type, ArrayType): - arrow_type = pa.list_(field("element", data_type.element_type)) - elif isinstance(data_type, MapType): - arrow_type = pa.map_(field("key", data_type.key_type).with_nullable(False), - field("value", data_type.value_type)) - else: - arrow_type = to_arrow_type(data_type) - return pa.field(name, arrow_type, nullable=data_type._nullable) - - return pa.schema([field(f.name, f.data_type) for f in row_type.fields]) - - def validate_arrow_batch(batch, schema, field_types): import pyarrow as pa @@ -89,7 +70,7 @@ def wrong_type(): wrong_type() -def check_arrow_udf_result(func, *args, result_type=None, arrow_type=None): +def check_arrow_udf_result(func, *args): import pyarrow as pa result = func(*args) @@ -102,15 +83,10 @@ def check_arrow_udf_result(func, *args, result_type=None, arrow_type=None): if isinstance(arg, (pa.Array, pa.ChunkedArray)) and len(result) != len(arg): raise ValueError( f"Arrow UDF '{name}' returned {len(result)} rows, expected {len(arg)}.") - if result_type is not None: - chunks = result.chunks if isinstance(result, pa.ChunkedArray) else [result] - # An empty ChunkedArray still has an element type that must match the declaration. - for chunk in chunks or [pa.array([], type=result.type)]: - _validate_array(chunk, arrow_type, result_type, name) return result -def create_arrow_batch(results, row_count): +def create_record_batch(results, row_count): import pyarrow as pa columns = [] diff --git a/flink-python/pyflink/fn_execution/utils/operation_utils.py b/flink-python/pyflink/fn_execution/utils/operation_utils.py index e47827d78f397..544d2997513d0 100644 --- a/flink-python/pyflink/fn_execution/utils/operation_utils.py +++ b/flink-python/pyflink/fn_execution/utils/operation_utils.py @@ -161,17 +161,8 @@ def _extract_input(args) -> Tuple[str, Dict, List]: else: variable_dict[func_name] = user_defined_func.eval if user_defined_function_proto.is_arrow_udf: - from pyflink.fn_execution.coders import LengthPrefixBaseCoder - from pyflink.fn_execution.utils.arrow_utils import check_arrow_udf_result, to_arrow_schema - from pyflink.table.types import RowField, RowType - result_type = None - arrow_type = None - if user_defined_function_proto.HasField('output_type'): - result_type = LengthPrefixBaseCoder._to_data_type( - user_defined_function_proto.output_type) - arrow_type = to_arrow_schema(RowType([RowField('result', result_type)]))[0].type - variable_dict[func_name] = partial(check_arrow_udf_result, variable_dict[func_name], - result_type=result_type, arrow_type=arrow_type) + from pyflink.fn_execution.utils.arrow_utils import check_arrow_udf_result + variable_dict[func_name] = partial(check_arrow_udf_result, variable_dict[func_name]) user_defined_funcs.append(user_defined_func) func_args, input_variable_dict, input_funcs = _extract_input(user_defined_function_proto.inputs) diff --git a/flink-python/pyflink/proto/flink-fn-execution.proto b/flink-python/pyflink/proto/flink-fn-execution.proto index e7abbed1eae9d..a9300ba494956 100644 --- a/flink-python/pyflink/proto/flink-fn-execution.proto +++ b/flink-python/pyflink/proto/flink-fn-execution.proto @@ -67,7 +67,6 @@ message UserDefinedFunction { // Whether the scalar UDF consumes and returns Arrow arrays directly bool is_arrow_udf = 6; - Schema.FieldType output_type = 7; } // Async execution configuration for async functions diff --git a/flink-python/pyflink/table/tests/test_udf.py b/flink-python/pyflink/table/tests/test_udf.py index eb3e1447c947a..a890b35b68f23 100644 --- a/flink-python/pyflink/table/tests/test_udf.py +++ b/flink-python/pyflink/table/tests/test_udf.py @@ -17,15 +17,21 @@ ################################################################################ import datetime import os +import pickle import unittest import uuid +import cloudpickle +import pyarrow as pa +import pyarrow.compute as pc import pytz from pyflink.common import Row +from pyflink.fn_execution import flink_fn_execution_pb2 as proto +from pyflink.fn_execution.table.operations import ScalarFunctionOperation from pyflink.table import DataTypes, expressions as expr from pyflink.table.expressions import call -from pyflink.table.udf import ScalarFunction, udf, FunctionContext +from pyflink.table.udf import DelegatingScalarFunction, ScalarFunction, udf, FunctionContext from pyflink.testing import source_sink_utils from pyflink.testing.test_case_utils import PyFlinkStreamTableTestCase, \ PyFlinkBatchTableTestCase @@ -35,6 +41,74 @@ def generate_random_table_name(): return "Table{0}".format(str(uuid.uuid1()).replace("-", "_")) +class ArrowScalarOperationTests(unittest.TestCase): + def operation(self, func, inputs): + function = proto.UserDefinedFunction( + payload=cloudpickle.dumps(DelegatingScalarFunction(func)), + is_arrow_udf=True, inputs=inputs) + operation = ScalarFunctionOperation(proto.UserDefinedFunctions(udfs=[function])) + operation.open() + self.addCleanup(operation.close) + return operation + + def test_literals_columns_and_chunked_results(self): + def add(offset, left, right): + if not isinstance(offset, int) or not isinstance(left, pa.Array) \ + or not isinstance(right, pa.Array): + raise TypeError("Expected a scalar literal followed by two Arrow arrays.") + values = pc.add(pc.add(left, right), offset) + return pa.chunked_array([values.slice(0, 1), values.slice(1)]) + + operation = self.operation(add, [ + proto.Input(inputConstant=b"\x00" + pickle.dumps(10)), + proto.Input(inputOffset=0), proto.Input(inputOffset=1)]) + result = operation.process_element(pa.record_batch( + [pa.array([1, None, 3]), pa.array([4, 5, 6])], names=["a", "b"])) + self.assertEqual(result.column(0).to_pylist(), [15, None, 19]) + + def test_invalid_scalar_results(self): + batch = pa.record_batch([pa.array([1, 2, 3])], names=["value"]) + for result, error, message in ( + ([1, 2, 3], TypeError, "Array or pyarrow.ChunkedArray"), + (None, TypeError, "NoneType"), + (pa.scalar(1), TypeError, "Scalar"), + (batch, TypeError, "RecordBatch"), + (pa.Table.from_batches([batch]), TypeError, "Table"), + (pa.array([1]), ValueError, "returned 1 rows, expected 3"), + (pa.chunked_array([[1], [2]]), ValueError, "returned 2 rows, expected 3"), + ): + with self.subTest(result=result): + operation = self.operation(lambda values: result, [proto.Input(inputOffset=0)]) + with self.assertRaisesRegex(error, message): + operation.process_element(batch) + + def test_invalid_intermediate_result_is_not_consumed(self): + inner = proto.UserDefinedFunction( + payload=cloudpickle.dumps(DelegatingScalarFunction(lambda values: values.slice(0, 1))), + is_arrow_udf=True, inputs=[proto.Input(inputOffset=0)]) + # The outer result has the correct batch length, but must not hide the invalid inner result. + operation = self.operation(lambda values: pa.array([1, 2, 3]), [proto.Input(udf=inner)]) + with self.assertRaisesRegex(ValueError, "returned 1 rows, expected 3"): + operation.process_element(pa.record_batch([pa.array([1, 2, 3])], names=["value"])) + + def test_arrow_scalar_operation(self): + class Uppercase(ScalarFunction): + def eval(self, values): + return pc.utf8_upper(values) + + function = proto.UserDefinedFunction( + payload=cloudpickle.dumps(Uppercase()), is_arrow_udf=True, + inputs=[proto.Input(inputOffset=0)]) + operation = ScalarFunctionOperation( + proto.UserDefinedFunctions(udfs=[function]), one_arg_optimization=True) + operation.open() + self.addCleanup(operation.close) + result = operation.process_element(pa.record_batch( + [pa.array(["alice", None, "Bob"])], names=["name"])) + self.assertIsInstance(result, pa.RecordBatch) + self.assertEqual(result.column(0).to_pylist(), ["ALICE", None, "BOB"]) + + class UserDefinedFunctionTests(object): def test_scalar_function(self): diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index ad07937f38618..4a2d61363f449 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -2213,12 +2213,16 @@ def verify(obj): return verify -def create_arrow_schema(field_names: List[str], field_types: List[DataType]): +def create_arrow_schema(field_names: List[str], field_types: List[DataType], *, allow_nested=False): """ - Create an Arrow schema with the specified filed names and types. + Create an Arrow schema with the specified field names and types. + + By default, retain the nested-type restrictions of the pandas conversion path. + Arrow-native transport can set ``allow_nested`` to include nested rows and timestamps. """ import pyarrow as pa - fields = [pa.field(field_name, to_arrow_type(field_type), field_type._nullable) + fields = [pa.field(field_name, to_arrow_type(field_type, allow_nested=allow_nested), + field_type._nullable) for field_name, field_type in zip(field_names, field_types)] return pa.schema(fields) @@ -2296,7 +2300,7 @@ def from_arrow_type(arrow_type, nullable: bool = True) -> DataType: raise TypeError("Unsupported data type to convert from Arrow type: " + str(arrow_type)) -def to_arrow_type(data_type: DataType): +def to_arrow_type(data_type: DataType, *, allow_nested=False): """ Converts the specified Flink data type to pyarrow data type. """ @@ -2345,21 +2349,24 @@ def to_arrow_type(data_type: DataType): return pa.timestamp('ns') elif isinstance(data_type, MapType): return pa.map_( - pa.field("key", to_arrow_type(data_type.key_type), nullable=False), - pa.field("value", to_arrow_type(data_type.value_type), + pa.field("key", to_arrow_type(data_type.key_type, allow_nested=allow_nested), + nullable=False), + pa.field("value", to_arrow_type(data_type.value_type, allow_nested=allow_nested), nullable=data_type.value_type._nullable)) elif isinstance(data_type, ArrayType): - if type(data_type.element_type) in [LocalZonedTimestampType, RowType]: + if not allow_nested and type(data_type.element_type) in [LocalZonedTimestampType, RowType]: raise ValueError("%s is not supported to be used as the element type of ArrayType." % data_type.element_type) - return pa.list_(pa.field("item", to_arrow_type(data_type.element_type), - nullable=data_type.element_type._nullable)) + return pa.list_(pa.field( + "item", to_arrow_type(data_type.element_type, allow_nested=allow_nested), + nullable=data_type.element_type._nullable)) elif isinstance(data_type, RowType): for field in data_type: - if type(field.data_type) in [LocalZonedTimestampType, RowType]: + if not allow_nested and type(field.data_type) in [LocalZonedTimestampType, RowType]: raise TypeError("%s is not supported to be used as the field type of RowType" % field.data_type) - fields = [pa.field(field.name, to_arrow_type(field.data_type), field.data_type._nullable) + fields = [pa.field(field.name, to_arrow_type(field.data_type, allow_nested=allow_nested), + field.data_type._nullable) for field in data_type] return pa.struct(fields) elif isinstance(data_type, NullType): diff --git a/flink-python/pyflink/table/udf.py b/flink-python/pyflink/table/udf.py index 92dee5ccbf3a5..7035b6ac91dc0 100644 --- a/flink-python/pyflink/table/udf.py +++ b/flink-python/pyflink/table/udf.py @@ -842,20 +842,6 @@ def udf(f: Union[Callable, ScalarFunction, AsyncScalarFunction, Type] = None, ... return f"value_for_{key}" >>> async_lookup = udf(AsyncLookup(), result_type=DataTypes.STRING()) - Arrow vectorized scalar functions use ``func_type="arrow"`` and operate - directly on Arrow arrays, without converting to pandas:: - - >>> import pyarrow.compute as pc - >>> @udf(result_type=DataTypes.STRING(), func_type="arrow") - ... def uppercase(values): - ... return pc.utf8_upper(values) - - Arrow column arguments are ``pyarrow.Array`` values (``StructArray`` for - ROW columns); literal arguments remain Python scalars. Supply at least one - column-valued argument and return an ``Array`` or ``ChunkedArray`` with the - same row count and declared logical result type. Results are validated - without implicit element-type casts. Async Arrow functions are not supported. - :param f: lambda function, user-defined function, or async function. :param input_types: optional, the input data types. :param result_type: the result data type. diff --git a/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java b/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java index 86159036b55d6..6da72d75bfc9f 100644 --- a/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java +++ b/flink-python/src/main/java/org/apache/flink/python/util/ProtoUtils.java @@ -245,9 +245,6 @@ public static FlinkFnApi.UserDefinedFunction createUserDefinedFunctionProto( builder.setIsArrowUdf( pythonFunctionInfo.getPythonFunction().getPythonFunctionKind() == PythonFunctionKind.ARROW); - if (pythonFunctionInfo.getOutputType() != null) { - builder.setOutputType(toProtoType(pythonFunctionInfo.getOutputType())); - } return builder.build(); } diff --git a/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java b/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java index 78a10489b3123..9eb6029ee57aa 100644 --- a/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java +++ b/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java @@ -21,8 +21,18 @@ import org.apache.flink.api.common.state.StateTtlConfig; import org.apache.flink.fnexecution.v1.FlinkFnApi; import org.apache.flink.python.util.ProtoUtils; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.functions.python.PythonEnv; +import org.apache.flink.table.functions.python.InputRef; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.functions.python.PythonFunctionInput; +import org.apache.flink.table.functions.python.PythonFunctionKind; +import org.apache.flink.table.functions.python.PythonScalarFunction; +import org.apache.flink.table.types.logical.RowType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.time.Duration; @@ -33,6 +43,47 @@ * protobuf objects. */ class ProtoUtilsTest { + @Test + void testArrowCoderBatchFormat() { + final RowType rowType = RowType.of(DataTypes.INT().getLogicalType()); + assertThat(FlinkFnApi.CoderInfoDescriptor.ArrowType.getDefaultInstance().getBatchFormat()) + .isEqualTo(FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.PANDAS); + assertThat( + ProtoUtils.createArrowTypeCoderInfoDescriptorProto( + rowType, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false) + .getArrowType() + .getBatchFormat()) + .isEqualTo(FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.PANDAS); + assertThat( + ProtoUtils.createArrowTypeCoderInfoDescriptorProto( + rowType, + FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, + false, + FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.ARROW) + .getArrowType() + .getBatchFormat()) + .isEqualTo(FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.ARROW); + } + + @ParameterizedTest + @EnumSource(PythonFunctionKind.class) + void testScalarFunctionKind(PythonFunctionKind kind) { + final PythonScalarFunction function = + new PythonScalarFunction( + "identity", + new byte[0], + kind, + true, + false, + new PythonEnv(PythonEnv.ExecType.PROCESS)); + final FlinkFnApi.UserDefinedFunction proto = + ProtoUtils.createUserDefinedFunctionProto( + new PythonFunctionInfo( + function, new PythonFunctionInput[] {new InputRef(0)})); + assertThat(proto.getIsArrowUdf()).isEqualTo(kind == PythonFunctionKind.ARROW); + assertThat(proto.getIsPandasUdf()).isEqualTo(kind == PythonFunctionKind.PANDAS); + } + @Test void testParseStateTtlConfigFromProto() { FlinkFnApi.StateDescriptor.StateTTLConfig.CleanupStrategies cleanupStrategiesProto = diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java index c73ea4e7b83cc..5aa7dd76e65aa 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java @@ -20,18 +20,13 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.configuration.Configuration; -import org.apache.flink.fnexecution.v1.FlinkFnApi; import org.apache.flink.python.PythonFunctionRunner; -import org.apache.flink.python.util.ProtoUtils; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.connector.Projection; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.functions.python.PythonEnv; import org.apache.flink.table.functions.python.PythonFunctionInfo; -import org.apache.flink.table.functions.python.PythonFunctionKind; -import org.apache.flink.table.functions.python.PythonScalarFunction; import org.apache.flink.table.planner.codegen.CodeGeneratorContext; import org.apache.flink.table.planner.codegen.ProjectionCodeGenerator; import org.apache.flink.table.runtime.generated.GeneratedProjection; @@ -44,14 +39,10 @@ import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.RowKind; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; - import java.io.IOException; import java.util.Collection; import static org.apache.flink.table.runtime.util.StreamRecordUtils.row; -import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link ArrowPythonScalarFunctionOperator}. */ public class ArrowPythonScalarFunctionOperatorTest @@ -65,60 +56,6 @@ public class ArrowPythonScalarFunctionOperatorTest DataTypes.BIGINT().getLogicalType() }); - @ParameterizedTest - @EnumSource( - value = PythonFunctionKind.class, - names = {"PANDAS", "ARROW"}) - void testScalarBatchFormat(PythonFunctionKind kind) { - final RowType rowType = RowType.of(DataTypes.STRING().getLogicalType()); - final PythonFunctionInfo function = - new PythonFunctionInfo( - new PythonScalarFunction( - "identity", - new byte[0], - kind, - true, - false, - new PythonEnv(PythonEnv.ExecType.PROCESS)), - new Object[] {0}, - kind == PythonFunctionKind.ARROW - ? DataTypes.STRING().notNull().getLogicalType() - : null); - final ArrowPythonScalarFunctionOperator operator = - getTestOperator( - new Configuration(), - new PythonFunctionInfo[] {function}, - rowType, - rowType, - new int[] {0}, - new int[0]); - final FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat expected = - FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.valueOf(kind.name()); - assertThat(operator.createInputCoderInfoDescriptor(rowType).getArrowType().getBatchFormat()) - .isEqualTo(expected); - assertThat( - operator.createOutputCoderInfoDescriptor(rowType) - .getArrowType() - .getBatchFormat()) - .isEqualTo(expected); - assertThat(ProtoUtils.createUserDefinedFunctionProto(function).getIsArrowUdf()) - .isEqualTo(kind == PythonFunctionKind.ARROW); - final FlinkFnApi.UserDefinedFunction functionProto = - ProtoUtils.createUserDefinedFunctionProto(function); - assertThat(functionProto.hasOutputType()).isEqualTo(kind == PythonFunctionKind.ARROW); - if (kind == PythonFunctionKind.ARROW) { - assertThat(functionProto.getOutputType().getTypeName()) - .isEqualTo(FlinkFnApi.Schema.TypeName.VARCHAR); - assertThat(functionProto.getOutputType().getNullable()).isFalse(); - } - assertThat( - ProtoUtils.createArrowTypeCoderInfoDescriptorProto( - rowType, FlinkFnApi.CoderInfoDescriptor.Mode.SINGLE, false) - .getArrowType() - .getBatchFormat()) - .isEqualTo(FlinkFnApi.CoderInfoDescriptor.ArrowType.BatchFormat.PANDAS); - } - @Override public ArrowPythonScalarFunctionOperator getTestOperator( Configuration config, diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java index 8400a8300aca5..e72e6b403c864 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonFunctionInfo.java @@ -19,11 +19,8 @@ package org.apache.flink.table.functions.python; import org.apache.flink.annotation.Internal; -import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.util.Preconditions; -import javax.annotation.Nullable; - /** * PythonFunctionInfo contains the execution information of a Python function, such as: the actual * Python function, the input arguments, etc. @@ -42,23 +39,9 @@ public class PythonFunctionInfo implements PythonFunctionInput { /** The input arguments of this function. */ private PythonFunctionInput[] inputs; - @Nullable private final LogicalType outputType; - public PythonFunctionInfo(PythonFunction pythonFunction, PythonFunctionInput[] inputs) { - this(pythonFunction, inputs, null); - } - - public PythonFunctionInfo( - PythonFunction pythonFunction, PythonFunctionInput[] inputs, - @Nullable LogicalType outputType) { this.pythonFunction = Preconditions.checkNotNull(pythonFunction); this.inputs = Preconditions.checkNotNull(inputs); - this.outputType = outputType; - } - - @Nullable - public LogicalType getOutputType() { - return outputType; } public PythonFunction getPythonFunction() { diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java index 371708dd63718..795e50b152692 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/utils/CommonPythonUtil.java @@ -36,8 +36,6 @@ import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.functions.python.PythonFunctionInput; import org.apache.flink.table.functions.python.ResultRef; -import org.apache.flink.table.functions.python.PythonFunctionKind; -import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.functions.aggfunctions.AvgAggFunction; import org.apache.flink.table.planner.functions.aggfunctions.Count1AggFunction; import org.apache.flink.table.planner.functions.aggfunctions.CountAggFunction; @@ -510,13 +508,8 @@ private static PythonFunctionInfo createPythonFunctionInfo( inputNodes.put(operand, inputOffset); } } - final PythonFunction pythonFunction = (PythonFunction) functionDefinition; return new PythonFunctionInfo( - pythonFunction, - inputs.toArray(new PythonFunctionInput[0]), - pythonFunction.getPythonFunctionKind() == PythonFunctionKind.ARROW - ? FlinkTypeFactory.toLogicalType(pythonRexCall.getType()) - : null); + (PythonFunction) functionDefinition, inputs.toArray(new PythonFunctionInput[0])); } private static BuiltInPythonAggregateFunction getBuiltInPythonAggregateFunction( diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.java similarity index 97% rename from flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.java rename to flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.java index f2717a2ef3c11..14664121e57ea 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; /** Plans compositions of scalar UDFs with different Python argument representations. */ -class PythonArrowCalcTest extends TableTestBase { +class PythonCalcSplitFunctionKindRuleTest extends TableTestBase { @Test void testStreamingComposition() { diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.xml similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonArrowCalcTest.xml rename to flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.xml From 5cc4d7018cc645630f12c672e10ba30520928e4c Mon Sep 17 00:00:00 2001 From: auroflow Date: Tue, 15 Sep 2026 12:00:31 +0800 Subject: [PATCH 04/14] [FLINK-40472][python] Fix Arrow UDF map calls and reuse single-chunk results Generated-by: Codex (GPT-6) --- .../pyflink/fn_execution/utils/arrow_utils.py | 4 +- .../fn_execution/utils/operation_utils.py | 5 ++ flink-python/pyflink/table/tests/test_udf.py | 45 +++++++++++++- .../rules/logical/PythonMapMergeRule.java | 8 ++- .../PythonCalcSplitFunctionKindRuleTest.java | 59 ++++++++++++++++--- 5 files changed, 109 insertions(+), 12 deletions(-) diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py index 399eeaebeaaed..7124e1fcaccd7 100644 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -93,5 +93,7 @@ def create_record_batch(results, row_count): for result in results: if len(result) != row_count: raise ValueError(f"Arrow UDF returned {len(result)} rows, expected {row_count}.") - columns.append(result.combine_chunks() if isinstance(result, pa.ChunkedArray) else result) + if isinstance(result, pa.ChunkedArray): + result = result.chunk(0) if result.num_chunks == 1 else result.combine_chunks() + columns.append(result) return pa.RecordBatch.from_arrays(columns, names=[f"f{i}" for i in range(len(columns))]) diff --git a/flink-python/pyflink/fn_execution/utils/operation_utils.py b/flink-python/pyflink/fn_execution/utils/operation_utils.py index 544d2997513d0..4d59af84f9793 100644 --- a/flink-python/pyflink/fn_execution/utils/operation_utils.py +++ b/flink-python/pyflink/fn_execution/utils/operation_utils.py @@ -182,6 +182,11 @@ def _extract_input(args) -> Tuple[str, Dict, List]: # receives a previously computed intermediate result rather than # the original input row. We must use func_args instead of `value`. func_str = "%s(%s)" % (func_name, func_args) + elif user_defined_function_proto.is_arrow_udf: + import pyarrow as pa + + variable_dict['create_struct_array'] = pa.StructArray.from_arrays + func_str = f"{func_name}(create_struct_array(value.columns, fields=value.schema))" else: # directly use `value` as input argument # e.g. diff --git a/flink-python/pyflink/table/tests/test_udf.py b/flink-python/pyflink/table/tests/test_udf.py index a890b35b68f23..64dd552a67b99 100644 --- a/flink-python/pyflink/table/tests/test_udf.py +++ b/flink-python/pyflink/table/tests/test_udf.py @@ -42,10 +42,10 @@ def generate_random_table_name(): class ArrowScalarOperationTests(unittest.TestCase): - def operation(self, func, inputs): + def operation(self, func, inputs, takes_row_as_input=False): function = proto.UserDefinedFunction( payload=cloudpickle.dumps(DelegatingScalarFunction(func)), - is_arrow_udf=True, inputs=inputs) + is_arrow_udf=True, inputs=inputs, takes_row_as_input=takes_row_as_input) operation = ScalarFunctionOperation(proto.UserDefinedFunctions(udfs=[function])) operation.open() self.addCleanup(operation.close) @@ -66,6 +66,47 @@ def add(offset, left, right): [pa.array([1, None, 3]), pa.array([4, 5, 6])], names=["a", "b"])) self.assertEqual(result.column(0).to_pylist(), [15, None, 19]) + def test_single_chunk_result_preserves_buffers(self): + values = pa.array([0, 1, None, 3]).slice(1, 2) + operation = self.operation(lambda column: pa.chunked_array([column]), + [proto.Input(inputOffset=0)]) + result = operation.process_element(pa.record_batch([values], names=["value"])) + column = result.column(0) + self.assertEqual(column.to_pylist(), [1, None]) + self.assertEqual(column.offset, values.offset) + self.assertEqual([buffer.address for buffer in column.buffers()], + [buffer.address for buffer in values.buffers()]) + + def test_empty_chunked_results(self): + values = pa.array([], type=pa.int64()) + for chunks in ([], [values]): + with self.subTest(chunks=len(chunks)): + operation = self.operation( + lambda column: pa.chunked_array(chunks, type=pa.int64()), + [proto.Input(inputOffset=0)]) + result = operation.process_element(pa.record_batch([values], names=["value"])) + self.assertEqual(result.column(0), values) + + def test_whole_row_input(self): + batch = pa.record_batch([pa.array(["alice", None]), pa.array([1, 2])], + names=["name", "count"]) + + def increment(row): + return pa.StructArray.from_arrays( + [pc.struct_field(row, "name"), pc.add(pc.struct_field(row, "count"), 1)], + names=["name", "count"]) + + inputs = [proto.Input(inputOffset=0), proto.Input(inputOffset=1)] + identity = proto.UserDefinedFunction( + payload=cloudpickle.dumps(DelegatingScalarFunction(lambda row: row)), + is_arrow_udf=True, inputs=inputs, takes_row_as_input=True) + for arguments in (inputs, [proto.Input(udf=identity)]): + with self.subTest(nested=arguments[0].HasField("udf")): + operation = self.operation(increment, arguments, takes_row_as_input=True) + result = operation.process_element(batch) + self.assertEqual(result.column(0).to_pylist(), + [{"name": "alice", "count": 2}, {"name": None, "count": 3}]) + def test_invalid_scalar_results(self): batch = pa.record_batch([pa.array([1, 2, 3])], names=["value"]) for result, error, message in ( diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java index 57d2f2cea6881..c93c643301e6b 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/PythonMapMergeRule.java @@ -32,6 +32,7 @@ import org.apache.calcite.rex.RexProgram; import org.apache.calcite.rex.RexProgramBuilder; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -87,8 +88,11 @@ public boolean matches(RelOptRuleCall call) { } // only Python functions with same Python function kind can be merged together. - if (PythonUtil.isPythonCall(topProjects.get(0), PythonFunctionKind.GENERAL) - ^ PythonUtil.isPythonCall(bottomProjects.get(0), PythonFunctionKind.GENERAL)) { + if (Arrays.stream(PythonFunctionKind.values()) + .noneMatch( + kind -> + PythonUtil.isPythonCall(topProjects.get(0), kind) + && PythonUtil.isPythonCall(bottomProjects.get(0), kind))) { return false; } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.java index 14664121e57ea..09cc9d10dd85a 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitFunctionKindRuleTest.java @@ -20,15 +20,22 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; import org.apache.flink.table.functions.python.PythonEnv; import org.apache.flink.table.functions.python.PythonFunctionKind; import org.apache.flink.table.functions.python.PythonScalarFunction; +import org.apache.flink.table.planner.utils.JavaScalaConversionUtil; import org.apache.flink.table.planner.utils.JavaTableTestUtil; import org.apache.flink.table.planner.utils.TableTestBase; import org.apache.flink.table.types.DataType; import org.junit.jupiter.api.Test; +import java.util.Arrays; + +import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.call; + /** Plans compositions of scalar UDFs with different Python argument representations. */ class PythonCalcSplitFunctionKindRuleTest extends TableTestBase { @@ -42,8 +49,51 @@ void testBatchComposition() { verifyComposition(javaBatchTestUtil()); } + @Test + void testStreamingMapComposition() { + verifyMapComposition(javaStreamTestUtil()); + } + + @Test + void testBatchMapComposition() { + verifyMapComposition(javaBatchTestUtil()); + } + private void verifyComposition(JavaTableTestUtil util) { util.addTableSource("T", Schema.newBuilder().column("a", DataTypes.INT()).build()); + registerFunctions(util, DataTypes.INT(), false); + util.verifyExecPlan( + "SELECT arrow_udf(a), pandas_udf(a), general_udf(a), " + + "arrow_udf(arrow_udf(a)), arrow_udf(pandas_udf(a)), " + + "pandas_udf(arrow_udf(a)), arrow_udf(general_udf(a)), " + + "general_udf(arrow_udf(a)) FROM T"); + } + + private void verifyMapComposition(JavaTableTestUtil util) { + util.addTableSource("T", Schema.newBuilder().column("a", DataTypes.INT()).build()); + registerFunctions(util, DataTypes.ROW(DataTypes.FIELD("a", DataTypes.INT())), true); + final Table result = + util.tableEnv() + .from("T") + .map(call("general_udf", $("a"))) + .map(call("arrow_udf", $("a"))) + .map(call("arrow_udf", $("a"))) + .map(call("pandas_udf", $("a"))) + .map(call("pandas_udf", $("a"))) + .map(call("arrow_udf", $("a"))) + .map(call("general_udf", $("a"))); + util.verifyRelPlanExpected( + result, + JavaScalaConversionUtil.toScala( + Arrays.asList( + "PythonCalc(select=[arrow_udf(arrow_udf(a)) AS f0])", + "PythonCalc(select=[pandas_udf(pandas_udf(a)) AS f0])", + "PythonCalc(select=[arrow_udf(a) AS f0])", + "PythonCalc(select=[general_udf(a) AS f0])"))); + } + + private void registerFunctions( + JavaTableTestUtil util, DataType resultType, boolean takesRowAsInput) { for (PythonFunctionKind kind : PythonFunctionKind.values()) { util.tableEnv() .createTemporarySystemFunction( @@ -52,16 +102,11 @@ private void verifyComposition(JavaTableTestUtil util) { kind.name(), new byte[0], new DataType[] {DataTypes.INT()}, - DataTypes.INT(), + resultType, kind, true, - false, + takesRowAsInput, new PythonEnv(PythonEnv.ExecType.PROCESS))); } - util.verifyExecPlan( - "SELECT arrow_udf(a), pandas_udf(a), general_udf(a), " - + "arrow_udf(arrow_udf(a)), arrow_udf(pandas_udf(a)), " - + "pandas_udf(arrow_udf(a)), arrow_udf(general_udf(a)), " - + "general_udf(arrow_udf(a)) FROM T"); } } From 6304ff986c1e6a5d414c785a2ebc125484426157 Mon Sep 17 00:00:00 2001 From: auroflow Date: Tue, 15 Sep 2026 15:57:24 +0800 Subject: [PATCH 05/14] [FLINK-40472][python] Fix Arrow UDF argument handling and validation Respect whole-row argument offsets, normalize chunked results before nested calls, skip unused pandas timezone conversion for native Arrow, and validate container nullability without copying payloads. Generated-by: Codex (GPT-6) --- flink-python/pyflink/fn_execution/coders.py | 5 +- .../pyflink/fn_execution/tests/test_coders.py | 107 ++++++++++++++++++ .../pyflink/fn_execution/utils/arrow_utils.py | 59 +++++++--- .../fn_execution/utils/operation_utils.py | 4 +- flink-python/pyflink/table/tests/test_udf.py | 34 ++++++ 5 files changed, 190 insertions(+), 19 deletions(-) diff --git a/flink-python/pyflink/fn_execution/coders.py b/flink-python/pyflink/fn_execution/coders.py index e2e7e2c36d611..d75afcc1ff9d6 100644 --- a/flink-python/pyflink/fn_execution/coders.py +++ b/flink-python/pyflink/fn_execution/coders.py @@ -84,11 +84,14 @@ def _to_field_coder(cls, coder_info_descriptor_proto): field_names = [f.name for f in schema_proto.fields] return RowCoder(field_coders, field_names) elif coder_info_descriptor_proto.HasField('arrow_type'): - timezone = pytz.timezone(os.environ['TABLE_LOCAL_TIME_ZONE']) schema_proto = coder_info_descriptor_proto.arrow_type.schema row_type = cls._to_row_type(schema_proto) batch_format = coder_info_descriptor_proto.arrow_type.BatchFormat.Name( coder_info_descriptor_proto.arrow_type.batch_format) + # Native Arrow does not use pandas timezone conversion. Some valid JVM zone IDs + # are not recognized by pytz, so resolve the timezone only for pandas batches. + timezone = (None if batch_format == "ARROW" + else pytz.timezone(os.environ['TABLE_LOCAL_TIME_ZONE'])) schema = create_arrow_schema(row_type.field_names(), row_type.field_types(), allow_nested=batch_format == "ARROW") return ArrowCoder(schema, row_type, timezone, batch_format) diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index 312bdc2fc9073..b17c9c3eb2a32 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -120,6 +120,42 @@ class ArrowCodersTests(unittest.TestCase): def arrow_coder(self, schema, row_type): return self.implementation.ArrowCoderImpl(schema, row_type, pytz.UTC, "ARROW") + @staticmethod + def with_parent_nulls(column, nulls): + validity = pa.array([not value for value in nulls]).buffers()[1] + if pa.types.is_struct(column.type): + buffers = [validity] + children = [column.field(index) for index in range(column.type.num_fields)] + else: + buffers = [validity, column.offsets.buffers()[1]] + children = [column.values] + return pa.Array.from_buffers(column.type, len(column), buffers, children=children) + + def test_native_arrow_timezone(self): + from pyflink.fn_execution import flink_fn_execution_pb2 as proto + from pyflink.fn_execution.coders import LengthPrefixBaseCoder + + schema_proto = proto.Schema(fields=[ + proto.Schema.Field(name="number", type=proto.Schema.FieldType( + type_name=proto.Schema.BIGINT, nullable=True)), + proto.Schema.Field(name="timestamp", type=proto.Schema.FieldType( + type_name=proto.Schema.TIMESTAMP, nullable=True, + timestamp_info=proto.Schema.TimestampInfo(precision=3))), + proto.Schema.Field(name="local_timestamp", type=proto.Schema.FieldType( + type_name=proto.Schema.LOCAL_ZONED_TIMESTAMP, nullable=True, + local_zoned_timestamp_info=proto.Schema.LocalZonedTimestampInfo(precision=3)))]) + descriptor = proto.CoderInfoDescriptor(arrow_type=proto.CoderInfoDescriptor.ArrowType( + schema=schema_proto, batch_format=proto.CoderInfoDescriptor.ArrowType.ARROW)) + batch = pa.record_batch([pa.array([1, None]), pa.array([0, None], type=pa.timestamp('ms')), + pa.array([0, None], type=pa.timestamp('ms'))], + names=["number", "timestamp", "local_timestamp"]) + for timezone in ("UTC", "GMT+08:00", "SystemV/PST8PDT"): + with self.subTest(timezone=timezone): + with mock.patch.dict(os.environ, {"TABLE_LOCAL_TIME_ZONE": timezone}), \ + mock.patch('pyflink.fn_execution.coders.coder_impl', self.implementation): + coder = LengthPrefixBaseCoder._to_field_coder(descriptor).get_impl() + self.assertEqual(coder.decode(coder.encode(batch)), batch) + def test_arrow_nested_nullability(self): from pyflink.table import DataTypes from pyflink.table.types import to_arrow_type @@ -190,6 +226,77 @@ def test_native_arrow_round_trip(self): with self.assertRaisesRegex(TypeError, "name.*string"): coder.encode(pa.record_batch([pa.array([1, 2])], names=["name"])) + def test_sliced_container_nullability(self): + from pyflink.table import DataTypes + from pyflink.table.types import create_arrow_schema, to_arrow_type + + item_type = DataTypes.ROW([DataTypes.FIELD("required", DataTypes.INT().not_null())]) + items = pa.array([{"required": None}, {"required": 1}, {"required": None}, + {"required": None}, None, {"required": None}], + type=to_arrow_type(item_type)) + nulls = [False, False, True, False, False, False] + offsets = pa.array(range(7), type=pa.int32()) + for data_type, column, first, last in ( + (DataTypes.ROW([DataTypes.FIELD("value", item_type)]), + pa.StructArray.from_arrays([items], names=["value"]), + {"value": {"required": 1}}, {"value": None}), + (DataTypes.ARRAY(item_type), pa.ListArray.from_arrays(offsets, items), + [{"required": 1}], [None]), + (DataTypes.MAP(DataTypes.STRING().not_null(), item_type), + pa.MapArray.from_arrays(offsets, pa.array(['k'] * 6), items), + [('k', {"required": 1})], [('k', None)]), + ): + with self.subTest(data_type=data_type): + column = self.with_parent_nulls(column, nulls) + row_type = DataTypes.ROW([DataTypes.FIELD("record", DataTypes.ROW([ + DataTypes.FIELD("container", data_type)]))]) + schema = create_arrow_schema(row_type.field_names(), row_type.field_types(), + allow_nested=True) + outer = self.with_parent_nulls( + pa.StructArray.from_arrays([column], names=["container"]), + [False, False, False, True, False, False]) + batch = pa.record_batch([outer], names=["record"]) + coder = self.arrow_coder(schema, row_type) + self.assertEqual(coder.decode(coder.encode(batch.slice(1, 4))).to_pylist(), [ + {"record": {"container": first}}, {"record": {"container": None}}, + {"record": None}, {"record": {"container": last}}]) + self.assertEqual(coder.decode(coder.encode(batch.slice(0, 0))).num_rows, 0) + with self.assertRaisesRegex(ValueError, "required.*not nullable"): + coder.encode(batch.slice(0, 1)) + + def test_nullable_container_validation_memory(self): + from pyflink.table import DataTypes + from pyflink.table.types import create_arrow_schema + + count = 512 + payload = pa.array([b'x' * 2048] * count) + nulls = [index == count // 2 for index in range(count)] + offsets = pa.array(range(count + 1), type=pa.int32()) + for data_type, column in ( + (DataTypes.ROW([DataTypes.FIELD("payload", DataTypes.BYTES())]), + pa.StructArray.from_arrays([payload], names=["payload"])), + (DataTypes.ARRAY(DataTypes.BYTES()), + pa.ListArray.from_arrays(offsets, payload)), + (DataTypes.MAP(DataTypes.STRING().not_null(), DataTypes.BYTES()), + pa.MapArray.from_arrays(offsets, pa.array(['k'] * count), payload)), + ): + with self.subTest(data_type=data_type): + column = self.with_parent_nulls(column, nulls) + row_type = DataTypes.ROW([DataTypes.FIELD("value", data_type)]) + schema = create_arrow_schema(["value"], [data_type], allow_nested=True) + batch = pa.record_batch([column], names=["value"]) + coder = self.arrow_coder(schema, row_type) + default_pool = pa.default_memory_pool() + pool = pa.proxy_memory_pool(default_pool) + try: + pa.set_memory_pool(pool) + encoded = coder.encode(batch) + finally: + pa.set_memory_pool(default_pool) + # Allow masks, indices and IPC metadata, but not a copy of the binary payload. + self.assertLess(pool.max_memory(), payload.nbytes // 4) + self.assertEqual(coder.decode(encoded).to_pylist(), batch.to_pylist()) + try: from pyflink.fn_execution import coder_impl_fast diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py index 7124e1fcaccd7..c5b78d8bcea36 100644 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -31,8 +31,9 @@ def validate_arrow_batch(batch, schema, field_types): return pa.RecordBatch.from_arrays(batch.columns, schema=schema) -def _validate_array(column, expected_type, data_type, path): +def _validate_array(column, expected_type, data_type, path, parent_validity=None): import pyarrow as pa + import pyarrow.compute as pc from pyflink.table.types import ArrayType, MapType, RowType def wrong_type(): @@ -40,36 +41,60 @@ def wrong_type(): f"Arrow result field '{path}' has type {column.type}, expected {expected_type}.") if not data_type._nullable and column.null_count: - raise ValueError(f"Arrow result field '{path}' is not nullable.") + if parent_validity is None or pc.any( + pc.and_(parent_validity, column.is_null())).as_py(): + raise ValueError(f"Arrow result field '{path}' is not nullable.") if isinstance(data_type, RowType): if not pa.types.is_struct(column.type) or column.type.num_fields != len(data_type.fields): wrong_type() - # Children hidden by a null parent are not logical values and may contain nulls. - visible = column.filter(column.is_valid()) if column.null_count else column + validity = _get_validity(column, parent_validity) for index, field in enumerate(data_type.fields): if column.type[index].name != field.name: wrong_type() - _validate_array(visible.field(index), expected_type[index].type, - field.data_type, f"{path}.{field.name}") + _validate_array(column.field(index), expected_type[index].type, + field.data_type, f"{path}.{field.name}", validity) elif isinstance(data_type, ArrayType): if not pa.types.is_list(column.type): wrong_type() - # flatten respects the slice offsets and excludes values under null lists. - _validate_array(column.flatten(), expected_type.value_type, - data_type.element_type, f"{path}[]") + start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() + _validate_array(column.values.slice(start, end - start), expected_type.value_type, + data_type.element_type, f"{path}[]", + _get_child_validity(column, parent_validity)) elif isinstance(data_type, MapType): if not pa.types.is_map(column.type): wrong_type() - visible = column.filter(column.is_valid()) if column.null_count else column - start, end = visible.offsets[0].as_py(), visible.offsets[-1].as_py() - _validate_array(visible.keys.slice(start, end - start), expected_type.key_type, - data_type.key_type.not_null(), f"{path}.key") - _validate_array(visible.items.slice(start, end - start), expected_type.item_type, - data_type.value_type, f"{path}.value") + validity = _get_child_validity(column, parent_validity) + start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() + _validate_array(column.keys.slice(start, end - start), expected_type.key_type, + data_type.key_type.not_null(), f"{path}.key", validity) + _validate_array(column.items.slice(start, end - start), expected_type.item_type, + data_type.value_type, f"{path}.value", validity) elif column.type != expected_type: wrong_type() +def _get_validity(column, parent_validity): + import pyarrow.compute as pc + + # Hidden child nulls are valid; propagate visibility instead of filtering the payload. + if column.null_count: + validity = column.is_valid() + return validity if parent_validity is None else pc.and_(parent_validity, validity) + return parent_validity + + +def _get_child_validity(column, parent_validity): + import pyarrow as pa + import pyarrow.compute as pc + + validity = _get_validity(column, parent_validity) + if validity is None: + return None + # A list view also handles map entries and keeps the original offsets and value buffers. + entries = pa.ListArray.from_arrays(column.offsets, column.values) + return pc.take(validity, pc.list_parent_indices(entries)) + + def check_arrow_udf_result(func, *args): import pyarrow as pa @@ -83,6 +108,8 @@ def check_arrow_udf_result(func, *args): if isinstance(arg, (pa.Array, pa.ChunkedArray)) and len(result) != len(arg): raise ValueError( f"Arrow UDF '{name}' returned {len(result)} rows, expected {len(arg)}.") + if isinstance(result, pa.ChunkedArray): + result = result.chunk(0) if result.num_chunks == 1 else result.combine_chunks() return result @@ -93,7 +120,5 @@ def create_record_batch(results, row_count): for result in results: if len(result) != row_count: raise ValueError(f"Arrow UDF returned {len(result)} rows, expected {row_count}.") - if isinstance(result, pa.ChunkedArray): - result = result.chunk(0) if result.num_chunks == 1 else result.combine_chunks() columns.append(result) return pa.RecordBatch.from_arrays(columns, names=[f"f{i}" for i in range(len(columns))]) diff --git a/flink-python/pyflink/fn_execution/utils/operation_utils.py b/flink-python/pyflink/fn_execution/utils/operation_utils.py index 4d59af84f9793..c19422af688e5 100644 --- a/flink-python/pyflink/fn_execution/utils/operation_utils.py +++ b/flink-python/pyflink/fn_execution/utils/operation_utils.py @@ -186,7 +186,9 @@ def _extract_input(args) -> Tuple[str, Dict, List]: import pyarrow as pa variable_dict['create_struct_array'] = pa.StructArray.from_arrays - func_str = f"{func_name}(create_struct_array(value.columns, fields=value.schema))" + offsets = [arg.inputOffset for arg in user_defined_function_proto.inputs] + func_str = (f"{func_name}(create_struct_array([{func_args}], " + f"fields=[value.schema[i] for i in {offsets}]))") else: # directly use `value` as input argument # e.g. diff --git a/flink-python/pyflink/table/tests/test_udf.py b/flink-python/pyflink/table/tests/test_udf.py index 64dd552a67b99..bb969cda5ece6 100644 --- a/flink-python/pyflink/table/tests/test_udf.py +++ b/flink-python/pyflink/table/tests/test_udf.py @@ -87,6 +87,28 @@ def test_empty_chunked_results(self): result = operation.process_element(pa.record_batch([values], names=["value"])) self.assertEqual(result.column(0), values) + def test_nested_chunked_results(self): + values = pa.StructArray.from_arrays([pa.array([0, 1, None, 3])], names=["value"]) + for chunks in (0, 1, 2): + with self.subTest(chunks=chunks): + column = values.slice(1, 0 if chunks == 0 else 3) + + def chunk_result(array): + parts = [] if chunks == 0 else [array] if chunks == 1 else [ + array.slice(0, 1), array.slice(1)] + return pa.chunked_array(parts, type=array.type) + + inner = proto.UserDefinedFunction( + payload=cloudpickle.dumps(DelegatingScalarFunction(chunk_result)), + is_arrow_udf=True, inputs=[proto.Input(inputOffset=0)]) + operation = self.operation(lambda row: row.field("value"), [proto.Input(udf=inner)]) + result = operation.process_element(pa.record_batch([column], names=["record"])) + self.assertEqual(result.column(0).to_pylist(), [] if chunks == 0 else [1, None, 3]) + if chunks == 1: + self.assertEqual(result.column(0).offset, column.field(0).offset) + self.assertEqual([buffer.address for buffer in result.column(0).buffers()], + [buffer.address for buffer in column.field(0).buffers()]) + def test_whole_row_input(self): batch = pa.record_batch([pa.array(["alice", None]), pa.array([1, 2])], names=["name", "count"]) @@ -107,6 +129,18 @@ def increment(row): self.assertEqual(result.column(0).to_pylist(), [{"name": "alice", "count": 2}, {"name": None, "count": 3}]) + def test_whole_row_argument_offsets(self): + batch = pa.record_batch([pa.array([0, 1, None, 3]), pa.array([0, 100, 200, 300])], + names=["a", "b"]).slice(1) + for offsets in ([0, 0, 1], [1, 0]): + with self.subTest(offsets=offsets): + operation = self.operation( + lambda row: row.field(1), + [proto.Input(inputOffset=offset) for offset in offsets], + takes_row_as_input=True) + result = operation.process_element(batch) + self.assertEqual(result.column(0).to_pylist(), [1, None, 3]) + def test_invalid_scalar_results(self): batch = pa.record_batch([pa.array([1, 2, 3])], names=["value"]) for result, error, message in ( From 78e94308437f976150df75d6ec64b95e12108f0d Mon Sep 17 00:00:00 2001 From: auroflow Date: Tue, 15 Sep 2026 19:40:26 +0800 Subject: [PATCH 06/14] [FLINK-40472][python] Compute Arrow nullability masks only when needed Defer and memoize ancestor visibility until a NOT NULL descendant contains physical nulls. Extend the existing coder round-trip and allocation test with Boolean lists. Generated-by: Codex (GPT-6) --- .../pyflink/fn_execution/tests/test_coders.py | 21 ++++++++++++------- .../pyflink/fn_execution/utils/arrow_utils.py | 15 +++++++------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index b17c9c3eb2a32..dc4280ea314de 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -272,16 +272,23 @@ def test_nullable_container_validation_memory(self): payload = pa.array([b'x' * 2048] * count) nulls = [index == count // 2 for index in range(count)] offsets = pa.array(range(count + 1), type=pa.int32()) - for data_type, column in ( + booleans = pa.array([False] * 65536) + boolean_lists = pa.ListArray.from_arrays([0, len(booleans), len(booleans)], booleans) + for data_type, column, parent_nulls, memory_limit in ( (DataTypes.ROW([DataTypes.FIELD("payload", DataTypes.BYTES())]), - pa.StructArray.from_arrays([payload], names=["payload"])), + pa.StructArray.from_arrays([payload], names=["payload"]), + nulls, payload.nbytes // 4), (DataTypes.ARRAY(DataTypes.BYTES()), - pa.ListArray.from_arrays(offsets, payload)), + pa.ListArray.from_arrays(offsets, payload), nulls, payload.nbytes // 4), (DataTypes.MAP(DataTypes.STRING().not_null(), DataTypes.BYTES()), - pa.MapArray.from_arrays(offsets, pa.array(['k'] * count), payload)), + pa.MapArray.from_arrays(offsets, pa.array(['k'] * count), payload), + nulls, payload.nbytes // 4), + (DataTypes.ARRAY(DataTypes.BOOLEAN()), boolean_lists, [False, True], 4096), + (DataTypes.ARRAY(DataTypes.BOOLEAN().not_null()), + boolean_lists, [False, True], 4096), ): with self.subTest(data_type=data_type): - column = self.with_parent_nulls(column, nulls) + column = self.with_parent_nulls(column, parent_nulls) row_type = DataTypes.ROW([DataTypes.FIELD("value", data_type)]) schema = create_arrow_schema(["value"], [data_type], allow_nested=True) batch = pa.record_batch([column], names=["value"]) @@ -293,8 +300,8 @@ def test_nullable_container_validation_memory(self): encoded = coder.encode(batch) finally: pa.set_memory_pool(default_pool) - # Allow masks, indices and IPC metadata, but not a copy of the binary payload. - self.assertLess(pool.max_memory(), payload.nbytes // 4) + # Allow IPC metadata, but not payload copies or element-sized temporary arrays. + self.assertLess(pool.max_memory(), memory_limit) self.assertEqual(coder.decode(encoded).to_pylist(), batch.to_pylist()) diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py index c5b78d8bcea36..86a6cf42e4d0a 100644 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -18,6 +18,8 @@ """Arrow-native scalar UDF result contracts shared by the Python and compiled coders.""" +from functools import cache + def validate_arrow_batch(batch, schema, field_types): import pyarrow as pa @@ -41,13 +43,13 @@ def wrong_type(): f"Arrow result field '{path}' has type {column.type}, expected {expected_type}.") if not data_type._nullable and column.null_count: - if parent_validity is None or pc.any( - pc.and_(parent_validity, column.is_null())).as_py(): + validity = parent_validity() if parent_validity is not None else None + if validity is None or pc.any(pc.and_(validity, column.is_null())).as_py(): raise ValueError(f"Arrow result field '{path}' is not nullable.") if isinstance(data_type, RowType): if not pa.types.is_struct(column.type) or column.type.num_fields != len(data_type.fields): wrong_type() - validity = _get_validity(column, parent_validity) + validity = cache(lambda: _get_validity(column, parent_validity)) for index, field in enumerate(data_type.fields): if column.type[index].name != field.name: wrong_type() @@ -59,11 +61,11 @@ def wrong_type(): start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() _validate_array(column.values.slice(start, end - start), expected_type.value_type, data_type.element_type, f"{path}[]", - _get_child_validity(column, parent_validity)) + cache(lambda: _get_child_validity(column, parent_validity))) elif isinstance(data_type, MapType): if not pa.types.is_map(column.type): wrong_type() - validity = _get_child_validity(column, parent_validity) + validity = cache(lambda: _get_child_validity(column, parent_validity)) start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() _validate_array(column.keys.slice(start, end - start), expected_type.key_type, data_type.key_type.not_null(), f"{path}.key", validity) @@ -76,7 +78,8 @@ def wrong_type(): def _get_validity(column, parent_validity): import pyarrow.compute as pc - # Hidden child nulls are valid; propagate visibility instead of filtering the payload. + # Resolve ancestor visibility only when a NOT NULL descendant contains physical nulls. + parent_validity = parent_validity() if parent_validity is not None else None if column.null_count: validity = column.is_valid() return validity if parent_validity is None else pc.and_(parent_validity, validity) From abff795c4190692a72fdba1bcd1b384f730a4ae6 Mon Sep 17 00:00:00 2001 From: auroflow Date: Tue, 15 Sep 2026 19:40:38 +0800 Subject: [PATCH 07/14] [FLINK-40472][python] Validate Arrow UDF results against schema fields Use expected Arrow fields for structure and nullability checks in both coder implementations, while retaining Flink types for pandas conversion. Generated-by: Codex (GPT-6) --- .../pyflink/fn_execution/coder_impl_fast.pyx | 2 +- .../pyflink/fn_execution/coder_impl_slow.py | 2 +- .../pyflink/fn_execution/utils/arrow_utils.py | 41 ++++++++++--------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx index f4d6fb7b85565..e0836af6142e9 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx +++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx @@ -446,7 +446,7 @@ cdef class ArrowCoderImpl(FieldCoderImpl): self._resettable_io.set_output_stream(out_stream) batch_writer = pa.RecordBatchStreamWriter(self._resettable_io, self._schema) if self._batch_format == "ARROW": - batch = validate_arrow_batch(cols, self._schema, self._field_types) + batch = validate_arrow_batch(cols, self._schema) else: batch = pandas_to_arrow(self._schema, self._timezone, self._field_types, cols) batch_writer.write_batch(batch) diff --git a/flink-python/pyflink/fn_execution/coder_impl_slow.py b/flink-python/pyflink/fn_execution/coder_impl_slow.py index 7a77cc566167d..cb982b02e293a 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_slow.py +++ b/flink-python/pyflink/fn_execution/coder_impl_slow.py @@ -293,7 +293,7 @@ def encode_to_stream(self, cols, out_stream: OutputStream): self._resettable_io.set_output_stream(out_stream) batch_writer = pa.RecordBatchStreamWriter(self._resettable_io, self._schema) if self._batch_format == "ARROW": - batch = validate_arrow_batch(cols, self._schema, self._field_types) + batch = validate_arrow_batch(cols, self._schema) else: batch = pandas_to_arrow(self._schema, self._timezone, self._field_types, cols) batch_writer.write_batch(batch) diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py index 86a6cf42e4d0a..ea5fba6cda8f3 100644 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -21,56 +21,57 @@ from functools import cache -def validate_arrow_batch(batch, schema, field_types): +def validate_arrow_batch(batch, schema): import pyarrow as pa if not isinstance(batch, pa.RecordBatch): raise TypeError("Arrow transport expects a pyarrow.RecordBatch.") if batch.num_columns != len(schema): raise ValueError(f"Arrow result has {batch.num_columns} columns, expected {len(schema)}.") - for column, field, data_type in zip(batch.columns, schema, field_types): - _validate_array(column, field.type, data_type, field.name) + for column, field in zip(batch.columns, schema): + _validate_array(column, field, field.name) return pa.RecordBatch.from_arrays(batch.columns, schema=schema) -def _validate_array(column, expected_type, data_type, path, parent_validity=None): +def _validate_array(column, field, path, parent_validity=None): import pyarrow as pa import pyarrow.compute as pc - from pyflink.table.types import ArrayType, MapType, RowType + expected_type = field.type def wrong_type(): raise TypeError( f"Arrow result field '{path}' has type {column.type}, expected {expected_type}.") - if not data_type._nullable and column.null_count: + if not field.nullable and column.null_count: validity = parent_validity() if parent_validity is not None else None if validity is None or pc.any(pc.and_(validity, column.is_null())).as_py(): raise ValueError(f"Arrow result field '{path}' is not nullable.") - if isinstance(data_type, RowType): - if not pa.types.is_struct(column.type) or column.type.num_fields != len(data_type.fields): + if pa.types.is_struct(expected_type): + if (not pa.types.is_struct(column.type) + or column.type.num_fields != expected_type.num_fields): wrong_type() validity = cache(lambda: _get_validity(column, parent_validity)) - for index, field in enumerate(data_type.fields): - if column.type[index].name != field.name: + for index, child_field in enumerate(expected_type): + if column.type[index].name != child_field.name: wrong_type() - _validate_array(column.field(index), expected_type[index].type, - field.data_type, f"{path}.{field.name}", validity) - elif isinstance(data_type, ArrayType): + _validate_array(column.field(index), child_field, + f"{path}.{child_field.name}", validity) + elif pa.types.is_list(expected_type): if not pa.types.is_list(column.type): wrong_type() start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() - _validate_array(column.values.slice(start, end - start), expected_type.value_type, - data_type.element_type, f"{path}[]", + _validate_array(column.values.slice(start, end - start), + expected_type.value_field, f"{path}[]", cache(lambda: _get_child_validity(column, parent_validity))) - elif isinstance(data_type, MapType): + elif pa.types.is_map(expected_type): if not pa.types.is_map(column.type): wrong_type() validity = cache(lambda: _get_child_validity(column, parent_validity)) start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() - _validate_array(column.keys.slice(start, end - start), expected_type.key_type, - data_type.key_type.not_null(), f"{path}.key", validity) - _validate_array(column.items.slice(start, end - start), expected_type.item_type, - data_type.value_type, f"{path}.value", validity) + _validate_array(column.keys.slice(start, end - start), expected_type.key_field, + f"{path}.key", validity) + _validate_array(column.items.slice(start, end - start), expected_type.item_field, + f"{path}.value", validity) elif column.type != expected_type: wrong_type() From 74a0f5e022d03c38e42839d0d6825988d1a12f12 Mon Sep 17 00:00:00 2001 From: auroflow Date: Tue, 15 Sep 2026 20:29:33 +0800 Subject: [PATCH 08/14] [FLINK-40472][python] Cover Arrow UDF result reuse after rebase Extend existing worker tests for normalized intermediate results, selected and repeated outputs, and whole-row result references. Apply import ordering to the rebase resolutions. Generated-by: Codex (GPT-6) --- flink-python/pyflink/table/tests/test_udf.py | 42 +++++++++++++------ .../streaming/api/utils/ProtoUtilsTest.java | 2 +- .../exec/common/CommonExecPythonCalc.java | 2 +- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/flink-python/pyflink/table/tests/test_udf.py b/flink-python/pyflink/table/tests/test_udf.py index bb969cda5ece6..3a5d42107e113 100644 --- a/flink-python/pyflink/table/tests/test_udf.py +++ b/flink-python/pyflink/table/tests/test_udf.py @@ -42,11 +42,12 @@ def generate_random_table_name(): class ArrowScalarOperationTests(unittest.TestCase): - def operation(self, func, inputs, takes_row_as_input=False): + def operation(self, func, inputs, takes_row_as_input=False, preceding=(), output_indices=()): function = proto.UserDefinedFunction( payload=cloudpickle.dumps(DelegatingScalarFunction(func)), is_arrow_udf=True, inputs=inputs, takes_row_as_input=takes_row_as_input) - operation = ScalarFunctionOperation(proto.UserDefinedFunctions(udfs=[function])) + operation = ScalarFunctionOperation(proto.UserDefinedFunctions( + udfs=[*preceding, function], output_indices=output_indices)) operation.open() self.addCleanup(operation.close) return operation @@ -101,13 +102,26 @@ def chunk_result(array): inner = proto.UserDefinedFunction( payload=cloudpickle.dumps(DelegatingScalarFunction(chunk_result)), is_arrow_udf=True, inputs=[proto.Input(inputOffset=0)]) - operation = self.operation(lambda row: row.field("value"), [proto.Input(udf=inner)]) - result = operation.process_element(pa.record_batch([column], names=["record"])) - self.assertEqual(result.column(0).to_pylist(), [] if chunks == 0 else [1, None, 3]) - if chunks == 1: - self.assertEqual(result.column(0).offset, column.field(0).offset) - self.assertEqual([buffer.address for buffer in result.column(0).buffers()], - [buffer.address for buffer in column.field(0).buffers()]) + for shared in (False, True): + with self.subTest(shared=shared): + operation = self.operation( + lambda row: row.field("value"), + [proto.Input(refIndex=0)] if shared else [proto.Input(udf=inner)], + preceding=[inner] if shared else [], + output_indices=[1, 0, 1] if shared else []) + result = operation.process_element( + pa.record_batch([column], names=["record"])) + self.assertEqual(result.num_columns, 3 if shared else 1) + self.assertEqual(result.column(0).to_pylist(), + [] if chunks == 0 else [1, None, 3]) + if shared: + self.assertEqual(result.column(1), column) + self.assertEqual(result.column(2), result.column(0)) + if chunks == 1: + self.assertEqual(result.column(0).offset, column.field(0).offset) + self.assertEqual( + [buffer.address for buffer in result.column(0).buffers()], + [buffer.address for buffer in column.field(0).buffers()]) def test_whole_row_input(self): batch = pa.record_batch([pa.array(["alice", None]), pa.array([1, 2])], @@ -122,10 +136,14 @@ def increment(row): identity = proto.UserDefinedFunction( payload=cloudpickle.dumps(DelegatingScalarFunction(lambda row: row)), is_arrow_udf=True, inputs=inputs, takes_row_as_input=True) - for arguments in (inputs, [proto.Input(udf=identity)]): - with self.subTest(nested=arguments[0].HasField("udf")): - operation = self.operation(increment, arguments, takes_row_as_input=True) + for arguments in (inputs, [proto.Input(udf=identity)], [proto.Input(refIndex=0)]): + with self.subTest(input_kind=arguments[0].WhichOneof("input")): + shared = arguments[0].HasField("refIndex") + operation = self.operation( + increment, arguments, takes_row_as_input=True, + preceding=[identity] if shared else [], output_indices=[1] if shared else []) result = operation.process_element(batch) + self.assertEqual(result.num_columns, 1) self.assertEqual(result.column(0).to_pylist(), [{"name": "alice", "count": 2}, {"name": None, "count": 3}]) diff --git a/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java b/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java index 9eb6029ee57aa..1331226b200ab 100644 --- a/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java +++ b/flink-python/src/test/java/org/apache/flink/streaming/api/utils/ProtoUtilsTest.java @@ -22,8 +22,8 @@ import org.apache.flink.fnexecution.v1.FlinkFnApi; import org.apache.flink.python.util.ProtoUtils; import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.functions.python.PythonEnv; import org.apache.flink.table.functions.python.InputRef; +import org.apache.flink.table.functions.python.PythonEnv; import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.functions.python.PythonFunctionInput; import org.apache.flink.table.functions.python.PythonFunctionKind; diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java index 83c8f11a36713..8a15da8d3c50e 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecPythonCalc.java @@ -31,8 +31,8 @@ import org.apache.flink.table.functions.python.InputRef; import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.functions.python.PythonFunctionInput; -import org.apache.flink.table.functions.python.ResultRef; import org.apache.flink.table.functions.python.PythonFunctionKind; +import org.apache.flink.table.functions.python.ResultRef; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.codegen.CodeGeneratorContext; import org.apache.flink.table.planner.codegen.ProjectionCodeGenerator; From ee3900c8464335f79f58afc94fa1ccc537b721f6 Mon Sep 17 00:00:00 2001 From: auroflow Date: Wed, 16 Sep 2026 16:09:07 +0800 Subject: [PATCH 09/14] [FLINK-40472][python] Reset Arrow map child writers between batches Reset key and value writers together with map vectors so consecutive batches reuse aligned offsets. Cover direct and nested maps, nulls, empty maps, and partial batches through Arrow IPC round trips. Generated-by: Codex (GPT-6) --- .../runtime/arrow/writers/MapWriter.java | 7 ++ .../runtime/arrow/ArrowReaderWriterTest.java | 92 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java b/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java index c2c960c332873..d9d03f32449da 100644 --- a/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java @@ -84,6 +84,13 @@ public void doWrite(T in, int ordinal) { } } + @Override + public void reset() { + super.reset(); + keyWriter.reset(); + valueWriter.reset(); + } + // ------------------------------------------------------------------------------------------ /** {@link MapWriter} for {@link RowData} input. */ diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java index 6441246a9be45..2e4c18a3a4794 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericMapData; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; @@ -39,6 +40,7 @@ import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.SmallIntType; import org.apache.flink.table.types.logical.TimeType; @@ -53,15 +55,24 @@ import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import static org.assertj.core.api.Assertions.assertThat; + /** Tests for {@link ArrowReader} and {@link ArrowWriter} of RowData. */ class ArrowReaderWriterTest extends ArrowReaderWriterTestBase { private static List fieldTypes = new ArrayList<>(); @@ -168,6 +179,87 @@ public Tuple2, ArrowStreamWriter> createArrowWriter( return Tuple2.of(arrowWriter, arrowStreamWriter); } + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void testMapsAcrossBatches(int batchSize) throws IOException { + final MapType mapType = new MapType(new IntType(false), new IntType()); + final MapType nestedMapType = + new MapType(new IntType(false), RowType.of(RowType.of(new IntType()))); + final RowType type = + RowType.of( + mapType, new ArrayType(mapType), new ArrayType(RowType.of(nestedMapType))); + final List rows = new ArrayList<>(); + for (Map values : + Arrays.asList( + Map.of(1, 11, 2, 22), + Collections.emptyMap(), + null, + Collections.singletonMap(3, null), + Map.of(4, 44))) { + final GenericMapData map = values == null ? null : new GenericMapData(values); + final Map nestedValues = new LinkedHashMap<>(); + if (values != null) { + values.forEach( + (key, value) -> + nestedValues.put( + key, + value == null + ? null + : GenericRowData.of(GenericRowData.of(value)))); + } + rows.add( + GenericRowData.of( + map, + new GenericArrayData(new Object[] {map}), + new GenericArrayData( + new Object[] { + GenericRowData.of( + values == null + ? null + : new GenericMapData(nestedValues)) + }))); + } + + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (BufferAllocator batchAllocator = + ArrowUtils.getRootAllocator() + .newChildAllocator("map-batches", 0, Long.MAX_VALUE); + VectorSchemaRoot root = + VectorSchemaRoot.create(ArrowUtils.toArrowSchema(type), batchAllocator); + ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, output)) { + final ArrowWriter writer = ArrowUtils.createRowDataArrowWriter(root, type); + streamWriter.start(); + for (int start = 0; start < rows.size(); start += batchSize) { + for (RowData row : rows.subList(start, Math.min(start + batchSize, rows.size()))) { + writer.write(row); + } + writer.finish(); + streamWriter.writeBatch(); + writer.reset(); + } + streamWriter.end(); + + try (ArrowStreamReader streamReader = + new ArrowStreamReader( + new ByteArrayInputStream(output.toByteArray()), batchAllocator)) { + final RowDataSerializer serializer = new RowDataSerializer(type); + final ArrowReader reader = + ArrowUtils.createArrowReader(streamReader.getVectorSchemaRoot(), type); + int rowIndex = 0; + while (streamReader.loadNextBatch()) { + final int rowCount = streamReader.getVectorSchemaRoot().getRowCount(); + assertThat(rowCount).isEqualTo(Math.min(batchSize, rows.size() - rowIndex)); + for (int i = 0; i < rowCount; i++) { + assertThat(serializer.toBinaryRow(reader.read(i)).copy()) + .as("row %s", rowIndex) + .isEqualTo(serializer.toBinaryRow(rows.get(rowIndex++)).copy()); + } + } + assertThat(rowIndex).isEqualTo(rows.size()); + } + } + } + @Override public RowData[] getTestData() { RowData row1 = From 14cd413fa7f8f47813886e926a37c93cc1ce49ac Mon Sep 17 00:00:00 2001 From: auroflow Date: Wed, 16 Sep 2026 16:54:25 +0800 Subject: [PATCH 10/14] [FLINK-40472][python] Preserve PyArrow 5 and struct result compatibility Use the legacy MAP constructor on PyArrow 5 while retaining logical map-value null checks. Apply validated native output schemas by rebuilding container metadata over existing buffers, preserving sliced and nested results without version-dependent casts. Generated-by: Codex (GPT-6) --- .../pyflink/fn_execution/coder_impl_fast.pyx | 2 +- .../pyflink/fn_execution/coder_impl_slow.py | 2 +- .../pyflink/fn_execution/tests/test_coders.py | 83 +++++++++++++++---- .../pyflink/fn_execution/utils/arrow_utils.py | 78 ++++++++++++++--- flink-python/pyflink/table/types.py | 11 +-- 5 files changed, 140 insertions(+), 36 deletions(-) diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx index e0836af6142e9..f4d6fb7b85565 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx +++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx @@ -446,7 +446,7 @@ cdef class ArrowCoderImpl(FieldCoderImpl): self._resettable_io.set_output_stream(out_stream) batch_writer = pa.RecordBatchStreamWriter(self._resettable_io, self._schema) if self._batch_format == "ARROW": - batch = validate_arrow_batch(cols, self._schema) + batch = validate_arrow_batch(cols, self._schema, self._field_types) else: batch = pandas_to_arrow(self._schema, self._timezone, self._field_types, cols) batch_writer.write_batch(batch) diff --git a/flink-python/pyflink/fn_execution/coder_impl_slow.py b/flink-python/pyflink/fn_execution/coder_impl_slow.py index cb982b02e293a..7a77cc566167d 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_slow.py +++ b/flink-python/pyflink/fn_execution/coder_impl_slow.py @@ -293,7 +293,7 @@ def encode_to_stream(self, cols, out_stream: OutputStream): self._resettable_io.set_output_stream(out_stream) batch_writer = pa.RecordBatchStreamWriter(self._resettable_io, self._schema) if self._batch_format == "ARROW": - batch = validate_arrow_batch(cols, self._schema) + batch = validate_arrow_batch(cols, self._schema, self._field_types) else: batch = pandas_to_arrow(self._schema, self._timezone, self._field_types, cols) batch_writer.write_batch(batch) diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index dc4280ea314de..8aa99c45a77ef 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -70,8 +70,9 @@ def test_pandas_collection_schema_and_round_trip(self): schema = create_arrow_schema(names, types) self.assertEqual(schema.field("values").type.value_field.name, "item") self.assertFalse(schema.field("values").type.value_field.nullable) - self.assertFalse(schema.field("lookup").type.key_field.nullable) - self.assertFalse(schema.field("lookup").type.item_field.nullable) + if hasattr(pa.MapType, 'item_field'): + self.assertFalse(schema.field("lookup").type.key_field.nullable) + self.assertFalse(schema.field("lookup").type.item_field.nullable) self.assertTrue(schema.field("lookup").type.item_type.value_field.nullable) self.assertFalse(schema.field("record").type[0].type.value_field.nullable) @@ -89,9 +90,10 @@ def test_pandas_collection_schema_and_round_trip(self): {"values": [], "lookup": [], "record": {"values": []}}, {"values": None, "lookup": None, "record": {"values": None}}, {"values": [None], "lookup": [('b', None)], "record": {"values": [None]}}] - self.assertEqual(decoded.to_pylist(), expected) + expected = {name: [row[name] for row in expected] for name in names} + self.assertEqual(decoded.to_pydict(), expected) restored = arrow_to_pandas(pytz.UTC, types, [decoded]) - self.assertEqual(pandas_to_arrow(schema, pytz.UTC, types, restored).to_pylist(), expected) + self.assertEqual(pandas_to_arrow(schema, pytz.UTC, types, restored).to_pydict(), expected) def test_arrow_descriptor_preserves_pandas_default(self): from pyflink.fn_execution import flink_fn_execution_pb2 as proto @@ -127,8 +129,14 @@ def with_parent_nulls(column, nulls): buffers = [validity] children = [column.field(index) for index in range(column.type.num_fields)] else: - buffers = [validity, column.offsets.buffers()[1]] - children = [column.values] + buffers = [validity, column.buffers()[1]] + if pa.types.is_map(column.type): + children = [pa.StructArray.from_arrays( + [column.keys, column.items], + fields=[pa.field("key", column.type.key_type, nullable=False), + pa.field("value", column.type.item_type)])] + else: + children = [column.values] return pa.Array.from_buffers(column.type, len(column), buffers, children=children) def test_native_arrow_timezone(self): @@ -194,16 +202,22 @@ def test_struct_map_and_temporal_results(self): {"record": None, "lookup": None, "amount": None, "time": None}, {"record": {"inner": None}, "lookup": [], "amount": decimal.Decimal("-0.50"), "time": datetime.datetime(2021, 3, 4)}] - batch = pa.RecordBatch.from_pylist(rows, schema=schema) - self.assertEqual(coder.decode(coder.encode(batch)).to_pylist(), rows) - self.assertEqual(coder.decode(coder.encode(batch.slice(1))).to_pylist(), rows[1:]) + expected = {name: [row[name] for row in rows] for name in schema.names} + batch = pa.record_batch([pa.array(expected[field.name], type=field.type) + for field in schema], schema=schema) + self.assertEqual(coder.decode(coder.encode(batch)).to_pydict(), expected) + self.assertEqual(coder.decode(coder.encode(batch.slice(1))).to_pydict(), + {name: values[1:] for name, values in expected.items()}) for field, value, message in ( ("record", {"inner": {"value": None}}, "record.inner.value.*not nullable"), ("lookup", [("a", None)], "lookup.value.*not nullable"), ): with self.subTest(field=field): - invalid = pa.RecordBatch.from_pylist([{**rows[0], field: value}], schema=schema) + invalid_values = {**rows[0], field: value} + invalid = pa.record_batch([ + pa.array([invalid_values[child.name]], type=child.type) for child in schema], + schema=schema) with self.assertRaisesRegex(ValueError, message): coder.encode(invalid) @@ -226,14 +240,48 @@ def test_native_arrow_round_trip(self): with self.assertRaisesRegex(TypeError, "name.*string"): coder.encode(pa.record_batch([pa.array([1, 2])], names=["name"])) + def test_pandas_map_round_trip(self): + import pandas as pd + from pyflink.table import DataTypes + from pyflink.table.types import create_arrow_schema + + row_type = DataTypes.ROW([ + DataTypes.FIELD("lookup", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT()))]) + schema = create_arrow_schema(row_type.field_names(), row_type.field_types()) + coder = self.implementation.ArrowCoderImpl(schema, row_type, pytz.UTC) + values = pd.Series([[("value", 10)], [("value", 20)], None, []]) + result = coder.decode(coder.encode([values])) + self.assertEqual(result[0].tolist(), [[("value", 10)], [("value", 20)], None, []]) + + def test_inferred_struct_nullability(self): + from pyflink.table import DataTypes + from pyflink.table.types import create_arrow_schema + + row_type = DataTypes.ROW([DataTypes.FIELD("wrapped", DataTypes.ROW([ + DataTypes.FIELD("v", DataTypes.BIGINT().not_null())]))]) + schema = create_arrow_schema(row_type.field_names(), row_type.field_types(), + allow_nested=True) + coder = self.arrow_coder(schema, row_type) + column = pa.StructArray.from_arrays([pa.array([0, 1, 2])], names=["v"]) + with_nulls = pa.StructArray.from_arrays([pa.array([None, 1, 2])], names=["v"]) + for values in (column, column.slice(1), column.slice(1, 0), + with_nulls.slice(1), with_nulls.slice(1, 0)): + with self.subTest(offset=values.offset, length=len(values)): + result = coder.decode(coder.encode(pa.record_batch([values], names=["wrapped"]))) + self.assertEqual(result.schema, schema) + self.assertEqual(result.column(0).to_pylist(), values.to_pylist()) + invalid = pa.StructArray.from_arrays([pa.array([None], type=pa.int64())], names=["v"]) + with self.assertRaisesRegex(ValueError, "wrapped.v.*not nullable"): + coder.encode(pa.record_batch([invalid], names=["wrapped"])) + def test_sliced_container_nullability(self): from pyflink.table import DataTypes - from pyflink.table.types import create_arrow_schema, to_arrow_type + from pyflink.table.types import create_arrow_schema item_type = DataTypes.ROW([DataTypes.FIELD("required", DataTypes.INT().not_null())]) items = pa.array([{"required": None}, {"required": 1}, {"required": None}, {"required": None}, None, {"required": None}], - type=to_arrow_type(item_type)) + type=pa.struct([pa.field("required", pa.int32())])) nulls = [False, False, True, False, False, False] offsets = pa.array(range(7), type=pa.int32()) for data_type, column, first, last in ( @@ -257,9 +305,9 @@ def test_sliced_container_nullability(self): [False, False, False, True, False, False]) batch = pa.record_batch([outer], names=["record"]) coder = self.arrow_coder(schema, row_type) - self.assertEqual(coder.decode(coder.encode(batch.slice(1, 4))).to_pylist(), [ - {"record": {"container": first}}, {"record": {"container": None}}, - {"record": None}, {"record": {"container": last}}]) + self.assertEqual(coder.decode(coder.encode(batch.slice(1, 4))).to_pydict(), { + "record": [{"container": first}, {"container": None}, + None, {"container": last}]}) self.assertEqual(coder.decode(coder.encode(batch.slice(0, 0))).num_rows, 0) with self.assertRaisesRegex(ValueError, "required.*not nullable"): coder.encode(batch.slice(0, 1)) @@ -278,6 +326,9 @@ def test_nullable_container_validation_memory(self): (DataTypes.ROW([DataTypes.FIELD("payload", DataTypes.BYTES())]), pa.StructArray.from_arrays([payload], names=["payload"]), nulls, payload.nbytes // 4), + (DataTypes.ROW([DataTypes.FIELD("payload", DataTypes.BYTES().not_null())]), + pa.StructArray.from_arrays([payload], names=["payload"]), + nulls, payload.nbytes // 4), (DataTypes.ARRAY(DataTypes.BYTES()), pa.ListArray.from_arrays(offsets, payload), nulls, payload.nbytes // 4), (DataTypes.MAP(DataTypes.STRING().not_null(), DataTypes.BYTES()), @@ -302,7 +353,7 @@ def test_nullable_container_validation_memory(self): pa.set_memory_pool(default_pool) # Allow IPC metadata, but not payload copies or element-sized temporary arrays. self.assertLess(pool.max_memory(), memory_limit) - self.assertEqual(coder.decode(encoded).to_pylist(), batch.to_pylist()) + self.assertEqual(coder.decode(encoded).to_pydict(), batch.to_pydict()) try: diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py index ea5fba6cda8f3..a1f13275147fc 100644 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -21,19 +21,50 @@ from functools import cache -def validate_arrow_batch(batch, schema): +def validate_arrow_batch(batch, schema, field_types): import pyarrow as pa if not isinstance(batch, pa.RecordBatch): raise TypeError("Arrow transport expects a pyarrow.RecordBatch.") if batch.num_columns != len(schema): raise ValueError(f"Arrow result has {batch.num_columns} columns, expected {len(schema)}.") - for column, field in zip(batch.columns, schema): - _validate_array(column, field, field.name) - return pa.RecordBatch.from_arrays(batch.columns, schema=schema) + # PyArrow 5 cannot express map-value nullability, so retain the logical constraint there. + legacy_map_fields = not hasattr(pa.MapType, 'item_field') + columns = [] + for index, (column, field) in enumerate(zip(batch.columns, schema)): + _validate_array(column, field, field.name, + data_type=field_types[index] if legacy_map_fields else None) + columns.append(_apply_arrow_type(column, field.type)) + return pa.RecordBatch.from_arrays(columns, schema=schema) -def _validate_array(column, field, path, parent_validity=None): +def _apply_arrow_type(column, expected_type): + import pyarrow as pa + + if column.type == expected_type: + return column + # Old Arrow casts reject valid nullability changes, and views inspect unsliced children. + # After validation, rebuild only container metadata while keeping the payload buffers. + if pa.types.is_struct(expected_type): + return pa.StructArray.from_arrays( + [_apply_arrow_type(column.field(index), field.type) + for index, field in enumerate(expected_type)], + fields=list(expected_type), mask=column.is_null() if column.null_count else None) + if pa.types.is_list(expected_type): + children = [_apply_arrow_type(column.values, expected_type.value_type)] + else: + value_field = (expected_type.item_field if hasattr(expected_type, 'item_field') + else pa.field("value", expected_type.item_type)) + children = [pa.StructArray.from_arrays( + [_apply_arrow_type(column.keys, expected_type.key_type), + _apply_arrow_type(column.items, expected_type.item_type)], + fields=[pa.field("key", expected_type.key_type, nullable=False), value_field])] + return pa.Array.from_buffers( + expected_type, len(column), column.buffers()[:expected_type.num_buffers], + null_count=column.null_count, offset=column.offset, children=children) + + +def _validate_array(column, field, path, parent_validity=None, data_type=None): import pyarrow as pa import pyarrow.compute as pc expected_type = field.type @@ -55,23 +86,33 @@ def wrong_type(): if column.type[index].name != child_field.name: wrong_type() _validate_array(column.field(index), child_field, - f"{path}.{child_field.name}", validity) + f"{path}.{child_field.name}", validity, + data_type.fields[index].data_type if data_type is not None else None) elif pa.types.is_list(expected_type): if not pa.types.is_list(column.type): wrong_type() start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() _validate_array(column.values.slice(start, end - start), expected_type.value_field, f"{path}[]", - cache(lambda: _get_child_validity(column, parent_validity))) + cache(lambda: _get_child_validity(column, parent_validity)), + data_type.element_type if data_type is not None else None) elif pa.types.is_map(expected_type): if not pa.types.is_map(column.type): wrong_type() validity = cache(lambda: _get_child_validity(column, parent_validity)) - start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() - _validate_array(column.keys.slice(start, end - start), expected_type.key_field, - f"{path}.key", validity) - _validate_array(column.items.slice(start, end - start), expected_type.item_field, - f"{path}.value", validity) + offsets = _get_offsets(column) + start, end = offsets[0].as_py(), offsets[-1].as_py() + key_field = (expected_type.key_field if data_type is None + else pa.field("key", expected_type.key_type, nullable=False)) + value_field = (expected_type.item_field if data_type is None + else pa.field("value", expected_type.item_type, + nullable=data_type.value_type._nullable)) + _validate_array(column.keys.slice(start, end - start), key_field, + f"{path}.key", validity, + data_type.key_type if data_type is not None else None) + _validate_array(column.items.slice(start, end - start), value_field, + f"{path}.value", validity, + data_type.value_type if data_type is not None else None) elif column.type != expected_type: wrong_type() @@ -95,10 +136,21 @@ def _get_child_validity(column, parent_validity): if validity is None: return None # A list view also handles map entries and keeps the original offsets and value buffers. - entries = pa.ListArray.from_arrays(column.offsets, column.values) + values = column.keys if pa.types.is_map(column.type) else column.values + entries = pa.ListArray.from_arrays(_get_offsets(column), values) return pc.take(validity, pc.list_parent_indices(entries)) +def _get_offsets(column): + import pyarrow as pa + + if hasattr(column, 'offsets'): + return column.offsets + # PyArrow 5 does not expose MapArray.offsets, but uses the same int32 offsets as lists. + return pa.Array.from_buffers(pa.int32(), len(column) + 1, + [None, column.buffers()[1]], offset=column.offset) + + def check_arrow_udf_result(func, *args): import pyarrow as pa diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index 4a2d61363f449..ac8397e007d66 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -2348,11 +2348,12 @@ def to_arrow_type(data_type: DataType, *, allow_nested=False): else: return pa.timestamp('ns') elif isinstance(data_type, MapType): - return pa.map_( - pa.field("key", to_arrow_type(data_type.key_type, allow_nested=allow_nested), - nullable=False), - pa.field("value", to_arrow_type(data_type.value_type, allow_nested=allow_nested), - nullable=data_type.value_type._nullable)) + key_type = to_arrow_type(data_type.key_type, allow_nested=allow_nested) + value_type = to_arrow_type(data_type.value_type, allow_nested=allow_nested) + # PyArrow 5 only accepts data types and always makes map values nullable. + if hasattr(pa.MapType, 'item_field'): + value_type = pa.field("value", value_type, nullable=data_type.value_type._nullable) + return pa.map_(key_type, value_type) elif isinstance(data_type, ArrayType): if not allow_nested and type(data_type.element_type) in [LocalZonedTimestampType, RowType]: raise ValueError("%s is not supported to be used as the element type of ArrayType." % From c3b03f999a548d455e6bd82e6835a744e5716a3c Mon Sep 17 00:00:00 2001 From: auroflow Date: Wed, 16 Sep 2026 20:47:06 +0800 Subject: [PATCH 11/14] [FLINK-40472][python] Reduce temporary memory for nested Arrow validation Expand parent visibility as booleans using child counts from list/map offsets. Reuse coder round-trip tests for slices, varying child counts and nullability, and remove memory-usage assertions. Generated-by: Codex (GPT-6) --- .../pyflink/fn_execution/tests/test_coders.py | 55 +++---------------- .../pyflink/fn_execution/utils/arrow_utils.py | 10 ++-- 2 files changed, 14 insertions(+), 51 deletions(-) diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index 8aa99c45a77ef..8cdb473b623be 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -282,8 +282,9 @@ def test_sliced_container_nullability(self): items = pa.array([{"required": None}, {"required": 1}, {"required": None}, {"required": None}, None, {"required": None}], type=pa.struct([pa.field("required", pa.int32())])) + booleans = pa.array([None, True, None, None, False, None]) nulls = [False, False, True, False, False, False] - offsets = pa.array(range(7), type=pa.int32()) + offsets = pa.array([0, 1, 2, 2, 4, 5, 6], type=pa.int32()) for data_type, column, first, last in ( (DataTypes.ROW([DataTypes.FIELD("value", item_type)]), pa.StructArray.from_arrays([items], names=["value"]), @@ -293,6 +294,11 @@ def test_sliced_container_nullability(self): (DataTypes.MAP(DataTypes.STRING().not_null(), item_type), pa.MapArray.from_arrays(offsets, pa.array(['k'] * 6), items), [('k', {"required": 1})], [('k', None)]), + (DataTypes.ARRAY(DataTypes.BOOLEAN().not_null()), + pa.ListArray.from_arrays(offsets, booleans), [True], [False]), + (DataTypes.MAP(DataTypes.STRING().not_null(), DataTypes.BOOLEAN().not_null()), + pa.MapArray.from_arrays(offsets, pa.array(['k'] * 6), booleans), + [('k', True)], [('k', False)]), ): with self.subTest(data_type=data_type): column = self.with_parent_nulls(column, nulls) @@ -309,51 +315,8 @@ def test_sliced_container_nullability(self): "record": [{"container": first}, {"container": None}, None, {"container": last}]}) self.assertEqual(coder.decode(coder.encode(batch.slice(0, 0))).num_rows, 0) - with self.assertRaisesRegex(ValueError, "required.*not nullable"): - coder.encode(batch.slice(0, 1)) - - def test_nullable_container_validation_memory(self): - from pyflink.table import DataTypes - from pyflink.table.types import create_arrow_schema - - count = 512 - payload = pa.array([b'x' * 2048] * count) - nulls = [index == count // 2 for index in range(count)] - offsets = pa.array(range(count + 1), type=pa.int32()) - booleans = pa.array([False] * 65536) - boolean_lists = pa.ListArray.from_arrays([0, len(booleans), len(booleans)], booleans) - for data_type, column, parent_nulls, memory_limit in ( - (DataTypes.ROW([DataTypes.FIELD("payload", DataTypes.BYTES())]), - pa.StructArray.from_arrays([payload], names=["payload"]), - nulls, payload.nbytes // 4), - (DataTypes.ROW([DataTypes.FIELD("payload", DataTypes.BYTES().not_null())]), - pa.StructArray.from_arrays([payload], names=["payload"]), - nulls, payload.nbytes // 4), - (DataTypes.ARRAY(DataTypes.BYTES()), - pa.ListArray.from_arrays(offsets, payload), nulls, payload.nbytes // 4), - (DataTypes.MAP(DataTypes.STRING().not_null(), DataTypes.BYTES()), - pa.MapArray.from_arrays(offsets, pa.array(['k'] * count), payload), - nulls, payload.nbytes // 4), - (DataTypes.ARRAY(DataTypes.BOOLEAN()), boolean_lists, [False, True], 4096), - (DataTypes.ARRAY(DataTypes.BOOLEAN().not_null()), - boolean_lists, [False, True], 4096), - ): - with self.subTest(data_type=data_type): - column = self.with_parent_nulls(column, parent_nulls) - row_type = DataTypes.ROW([DataTypes.FIELD("value", data_type)]) - schema = create_arrow_schema(["value"], [data_type], allow_nested=True) - batch = pa.record_batch([column], names=["value"]) - coder = self.arrow_coder(schema, row_type) - default_pool = pa.default_memory_pool() - pool = pa.proxy_memory_pool(default_pool) - try: - pa.set_memory_pool(pool) - encoded = coder.encode(batch) - finally: - pa.set_memory_pool(default_pool) - # Allow IPC metadata, but not payload copies or element-sized temporary arrays. - self.assertLess(pool.max_memory(), memory_limit) - self.assertEqual(coder.decode(encoded).to_pydict(), batch.to_pydict()) + with self.assertRaisesRegex(ValueError, "not nullable"): + coder.encode(batch.slice(0, 4)) try: diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py index a1f13275147fc..ccca1c82d6eb7 100644 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ b/flink-python/pyflink/fn_execution/utils/arrow_utils.py @@ -129,16 +129,16 @@ def _get_validity(column, parent_validity): def _get_child_validity(column, parent_validity): + import numpy as np import pyarrow as pa - import pyarrow.compute as pc validity = _get_validity(column, parent_validity) if validity is None: return None - # A list view also handles map entries and keeps the original offsets and value buffers. - values = column.keys if pa.types.is_map(column.type) else column.values - entries = pa.ListArray.from_arrays(_get_offsets(column), values) - return pc.take(validity, pc.list_parent_indices(entries)) + # Expand visibility directly instead of allocating an integer parent index for every child. + offsets = _get_offsets(column).to_numpy(zero_copy_only=True) + visible = validity.to_numpy(zero_copy_only=False) + return pa.array(np.repeat(visible, np.diff(offsets)), type=pa.bool_()) def _get_offsets(column): From 80451d84c1f7cadd2b1a77a6d57645a4cc11d35c Mon Sep 17 00:00:00 2001 From: auroflow Date: Thu, 17 Sep 2026 10:23:06 +0800 Subject: [PATCH 12/14] [FLINK-40472][python] Clarify Arrow UDF docs and consolidate helpers Move Arrow schema validation beside existing serialization utilities and UDF result handling beside pandas execution helpers. Generated-by: Codex (GPT-6) --- flink-python/pyflink/dataframe/udf.py | 20 +- .../pyflink/fn_execution/coder_impl_fast.pyx | 3 +- .../pyflink/fn_execution/coder_impl_slow.py | 3 +- .../pyflink/fn_execution/table/operations.py | 3 +- .../pyflink/fn_execution/utils/arrow_utils.py | 180 ------------------ .../fn_execution/utils/operation_utils.py | 30 ++- flink-python/pyflink/table/utils.py | 132 +++++++++++++ 7 files changed, 178 insertions(+), 193 deletions(-) delete mode 100644 flink-python/pyflink/fn_execution/utils/arrow_utils.py diff --git a/flink-python/pyflink/dataframe/udf.py b/flink-python/pyflink/dataframe/udf.py index f9e14c893b601..065db5d66ab42 100644 --- a/flink-python/pyflink/dataframe/udf.py +++ b/flink-python/pyflink/dataframe/udf.py @@ -410,14 +410,23 @@ def udf( ... def inferred_pandas_add_one(values: pd.Series) -> pd.Series: ... return values + 1 - Synchronous Arrow UDFs operate on ``pyarrow.Array`` columns and return an - ``Array`` or ``ChunkedArray``. They require ``return_dtype``. Select - ``func_type="arrow"`` explicitly, or infer it from an Arrow container - annotation on an unbound parameter or the return value:: + Arrow UDFs always require an explicit logical ``return_dtype`` and support + synchronous functions. Each column argument is received as a ``pyarrow.Array``; + a ``ROW``-typed column is received as a ``pyarrow.StructArray`` with one child + array per field. Results should be returned as a ``pyarrow.Array`` or + ``pyarrow.ChunkedArray`` of the declared logical type, with the same number of + rows as the input batch. A ``ROW``-typed result uses a ``pyarrow.StructArray`` + or a chunked array of structs. Arrow mode can be selected explicitly, or + inferred from an Arrow container annotation on any unbound parameter or the + return value:: >>> import pyarrow as pa >>> import pyarrow.compute as pc + >>> @pf.udf(return_dtype=pf.DataType.int64(), func_type="arrow") + ... def arrow_add_one(values): + ... return pc.add(values, 1) + >>> @pf.udf(return_dtype=pf.DataType.string()) ... def normalize_name(names: pa.Array) -> pa.Array: ... return pc.utf8_upper(names) @@ -442,8 +451,7 @@ def udf( :param name: Non-empty function identity used by the Table planner. :param func_type: ``"general"``, ``"pandas"``, or ``"arrow"``. If omitted, unbound container annotations select pandas or Arrow mode; - otherwise general mode is used. Mixed pandas and Arrow hints - require an explicit mode. + otherwise general mode is used. :return: A callable that accepts DataFrame expressions or Python literals and returns an :class:`~pyflink.table.expression.Expression`, or a decorator producing such a callable when ``func`` is omitted. diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx index f4d6fb7b85565..c25ac8cfdd729 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx +++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx @@ -36,8 +36,7 @@ from pyflink.datastream.window import CountWindow, TimeWindow, GlobalWindow from pyflink.fn_execution.formats.avro import FlinkAvroDecoder, FlinkAvroDatumReader, \ FlinkAvroBufferWrapper, FlinkAvroEncoder, FlinkAvroDatumWriter from pyflink.fn_execution.ResettableIO import ResettableIO -from pyflink.table.utils import pandas_to_arrow, arrow_to_pandas -from pyflink.fn_execution.utils.arrow_utils import validate_arrow_batch +from pyflink.table.utils import pandas_to_arrow, arrow_to_pandas, validate_arrow_batch ROW_KIND_BIT_SIZE = 2 diff --git a/flink-python/pyflink/fn_execution/coder_impl_slow.py b/flink-python/pyflink/fn_execution/coder_impl_slow.py index 7a77cc566167d..a7df600d8fb6a 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_slow.py +++ b/flink-python/pyflink/fn_execution/coder_impl_slow.py @@ -31,8 +31,7 @@ from pyflink.fn_execution.formats.avro import FlinkAvroDecoder, FlinkAvroDatumReader, \ FlinkAvroBufferWrapper, FlinkAvroEncoder, FlinkAvroDatumWriter from pyflink.fn_execution.stream_slow import InputStream, OutputStream -from pyflink.table.utils import pandas_to_arrow, arrow_to_pandas -from pyflink.fn_execution.utils.arrow_utils import validate_arrow_batch +from pyflink.table.utils import pandas_to_arrow, arrow_to_pandas, validate_arrow_batch ROW_KIND_BIT_SIZE = 2 diff --git a/flink-python/pyflink/fn_execution/table/operations.py b/flink-python/pyflink/fn_execution/table/operations.py index e2d5786011abe..6459602aa313d 100644 --- a/flink-python/pyflink/fn_execution/table/operations.py +++ b/flink-python/pyflink/fn_execution/table/operations.py @@ -157,8 +157,7 @@ def generate_func(self, serialized_fn): func_strs.append(func_str) if is_arrow: - from pyflink.fn_execution.utils.arrow_utils import create_record_batch - variable_dict['create_record_batch'] = create_record_batch + variable_dict['create_record_batch'] = operation_utils.create_record_batch output_indices = list(serialized_fn.output_indices) # Result references require sequential evaluation. A non-empty output_indices does too: diff --git a/flink-python/pyflink/fn_execution/utils/arrow_utils.py b/flink-python/pyflink/fn_execution/utils/arrow_utils.py deleted file mode 100644 index ccca1c82d6eb7..0000000000000 --- a/flink-python/pyflink/fn_execution/utils/arrow_utils.py +++ /dev/null @@ -1,180 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -"""Arrow-native scalar UDF result contracts shared by the Python and compiled coders.""" - -from functools import cache - - -def validate_arrow_batch(batch, schema, field_types): - import pyarrow as pa - - if not isinstance(batch, pa.RecordBatch): - raise TypeError("Arrow transport expects a pyarrow.RecordBatch.") - if batch.num_columns != len(schema): - raise ValueError(f"Arrow result has {batch.num_columns} columns, expected {len(schema)}.") - # PyArrow 5 cannot express map-value nullability, so retain the logical constraint there. - legacy_map_fields = not hasattr(pa.MapType, 'item_field') - columns = [] - for index, (column, field) in enumerate(zip(batch.columns, schema)): - _validate_array(column, field, field.name, - data_type=field_types[index] if legacy_map_fields else None) - columns.append(_apply_arrow_type(column, field.type)) - return pa.RecordBatch.from_arrays(columns, schema=schema) - - -def _apply_arrow_type(column, expected_type): - import pyarrow as pa - - if column.type == expected_type: - return column - # Old Arrow casts reject valid nullability changes, and views inspect unsliced children. - # After validation, rebuild only container metadata while keeping the payload buffers. - if pa.types.is_struct(expected_type): - return pa.StructArray.from_arrays( - [_apply_arrow_type(column.field(index), field.type) - for index, field in enumerate(expected_type)], - fields=list(expected_type), mask=column.is_null() if column.null_count else None) - if pa.types.is_list(expected_type): - children = [_apply_arrow_type(column.values, expected_type.value_type)] - else: - value_field = (expected_type.item_field if hasattr(expected_type, 'item_field') - else pa.field("value", expected_type.item_type)) - children = [pa.StructArray.from_arrays( - [_apply_arrow_type(column.keys, expected_type.key_type), - _apply_arrow_type(column.items, expected_type.item_type)], - fields=[pa.field("key", expected_type.key_type, nullable=False), value_field])] - return pa.Array.from_buffers( - expected_type, len(column), column.buffers()[:expected_type.num_buffers], - null_count=column.null_count, offset=column.offset, children=children) - - -def _validate_array(column, field, path, parent_validity=None, data_type=None): - import pyarrow as pa - import pyarrow.compute as pc - expected_type = field.type - - def wrong_type(): - raise TypeError( - f"Arrow result field '{path}' has type {column.type}, expected {expected_type}.") - - if not field.nullable and column.null_count: - validity = parent_validity() if parent_validity is not None else None - if validity is None or pc.any(pc.and_(validity, column.is_null())).as_py(): - raise ValueError(f"Arrow result field '{path}' is not nullable.") - if pa.types.is_struct(expected_type): - if (not pa.types.is_struct(column.type) - or column.type.num_fields != expected_type.num_fields): - wrong_type() - validity = cache(lambda: _get_validity(column, parent_validity)) - for index, child_field in enumerate(expected_type): - if column.type[index].name != child_field.name: - wrong_type() - _validate_array(column.field(index), child_field, - f"{path}.{child_field.name}", validity, - data_type.fields[index].data_type if data_type is not None else None) - elif pa.types.is_list(expected_type): - if not pa.types.is_list(column.type): - wrong_type() - start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() - _validate_array(column.values.slice(start, end - start), - expected_type.value_field, f"{path}[]", - cache(lambda: _get_child_validity(column, parent_validity)), - data_type.element_type if data_type is not None else None) - elif pa.types.is_map(expected_type): - if not pa.types.is_map(column.type): - wrong_type() - validity = cache(lambda: _get_child_validity(column, parent_validity)) - offsets = _get_offsets(column) - start, end = offsets[0].as_py(), offsets[-1].as_py() - key_field = (expected_type.key_field if data_type is None - else pa.field("key", expected_type.key_type, nullable=False)) - value_field = (expected_type.item_field if data_type is None - else pa.field("value", expected_type.item_type, - nullable=data_type.value_type._nullable)) - _validate_array(column.keys.slice(start, end - start), key_field, - f"{path}.key", validity, - data_type.key_type if data_type is not None else None) - _validate_array(column.items.slice(start, end - start), value_field, - f"{path}.value", validity, - data_type.value_type if data_type is not None else None) - elif column.type != expected_type: - wrong_type() - - -def _get_validity(column, parent_validity): - import pyarrow.compute as pc - - # Resolve ancestor visibility only when a NOT NULL descendant contains physical nulls. - parent_validity = parent_validity() if parent_validity is not None else None - if column.null_count: - validity = column.is_valid() - return validity if parent_validity is None else pc.and_(parent_validity, validity) - return parent_validity - - -def _get_child_validity(column, parent_validity): - import numpy as np - import pyarrow as pa - - validity = _get_validity(column, parent_validity) - if validity is None: - return None - # Expand visibility directly instead of allocating an integer parent index for every child. - offsets = _get_offsets(column).to_numpy(zero_copy_only=True) - visible = validity.to_numpy(zero_copy_only=False) - return pa.array(np.repeat(visible, np.diff(offsets)), type=pa.bool_()) - - -def _get_offsets(column): - import pyarrow as pa - - if hasattr(column, 'offsets'): - return column.offsets - # PyArrow 5 does not expose MapArray.offsets, but uses the same int32 offsets as lists. - return pa.Array.from_buffers(pa.int32(), len(column) + 1, - [None, column.buffers()[1]], offset=column.offset) - - -def check_arrow_udf_result(func, *args): - import pyarrow as pa - - result = func(*args) - name = getattr(func, "__qualname__", type(func).__name__) - if not isinstance(result, (pa.Array, pa.ChunkedArray)): - raise TypeError( - f"Arrow UDF '{name}' must return a pyarrow.Array or pyarrow.ChunkedArray, " - f"got {type(result).__name__}.") - for arg in args: - if isinstance(arg, (pa.Array, pa.ChunkedArray)) and len(result) != len(arg): - raise ValueError( - f"Arrow UDF '{name}' returned {len(result)} rows, expected {len(arg)}.") - if isinstance(result, pa.ChunkedArray): - result = result.chunk(0) if result.num_chunks == 1 else result.combine_chunks() - return result - - -def create_record_batch(results, row_count): - import pyarrow as pa - - columns = [] - for result in results: - if len(result) != row_count: - raise ValueError(f"Arrow UDF returned {len(result)} rows, expected {row_count}.") - columns.append(result) - return pa.RecordBatch.from_arrays(columns, names=[f"f{i}" for i in range(len(columns))]) diff --git a/flink-python/pyflink/fn_execution/utils/operation_utils.py b/flink-python/pyflink/fn_execution/utils/operation_utils.py index c19422af688e5..5bb4ff182260c 100644 --- a/flink-python/pyflink/fn_execution/utils/operation_utils.py +++ b/flink-python/pyflink/fn_execution/utils/operation_utils.py @@ -93,6 +93,35 @@ def check_pandas_udf_result(f, *input_args): return output +def check_arrow_udf_result(func, *args): + import pyarrow as pa + + result = func(*args) + name = getattr(func, "__qualname__", type(func).__name__) + if not isinstance(result, (pa.Array, pa.ChunkedArray)): + raise TypeError( + f"Arrow UDF '{name}' must return a pyarrow.Array or pyarrow.ChunkedArray, " + f"got {type(result).__name__}.") + for arg in args: + if isinstance(arg, (pa.Array, pa.ChunkedArray)) and len(result) != len(arg): + raise ValueError( + f"Arrow UDF '{name}' returned {len(result)} rows, expected {len(arg)}.") + if isinstance(result, pa.ChunkedArray): + result = result.chunk(0) if result.num_chunks == 1 else result.combine_chunks() + return result + + +def create_record_batch(results, row_count): + import pyarrow as pa + + columns = [] + for result in results: + if len(result) != row_count: + raise ValueError(f"Arrow UDF returned {len(result)} rows, expected {row_count}.") + columns.append(result) + return pa.RecordBatch.from_arrays(columns, names=[f"f{i}" for i in range(len(columns))]) + + def extract_over_window_user_defined_function(user_defined_function_proto): window_index = user_defined_function_proto.window_index return (*extract_user_defined_function(user_defined_function_proto, True), window_index) @@ -161,7 +190,6 @@ def _extract_input(args) -> Tuple[str, Dict, List]: else: variable_dict[func_name] = user_defined_func.eval if user_defined_function_proto.is_arrow_udf: - from pyflink.fn_execution.utils.arrow_utils import check_arrow_udf_result variable_dict[func_name] = partial(check_arrow_udf_result, variable_dict[func_name]) user_defined_funcs.append(user_defined_func) diff --git a/flink-python/pyflink/table/utils.py b/flink-python/pyflink/table/utils.py index bc4444e66ebb9..be1e71f64b256 100644 --- a/flink-python/pyflink/table/utils.py +++ b/flink-python/pyflink/table/utils.py @@ -16,6 +16,7 @@ # limitations under the License. ################################################################################ import ast +from functools import cache from pyflink.common.types import RowKind @@ -28,6 +29,137 @@ import pickle +@Internal() +def validate_arrow_batch(batch, schema, field_types): + import pyarrow as pa + + if not isinstance(batch, pa.RecordBatch): + raise TypeError("Arrow transport expects a pyarrow.RecordBatch.") + if batch.num_columns != len(schema): + raise ValueError(f"Arrow result has {batch.num_columns} columns, expected {len(schema)}.") + # PyArrow 5 cannot express map-value nullability, so retain the logical constraint there. + legacy_map_fields = not hasattr(pa.MapType, 'item_field') + columns = [] + for index, (column, field) in enumerate(zip(batch.columns, schema)): + _validate_array(column, field, field.name, + data_type=field_types[index] if legacy_map_fields else None) + columns.append(_apply_arrow_type(column, field.type)) + return pa.RecordBatch.from_arrays(columns, schema=schema) + + +def _apply_arrow_type(column, expected_type): + import pyarrow as pa + + if column.type == expected_type: + return column + # Old Arrow casts reject valid nullability changes, and views inspect unsliced children. + # After validation, rebuild only container metadata while keeping the payload buffers. + if pa.types.is_struct(expected_type): + return pa.StructArray.from_arrays( + [_apply_arrow_type(column.field(index), field.type) + for index, field in enumerate(expected_type)], + fields=list(expected_type), mask=column.is_null() if column.null_count else None) + if pa.types.is_list(expected_type): + children = [_apply_arrow_type(column.values, expected_type.value_type)] + else: + value_field = (expected_type.item_field if hasattr(expected_type, 'item_field') + else pa.field("value", expected_type.item_type)) + children = [pa.StructArray.from_arrays( + [_apply_arrow_type(column.keys, expected_type.key_type), + _apply_arrow_type(column.items, expected_type.item_type)], + fields=[pa.field("key", expected_type.key_type, nullable=False), value_field])] + return pa.Array.from_buffers( + expected_type, len(column), column.buffers()[:expected_type.num_buffers], + null_count=column.null_count, offset=column.offset, children=children) + + +def _validate_array(column, field, path, parent_validity=None, data_type=None): + import pyarrow as pa + import pyarrow.compute as pc + expected_type = field.type + + def wrong_type(): + raise TypeError( + f"Arrow result field '{path}' has type {column.type}, expected {expected_type}.") + + if not field.nullable and column.null_count: + validity = parent_validity() if parent_validity is not None else None + if validity is None or pc.any(pc.and_(validity, column.is_null())).as_py(): + raise ValueError(f"Arrow result field '{path}' is not nullable.") + if pa.types.is_struct(expected_type): + if (not pa.types.is_struct(column.type) + or column.type.num_fields != expected_type.num_fields): + wrong_type() + validity = cache(lambda: _get_validity(column, parent_validity)) + for index, child_field in enumerate(expected_type): + if column.type[index].name != child_field.name: + wrong_type() + _validate_array(column.field(index), child_field, + f"{path}.{child_field.name}", validity, + data_type.fields[index].data_type if data_type is not None else None) + elif pa.types.is_list(expected_type): + if not pa.types.is_list(column.type): + wrong_type() + start, end = column.offsets[0].as_py(), column.offsets[-1].as_py() + _validate_array(column.values.slice(start, end - start), + expected_type.value_field, f"{path}[]", + cache(lambda: _get_child_validity(column, parent_validity)), + data_type.element_type if data_type is not None else None) + elif pa.types.is_map(expected_type): + if not pa.types.is_map(column.type): + wrong_type() + validity = cache(lambda: _get_child_validity(column, parent_validity)) + offsets = _get_offsets(column) + start, end = offsets[0].as_py(), offsets[-1].as_py() + key_field = (expected_type.key_field if data_type is None + else pa.field("key", expected_type.key_type, nullable=False)) + value_field = (expected_type.item_field if data_type is None + else pa.field("value", expected_type.item_type, + nullable=data_type.value_type._nullable)) + _validate_array(column.keys.slice(start, end - start), key_field, + f"{path}.key", validity, + data_type.key_type if data_type is not None else None) + _validate_array(column.items.slice(start, end - start), value_field, + f"{path}.value", validity, + data_type.value_type if data_type is not None else None) + elif column.type != expected_type: + wrong_type() + + +def _get_validity(column, parent_validity): + import pyarrow.compute as pc + + # Resolve ancestor visibility only when a NOT NULL descendant contains physical nulls. + parent_validity = parent_validity() if parent_validity is not None else None + if column.null_count: + validity = column.is_valid() + return validity if parent_validity is None else pc.and_(parent_validity, validity) + return parent_validity + + +def _get_child_validity(column, parent_validity): + import numpy as np + import pyarrow as pa + + validity = _get_validity(column, parent_validity) + if validity is None: + return None + # Expand visibility directly instead of allocating an integer parent index for every child. + offsets = _get_offsets(column).to_numpy(zero_copy_only=True) + visible = validity.to_numpy(zero_copy_only=False) + return pa.array(np.repeat(visible, np.diff(offsets)), type=pa.bool_()) + + +def _get_offsets(column): + import pyarrow as pa + + if hasattr(column, 'offsets'): + return column.offsets + # PyArrow 5 does not expose MapArray.offsets, but uses the same int32 offsets as lists. + return pa.Array.from_buffers(pa.int32(), len(column) + 1, + [None, column.buffers()[1]], offset=column.offset) + + @Internal() def pandas_to_arrow(schema, timezone, field_types, series): import pyarrow as pa From b63a1288f26c9018157530005a4295b334b6f88f Mon Sep 17 00:00:00 2001 From: auroflow Date: Fri, 18 Sep 2026 12:20:02 +0800 Subject: [PATCH 13/14] [FLINK-40472][python] Separate map writer fix from Arrow UDF support Remove the independent map-writer fix and its regression test from this feature change so they can be submitted and backported separately. Generated-by: Codex (GPT-6) --- .../runtime/arrow/writers/MapWriter.java | 7 -- .../runtime/arrow/ArrowReaderWriterTest.java | 92 ------------------- 2 files changed, 99 deletions(-) diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java b/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java index d9d03f32449da..c2c960c332873 100644 --- a/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/writers/MapWriter.java @@ -84,13 +84,6 @@ public void doWrite(T in, int ordinal) { } } - @Override - public void reset() { - super.reset(); - keyWriter.reset(); - valueWriter.reset(); - } - // ------------------------------------------------------------------------------------------ /** {@link MapWriter} for {@link RowData} input. */ diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java index 2e4c18a3a4794..6441246a9be45 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/ArrowReaderWriterTest.java @@ -21,7 +21,6 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.GenericArrayData; -import org.apache.flink.table.data.GenericMapData; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; @@ -40,7 +39,6 @@ import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.types.logical.SmallIntType; import org.apache.flink.table.types.logical.TimeType; @@ -55,24 +53,15 @@ import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Objects; -import static org.assertj.core.api.Assertions.assertThat; - /** Tests for {@link ArrowReader} and {@link ArrowWriter} of RowData. */ class ArrowReaderWriterTest extends ArrowReaderWriterTestBase { private static List fieldTypes = new ArrayList<>(); @@ -179,87 +168,6 @@ public Tuple2, ArrowStreamWriter> createArrowWriter( return Tuple2.of(arrowWriter, arrowStreamWriter); } - @ParameterizedTest - @ValueSource(ints = {1, 2}) - void testMapsAcrossBatches(int batchSize) throws IOException { - final MapType mapType = new MapType(new IntType(false), new IntType()); - final MapType nestedMapType = - new MapType(new IntType(false), RowType.of(RowType.of(new IntType()))); - final RowType type = - RowType.of( - mapType, new ArrayType(mapType), new ArrayType(RowType.of(nestedMapType))); - final List rows = new ArrayList<>(); - for (Map values : - Arrays.asList( - Map.of(1, 11, 2, 22), - Collections.emptyMap(), - null, - Collections.singletonMap(3, null), - Map.of(4, 44))) { - final GenericMapData map = values == null ? null : new GenericMapData(values); - final Map nestedValues = new LinkedHashMap<>(); - if (values != null) { - values.forEach( - (key, value) -> - nestedValues.put( - key, - value == null - ? null - : GenericRowData.of(GenericRowData.of(value)))); - } - rows.add( - GenericRowData.of( - map, - new GenericArrayData(new Object[] {map}), - new GenericArrayData( - new Object[] { - GenericRowData.of( - values == null - ? null - : new GenericMapData(nestedValues)) - }))); - } - - final ByteArrayOutputStream output = new ByteArrayOutputStream(); - try (BufferAllocator batchAllocator = - ArrowUtils.getRootAllocator() - .newChildAllocator("map-batches", 0, Long.MAX_VALUE); - VectorSchemaRoot root = - VectorSchemaRoot.create(ArrowUtils.toArrowSchema(type), batchAllocator); - ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, output)) { - final ArrowWriter writer = ArrowUtils.createRowDataArrowWriter(root, type); - streamWriter.start(); - for (int start = 0; start < rows.size(); start += batchSize) { - for (RowData row : rows.subList(start, Math.min(start + batchSize, rows.size()))) { - writer.write(row); - } - writer.finish(); - streamWriter.writeBatch(); - writer.reset(); - } - streamWriter.end(); - - try (ArrowStreamReader streamReader = - new ArrowStreamReader( - new ByteArrayInputStream(output.toByteArray()), batchAllocator)) { - final RowDataSerializer serializer = new RowDataSerializer(type); - final ArrowReader reader = - ArrowUtils.createArrowReader(streamReader.getVectorSchemaRoot(), type); - int rowIndex = 0; - while (streamReader.loadNextBatch()) { - final int rowCount = streamReader.getVectorSchemaRoot().getRowCount(); - assertThat(rowCount).isEqualTo(Math.min(batchSize, rows.size() - rowIndex)); - for (int i = 0; i < rowCount; i++) { - assertThat(serializer.toBinaryRow(reader.read(i)).copy()) - .as("row %s", rowIndex) - .isEqualTo(serializer.toBinaryRow(rows.get(rowIndex++)).copy()); - } - } - assertThat(rowIndex).isEqualTo(rows.size()); - } - } - } - @Override public RowData[] getTestData() { RowData row1 = From c410f540729aef49246d943fd999fa0d1599b8b0 Mon Sep 17 00:00:00 2001 From: auroflow Date: Fri, 18 Sep 2026 12:20:06 +0800 Subject: [PATCH 14/14] [FLINK-40472][python] Reject async callables for pandas UDFs Apply the Arrow declaration check to pandas mode as well, catching asynchronous callable objects and scalar eval methods before synchronous execution. Generated-by: Codex (GPT-6) --- flink-python/pyflink/table/udf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-python/pyflink/table/udf.py b/flink-python/pyflink/table/udf.py index 7035b6ac91dc0..733b8a7917806 100644 --- a/flink-python/pyflink/table/udf.py +++ b/flink-python/pyflink/table/udf.py @@ -762,7 +762,7 @@ def _get_python_env(): def _create_udf(f, input_types, result_type, func_type, deterministic, name): - if func_type == 'arrow': + if func_type in ('pandas', 'arrow'): target = f while isinstance(target, functools.partial): target = target.func @@ -770,7 +770,7 @@ def _create_udf(f, input_types, result_type, func_type, deterministic, name): target = target.eval if inspect.iscoroutinefunction(target) or inspect.iscoroutinefunction( getattr(target, '__call__', None)): - raise ValueError("Async scalar functions do not support arrow func_type.") + raise ValueError(f"Async scalar functions do not support {func_type} func_type.") if isinstance(f, AsyncScalarFunction) or inspect.iscoroutinefunction(f): if func_type in ('pandas', 'arrow'): raise ValueError(