Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions cpp/src/gandiva/gdv_function_stubs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,7 @@ const char* gdv_mask_first_n_utf8_int32(int64_t context, const char* data,
while ((chars_masked < n_to_mask) && (bytes_masked < data_len)) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data + bytes_masked),
data_len, &utf8_char);
data_len - bytes_masked, &utf8_char);

if (char_len < 0) {
gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len));
Expand Down Expand Up @@ -592,7 +592,7 @@ const char* gdv_mask_last_n_utf8_int32(int64_t context, const char* data,
while ((bytes_read < data_len) && (chars_counter < (num_of_chars - n_to_mask))) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data + bytes_read),
data_len, &utf8_char);
data_len - bytes_read, &utf8_char);
chars_counter++;
bytes_read += static_cast<int>(char_len);
}
Expand All @@ -606,7 +606,7 @@ const char* gdv_mask_last_n_utf8_int32(int64_t context, const char* data,
while (bytes_read < data_len) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data + bytes_read),
data_len, &utf8_char);
data_len - bytes_read, &utf8_char);
switch (utf8proc_category(utf8_char)) {
case 1:
out[out_idx] = 'X';
Expand Down Expand Up @@ -695,7 +695,12 @@ const char* mask_utf8_utf8_utf8_utf8(int64_t context, const char* data, int32_t
while (bytes_read < data_len) {
auto char_len =
utf8proc_iterate(reinterpret_cast<const utf8proc_uint8_t*>(data + bytes_read),
data_len, &utf8_char);
data_len - bytes_read, &utf8_char);
if (char_len < 0) {
gdv_fn_context_set_error_msg(context, utf8proc_errmsg(char_len));
*out_len = 0;
return nullptr;
}
switch (utf8proc_category(utf8_char)) {
case UTF8PROC_CATEGORY_LU:
memcpy(out + out_index, upper, upper_length);
Expand Down
28 changes: 28 additions & 0 deletions cpp/src/gandiva/gdv_function_stubs_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1151,6 +1151,34 @@ TEST(TestGdvFnStubs, TestMaskLastN) {
EXPECT_EQ(expected, std::string(result, out_len));
}

TEST(TestGdvFnStubs, TestMaskTruncatedUtf8NoOverread) {
gandiva::ExecutionContext ctx;
int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = -1;

// A byte > 127 routes the mask functions through the utf8proc multi-byte path.
// The buffer holds a complete euro sign (0xE2 0x82 0xAC) but the reported
// length stops one byte short, so the trailing glyph is truncated. The
// functions must not read the byte past data_len: utf8proc_iterate has to be
// told only data_len - offset bytes remain, otherwise it consumes the
// out-of-range continuation byte and decodes a full glyph instead of
// reporting the truncated input.
const char buf[] = {'a', static_cast<char>(0xE2), static_cast<char>(0x82),
static_cast<char>(0xAC)};
const int32_t truncated_len = 3; // 'a' + first two bytes of the euro sign

ctx.Reset();
gdv_mask_first_n_utf8_int32(ctx_ptr, buf, truncated_len, 4, &out_len);
EXPECT_EQ(out_len, 0);
EXPECT_TRUE(ctx.has_error());

out_len = -1;
ctx.Reset();
mask_utf8(ctx_ptr, buf, truncated_len, &out_len);
EXPECT_EQ(out_len, 0);
EXPECT_TRUE(ctx.has_error());
}

TEST(TestGdvFnStubs, TestTranslate) {
gandiva::ExecutionContext ctx;
int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
Expand Down
Loading