diff --git a/cpp/src/arrow/json/from_string.cc b/cpp/src/arrow/json/from_string.cc index 5694fbde4c53..c9d910667133 100644 --- a/cpp/src/arrow/json/from_string.cc +++ b/cpp/src/arrow/json/from_string.cc @@ -47,6 +47,7 @@ #include "arrow/util/decimal.h" #include "arrow/util/float16.h" #include "arrow/util/logging_internal.h" +#include "arrow/util/simdjson_internal.h" #include "arrow/util/unreachable.h" #include "arrow/util/value_parsing.h" @@ -66,116 +67,6 @@ using ::arrow::internal::checked_pointer_cast; namespace { -const char* JsonTypeName(sj::json_type type) { - switch (type) { - case sj::json_type::array: - return "array"; - case sj::json_type::object: - return "object"; - case sj::json_type::number: - return "number"; - case sj::json_type::string: - return "string"; - case sj::json_type::boolean: - return "boolean"; - case sj::json_type::null: - return "null"; - default: - return "unknown"; - } -} - -// Empty struct to represent the type of a simdjson null value -struct SimdjsonNull {}; - -template -struct JsonTypeNameOf; - -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "array"; -}; -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "object"; -}; -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "string"; -}; -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "boolean"; -}; -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "null"; -}; -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "number"; -}; -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "number"; -}; -template <> -struct JsonTypeNameOf { - static constexpr const char* kValue = "number"; -}; - -template -constexpr const char* JsonTypeName() { - return JsonTypeNameOf::kValue; -} - -template -Result GetJsonAs(sj::value& value) { - SimdjsonValueType typed_value{}; - simdjson::error_code error_code; - if constexpr (std::is_same_v) { - // simdjson has no get<>() for null; probe it explicitly - bool is_null; - error_code = value.is_null().get(is_null); - if (error_code == simdjson::SUCCESS && !is_null) { - error_code = simdjson::INCORRECT_TYPE; - } - } else { - error_code = value.get(typed_value); - } - if (error_code != simdjson::SUCCESS) { - sj::json_type json_type; - if (value.type().get(json_type) != simdjson::SUCCESS) { - return Status::Invalid("Expected ", JsonTypeName(), - " or null, got malformed JSON value"); - } - return Status::Invalid("Expected ", JsonTypeName(), - " or null, got JSON type ", JsonTypeName(json_type)); - } - return typed_value; -} - -template -Result GetJsonResult( - simdjson::simdjson_result element, std::string_view error) { - SimdjsonValueType typed_value; - if (auto error_code = std::move(element).get(typed_value); - error_code != simdjson::SUCCESS) { - return Status::Invalid(error, simdjson::error_message(error_code)); - } - return typed_value; -} - -// Result because peeking the nonRootScalar can fail (parsed lazily) -Result IsJsonNull(sj::value& value) { - bool is_null; - if (auto error_code = value.is_null().get(is_null); error_code != simdjson::SUCCESS) { - return Status::Invalid("Error checking for JSON null: ", - simdjson::error_message(error_code)); - } - return is_null; -} - class JSONConverter { public: virtual ~JSONConverter() = default; @@ -215,7 +106,7 @@ class ConcreteConverter : public JSONConverter { int32_t num_elements = 0; for (auto element : json_array) { ARROW_ASSIGN_OR_RAISE(auto value, - GetJsonResult( + internal::GetSimdjsonResult( element, "Could not iterate elements of JSON array: ")); RETURN_NOT_OK(self->AppendValue(value)); num_elements++; @@ -251,7 +142,7 @@ class NullConverter final : public ConcreteConverter { } Status AppendValue(sj::value& json_obj) override { - ARROW_RETURN_NOT_OK(GetJsonAs(json_obj)); + ARROW_RETURN_NOT_OK(internal::GetJsonAs(json_obj)); return AppendNull(); } @@ -272,7 +163,7 @@ class BooleanConverter final : public ConcreteConverter { } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return AppendNull(); } @@ -280,7 +171,7 @@ class BooleanConverter final : public ConcreteConverter { if (json_obj.get(int_value) == simdjson::SUCCESS) { return builder_->Append(int_value != 0); } - ARROW_ASSIGN_OR_RAISE(bool bool_value, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool bool_value, internal::GetJsonAs(json_obj)); return builder_->Append(bool_value); } @@ -299,7 +190,7 @@ enable_if_physical_signed_integer ConvertNumber(sj::value& json_obj, const DataType& type, typename T::c_type* out) { *out = static_cast(0); - ARROW_ASSIGN_OR_RAISE(int64_t v64, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(int64_t v64, internal::GetJsonAs(json_obj)); *out = static_cast(v64); if (*out == v64) { return Status::OK(); @@ -314,7 +205,7 @@ enable_if_unsigned_integer ConvertNumber(sj::value& json_obj, const DataType& type, typename T::c_type* out) { *out = static_cast(0); - ARROW_ASSIGN_OR_RAISE(uint64_t v64, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(uint64_t v64, internal::GetJsonAs(json_obj)); *out = static_cast(v64); if (*out == v64) { return Status::OK(); @@ -356,7 +247,7 @@ enable_if_half_float ConvertNumber(sj::value& json_obj, const DataTyp *out = Float16(f64.value()).bits(); return Status::OK(); } - ARROW_ASSIGN_OR_RAISE(auto f64, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto f64, internal::GetJsonAs(json_obj)); *out = Float16(f64).bits(); return arrow::Status::OK(); } @@ -371,7 +262,7 @@ enable_if_physical_floating_point ConvertNumber(sj::value& json_obj, *out = static_cast(f64.value()); return Status::OK(); } - ARROW_ASSIGN_OR_RAISE(auto f64, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto f64, internal::GetJsonAs(json_obj)); *out = static_cast(f64); return arrow::Status::OK(); } @@ -395,9 +286,9 @@ Status ProcessJsonArrayElements( " elements, had ", index); } - ARROW_ASSIGN_OR_RAISE( - sj::value element, - GetJsonResult(*it, "Could not iterate elements of JSON array: ")); + ARROW_ASSIGN_OR_RAISE(sj::value element, + internal::GetSimdjsonResult( + *it, "Could not iterate elements of JSON array: ")); RETURN_NOT_OK(handler(element)); ++it; ++index; @@ -426,7 +317,7 @@ class IntegerConverter final Status Init() override { return this->MakeConcreteBuilder(&builder_); } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } @@ -454,7 +345,7 @@ class FloatConverter final : public ConcreteConverterMakeConcreteBuilder(&builder_); } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } @@ -485,11 +376,12 @@ class DecimalConverter final Status Init() override { return this->MakeConcreteBuilder(&builder_); } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } - ARROW_ASSIGN_OR_RAISE(auto string_value, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto string_value, + internal::GetJsonAs(json_obj)); int32_t precision, scale; DecimalValue d; RETURN_NOT_OK(DecimalValue::FromString(string_value, &d, &precision, &scale)); @@ -528,7 +420,7 @@ class TimestampConverter final : public ConcreteConverter { } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } @@ -563,12 +455,12 @@ class DayTimeIntervalConverter final } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs(json_obj)); DayTimeIntervalType::DayMilliseconds value; RETURN_NOT_OK(ProcessJsonArrayElements<2>( @@ -597,12 +489,12 @@ class MonthDayNanoIntervalConverter final } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs(json_obj)); MonthDayNanoIntervalType::MonthDayNanos value; RETURN_NOT_OK(ProcessJsonArrayElements<3>( @@ -637,12 +529,12 @@ class StringConverter final Status Init() override { return this->MakeConcreteBuilder(&builder_); } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } - ARROW_ASSIGN_OR_RAISE(auto view, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto view, internal::GetJsonAs(json_obj)); return builder_->Append(view); } @@ -666,11 +558,11 @@ class FixedSizeBinaryConverter final Status Init() override { return this->MakeConcreteBuilder(&builder_); } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } - ARROW_ASSIGN_OR_RAISE(auto view, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto view, internal::GetJsonAs(json_obj)); if (view.length() != static_cast(builder_->byte_width())) { std::stringstream ss; ss << "Invalid string length " << view.length() << " in JSON input for " @@ -710,11 +602,11 @@ class VarLengthListLikeConverter final } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs(json_obj)); size_t num_elements; if (array.count_elements().get(num_elements) != simdjson::SUCCESS) { return Status::Invalid("Malformed JSON array for type ", this->type_->ToString()); @@ -750,24 +642,25 @@ class MapConverter final : public ConcreteConverter { } Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } RETURN_NOT_OK(builder_->Append()); - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs(json_obj)); for (auto json_pair_result : array) { ARROW_ASSIGN_OR_RAISE( auto json_pair, - GetJsonResult(json_pair_result, - "Could not iterate elements of JSON array: ")); - ARROW_ASSIGN_OR_RAISE(auto json_pair_array, GetJsonAs(json_pair)); + internal::GetSimdjsonResult( + json_pair_result, "Could not iterate elements of JSON array: ")); + ARROW_ASSIGN_OR_RAISE(auto json_pair_array, + internal::GetJsonAs(json_pair)); RETURN_NOT_OK(ProcessJsonArrayElements<2>( json_pair_array, "key-item pair", {[this](sj::value& key) { - ARROW_ASSIGN_OR_RAISE(bool key_is_null, IsJsonNull(key)); + ARROW_ASSIGN_OR_RAISE(bool key_is_null, internal::IsJsonNull(key)); if (key_is_null) { return Status::Invalid("null key is invalid"); } @@ -803,13 +696,13 @@ class FixedSizeListConverter final : public ConcreteConverterAppendNull(); } RETURN_NOT_OK(builder_->Append()); // Extend the child converter with this JSON array - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs(json_obj)); ARROW_ASSIGN_OR_RAISE(int32_t size, child_converter_->AppendValues(array)); if (size != list_size_) { return Status::Invalid("incorrect list size ", size); @@ -852,7 +745,7 @@ class StructConverter final : public ConcreteConverter { // or an object mapping struct names to values (omitted struct members // are mapped to null). Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } @@ -870,14 +763,14 @@ class StructConverter final : public ConcreteConverter { size_t i = 0; for (auto child : array) { ARROW_ASSIGN_OR_RAISE(auto child_value, - GetJsonResult( + internal::GetSimdjsonResult( child, "Could not iterate elements of JSON array: ")); RETURN_NOT_OK(child_converters_[i]->AppendValue(child_value)); ++i; } return builder_->Append(); } - ARROW_ASSIGN_OR_RAISE(auto object, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto object, internal::GetJsonAs(json_obj)); // Iterate the object fields in JSON order (the on-demand API is // forward-only, so per-field lookups would be quadratic and would also // compare against raw, still-escaped keys). Fields absent from the JSON @@ -885,9 +778,9 @@ class StructConverter final : public ConcreteConverter { auto num_fields = type_->num_fields(); std::vector field_seen(num_fields, false); for (auto field_result : object) { - ARROW_ASSIGN_OR_RAISE( - auto field, - GetJsonResult(field_result, "Error getting field of object: ")); + ARROW_ASSIGN_OR_RAISE(auto field, + internal::GetSimdjsonResult( + field_result, "Error getting field of object: ")); std::string_view key; if (field.unescaped_key(/*allow_replacement=*/false).get(key) != simdjson::SUCCESS) { @@ -961,12 +854,12 @@ class UnionConverter final : public ConcreteConverter { // Append a JSON value that must be a 2-long array, containing the type_id // and value of the UnionArray's slot. Status AppendValue(sj::value& json_obj) override { - ARROW_ASSIGN_OR_RAISE(bool is_null, IsJsonNull(json_obj)); + ARROW_ASSIGN_OR_RAISE(bool is_null, internal::IsJsonNull(json_obj)); if (is_null) { return this->AppendNull(); } - ARROW_ASSIGN_OR_RAISE(auto array, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs(json_obj)); int8_t id = 0; std::shared_ptr child_converter; @@ -974,7 +867,7 @@ class UnionConverter final : public ConcreteConverter { RETURN_NOT_OK(ProcessJsonArrayElements<2>( array, "[type_id, value] pair", {[this, &id, &child_converter](sj::value& id_elem) { - ARROW_ASSIGN_OR_RAISE(auto id_value, GetJsonAs(id_elem)); + ARROW_ASSIGN_OR_RAISE(auto id_value, internal::GetJsonAs(id_elem)); id = static_cast(id_value); auto child_num = type_id_to_child_num_[id]; if (child_num == -1) { @@ -1156,7 +1049,7 @@ Result> ArrayFromJSONString(const std::shared_ptr(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto array, internal::GetJsonAs(json_obj)); // The JSON document should be an array, append it RETURN_NOT_OK(converter->AppendValues(array)); @@ -1226,7 +1119,7 @@ Result> ScalarFromJSONString( error_code != simdjson::SUCCESS) { return Status::Invalid("JSON parse error: ", simdjson::error_message(error_code)); } - ARROW_ASSIGN_OR_RAISE(auto singleton_array, GetJsonAs(json_obj)); + ARROW_ASSIGN_OR_RAISE(auto singleton_array, internal::GetJsonAs(json_obj)); ARROW_ASSIGN_OR_RAISE(int32_t num_elements, converter->AppendValues(singleton_array)); if (num_elements != 1) { diff --git a/cpp/src/arrow/json/json_writer_internal.cc b/cpp/src/arrow/json/json_writer_internal.cc index 446c7f06f4bf..567694902497 100644 --- a/cpp/src/arrow/json/json_writer_internal.cc +++ b/cpp/src/arrow/json/json_writer_internal.cc @@ -16,9 +16,12 @@ // under the License. #include "arrow/json/json_writer_internal.h" +#include "arrow/util/simdjson_internal.h" namespace arrow::json { +namespace sj = simdjson::ondemand; + void JsonWriter::StartObject() { MaybeComma(); builder_.start_object(); @@ -96,6 +99,85 @@ void JsonWriter::Double(double value) { needs_comma_ = true; } +Status JsonWriter::WriteValue(sj::value value) { + return internal::VisitJsonValue( + value, + + [&](sj::object object) -> Status { + StartObject(); + + for (auto field : object) { + ARROW_ASSIGN_OR_RAISE( + auto key, internal::GetSimdjsonResult(field.unescaped_key(), + "Failed to get object key: ")); + + Key(key); + + ARROW_ASSIGN_OR_RAISE( + auto field_value, + internal::GetSimdjsonResult(field.value(), "Failed to get object value: ")); + + RETURN_NOT_OK(WriteValue(field_value)); + } + + EndObject(); + return Status::OK(); + }, + + [&](sj::array array) -> Status { + StartArray(); + + for (auto element : array) { + ARROW_ASSIGN_OR_RAISE( + auto element_value, + internal::GetSimdjsonResult(element, "Failed to iterate JSON array: ")); + + RETURN_NOT_OK(WriteValue(element_value)); + } + + EndArray(); + return Status::OK(); + }, + + [&](std::string_view string_value) -> Status { + String(string_value); + return Status::OK(); + }, + + [&](bool bool_value) -> Status { + Bool(bool_value); + return Status::OK(); + }, + + [&]() -> Status { + Null(); + return Status::OK(); + }, + + [&](int64_t value) -> Status { + Int64(value); + return Status::OK(); + }, + + [&](uint64_t value) -> Status { + Uint64(value); + return Status::OK(); + }, + + [&](double value) -> Status { + Double(value); + return Status::OK(); + }, + + [&](sj::value value) -> Status { + ARROW_ASSIGN_OR_RAISE(auto raw_json, + internal::GetSimdjsonResult(simdjson::to_json_string(value), + "Failed to get raw JSON: ")); + RawValue(raw_json); + return Status::OK(); + }); +} + void JsonWriter::Null() { MaybeComma(); builder_.append_null(); diff --git a/cpp/src/arrow/json/json_writer_internal.h b/cpp/src/arrow/json/json_writer_internal.h index e6043bd7e5ac..4a8200f1df30 100644 --- a/cpp/src/arrow/json/json_writer_internal.h +++ b/cpp/src/arrow/json/json_writer_internal.h @@ -23,6 +23,7 @@ #include #include "arrow/result.h" +#include "arrow/status.h" #include "arrow/util/visibility.h" namespace arrow::json { @@ -51,6 +52,8 @@ class ARROW_EXPORT JsonWriter { void Double(double value); + Status WriteValue(simdjson::ondemand::value value); + void Null(); void StringField(std::string_view key, std::string_view value); diff --git a/cpp/src/arrow/json/json_writer_internal_test.cc b/cpp/src/arrow/json/json_writer_internal_test.cc index abdf9be7af09..7fc12e8adbe6 100644 --- a/cpp/src/arrow/json/json_writer_internal_test.cc +++ b/cpp/src/arrow/json/json_writer_internal_test.cc @@ -20,6 +20,8 @@ #include "arrow/json/json_writer_internal.h" #include "arrow/testing/gtest_util.h" +namespace sj = simdjson::ondemand; + namespace arrow::json { TEST(JsonWriter, SimpleObject) { @@ -170,4 +172,122 @@ TEST(JsonWriter, StringWithExplicitLength) { EXPECT_EQ(json, R"({"value":"abc"})"); } +TEST(JsonWriter, WriteValueSimpleObject) { + sj::parser parser; + std::string json_str = R"({"a":42,"b":"hello"})"; + simdjson::padded_string json(json_str); + + sj::document doc; + ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS); + + sj::value value; + ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS); + + JsonWriter writer; + ASSERT_OK(writer.WriteValue(value)); + + ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString()); + EXPECT_EQ(out, R"({"a":42,"b":"hello"})"); +} + +TEST(JsonWriter, WriteValueNestedObject) { + sj::parser parser; + std::string json_str = R"({"child":{"x":true}})"; + simdjson::padded_string json(json_str); + + sj::document doc; + ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS); + + sj::value value; + ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS); + + JsonWriter writer; + ASSERT_OK(writer.WriteValue(value)); + + ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString()); + EXPECT_EQ(out, R"({"child":{"x":true}})"); +} + +TEST(JsonWriter, WriteValueObjectWithArray) { + sj::parser parser; + std::string json_str = R"({"values":[1,2,3]})"; + simdjson::padded_string json(json_str); + + sj::document doc; + ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS); + + sj::value value; + ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS); + + JsonWriter writer; + ASSERT_OK(writer.WriteValue(value)); + + ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString()); + EXPECT_EQ(out, R"({"values":[1,2,3]})"); +} + +TEST(JsonWriter, WriteValueComplexObject) { + sj::parser parser; + std::string json_str = + R"({"name":"arrow","version":1,"enabled":true,"values":[1,2.5,null,{"nested":[false,{"x":10}]}]})"; + simdjson::padded_string json(json_str); + + sj::document doc; + ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS); + + sj::value value; + ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS); + + JsonWriter writer; + ASSERT_OK(writer.WriteValue(value)); + + ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString()); + EXPECT_EQ( + out, + R"({"name":"arrow","version":1,"enabled":true,"values":[1,2.5,null,{"nested":[false,{"x":10}]}]})"); +} + +TEST(JsonWriter, WriteValueEmptyObject) { + sj::parser parser; + std::string json_str = "{}"; + simdjson::padded_string json(json_str); + + sj::document doc; + ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS); + + sj::value value; + ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS); + + JsonWriter writer; + ASSERT_OK(writer.WriteValue(value)); + + ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString()); + EXPECT_EQ(out, "{}"); +} + +TEST(JsonWriter, WriteValueAllNumberTypes) { + sj::parser parser; + std::string json_str = R"({ + "signed":-42, + "unsigned":18446744073709551615, + "double":2.5, + "big":184467440737095516161234567890 + })"; + simdjson::padded_string json(json_str); + + sj::document doc; + ASSERT_EQ(parser.iterate(json).get(doc), simdjson::SUCCESS); + + sj::value value; + ASSERT_EQ(doc.get_value().get(value), simdjson::SUCCESS); + + JsonWriter writer; + ASSERT_OK(writer.WriteValue(value)); + + ASSERT_OK_AND_ASSIGN(std::string_view out, writer.GetString()); + EXPECT_EQ( + out, + R"({"signed":-42,"unsigned":18446744073709551615,"double":2.5,"big":184467440737095516161234567890})"); +} + } // namespace arrow::json diff --git a/cpp/src/arrow/util/simdjson_internal.h b/cpp/src/arrow/util/simdjson_internal.h new file mode 100644 index 000000000000..8ffb741da42e --- /dev/null +++ b/cpp/src/arrow/util/simdjson_internal.h @@ -0,0 +1,244 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include + +#include "arrow/result.h" +#include "arrow/status.h" + +namespace arrow { +namespace internal { + +// Empty struct to represent the type of a simdjson null value +struct SimdjsonNull {}; + +template +struct JsonTypeNameOf; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "array"; +}; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "object"; +}; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "string"; +}; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "boolean"; +}; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "null"; +}; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "number"; +}; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "number"; +}; + +template <> +struct JsonTypeNameOf { + static constexpr const char* kValue = "number"; +}; + +template +constexpr const char* JsonTypeName() { + return JsonTypeNameOf::kValue; +} + +template +Result GetSimdjsonResult(simdjson::simdjson_result result, std::string_view error) { + T value; + if (auto error_code = std::move(result).get(value); error_code != simdjson::SUCCESS) { + return Status::Invalid(error, simdjson::error_message(error_code)); + } + return value; +} + +template +Status VisitJsonValue(simdjson::ondemand::value value, ObjectFn&& object_fn, + ArrayFn&& array_fn, StringFn&& string_fn, BoolFn&& bool_fn, + NullFn&& null_fn, Int64Fn&& int64_fn, Uint64Fn&& uint64_fn, + DoubleFn&& double_fn, BigIntegerFn&& big_integer_fn) { + ARROW_ASSIGN_OR_RAISE( + auto type, GetSimdjsonResult(value.type(), "Failed to determine JSON type: ")); + + switch (type) { + case simdjson::ondemand::json_type::object: { + ARROW_ASSIGN_OR_RAISE( + auto object, + GetSimdjsonResult(value.get_object(), "Failed to get JSON object: ")); + return object_fn(object); + } + + case simdjson::ondemand::json_type::array: { + ARROW_ASSIGN_OR_RAISE( + auto array, GetSimdjsonResult(value.get_array(), "Failed to get JSON array: ")); + return array_fn(array); + } + + case simdjson::ondemand::json_type::string: { + ARROW_ASSIGN_OR_RAISE( + auto string, + GetSimdjsonResult(value.get_string(), "Failed to get JSON string: ")); + return string_fn(string); + } + + case simdjson::ondemand::json_type::boolean: { + ARROW_ASSIGN_OR_RAISE( + auto boolean, + GetSimdjsonResult(value.get_bool(), "Failed to get JSON boolean: ")); + return bool_fn(boolean); + } + + case simdjson::ondemand::json_type::null: + return null_fn(); + + case simdjson::ondemand::json_type::number: { + ARROW_ASSIGN_OR_RAISE(auto number_type, + GetSimdjsonResult(value.get_number_type(), + "Failed to determine JSON number type: ")); + + switch (number_type) { + case simdjson::ondemand::number_type::signed_integer: { + ARROW_ASSIGN_OR_RAISE( + auto number, + GetSimdjsonResult(value.get_int64(), "Failed to get signed integer: ")); + return int64_fn(number); + } + + case simdjson::ondemand::number_type::unsigned_integer: { + ARROW_ASSIGN_OR_RAISE( + auto number, + GetSimdjsonResult(value.get_uint64(), "Failed to get unsigned integer: ")); + return uint64_fn(number); + } + + case simdjson::ondemand::number_type::floating_point_number: { + ARROW_ASSIGN_OR_RAISE( + auto number, GetSimdjsonResult(value.get_double(), + "Failed to get floating-point number: ")); + return double_fn(number); + } + + case simdjson::ondemand::number_type::big_integer: + return big_integer_fn(value); + } + + return Status::Invalid("Unknown JSON number type"); + } + + case simdjson::ondemand::json_type::unknown: + return Status::Invalid("Unknown JSON type"); + } + + return Status::Invalid("Unreachable"); +} + +inline const char* JsonTypeName(simdjson::ondemand::json_type type) { + switch (type) { + case simdjson::ondemand::json_type::array: + return "array"; + case simdjson::ondemand::json_type::object: + return "object"; + case simdjson::ondemand::json_type::number: + return "number"; + case simdjson::ondemand::json_type::string: + return "string"; + case simdjson::ondemand::json_type::boolean: + return "boolean"; + case simdjson::ondemand::json_type::null: + return "null"; + default: + return "unknown"; + } +} + +// Result because peeking the nonRootScalar can fail (parsed lazily) +inline Result IsJsonNull(simdjson::ondemand::value& value) { + bool is_null; + auto error_code = value.is_null().get(is_null); + if (error_code != simdjson::SUCCESS) { + return Status::Invalid("Error checking for JSON null: ", + simdjson::error_message(error_code)); + } + return is_null; +} + +template +Result GetJsonAs(simdjson::ondemand::value& value) { + SimdjsonValueType typed_value{}; + simdjson::error_code error_code; + + if constexpr (std::same_as) { + // simdjson has no get<>() for null; probe it explicitly + bool is_null; + error_code = value.is_null().get(is_null); + if (error_code == simdjson::SUCCESS && !is_null) { + error_code = simdjson::INCORRECT_TYPE; + } + } else { + error_code = value.get(typed_value); + } + + if (error_code != simdjson::SUCCESS) { + simdjson::ondemand::json_type json_type; + if (value.type().get(json_type) != simdjson::SUCCESS) { + if constexpr (std::same_as) { + return Status::Invalid("Expected null, got malformed JSON value"); + } else { + return Status::Invalid("Expected ", JsonTypeName(), + ", got malformed JSON value"); + } + } + + if constexpr (std::same_as) { + return Status::Invalid("Expected null, got JSON type ", JsonTypeName(json_type)); + } else { + return Status::Invalid("Expected ", JsonTypeName(), + ", got JSON type ", JsonTypeName(json_type)); + } + } + + return typed_value; +} + +} // namespace internal +} // namespace arrow