From e48a9327d9ff94ae816aa21e197e29c45c060714 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 11:56:54 +0200 Subject: [PATCH 1/8] Add map_unary --- CMakeLists.txt | 4 - include/xsimd_algorithm/builder.hpp | 176 ++++++++++++++++++++++++++++ include/xsimd_algorithm/macros.hpp | 20 ++++ test/CMakeLists.txt | 1 + test/test_builder.cpp | 78 ++++++++++++ 5 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 include/xsimd_algorithm/builder.hpp create mode 100644 include/xsimd_algorithm/macros.hpp create mode 100644 test/test_builder.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f7a6565..790e26f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,10 +22,6 @@ endif() # Build # ===== -set(XSIMDALGO_HEADERS - ${XSIMDALGO_INCLUDE_DIR}/xsimd_algorithm/algorithms.hpp -) - add_library(xsimd-algorithm INTERFACE) target_include_directories(xsimd-algorithm INTERFACE diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp new file mode 100644 index 0000000..ab6aa21 --- /dev/null +++ b/include/xsimd_algorithm/builder.hpp @@ -0,0 +1,176 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_BUILDER_HPP +#define XSIMD_ALGORITHM_BUILDER_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "./macros.hpp" + +namespace xsimd::builder +{ + struct alignment_options + { + bool start_aligned = false; + bool end_aligned = false; + }; + + template + auto prev_aligned(T* ptr, std::size_t alignment) -> T* + { + assert(std::has_single_bit(alignment)); + auto const address = reinterpret_cast(ptr); + return reinterpret_cast(address & ~(alignment - 1)); + } + + template + auto next_aligned(T* ptr, std::size_t alignment) -> T* + { + assert(std::has_single_bit(alignment)); + auto const address = reinterpret_cast(ptr); + return reinterpret_cast((address + alignment - 1) & ~(alignment - 1)); + } + + template + auto bytes_to_next_aligned(T* ptr, std::size_t alignment) -> std::size_t + { + assert(std::has_single_bit(alignment)); + auto const address = reinterpret_cast(ptr); + return (alignment - (address & (alignment - 1))) & (alignment - 1); + } + + template + auto are_aliased(std::span lhs, std::span rhs) -> bool + { + // Comparing pointers from unrelated objects is unspecified, integers are not. + auto const lhs_begin = reinterpret_cast(lhs.data()); + auto const rhs_begin = reinterpret_cast(rhs.data()); + return (lhs_begin < rhs_begin + rhs.size_bytes()) && (rhs_begin < lhs_begin + lhs.size_bytes()); + } + + template < + typename Arch = xsimd::default_arch, + typename T, typename U, typename Func> + void map_unary_batch( + T const* XSIMD_RESTRICT begin, + T const* XSIMD_RESTRICT end, + U* XSIMD_RESTRICT out, + Func&& func) + { + using input_batch = xsimd::batch; + using output_batch = xsimd::batch; + + assert(begin <= end); + assert(static_cast(end - begin) <= input_batch::size); + + if (begin == end) [[unlikely]] + { + return; + } + + alignas(Arch::alignment()) T input_buffer[input_batch::size] {}; + alignas(Arch::alignment()) U output_buffer[output_batch::size]; + + const std::size_t in_count = static_cast(end - begin); + std::memcpy(input_buffer, begin, in_count * sizeof(T)); + func(input_batch::load_aligned(input_buffer)).store_aligned(output_buffer); + std::memcpy(out, output_buffer, in_count * sizeof(T)); + } + + template < + alignment_options aligned = {}, + typename Arch = xsimd::default_arch, + typename T, typename U, typename Func> + void map_unary(std::span in, std::span out, Func&& func) + { + using input_batch = xsimd::batch; + using output_batch = xsimd::batch; + + // Both sides can only be split at an element boundary. + // If input is not guarenteed aligned, we will try to align preferably the + // output (more expensive unaligned stores) or otherwise the input. + constexpr bool align_output = sizeof(U) >= sizeof(T); + constexpr bool load_is_aligned = aligned.start_aligned || !align_output; + constexpr bool store_is_aligned = aligned.start_aligned || align_output; + + assert(in.size() * sizeof(T) == out.size() * sizeof(U)); + assert(!are_aliased(in, out)); + + if (in.empty()) [[unlikely]] + { + return; + } + + auto ot = out.data(); + auto it = in.data(); + auto const end = in.data() + in.size(); + + // Input and output may not have the same alignment so it may be impossible + // to get both aligned, so we align a single side. + if constexpr (!aligned.start_aligned) + { + // The span may be too short to reach the next alignment boundary. + const auto head_bytes = std::min( + align_output ? bytes_to_next_aligned(ot, Arch::alignment()) + : bytes_to_next_aligned(it, Arch::alignment()), + in.size_bytes()); + assert(head_bytes % sizeof(T) == 0); + assert(head_bytes % sizeof(U) == 0); + + map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + it += head_bytes / sizeof(T); + ot += head_bytes / sizeof(U); + } + + // No loop-carried dependencies and no aliasing, so we leave the compiler + // to unroll the loop. + while (static_cast(end - it) >= input_batch::size) + { + input_batch x; + if constexpr (load_is_aligned) + { + x = input_batch::load_aligned(it); + } + else + { + x = input_batch::load_unaligned(it); + } + + const auto y = func(x); + if constexpr (store_is_aligned) + { + y.store_aligned(ot); + } + else + { + y.store_unaligned(ot); + } + + it += input_batch::size; + ot += output_batch::size; + } + + // Unlikely to be skipped, meant for users that know they allocate + // a multiple of the batch size. + if constexpr (!aligned.end_aligned) + { + map_unary_batch(it, end, ot, func); + } + } +} + +#endif diff --git a/include/xsimd_algorithm/macros.hpp b/include/xsimd_algorithm/macros.hpp new file mode 100644 index 0000000..341f898 --- /dev/null +++ b/include/xsimd_algorithm/macros.hpp @@ -0,0 +1,20 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_MACRO_HPP +#define XSIMD_ALGORITHM_MACRO_HPP + +#if defined(_MSC_VER) && !defined(__clang__) +#define XSIMD_RESTRICT __restrict +#elif defined(__GNUC__) || defined(__clang__) +#define XSIMD_RESTRICT __restrict__ +#else +#define XSIMD_RESTRICT +#endif + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2285b23..b6af158 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,6 +48,7 @@ endif() set(XSIMD_ALGORITHM_TESTS main.cpp test_arange.cpp + test_builder.cpp test_iterator.cpp test_reduce.cpp test_transform.cpp diff --git a/test/test_builder.cpp b/test/test_builder.cpp new file mode 100644 index 0000000..6cd6236 --- /dev/null +++ b/test/test_builder.cpp @@ -0,0 +1,78 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include "xsimd_algorithm/builder.hpp" + +#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE + +#include "doctest/doctest.h" + +#include +#include +#include +#include +#include + +namespace +{ + template + using aligned_vector = std::vector>; + + template + std::span as_span(std::vector const& v) + { + return std::span { v.data(), v.size() }; + } + + template + std::span as_span(std::vector& v) + { + return std::span { v.data(), v.size() }; + } + + template + aligned_vector make_arange(std::size_t size, T start = T { 0 }) + { + aligned_vector data(size); + std::iota(data.begin(), data.end(), start); + return data; + } +} + +TEST_CASE("map_unary int32 to int64") +{ + using input_type = std::int32_t; + using output_type = std::int64_t; + + // Not a multiple of the batch size, to exercise the tail. + static constexpr std::size_t input_size = 94; + static constexpr std::size_t output_size = input_size * sizeof(input_type) / sizeof(output_type); + + const auto input = make_arange(input_size); + auto output = aligned_vector(output_size); + + const auto func = [](auto const& x) + { return xsimd::widen(x + input_type { 1 })[0]; }; + + xsimd::builder::map_unary(as_span(input), as_span(output), func); + + static constexpr std::size_t in_batch_size = xsimd::batch::size; + static constexpr std::size_t out_batch_size = xsimd::batch::size; + + for (std::size_t i = 0; i < output_size; ++i) + { + const auto in_index = (i / out_batch_size) * in_batch_size + (i % out_batch_size); + CAPTURE(i); + CHECK(output[i] == static_cast(input[in_index] + 1)); + } +} + +#endif From e73d9e0d651cf45d63c6570dc7fc91a5260d049a Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 13:54:38 +0200 Subject: [PATCH 2/8] Add sqrt and abs --- include/xsimd_algorithm/math.hpp | 41 +++++++++++ test/CMakeLists.txt | 1 + test/test_builder.cpp | 29 +------- test/test_math.cpp | 118 +++++++++++++++++++++++++++++++ test/utils.hpp | 81 +++++++++++++++++++++ 5 files changed, 243 insertions(+), 27 deletions(-) create mode 100644 include/xsimd_algorithm/math.hpp create mode 100644 test/test_math.cpp create mode 100644 test/utils.hpp diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp new file mode 100644 index 0000000..88d2726 --- /dev/null +++ b/include/xsimd_algorithm/math.hpp @@ -0,0 +1,41 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_MATH_HPP +#define XSIMD_ALGORITHM_MATH_HPP + +#include + +#include "./builder.hpp" + +namespace xsimd::algo +{ + template < + xsimd::builder::alignment_options aligned = {}, + typename Arch = xsimd::default_arch, + typename T> + void sqrt(std::span in, std::span out) + { + return xsimd::builder::map_unary( + in, out, [](auto x) + { return sqrt(x); }); + } + + template < + xsimd::builder::alignment_options aligned = {}, + typename Arch = xsimd::default_arch, + typename T> + void abs(std::span in, std::span out) + { + return xsimd::builder::map_unary( + in, out, [](auto x) + { return abs(x); }); + } +} + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b6af158..3ee3907 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -50,6 +50,7 @@ set(XSIMD_ALGORITHM_TESTS test_arange.cpp test_builder.cpp test_iterator.cpp + test_math.cpp test_reduce.cpp test_transform.cpp ) diff --git a/test/test_builder.cpp b/test/test_builder.cpp index 6cd6236..50c8808 100644 --- a/test/test_builder.cpp +++ b/test/test_builder.cpp @@ -17,35 +17,10 @@ #include #include -#include -#include -#include -namespace -{ - template - using aligned_vector = std::vector>; - - template - std::span as_span(std::vector const& v) - { - return std::span { v.data(), v.size() }; - } - - template - std::span as_span(std::vector& v) - { - return std::span { v.data(), v.size() }; - } +#include "utils.hpp" - template - aligned_vector make_arange(std::size_t size, T start = T { 0 }) - { - aligned_vector data(size); - std::iota(data.begin(), data.end(), start); - return data; - } -} +using namespace xsimd::test; TEST_CASE("map_unary int32 to int64") { diff --git a/test/test_math.cpp b/test/test_math.cpp new file mode 100644 index 0000000..45b0d99 --- /dev/null +++ b/test/test_math.cpp @@ -0,0 +1,118 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include "xsimd_algorithm/math.hpp" + +#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE + +#include "doctest/doctest.h" + +#include +#include +#include + +#include "utils.hpp" + +using namespace xsimd::test; + +namespace +{ + template + struct sqrt_op + { + using value_type = T; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::sqrt(in, out); + } + + static T scalar(T x) + { + return std::sqrt(x); + } + + template + static std::vector input(std::size_t size) + { + return make_arange(size); + } + }; + + template + struct abs_op + { + using value_type = T; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::abs(in, out); + } + + static T scalar(T x) + { + return std::abs(x); + } + + template + static std::vector input(std::size_t size) + { + return make_arange(size, -static_cast(size) / 2); + } + }; + + template + void check_unary_math() + { + using value_type = typename Op::value_type; + // Not a multiple of the batch size, to exercise the tail. + constexpr std::size_t test_size = 94; + + const auto input = Op::template input(test_size); + auto output = std::vector(input.size()); + + Op::template apply(as_span(input), as_span(output)); + + for (std::size_t i = 0; i < input.size(); ++i) + { + CAPTURE(i); + CHECK(output[i] == doctest::Approx(Op::scalar(input[i]))); + } + } +} + +TEST_CASE_TEMPLATE( + "unary math", + Op, + sqrt_op, sqrt_op, + abs_op, abs_op) +{ + using value_type = typename Op::value_type; + + SUBCASE("aligned without header") + { + using allocator = typename aligned_vector::allocator_type; + check_unary_math(); + } + + SUBCASE("aligned with header") + { + using allocator = typename aligned_vector::allocator_type; + check_unary_math(); + } + + SUBCASE("unaligned with header") + { + using allocator = typename unaligned_vector::allocator_type; + check_unary_math(); + } +} + +#endif diff --git a/test/utils.hpp b/test/utils.hpp new file mode 100644 index 0000000..db9b9ed --- /dev/null +++ b/test/utils.hpp @@ -0,0 +1,81 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_TEST_UTILS_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_HPP + +#include +#include +#include +#include + +#include + +namespace xsimd::test +{ + template + using aligned_vector = std::vector>; + + /// An allocator returning memory guaranteed not to be aligned on @p Align. + /// + /// Over-allocates by @p Offset elements and shifts the returned pointer, so that the data + /// starts @p Offset * sizeof(T) bytes past an aligned address. + template + struct unaligned_allocator : private xsimd::aligned_allocator + { + static_assert((Offset * sizeof(T)) % Align != 0, "Shifted pointer would still be aligned"); + + using base_type = xsimd::aligned_allocator; + using value_type = T; + + // The non-type Align parameter defeats the default allocator_traits rebind. + template + struct rebind + { + using other = unaligned_allocator; + }; + + unaligned_allocator() = default; + + template + unaligned_allocator(unaligned_allocator const&) + { + } + + T* allocate(std::size_t n) { return base_type::allocate(n + Offset) + Offset; } + + void deallocate(T* p, std::size_t n) { base_type::deallocate(p - Offset, n + Offset); } + + friend bool operator==(unaligned_allocator const&, unaligned_allocator const&) { return true; } + }; + + template + using unaligned_vector = std::vector>; + + template + std::span as_span(std::vector const& v) + { + return std::span { v.data(), v.size() }; + } + + template + std::span as_span(std::vector& v) + { + return std::span { v.data(), v.size() }; + } + + template ::allocator_type> + std::vector make_arange(std::size_t size, T start = T { 0 }) + { + std::vector data(size); + std::iota(data.begin(), data.end(), start); + return data; + } +} + +#endif From 678777e6c3a567e6b9b66040e9a38ee041acb052 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 15:01:57 +0200 Subject: [PATCH 3/8] Refactor data library --- CMakeLists.txt | 2 + test-utils/CMakeLists.txt | 23 +++++ .../include/xsimd_test_utils/math_data.hpp | 99 +++++++++++++++++++ .../include/xsimd_test_utils}/utils.hpp | 4 +- test/CMakeLists.txt | 4 +- test/test_builder.cpp | 20 ++-- test/test_math.cpp | 86 +++------------- 7 files changed, 150 insertions(+), 88 deletions(-) create mode 100644 test-utils/CMakeLists.txt create mode 100644 test-utils/include/xsimd_test_utils/math_data.hpp rename {test => test-utils/include/xsimd_test_utils}/utils.hpp (96%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 790e26f..85c1634 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,8 @@ target_link_libraries(xsimd-algorithm INTERFACE xsimd) OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF) +add_subdirectory(test-utils) + if(BUILD_TESTS) enable_testing() add_subdirectory(test) diff --git a/test-utils/CMakeLists.txt b/test-utils/CMakeLists.txt new file mode 100644 index 0000000..7f832e4 --- /dev/null +++ b/test-utils/CMakeLists.txt @@ -0,0 +1,23 @@ +############################################################################ +# Copyright (c) xsimd-algorithm contributors # +# # +# Distributed under the terms of the BSD 3-Clause License. # +# # +# The full license is in the file LICENSE, distributed with this software. # +############################################################################ + +cmake_minimum_required(VERSION 3.8) + +project(xsimd-algorithm-test-utils) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + find_package(xsimd-algorithm REQUIRED CONFIG) +endif () + +add_library(xsimd-algorithm-test-utils INTERFACE) +add_library(xsimd::test-utils ALIAS xsimd-algorithm-test-utils) + +target_include_directories(xsimd-algorithm-test-utils INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/include) + +target_link_libraries(xsimd-algorithm-test-utils INTERFACE xsimd-algorithm) diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp new file mode 100644 index 0000000..da79e1d --- /dev/null +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -0,0 +1,99 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_MATH_DATA_HPP + +#include +#include +#include +#include +#include + +#include "xsimd_algorithm/builder.hpp" +#include "xsimd_algorithm/math.hpp" + +#include "xsimd_test_utils/utils.hpp" + +namespace xsimd::test +{ + /// Derives the scalar range application from the element-wise Derived::apply. + template + struct unary_op + { + using value_type = T; + + static void apply_range_scalar(std::span in, std::span out) + { + for (std::size_t i = 0; i < in.size(); ++i) + { + out[i] = Derived::apply(in[i]); + } + } + + template + static std::pair, std::vector> make_input_output(std::size_t size) + { + auto input = Derived::template make_input(size); + auto output = std::vector(input.size()); + return { std::move(input), std::move(output) }; + } + }; + + /******************* + * Test fixtures * + *******************/ + + template + struct sqrt_op : unary_op, T> + { + static constexpr auto name = "sqrt"; + + static T apply(T x) + { + return std::sqrt(x); + } + + template + static void apply_range_simd(std::span in, std::span out) + { + xsimd::algo::sqrt(in, out); + } + + template + static std::vector make_input(std::size_t size) + { + return make_arange(size); + } + }; + + template + struct abs_op : unary_op, T> + { + static constexpr auto name = "abs"; + + static T apply(T x) + { + return std::abs(x); + } + + template + static void apply_range_simd(std::span in, std::span out) + { + xsimd::algo::abs(in, out); + } + + template + static std::vector make_input(std::size_t size) + { + return make_arange(size, -static_cast(size) / 2); + } + }; +} + +#endif diff --git a/test/utils.hpp b/test-utils/include/xsimd_test_utils/utils.hpp similarity index 96% rename from test/utils.hpp rename to test-utils/include/xsimd_test_utils/utils.hpp index db9b9ed..692885b 100644 --- a/test/utils.hpp +++ b/test-utils/include/xsimd_test_utils/utils.hpp @@ -6,8 +6,8 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef XSIMD_ALGORITHM_TEST_UTILS_HPP -#define XSIMD_ALGORITHM_TEST_UTILS_HPP +#ifndef XSIMD_ALGORITHM_TEST_UTILS_UTILS_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_UTILS_HPP #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3ee3907..a6d7e87 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -55,8 +55,8 @@ set(XSIMD_ALGORITHM_TESTS test_transform.cpp ) -add_executable(test_xsimd_algorithm ${XSIMD_ALGORITHM_TESTS})# ${XSIMD_ALGORITHM_HEADERS}) -target_link_libraries(test_xsimd_algorithm PRIVATE xsimd-algorithm) +add_executable(test_xsimd_algorithm ${XSIMD_ALGORITHM_TESTS}) +target_link_libraries(test_xsimd_algorithm PRIVATE xsimd-algorithm xsimd::test-utils) option(DOWNLOAD_DOCTEST OFF) find_package(doctest QUIET) diff --git a/test/test_builder.cpp b/test/test_builder.cpp index 50c8808..177716f 100644 --- a/test/test_builder.cpp +++ b/test/test_builder.cpp @@ -9,19 +9,15 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#include "xsimd_algorithm/builder.hpp" - -#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE - -#include "doctest/doctest.h" - #include #include -#include "utils.hpp" +#include +#include -using namespace xsimd::test; +#include "xsimd_algorithm/builder.hpp" +/// Map unary test that turned int32 into half has many int64. TEST_CASE("map_unary int32 to int64") { using input_type = std::int32_t; @@ -31,13 +27,13 @@ TEST_CASE("map_unary int32 to int64") static constexpr std::size_t input_size = 94; static constexpr std::size_t output_size = input_size * sizeof(input_type) / sizeof(output_type); - const auto input = make_arange(input_size); - auto output = aligned_vector(output_size); + const auto input = xsimd::test::make_arange(input_size); + auto output = xsimd::test::aligned_vector(output_size); const auto func = [](auto const& x) { return xsimd::widen(x + input_type { 1 })[0]; }; - xsimd::builder::map_unary(as_span(input), as_span(output), func); + xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); static constexpr std::size_t in_batch_size = xsimd::batch::size; static constexpr std::size_t out_batch_size = xsimd::batch::size; @@ -49,5 +45,3 @@ TEST_CASE("map_unary int32 to int64") CHECK(output[i] == static_cast(input[in_index] + 1)); } } - -#endif diff --git a/test/test_math.cpp b/test/test_math.cpp index 45b0d99..26ab5c8 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -6,84 +6,29 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#include "xsimd_algorithm/math.hpp" - -#ifndef XSIMD_NO_SUPPORTED_ARCHITECTURE - -#include "doctest/doctest.h" - -#include #include -#include -#include "utils.hpp" +#include -using namespace xsimd::test; +#include +#include namespace { - template - struct sqrt_op - { - using value_type = T; - - template - static void apply(std::span in, std::span out) - { - xsimd::algo::sqrt(in, out); - } - - static T scalar(T x) - { - return std::sqrt(x); - } - - template - static std::vector input(std::size_t size) - { - return make_arange(size); - } - }; - - template - struct abs_op - { - using value_type = T; - - template - static void apply(std::span in, std::span out) - { - xsimd::algo::abs(in, out); - } - - static T scalar(T x) - { - return std::abs(x); - } - - template - static std::vector input(std::size_t size) - { - return make_arange(size, -static_cast(size) / 2); - } - }; - template void check_unary_math() { - using value_type = typename Op::value_type; // Not a multiple of the batch size, to exercise the tail. constexpr std::size_t test_size = 94; - const auto input = Op::template input(test_size); - auto output = std::vector(input.size()); + auto [input, output] = Op::template make_input_output(test_size); - Op::template apply(as_span(input), as_span(output)); + Op::template apply_range_simd(xsimd::test::as_span(input), xsimd::test::as_span(output)); for (std::size_t i = 0; i < input.size(); ++i) { CAPTURE(i); - CHECK(output[i] == doctest::Approx(Op::scalar(input[i]))); + CHECK(output[i] == doctest::Approx(Op::apply(input[i]))); } } } @@ -91,28 +36,27 @@ namespace TEST_CASE_TEMPLATE( "unary math", Op, - sqrt_op, sqrt_op, - abs_op, abs_op) + xsimd::test::sqrt_op, + xsimd::test::sqrt_op, + xsimd::test::abs_op, + xsimd::test::abs_op) { using value_type = typename Op::value_type; + using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_allocator = typename xsimd::test::unaligned_vector::allocator_type; SUBCASE("aligned without header") { - using allocator = typename aligned_vector::allocator_type; - check_unary_math(); + check_unary_math(); } SUBCASE("aligned with header") { - using allocator = typename aligned_vector::allocator_type; - check_unary_math(); + check_unary_math(); } SUBCASE("unaligned with header") { - using allocator = typename unaligned_vector::allocator_type; - check_unary_math(); + check_unary_math(); } } - -#endif From bb86eeecbb727504585ff004df3d7fccfe69cbaf Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 16:24:03 +0200 Subject: [PATCH 4/8] Improve map_unary tail --- include/xsimd_algorithm/builder.hpp | 107 ++++++++++++------ include/xsimd_algorithm/math.hpp | 10 +- .../include/xsimd_test_utils/math_data.hpp | 4 +- .../include/xsimd_test_utils/math_ops.hpp | 75 ++++++++++++ 4 files changed, 158 insertions(+), 38 deletions(-) create mode 100644 test-utils/include/xsimd_test_utils/math_ops.hpp diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index ab6aa21..ddd90c4 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -23,14 +23,14 @@ namespace xsimd::builder { - struct alignment_options + struct alignment { bool start_aligned = false; bool end_aligned = false; }; template - auto prev_aligned(T* ptr, std::size_t alignment) -> T* + XSIMD_INLINE auto prev_aligned(T* ptr, std::size_t alignment) -> T* { assert(std::has_single_bit(alignment)); auto const address = reinterpret_cast(ptr); @@ -38,7 +38,7 @@ namespace xsimd::builder } template - auto next_aligned(T* ptr, std::size_t alignment) -> T* + XSIMD_INLINE auto next_aligned(T* ptr, std::size_t alignment) -> T* { assert(std::has_single_bit(alignment)); auto const address = reinterpret_cast(ptr); @@ -46,7 +46,7 @@ namespace xsimd::builder } template - auto bytes_to_next_aligned(T* ptr, std::size_t alignment) -> std::size_t + XSIMD_INLINE auto bytes_to_next_aligned(T* ptr, std::size_t alignment) -> std::size_t { assert(std::has_single_bit(alignment)); auto const address = reinterpret_cast(ptr); @@ -54,7 +54,7 @@ namespace xsimd::builder } template - auto are_aliased(std::span lhs, std::span rhs) -> bool + XSIMD_INLINE auto are_aliased(std::span lhs, std::span rhs) -> bool { // Comparing pointers from unrelated objects is unspecified, integers are not. auto const lhs_begin = reinterpret_cast(lhs.data()); @@ -62,10 +62,16 @@ namespace xsimd::builder return (lhs_begin < rhs_begin + rhs.size_bytes()) && (rhs_begin < lhs_begin + lhs.size_bytes()); } + struct unary_options + { + std::size_t unroll_factor = 4; + bool pure = false; + }; + template < typename Arch = xsimd::default_arch, typename T, typename U, typename Func> - void map_unary_batch( + XSIMD_INLINE void map_unary_batch( T const* XSIMD_RESTRICT begin, T const* XSIMD_RESTRICT end, U* XSIMD_RESTRICT out, @@ -91,11 +97,38 @@ namespace xsimd::builder std::memcpy(out, output_buffer, in_count * sizeof(T)); } + template + XSIMD_INLINE xsimd::batch load_batch(T const* ptr) + { + if constexpr (aligned) + { + return xsimd::batch::load_aligned(ptr); + } + else + { + return xsimd::batch::load_unaligned(ptr); + } + } + + template + XSIMD_INLINE void store_batch(xsimd::batch x, T* ptr) + { + if constexpr (aligned) + { + x.store_aligned(ptr); + } + else + { + x.store_unaligned(ptr); + } + } + template < - alignment_options aligned = {}, + alignment align = {}, + unary_options opts = {}, typename Arch = xsimd::default_arch, typename T, typename U, typename Func> - void map_unary(std::span in, std::span out, Func&& func) + XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) { using input_batch = xsimd::batch; using output_batch = xsimd::batch; @@ -104,8 +137,8 @@ namespace xsimd::builder // If input is not guarenteed aligned, we will try to align preferably the // output (more expensive unaligned stores) or otherwise the input. constexpr bool align_output = sizeof(U) >= sizeof(T); - constexpr bool load_is_aligned = aligned.start_aligned || !align_output; - constexpr bool store_is_aligned = aligned.start_aligned || align_output; + constexpr bool load_is_aligned = align.start_aligned || !align_output; + constexpr bool store_is_aligned = align.start_aligned || align_output; assert(in.size() * sizeof(T) == out.size() * sizeof(U)); assert(!are_aliased(in, out)); @@ -117,11 +150,11 @@ namespace xsimd::builder auto ot = out.data(); auto it = in.data(); - auto const end = in.data() + in.size(); + auto const iend = in.data() + in.size(); // Input and output may not have the same alignment so it may be impossible // to get both aligned, so we align a single side. - if constexpr (!aligned.start_aligned) + if constexpr (!align.start_aligned) { // The span may be too short to reach the next alignment boundary. const auto head_bytes = std::min( @@ -136,39 +169,49 @@ namespace xsimd::builder ot += head_bytes / sizeof(U); } - // No loop-carried dependencies and no aliasing, so we leave the compiler - // to unroll the loop. - while (static_cast(end - it) >= input_batch::size) + // Unrolled loop processing multiple batches at a time + while (static_cast(iend - it) >= opts.unroll_factor * input_batch::size) { - input_batch x; - if constexpr (load_is_aligned) + input_batch x[opts.unroll_factor]; + for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - x = input_batch::load_aligned(it); + x[u] = load_batch(it + u * input_batch::size); } - else + for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - x = input_batch::load_unaligned(it); + store_batch(func(x[u]), ot + u * output_batch::size); } - const auto y = func(x); - if constexpr (store_is_aligned) - { - y.store_aligned(ot); - } - else - { - y.store_unaligned(ot); - } + it += opts.unroll_factor * input_batch::size; + ot += opts.unroll_factor * output_batch::size; + } + while (static_cast(iend - it) >= input_batch::size) + { + const auto x = load_batch(it); + store_batch(func(x), ot); it += input_batch::size; ot += output_batch::size; } // Unlikely to be skipped, meant for users that know they allocate - // a multiple of the batch size. - if constexpr (!aligned.end_aligned) + // a multiple of the batch size, such as in a local buffer + if constexpr (!align.end_aligned) { - map_unary_batch(it, end, ot, func); + auto const oend = out.data() + out.size(); + // Stepping back shifts the batch boundary, which pairs lanes differently + // than starting from the front unless both sides have the same lane count. + constexpr bool can_step_back = input_batch::size == output_batch::size; + if (can_step_back && it != iend && (in.size() >= input_batch::size)) [[likely]] + { + // Recompute overlapping data, this time starting from the end. + const auto x = load_batch(iend - input_batch::size); + store_batch(func(x), oend - output_batch::size); + } + else + { + map_unary_batch(it, iend, ot, func); + } } } } diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 88d2726..369c940 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -16,23 +16,25 @@ namespace xsimd::algo { template < - xsimd::builder::alignment_options aligned = {}, + xsimd::builder::alignment align = {}, typename Arch = xsimd::default_arch, typename T> void sqrt(std::span in, std::span out) { - return xsimd::builder::map_unary( + constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::builder::map_unary( in, out, [](auto x) { return sqrt(x); }); } template < - xsimd::builder::alignment_options aligned = {}, + xsimd::builder::alignment align = {}, typename Arch = xsimd::default_arch, typename T> void abs(std::span in, std::span out) { - return xsimd::builder::map_unary( + constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::builder::map_unary( in, out, [](auto x) { return abs(x); }); } diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp index da79e1d..09f221e 100644 --- a/test-utils/include/xsimd_test_utils/math_data.hpp +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -59,7 +59,7 @@ namespace xsimd::test return std::sqrt(x); } - template + template static void apply_range_simd(std::span in, std::span out) { xsimd::algo::sqrt(in, out); @@ -82,7 +82,7 @@ namespace xsimd::test return std::abs(x); } - template + template static void apply_range_simd(std::span in, std::span out) { xsimd::algo::abs(in, out); diff --git a/test-utils/include/xsimd_test_utils/math_ops.hpp b/test-utils/include/xsimd_test_utils/math_ops.hpp new file mode 100644 index 0000000..497df13 --- /dev/null +++ b/test-utils/include/xsimd_test_utils/math_ops.hpp @@ -0,0 +1,75 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHM_TEST_UTILS_MATH_OPS_HPP +#define XSIMD_ALGORITHM_TEST_UTILS_MATH_OPS_HPP + +#include +#include +#include +#include + +#include +#include + +#include "xsimd_test_utils/utils.hpp" + +namespace xsimd::test +{ + template + struct sqrt_op + { + using value_type = T; + + static constexpr auto name = "sqrt"; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::sqrt(in, out); + } + + static T scalar(T x) + { + return std::sqrt(x); + } + + template + static std::vector input(std::size_t size) + { + return xsimd::test::make_arange(size); + } + }; + + template + struct abs_op + { + using value_type = T; + + static constexpr auto name = "abs"; + + template + static void apply(std::span in, std::span out) + { + xsimd::algo::abs(in, out); + } + + static T scalar(T x) + { + return std::abs(x); + } + + template + static std::vector input(std::size_t size) + { + return xsimd::test::make_arange(size, -static_cast(size) / 2); + } + }; +} + +#endif From 96f94d2f7a8b286149d4827627fa91a352c8a2f0 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 17:01:22 +0200 Subject: [PATCH 5/8] Improve map_unary header --- include/xsimd_algorithm/builder.hpp | 35 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index ddd90c4..8266227 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -10,6 +10,7 @@ #define XSIMD_ALGORITHM_BUILDER_HPP #include +#include #include #include #include @@ -88,13 +89,13 @@ namespace xsimd::builder return; } - alignas(Arch::alignment()) T input_buffer[input_batch::size] {}; - alignas(Arch::alignment()) U output_buffer[output_batch::size]; + alignas(Arch::alignment()) std::array input_buffer {}; + alignas(Arch::alignment()) std::array output_buffer; const std::size_t in_count = static_cast(end - begin); - std::memcpy(input_buffer, begin, in_count * sizeof(T)); - func(input_batch::load_aligned(input_buffer)).store_aligned(output_buffer); - std::memcpy(out, output_buffer, in_count * sizeof(T)); + std::memcpy(input_buffer.data(), begin, in_count * sizeof(T)); + func(input_batch::load_aligned(input_buffer.data())).store_aligned(output_buffer.data()); + std::memcpy(out, output_buffer.data(), in_count * sizeof(T)); } template @@ -140,6 +141,12 @@ namespace xsimd::builder constexpr bool load_is_aligned = align.start_aligned || !align_output; constexpr bool store_is_aligned = align.start_aligned || align_output; + // Edges can be handled by recomputing a region overlapping the aligned body, + // which is cheaper than a round trip through a scratch buffer. This requires + // func to be free of side effects, and both sides to have the same lane count + // since shifting the batch boundary otherwise pairs lanes differently. + constexpr bool can_overlap = opts.pure && (input_batch::size == output_batch::size); + assert(in.size() * sizeof(T) == out.size() * sizeof(U)); assert(!are_aliased(in, out)); @@ -164,7 +171,16 @@ namespace xsimd::builder assert(head_bytes % sizeof(T) == 0); assert(head_bytes % sizeof(U) == 0); - map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + if (can_overlap && (head_bytes != 0) && (in.size() >= input_batch::size)) + { + // Recompute the head as a full batch, the body overwrites the excess. + const auto x = load_batch(it); + store_batch(func(x), ot); + } + else + { + map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + } it += head_bytes / sizeof(T); ot += head_bytes / sizeof(U); } @@ -172,7 +188,7 @@ namespace xsimd::builder // Unrolled loop processing multiple batches at a time while (static_cast(iend - it) >= opts.unroll_factor * input_batch::size) { - input_batch x[opts.unroll_factor]; + std::array x; for (std::size_t u = 0; u < opts.unroll_factor; ++u) { x[u] = load_batch(it + u * input_batch::size); @@ -199,10 +215,7 @@ namespace xsimd::builder if constexpr (!align.end_aligned) { auto const oend = out.data() + out.size(); - // Stepping back shifts the batch boundary, which pairs lanes differently - // than starting from the front unless both sides have the same lane count. - constexpr bool can_step_back = input_batch::size == output_batch::size; - if (can_step_back && it != iend && (in.size() >= input_batch::size)) [[likely]] + if (can_overlap && (it != iend) && (in.size() >= input_batch::size)) [[likely]] { // Recompute overlapping data, this time starting from the end. const auto x = load_batch(iend - input_batch::size); From 85db5217503f885a967090fb9c825966b598cc65 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 17:02:25 +0200 Subject: [PATCH 6/8] Add exp unary function --- include/xsimd_algorithm/math.hpp | 12 ++++++++ .../include/xsimd_test_utils/math_data.hpp | 29 +++++++++++++++++++ test/test_math.cpp | 6 ++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/include/xsimd_algorithm/math.hpp b/include/xsimd_algorithm/math.hpp index 369c940..139016a 100644 --- a/include/xsimd_algorithm/math.hpp +++ b/include/xsimd_algorithm/math.hpp @@ -38,6 +38,18 @@ namespace xsimd::algo in, out, [](auto x) { return abs(x); }); } + + template < + xsimd::builder::alignment align = {}, + typename Arch = xsimd::default_arch, + typename T> + void exp(std::span in, std::span out) + { + constexpr builder::unary_options opts = { .unroll_factor = 4, .pure = true }; + return xsimd::builder::map_unary( + in, out, [](auto x) + { return exp(x); }); + } } #endif diff --git a/test-utils/include/xsimd_test_utils/math_data.hpp b/test-utils/include/xsimd_test_utils/math_data.hpp index 09f221e..736fb7e 100644 --- a/test-utils/include/xsimd_test_utils/math_data.hpp +++ b/test-utils/include/xsimd_test_utils/math_data.hpp @@ -94,6 +94,35 @@ namespace xsimd::test return make_arange(size, -static_cast(size) / 2); } }; + + template + struct exp_op : unary_op, T> + { + static constexpr auto name = "exp"; + + static T apply(T x) + { + return std::exp(x); + } + + template + static void apply_range_simd(std::span in, std::span out) + { + xsimd::algo::exp(in, out); + } + + template + static std::vector make_input(std::size_t size) + { + // exp overflows past a small range, so wrap the values back into [-10, 10). + auto input = make_arange(size); + for (auto& x : input) + { + x = std::fmod(x, T { 20 }) - T { 10 }; + } + return input; + } + }; } #endif diff --git a/test/test_math.cpp b/test/test_math.cpp index 26ab5c8..fa995b8 100644 --- a/test/test_math.cpp +++ b/test/test_math.cpp @@ -15,7 +15,7 @@ namespace { - template + template void check_unary_math() { // Not a multiple of the batch size, to exercise the tail. @@ -39,7 +39,9 @@ TEST_CASE_TEMPLATE( xsimd::test::sqrt_op, xsimd::test::sqrt_op, xsimd::test::abs_op, - xsimd::test::abs_op) + xsimd::test::abs_op, + xsimd::test::exp_op, + xsimd::test::exp_op) { using value_type = typename Op::value_type; using aligned_allocator = typename xsimd::test::aligned_vector::allocator_type; From 01eb843c5e64761a743772baa02facf47e18cfd3 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 10 Sep 2026 17:04:23 +0200 Subject: [PATCH 7/8] Add benchmarks --- CMakeLists.txt | 5 ++ benchmark/CMakeLists.txt | 26 ++++++++ benchmark/bench_math.cpp | 124 +++++++++++++++++++++++++++++++++++++++ benchmark/main.cpp | 11 ++++ environment-dev.yml | 3 +- 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/bench_math.cpp create mode 100644 benchmark/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 85c1634..9f1837d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ target_compile_features(xsimd-algorithm INTERFACE cxx_std_20) target_link_libraries(xsimd-algorithm INTERFACE xsimd) OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF) +OPTION(BUILD_BENCHMARK "xsimd-algorithm benchmark suite" OFF) add_subdirectory(test-utils) @@ -41,6 +42,10 @@ if(BUILD_TESTS) add_subdirectory(test) endif() +if(BUILD_BENCHMARK) + add_subdirectory(benchmark) +endif() + # Installation # ============ diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 0000000..5b70a13 --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,26 @@ +############################################################################ +# Copyright (c) xsimd-algorithm contributors # +# # +# Distributed under the terms of the BSD 3-Clause License. # +# # +# The full license is in the file LICENSE, distributed with this software. # +############################################################################ + +cmake_minimum_required(VERSION 3.8) + +project(xsimd-algorithm-benchmark) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + find_package(xsimd-algorithm REQUIRED CONFIG) +endif () + +find_package(benchmark REQUIRED) + +set(XSIMD_ALGORITHM_BENCHMARKS + main.cpp + bench_math.cpp +) + +add_executable(benchmark_xsimd_algorithm ${XSIMD_ALGORITHM_BENCHMARKS}) +target_link_libraries(benchmark_xsimd_algorithm + PRIVATE xsimd-algorithm xsimd::test-utils benchmark::benchmark) diff --git a/benchmark/bench_math.cpp b/benchmark/bench_math.cpp new file mode 100644 index 0000000..35694c6 --- /dev/null +++ b/benchmark/bench_math.cpp @@ -0,0 +1,124 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + + +#include +#include +#include +#include +#include + +#include + +#include "xsimd_test_utils/math_data.hpp" +#include "xsimd_test_utils/utils.hpp" + +using xsimd::builder::alignment; + +namespace +{ + template + void bench_unary(benchmark::State& state, Apply apply) + { + using value_type = typename Op::value_type; + + auto const size = static_cast(state.range(0)); + auto [input, output] = Op::template make_input_output(size); + + for (auto _ : state) + { + apply(xsimd::test::as_span(input), xsimd::test::as_span(output)); + benchmark::DoNotOptimize(output.data()); + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(static_cast(state.iterations() * size)); + state.SetBytesProcessed( + static_cast(state.iterations() * size * 2 * sizeof(value_type))); + } + + /// Sizes spanning L1-resident to memory-bound, each with and without a scalar tail. + template + std::vector bench_sizes() + { + constexpr auto batch_size = static_cast(xsimd::batch::size); + + auto sizes = std::vector {}; + for (std::int64_t size : { 64, 1024, 65536, 1 << 21 }) + { + auto const whole = size - (size % batch_size); + sizes.push_back(whole); + sizes.push_back(whole + batch_size / 2 + 1); + } + return sizes; + } + + template + constexpr auto type_name() + { + if constexpr (std::is_same_v) + { + return "f32"; + } + else if constexpr (std::is_same_v) + { + return "f64"; + } + } + + template + void bench_simd(benchmark::State& state) + { + bench_unary( + state, + [](auto in, auto out) { Op::template apply_range_simd(in, out); }); + } + + template + void bench_scalar(benchmark::State& state) + { + bench_unary(state, [](auto in, auto out) { Op::apply_range_scalar(in, out); }); + } + + template + void register_bench(std::string_view variant, Bench bench_fn) + { + using value_type = typename Op::value_type; + + auto* bench = benchmark::RegisterBenchmark( + std::format("{}/{}/{}", Op::name, type_name(), variant), + bench_fn); + for (auto const size : bench_sizes()) + { + bench->Arg(size); + } + } + + template + void register_benches() + { + using value_type = typename Op::value_type; + using aligned_alloc = typename xsimd::test::aligned_vector::allocator_type; + using unaligned_alloc = typename xsimd::test::unaligned_vector::allocator_type; + + register_bench("scalar/aligned", bench_scalar); + register_bench("simd/aligned", bench_simd); + register_bench("simd/unaligned", bench_simd); + } + + bool const registered = [] + { + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + register_benches>(); + return true; + }(); +} diff --git a/benchmark/main.cpp b/benchmark/main.cpp new file mode 100644 index 0000000..68e814d --- /dev/null +++ b/benchmark/main.cpp @@ -0,0 +1,11 @@ +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#include + +BENCHMARK_MAIN(); diff --git a/environment-dev.yml b/environment-dev.yml index 767ec13..bdbfb7a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -5,4 +5,5 @@ dependencies: - cmake - xsimd=14.3.0 - doctest -- ninja \ No newline at end of file +- benchmark +- ninja From b2fa5e3f548d281864890af8fff48286ae8db5e9 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 11 Sep 2026 10:30:13 +0200 Subject: [PATCH 8/8] Fixed element map_unary contract --- include/xsimd_algorithm/builder.hpp | 228 +++++++++++++++++++--------- test/test_builder.cpp | 66 ++++++-- 2 files changed, 207 insertions(+), 87 deletions(-) diff --git a/include/xsimd_algorithm/builder.hpp b/include/xsimd_algorithm/builder.hpp index 8266227..d8f45aa 100644 --- a/include/xsimd_algorithm/builder.hpp +++ b/include/xsimd_algorithm/builder.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -30,6 +31,7 @@ namespace xsimd::builder bool end_aligned = false; }; + /// Return the pointer before the input with the given alignment or itself if aligned. template XSIMD_INLINE auto prev_aligned(T* ptr, std::size_t alignment) -> T* { @@ -38,6 +40,7 @@ namespace xsimd::builder return reinterpret_cast(address & ~(alignment - 1)); } + /// Return the pointer after the input with the given alignment or itself if aligned. template XSIMD_INLINE auto next_aligned(T* ptr, std::size_t alignment) -> T* { @@ -54,6 +57,7 @@ namespace xsimd::builder return (alignment - (address & (alignment - 1))) & (alignment - 1); } + /// Check if two spans are aliasing each others (overlapping). template XSIMD_INLINE auto are_aliased(std::span lhs, std::span rhs) -> bool { @@ -69,35 +73,14 @@ namespace xsimd::builder bool pure = false; }; - template < - typename Arch = xsimd::default_arch, - typename T, typename U, typename Func> - XSIMD_INLINE void map_unary_batch( - T const* XSIMD_RESTRICT begin, - T const* XSIMD_RESTRICT end, - U* XSIMD_RESTRICT out, - Func&& func) - { - using input_batch = xsimd::batch; - using output_batch = xsimd::batch; - - assert(begin <= end); - assert(static_cast(end - begin) <= input_batch::size); - - if (begin == end) [[unlikely]] - { - return; - } - - alignas(Arch::alignment()) std::array input_buffer {}; - alignas(Arch::alignment()) std::array output_buffer; - - const std::size_t in_count = static_cast(end - begin); - std::memcpy(input_buffer.data(), begin, in_count * sizeof(T)); - func(input_batch::load_aligned(input_buffer.data())).store_aligned(output_buffer.data()); - std::memcpy(out, output_buffer.data(), in_count * sizeof(T)); - } + /// Number of batches of T spanning as many elements as one batch of the widest of T and U. + /// + /// Pairing that many batches on each side lets both sides advance by the same number of + /// elements, so a mapping stays elementwise regardless of the respective lane counts. + template + inline constexpr std::size_t batch_arity = sizeof(T) / std::min(sizeof(T), sizeof(U)); + /// Load batch wrapper with an alignment as template parameter. template XSIMD_INLINE xsimd::batch load_batch(T const* ptr) { @@ -111,6 +94,7 @@ namespace xsimd::builder } } + /// Store batch wrapper with an alignment as template parameter. template XSIMD_INLINE void store_batch(xsimd::batch x, T* ptr) { @@ -124,6 +108,113 @@ namespace xsimd::builder } } + /// Load an array of batches. + template + XSIMD_INLINE auto load_batches(T const* ptr) -> std::array, N> + { + std::array, N> x; + for (std::size_t i = 0; i < N; ++i) + { + x[i] = load_batch(ptr + i * xsimd::batch::size); + } + return x; + } + + /// Store an array of batches. + template + XSIMD_INLINE void store_batches(std::array, N> const& x, T* ptr) + { + for (std::size_t i = 0; i < N; ++i) + { + store_batch(x[i], ptr + i * xsimd::batch::size); + } + } + + namespace internal + { + template + inline constexpr bool is_array = false; + + template + inline constexpr bool is_array> = true; + + /// If an array contains only one element, return it. + template + XSIMD_INLINE auto const& unwrap_array(std::array const& x) + { + if constexpr (N == 1) + { + return x[0]; + } + else + { + return x; + } + } + + /// Wrap user function to handle ``xsimd::batch`` as 1D array. + /// + /// Transform 1D input array as batch from alogrithm functions to batch for to + /// the user function, and user batch result as 1D arrays for the algorithm + /// functions. + template + XSIMD_INLINE auto wrap_params_as_1d_arrays(Func&& func) + { + return [func = std::forward(func)](auto const&... x) + { + auto res = func(internal::unwrap_array(x)...); + if constexpr (internal::is_array) + { + return res; + } + else + { + return std::array { res }; + } + }; + } + } + + /// Map fewer elements than a full step through a scratch buffer. + template < + typename Arch = xsimd::default_arch, + typename T, typename U, typename Func> + XSIMD_INLINE void map_unary_batch( + T const* XSIMD_RESTRICT begin, + T const* XSIMD_RESTRICT end, + U* XSIMD_RESTRICT out, + Func&& func) + { + constexpr std::size_t in_arity = batch_arity; + constexpr std::size_t out_arity = batch_arity; + constexpr std::size_t step = in_arity * xsimd::batch::size; + static_assert(step == out_arity * xsimd::batch::size); + auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); + + assert(begin <= end); + assert(static_cast(end - begin) <= step); + + if (begin == end) [[unlikely]] + { + return; + } + + alignas(Arch::alignment()) std::array input_buffer {}; + alignas(Arch::alignment()) std::array output_buffer; + + const std::size_t count = static_cast(end - begin); + std::memcpy(input_buffer.data(), begin, count * sizeof(T)); + store_batches( + mapper(load_batches(input_buffer.data())), + output_buffer.data()); + std::memcpy(out, output_buffer.data(), count * sizeof(U)); + } + + /// Apply func elementwise over in, writing as many elements to out. + /// + /// Func maps a std::array, batch_arity> to a + /// std::array, batch_arity>, both spanning the same element count. + /// When arity is one, a callback over plain batches is accepted as well. template < alignment align = {}, unary_options opts = {}, @@ -131,23 +222,21 @@ namespace xsimd::builder typename T, typename U, typename Func> XSIMD_INLINE void map_unary(std::span in, std::span out, Func&& func) { - using input_batch = xsimd::batch; - using output_batch = xsimd::batch; + constexpr std::size_t in_arity = batch_arity; + constexpr std::size_t out_arity = batch_arity; + // Elements consumed and produced by a single call to func. + constexpr std::size_t step = in_arity * xsimd::batch::size; + static_assert(step == out_arity * xsimd::batch::size); + auto mapper = internal::wrap_params_as_1d_arrays(std::forward(func)); - // Both sides can only be split at an element boundary. - // If input is not guarenteed aligned, we will try to align preferably the - // output (more expensive unaligned stores) or otherwise the input. + // Input and output may not have the same alignment so it may be impossible + // to get both aligned. We align preferably the output (more expensive + // unaligned stores) or otherwise the input. constexpr bool align_output = sizeof(U) >= sizeof(T); constexpr bool load_is_aligned = align.start_aligned || !align_output; constexpr bool store_is_aligned = align.start_aligned || align_output; - // Edges can be handled by recomputing a region overlapping the aligned body, - // which is cheaper than a round trip through a scratch buffer. This requires - // func to be free of side effects, and both sides to have the same lane count - // since shifting the batch boundary otherwise pairs lanes differently. - constexpr bool can_overlap = opts.pure && (input_batch::size == output_batch::size); - - assert(in.size() * sizeof(T) == out.size() * sizeof(U)); + assert(in.size() == out.size()); assert(!are_aliased(in, out)); if (in.empty()) [[unlikely]] @@ -159,55 +248,52 @@ namespace xsimd::builder auto it = in.data(); auto const iend = in.data() + in.size(); - // Input and output may not have the same alignment so it may be impossible - // to get both aligned, so we align a single side. if constexpr (!align.start_aligned) { // The span may be too short to reach the next alignment boundary. - const auto head_bytes = std::min( - align_output ? bytes_to_next_aligned(ot, Arch::alignment()) - : bytes_to_next_aligned(it, Arch::alignment()), - in.size_bytes()); - assert(head_bytes % sizeof(T) == 0); - assert(head_bytes % sizeof(U) == 0); - - if (can_overlap && (head_bytes != 0) && (in.size() >= input_batch::size)) + const auto head = std::min( + align_output ? bytes_to_next_aligned(ot, Arch::alignment()) / sizeof(U) + : bytes_to_next_aligned(it, Arch::alignment()) / sizeof(T), + in.size()); + + if (opts.pure && (head != 0) && (in.size() >= step)) { - // Recompute the head as a full batch, the body overwrites the excess. - const auto x = load_batch(it); - store_batch(func(x), ot); + // Recompute the head as a full step, the body overwrites the excess. + store_batches( + mapper(load_batches(it)), + ot); } else { - map_unary_batch(it, it + head_bytes / sizeof(T), ot, func); + map_unary_batch(it, it + head, ot, func); } - it += head_bytes / sizeof(T); - ot += head_bytes / sizeof(U); + it += head; + ot += head; } - // Unrolled loop processing multiple batches at a time - while (static_cast(iend - it) >= opts.unroll_factor * input_batch::size) + // Unrolled loop processing multiple steps at a time + while (static_cast(iend - it) >= opts.unroll_factor * step) { - std::array x; + std::array, in_arity>, opts.unroll_factor> x; for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - x[u] = load_batch(it + u * input_batch::size); + x[u] = load_batches(it + u * step); } for (std::size_t u = 0; u < opts.unroll_factor; ++u) { - store_batch(func(x[u]), ot + u * output_batch::size); + store_batches(mapper(x[u]), ot + u * step); } - it += opts.unroll_factor * input_batch::size; - ot += opts.unroll_factor * output_batch::size; + it += opts.unroll_factor * step; + ot += opts.unroll_factor * step; } - while (static_cast(iend - it) >= input_batch::size) + while (static_cast(iend - it) >= step) { - const auto x = load_batch(it); - store_batch(func(x), ot); - it += input_batch::size; - ot += output_batch::size; + auto x = load_batches(it); + store_batches(mapper(x), ot); + it += step; + ot += step; } // Unlikely to be skipped, meant for users that know they allocate @@ -215,11 +301,11 @@ namespace xsimd::builder if constexpr (!align.end_aligned) { auto const oend = out.data() + out.size(); - if (can_overlap && (it != iend) && (in.size() >= input_batch::size)) [[likely]] + if (opts.pure && (it != iend) && (in.size() >= step)) [[likely]] { // Recompute overlapping data, this time starting from the end. - const auto x = load_batch(iend - input_batch::size); - store_batch(func(x), oend - output_batch::size); + auto x = load_batches(iend - step); + store_batches(mapper(x), oend - step); } else { diff --git a/test/test_builder.cpp b/test/test_builder.cpp index 177716f..5088ebd 100644 --- a/test/test_builder.cpp +++ b/test/test_builder.cpp @@ -1,14 +1,12 @@ -/*************************************************************************** - * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * - * Martin Renou * - * Copyright (c) QuantStack * - * Copyright (c) Serge Guelton * +/**************************************************************************** + * Copyright (c) xsimd-algorithm contributors * * * * Distributed under the terms of the BSD 3-Clause License. * * * * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ +#include #include #include @@ -17,31 +15,67 @@ #include "xsimd_algorithm/builder.hpp" -/// Map unary test that turned int32 into half has many int64. +/// Map unary test where one input batch pairs with two output batches. TEST_CASE("map_unary int32 to int64") { using input_type = std::int32_t; using output_type = std::int64_t; // Not a multiple of the batch size, to exercise the tail. - static constexpr std::size_t input_size = 94; - static constexpr std::size_t output_size = input_size * sizeof(input_type) / sizeof(output_type); + static constexpr std::size_t size = 94; - const auto input = xsimd::test::make_arange(input_size); - auto output = xsimd::test::aligned_vector(output_size); + const auto input = xsimd::test::make_arange(size); + auto output = xsimd::test::aligned_vector(size); const auto func = [](auto const& x) - { return xsimd::widen(x + input_type { 1 })[0]; }; + { return xsimd::widen(x + input_type { 1 }); }; xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); - static constexpr std::size_t in_batch_size = xsimd::batch::size; - static constexpr std::size_t out_batch_size = xsimd::batch::size; + for (std::size_t i = 0; i < size; ++i) + { + CAPTURE(i); + CHECK(output[i] == static_cast(input[i] + 1)); + } +} + +/// Map unary test where two input batches pair with one output batch. +TEST_CASE("map_unary int64 to int32") +{ + using input_type = std::int64_t; + using output_type = std::int32_t; + using input_batch = xsimd::batch; + using output_batch = xsimd::batch; + + // Not a multiple of the batch size, to exercise the tail. + static constexpr std::size_t size = 94; + + const auto input = xsimd::test::make_arange(size); + auto output = xsimd::test::aligned_vector(size); + + // xsimd has no narrowing counterpart to widen, so truncate by keeping the low + // half of each lane, those of the first batch followed by those of the second. + struct low_halves + { + static constexpr unsigned get(unsigned i, unsigned n) + { + return (i < n / 2) ? 2 * i : n + 2 * (i - n / 2); + } + }; + + const auto func = [](std::array const& x) -> output_batch + { + return xsimd::shuffle( + xsimd::bitwise_cast(x[0] + input_type { 1 }), + xsimd::bitwise_cast(x[1] + input_type { 1 }), + xsimd::make_batch_constant()); + }; + + xsimd::builder::map_unary(xsimd::test::as_span(input), xsimd::test::as_span(output), func); - for (std::size_t i = 0; i < output_size; ++i) + for (std::size_t i = 0; i < size; ++i) { - const auto in_index = (i / out_batch_size) * in_batch_size + (i % out_batch_size); CAPTURE(i); - CHECK(output[i] == static_cast(input[in_index] + 1)); + CHECK(output[i] == static_cast(input[i] + 1)); } }