Skip to content

Fix unreachable constant-mask load_masked kernels (sse2, neon) #1376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DiamonDinoia merged 1 commit into from
Jul 10, 2026
Merged

Fix unreachable constant-mask load_masked kernels (sse2, neon) #1376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions include/xsimd/arch/xsimd_avx_128.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,19 @@ namespace xsimd
return _mm_cmp_pd(self, other, _CMP_NEQ_UQ);
}

// constant masks gain nothing on a single register; forward to runtime
// Masks that lower to plain moves go to sse2; the rest gain nothing on a
// single register, so take the runtime path.
template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_floating_point<T>::value>>
XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, convert<T>, Mode, requires_arch<avx_128>) noexcept
{
return load_masked(mem, mask.as_batch_bool(), convert<T> {}, Mode {}, avx_128 {});
XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
{
return load_masked(mem, mask, convert<T> {}, Mode {}, sse2 {});
}
else
{
return load_masked(mem, mask.as_batch_bool(), convert<T> {}, Mode {}, avx_128 {});
}
}

// Runtime-mask load (float/double).
Expand Down Expand Up @@ -142,7 +150,14 @@ namespace xsimd
template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_floating_point<T>::value>>
XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool_constant<T, A, Values...> mask, Mode, requires_arch<avx_128>) noexcept
{
store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx_128 {});
XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
{
store_masked(mem, src, mask, Mode {}, sse2 {});
}
else
{
store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx_128 {});
}
}

// Runtime-mask store (float/double).
Expand Down
22 changes: 10 additions & 12 deletions include/xsimd/arch/xsimd_neon.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,8 +539,8 @@ namespace xsimd
template <>
struct load_masked<>
{
template <size_t I, class A, class T, bool Use>
static XSIMD_INLINE batch<T, A> apply(T const* /* mem */, batch<T, A> acc, std::integral_constant<bool, Use>) noexcept
template <size_t I, class A, class T>
static XSIMD_INLINE batch<T, A> apply(T const* /* mem */, batch<T, A> acc) noexcept
{
return acc;
}
Expand All @@ -549,23 +549,21 @@ namespace xsimd
struct load_masked<Value, Values...>
{
template <size_t I, class A, class T>
static XSIMD_INLINE batch<T, A> apply(T const* mem, batch<T, A> acc, std::true_type) noexcept
{
return load_masked<Values...>::template apply<I + 1>(mem, insert(acc, mem[I], index<I> {}), std::integral_constant<bool, Value> {});
}
template <size_t I, class A, class T>
static XSIMD_INLINE batch<T, A> apply(T const* mem, batch<T, A> acc, std::false_type) noexcept
static XSIMD_INLINE batch<T, A> apply(T const* mem, batch<T, A> acc) noexcept
{
return load_masked<Values...>::template apply<I + 1>(mem, acc, std::integral_constant<bool, Value> {});
XSIMD_IF_CONSTEXPR(Value)
{
acc = insert(acc, mem[I], index<I> {});
}
return load_masked<Values...>::template apply<I + 1>(mem, acc);
}
};
}

template <class A, class T, bool Value, bool... Values, class Mode>
XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Value, Values...> /* mask */, Mode, requires_arch<neon>) noexcept
XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Value, Values...> /* mask */, convert<T>, Mode, requires_arch<neon>) noexcept
{
// Call insert whenever Values... are true
return detail::load_masked<Values...>::template apply<0>(mem, broadcast(T(0), A {}), std::integral_constant<bool, Value> {});
return detail::load_masked<Value, Values...>::template apply<0>(mem, batch<T, A>(T(0)));
}

/*********
Expand Down
78 changes: 44 additions & 34 deletions include/xsimd/arch/xsimd_sse2.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1100,54 +1100,64 @@ namespace xsimd
return { load_unaligned(mem, batch_bool<char, A> {}, r).data };
}

namespace detail
{
// Plain moves store-forward; vmaskmov never does on Intel and its
// stores are microcoded on AMD. So wider archs delegate the masks
// that lower to plain moves here rather than taking their runtime path.
template <class T, class A, bool... Values>
constexpr bool lowers_to_plain_moves(batch_bool_constant<T, A, Values...> mask) noexcept
{
return (mask.is_prefix() && mask.any() && !mask.all())
|| mask.suffix() == mask.size / 2;
}
}

// load_masked
template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_integral<T>::value>>
XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, convert<T>, Mode, requires_arch<sse2>) noexcept
{
XSIMD_IF_CONSTEXPR(mask.mask() == 0x1)
XSIMD_IF_CONSTEXPR(sizeof(T) == 2 && mask.prefix() == 1)
{
XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
{
return mm_loadu_si16(mem);
}
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
{
return mm_loadu_si32(mem);
}
XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
{
return mm_loadu_si64(mem);
}
return _mm_loadu_si16(mem);
}
else XSIMD_IF_CONSTEXPR(sizeof(T) == 4 && mask.prefix() == 1)
{
return _mm_loadu_si32(mem);
}
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8 && mask.prefix() == 1)
{
return _mm_loadu_si64(mem);
}
else XSIMD_IF_CONSTEXPR(sizeof(T) == 2 && mask.mask() == 0x3)
else XSIMD_IF_CONSTEXPR(sizeof(T) == 2 && mask.prefix() == 2)
{
return mm_loadu_si32(mem);
return _mm_loadu_si32(mem);
}
else XSIMD_IF_CONSTEXPR(sizeof(T) == 4 && mask.mask() == 0x3)
else XSIMD_IF_CONSTEXPR(sizeof(T) == 4 && mask.prefix() == 2)
{
return mm_loadu_si64(mem);
return _mm_loadu_si64(mem);
}
else
{
return load_masked<A>(mem, mask, convert<T> {}, Mode {}, common {});
}
}
template <class A, bool... Values, class Mode>
XSIMD_INLINE batch<float, A> load_masked(float const* mem, batch_bool_constant<float, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
XSIMD_INLINE batch<float, A> load_masked(float const* mem, batch_bool_constant<float, A, Values...> mask, convert<float>, Mode, requires_arch<sse2>) noexcept
{
XSIMD_IF_CONSTEXPR(mask.mask() == 0x1)
XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
{
return _mm_load_ss(mem);
}
else XSIMD_IF_CONSTEXPR(mask.countr_one() == 2)
else XSIMD_IF_CONSTEXPR(mask.prefix() == 2)
{
return _mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<__m64 const*>(mem));
}
else XSIMD_IF_CONSTEXPR(mask.countl_one() == 2)
else XSIMD_IF_CONSTEXPR(mask.suffix() == 2)
{
return _mm_loadh_pi(_mm_setzero_ps(), reinterpret_cast<__m64 const*>(mem + 2));
}
else XSIMD_IF_CONSTEXPR(mask.countr_one() == 3)
else XSIMD_IF_CONSTEXPR(mask.prefix() == 3)
{
__m128 const lo2 = _mm_castsi128_ps(_mm_loadl_epi64(reinterpret_cast<__m128i const*>(mem)));
return _mm_shuffle_ps(lo2, _mm_load_ss(mem + 2), _MM_SHUFFLE(3, 0, 1, 0));
Expand All @@ -1158,13 +1168,13 @@ namespace xsimd
}
}
template <class A, bool... Values, class Mode>
XSIMD_INLINE batch<double, A> load_masked(double const* mem, batch_bool_constant<double, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
XSIMD_INLINE batch<double, A> load_masked(double const* mem, batch_bool_constant<double, A, Values...> mask, convert<double>, Mode, requires_arch<sse2>) noexcept
{
XSIMD_IF_CONSTEXPR(mask.countr_one() == 1)
XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
{
return _mm_load_sd(mem);
}
else XSIMD_IF_CONSTEXPR(mask.countl_one() == 1)
else XSIMD_IF_CONSTEXPR(mask.suffix() == 1)
{
return _mm_loadh_pd(_mm_setzero_pd(), mem + 1);
}
Expand All @@ -1178,19 +1188,19 @@ namespace xsimd
template <class A, bool... Values, class Mode>
XSIMD_INLINE void store_masked(float* mem, batch<float, A> const& src, batch_bool_constant<float, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
{
XSIMD_IF_CONSTEXPR(mask.mask() == 0x1)
XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
{
_mm_store_ss(mem, src);
}
else XSIMD_IF_CONSTEXPR(mask.countr_one() == 2)
else XSIMD_IF_CONSTEXPR(mask.prefix() == 2)
{
_mm_storel_pi(reinterpret_cast<__m64*>(mem), src);
}
else XSIMD_IF_CONSTEXPR(mask.countl_one() == 2)
else XSIMD_IF_CONSTEXPR(mask.suffix() == 2)
{
_mm_storeh_pi(reinterpret_cast<__m64*>(mem + 2), src);
}
else XSIMD_IF_CONSTEXPR(mask.countr_one() == 3)
else XSIMD_IF_CONSTEXPR(mask.prefix() == 3)
{
_mm_storel_pi(reinterpret_cast<__m64*>(mem), src);
_mm_store_ss(mem + 2, _mm_movehl_ps(src, src));
Expand All @@ -1204,11 +1214,11 @@ namespace xsimd
template <class A, bool... Values, class Mode>
XSIMD_INLINE void store_masked(double* mem, batch<double, A> const& src, batch_bool_constant<double, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
{
XSIMD_IF_CONSTEXPR(mask.countr_one() == 1)
XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
{
_mm_store_sd(mem, src);
}
else XSIMD_IF_CONSTEXPR(mask.countl_one() == 1)
else XSIMD_IF_CONSTEXPR(mask.suffix() == 1)
{
_mm_storeh_pd(mem + 1, src);
}
Expand Down Expand Up @@ -2331,11 +2341,11 @@ namespace xsimd
aligned_mode,
requires_arch<sse2>) noexcept
{
XSIMD_IF_CONSTEXPR(mask.countr_one() == 2)
XSIMD_IF_CONSTEXPR(mask.prefix() == 2)
{
_mm_storel_pi(reinterpret_cast<__m64*>(mem), src);
}
else XSIMD_IF_CONSTEXPR(mask.countl_one() == 2)
else XSIMD_IF_CONSTEXPR(mask.suffix() == 2)
{
_mm_storeh_pi(reinterpret_cast<__m64*>(mem + 2), src);
}
Expand Down
28 changes: 28 additions & 0 deletions include/xsimd/types/xsimd_batch_constant.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,34 @@ namespace xsimd
return countl_one_impl(truncated_mask(), size);
}

// true when the set lanes form one contiguous run starting at lane 0
// (the empty and full masks are prefixes)
static constexpr bool is_prefix() noexcept
{
return (truncated_mask() & (truncated_mask() + 1u)) == 0u;
}

// true when the set lanes form one contiguous run ending at the last
// lane (the empty and full masks are suffixes)
static constexpr bool is_suffix() noexcept
{
return ((truncated_mask() ^ low_mask(size)) & ((truncated_mask() ^ low_mask(size)) + 1u)) == 0u;
}

// length of the set run when the mask is a pure prefix (first k lanes
// set, rest clear), else size + 1 so `prefix() == k` stays exact
static constexpr std::size_t prefix() noexcept
{
return is_prefix() ? countr_one() : size + 1;
}

// length of the set run when the mask is a pure suffix (last k lanes
// set, rest clear), else size + 1 so `suffix() == k` stays exact
static constexpr std::size_t suffix() noexcept
{
return is_suffix() ? countl_one() : size + 1;
}

private:
static constexpr int mask_helper(int acc) noexcept { return acc; }

Expand Down
74 changes: 74 additions & 0 deletions test/test_batch_constant.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,76 @@ struct constant_bool_batch_test
constexpr auto inv_x = ~x;
static_assert(std::is_same<decltype(inv_x), decltype(y)>::value, "~x == y");
}

struct first_half
{
static constexpr bool get(size_t index, size_t size) { return index < size / 2; }
};
struct second_half
{
static constexpr bool get(size_t index, size_t size) { return index >= size / 2; }
};
struct ends
{
static constexpr bool get(size_t index, size_t size) { return index == 0 || index + 1 == size; }
};
struct all_but_last
{
static constexpr bool get(size_t index, size_t size) { return index + 1 < size; }
};
struct all_but_first
{
static constexpr bool get(size_t index, size_t) { return index != 0; }
};
struct first_only
{
static constexpr bool get(size_t index, size_t) { return index == 0; }
};
struct last_only
{
static constexpr bool get(size_t index, size_t size) { return index + 1 == size; }
};

void test_shape() const
{
constexpr auto all_true = xsimd::make_batch_bool_constant<value_type, constant<true>, arch_type>();
constexpr auto all_false = xsimd::make_batch_bool_constant<value_type, constant<false>, arch_type>();
constexpr auto lo = xsimd::make_batch_bool_constant<value_type, first_half, arch_type>();
constexpr auto hi = xsimd::make_batch_bool_constant<value_type, second_half, arch_type>();
constexpr auto edges = xsimd::make_batch_bool_constant<value_type, ends, arch_type>();
constexpr auto size = decltype(all_true)::size;

static_assert(all_true.is_prefix() && all_true.is_suffix(), "full mask is prefix and suffix");
static_assert(all_false.is_prefix() && all_false.is_suffix(), "empty mask is prefix and suffix");
static_assert(lo.is_prefix() && !lo.is_suffix(), "first half is a prefix only");
static_assert(hi.is_suffix() && !hi.is_prefix(), "second half is a suffix only");
// {first, last} lanes: contiguous only when that covers the whole batch
static_assert(edges.is_prefix() == (size <= 2), "non-contiguous mask is not a prefix");
static_assert(edges.is_suffix() == (size <= 2), "non-contiguous mask is not a suffix");

// prefix()/suffix() return the set-run length, or size+1 when not that shape
static_assert(lo.prefix() == size / 2 && lo.suffix() == size + 1, "lo is the size/2 prefix, no suffix");
static_assert(hi.suffix() == size / 2 && hi.prefix() == size + 1, "hi is the size/2 suffix, no prefix");
static_assert(all_true.prefix() == size && all_true.suffix() == size, "full mask runs the whole width");
static_assert(all_false.prefix() == 0 && all_false.suffix() == 0, "empty mask has zero-length runs");
static_assert(edges.prefix() == (size <= 2 ? size : size + 1), "non-contiguous mask has no prefix length");

// off-by-one boundary: the length adjacent to the sentinel (size-1) must
// count exactly, and the sentinel must be exactly size+1 (never size, size+2)
constexpr auto pre1 = xsimd::make_batch_bool_constant<value_type, all_but_last, arch_type>();
constexpr auto suf1 = xsimd::make_batch_bool_constant<value_type, all_but_first, arch_type>();
constexpr auto f1 = xsimd::make_batch_bool_constant<value_type, first_only, arch_type>();
constexpr auto l1 = xsimd::make_batch_bool_constant<value_type, last_only, arch_type>();
static_assert(size < 2 || pre1.prefix() == size - 1, "size-1 prefix counts exactly");
static_assert(size < 2 || pre1.suffix() == size + 1, "a size-1 prefix is not a suffix");
static_assert(size < 2 || suf1.suffix() == size - 1, "size-1 suffix counts exactly");
static_assert(size < 2 || suf1.prefix() == size + 1, "a size-1 suffix is not a prefix");
static_assert(size < 2 || (pre1.suffix() != size && pre1.suffix() != size + 2), "sentinel is exactly size+1");
static_assert(f1.prefix() == 1, "single first lane is the 1-prefix");
static_assert(size < 2 || f1.suffix() == size + 1, "single first lane is not a suffix");
static_assert(l1.suffix() == 1, "single last lane is the 1-suffix");
static_assert(size < 2 || l1.prefix() == size + 1, "single last lane is not a prefix");
}
};

TEST_CASE_TEMPLATE("[constant bool batch]", B, BATCH_INT_TYPES)
Expand All @@ -410,5 +480,9 @@ TEST_CASE_TEMPLATE("[constant bool batch]", B, BATCH_INT_TYPES)
{
Test.test_ops();
}
SUBCASE("shape")
{
Test.test_shape();
}
}
#endif
Loading
Loading