diff --git a/Cargo.toml b/Cargo.toml index ae164db13c2..0a5f0f21001 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -378,6 +378,7 @@ get_unwrap = "deny" host_endian_bytes = "deny" if_then_some_else_none = "deny" inconsistent_struct_constructor = "deny" +inline_always = "deny" manual_assert = "deny" manual_is_variant_and = "deny" many_single_char_names = "deny" diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index 3c77e146ad0..c1003ce86e3 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -33,6 +33,7 @@ pub trait UnpackStrategy { pub struct BitPackingStrategy; impl> UnpackStrategy for BitPackingStrategy { + #[allow(clippy::inline_always)] #[inline(always)] unsafe fn unpack_chunk( &self, @@ -154,6 +155,7 @@ impl> UnpackedChunks { }) } + #[allow(clippy::inline_always)] #[inline(always)] const fn elems_per_chunk(&self) -> usize { 128 * self.bit_width / size_of::() diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index cb94618c071..14aac075d37 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -32,6 +32,7 @@ struct FoRStrategy { } impl + FoR> UnpackStrategy for FoRStrategy { + #[allow(clippy::inline_always)] #[inline(always)] unsafe fn unpack_chunk( &self, diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index d2b778fdbbf..658138a905e 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -101,6 +101,7 @@ pub fn runend_decode_typed_bool( /// Fast path for few runs with no offset. Uses direct slice access to minimize overhead. /// This avoids the `trimmed_ends_iter` iterator chain which adds significant overhead /// for small numbers of runs. +#[allow(clippy::inline_always)] #[inline(always)] fn decode_few_runs_no_offset( ends: &[E], @@ -220,6 +221,7 @@ fn decode_bool_nullable( } /// Sequential decode for few runs - avoids prefill overhead. +#[allow(clippy::inline_always)] #[inline(always)] fn decode_nullable_sequential( run_ends: impl Iterator, diff --git a/vortex-array/src/aggregate_fn/fns/sum/mod.rs b/vortex-array/src/aggregate_fn/fns/sum/mod.rs index 1ceb529bb10..76eaba22c15 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/mod.rs @@ -370,6 +370,7 @@ pub(crate) fn make_zero_state(return_dtype: &DType) -> SumState { } /// Checked add for u64, returning true if overflow occurred. +#[allow(clippy::inline_always)] #[inline(always)] fn checked_add_u64(acc: &mut u64, val: u64) -> bool { match acc.checked_add(val) { @@ -382,6 +383,7 @@ fn checked_add_u64(acc: &mut u64, val: u64) -> bool { } /// Checked add for i64, returning true if overflow occurred. +#[allow(clippy::inline_always)] #[inline(always)] fn checked_add_i64(acc: &mut i64, val: i64) -> bool { match acc.checked_add(val) { diff --git a/vortex-array/src/aggregate_fn/typed.rs b/vortex-array/src/aggregate_fn/typed.rs index 65c4127a55b..b5e95febea9 100644 --- a/vortex-array/src/aggregate_fn/typed.rs +++ b/vortex-array/src/aggregate_fn/typed.rs @@ -63,11 +63,13 @@ pub(super) struct AggregateFnInner { } impl DynAggregateFn for AggregateFnInner { + #[allow(clippy::inline_always)] #[inline(always)] fn as_any(&self) -> &dyn Any { self } + #[allow(clippy::inline_always)] #[inline(always)] fn id(&self) -> AggregateFnId { V::id(&self.vtable) diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index a7fb4fc67dd..76c0b33cd18 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -84,12 +84,14 @@ impl ArrayRef { } /// Returns a reference to the `dyn DynArrayData` inside the inner. + #[allow(clippy::inline_always)] #[inline(always)] pub(crate) fn dyn_array(&self) -> &dyn DynArrayData { &self.0.data } /// Returns a mutable reference to the inner if this is the sole owner. + #[allow(clippy::inline_always)] #[inline(always)] pub(crate) fn inner_mut(&mut self) -> Option<&mut ArrayInner> { Arc::get_mut(&mut self.0) @@ -119,6 +121,7 @@ impl ArrayRef { /// /// # Safety /// The caller must guarantee the concrete type behind `dyn DynArrayData` is `ArrayData`. + #[allow(clippy::inline_always)] #[inline(always)] pub(crate) unsafe fn downcast_inner_unchecked( self, @@ -791,6 +794,7 @@ impl ArrayRef { } impl IntoArray for ArrayRef { + #[allow(clippy::inline_always)] #[inline(always)] fn into_array(self) -> ArrayRef { self diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 4769073b82c..a9def0318bf 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -348,6 +348,7 @@ impl Array { } /// Returns the internal [`ArrayRef`]. + #[allow(clippy::inline_always)] #[inline(always)] pub fn as_array(&self) -> &ArrayRef { &self.inner @@ -361,6 +362,7 @@ impl Array { } /// Downcast the inner `ArrayRef` to `&ArrayData`. + #[allow(clippy::inline_always)] #[inline(always)] fn downcast_inner(&self) -> &ArrayData { let any = self.inner.dyn_array().as_any(); @@ -508,6 +510,7 @@ impl AsRef for Array { } impl IntoArray for Array { + #[allow(clippy::inline_always)] #[inline(always)] fn into_array(self) -> ArrayRef { self.inner diff --git a/vortex-array/src/arrays/bool/compute/filter.rs b/vortex-array/src/arrays/bool/compute/filter.rs index 98f52ed11ee..106bd9650dd 100644 --- a/vortex-array/src/arrays/bool/compute/filter.rs +++ b/vortex-array/src/arrays/bool/compute/filter.rs @@ -129,6 +129,7 @@ fn filter_pext_fallback(src: &BitBuffer, mask_buf: &BitBuffer, true_count: usize /// Extracted so the same logic is shared between the software and hardware paths. /// Uses raw pointer writes instead of Vec::push to eliminate bounds checks /// in the hot loop — we know the exact output size from true_count. +#[allow(clippy::inline_always)] #[inline(always)] #[allow(clippy::cast_possible_truncation)] fn filter_inner( @@ -227,6 +228,7 @@ fn filter_inner( /// lookup table per mask byte. Each byte PEXT is a single table lookup with no /// data dependencies between bytes, making this faster than the parallel-prefix /// approach (~12ns vs ~18ns per word). +#[allow(clippy::inline_always)] #[inline(always)] pub fn pext_fallback(src: u64, mask: u64) -> u64 { pext_byte_lut(src, mask) @@ -268,6 +270,7 @@ static BYTE_PEXT_LUT: &[u8; 256 * 256] = &{ }; /// Byte-level PEXT using precomputed lookup table. +#[allow(clippy::inline_always)] #[inline(always)] fn pext_byte_lut(src: u64, mask: u64) -> u64 { let src_bytes = src.to_le_bytes(); diff --git a/vortex-array/src/arrays/filter/execute/simd_compress/mod.rs b/vortex-array/src/arrays/filter/execute/simd_compress/mod.rs index bb9ebea63a7..807259cd5e7 100644 --- a/vortex-array/src/arrays/filter/execute/simd_compress/mod.rs +++ b/vortex-array/src/arrays/filter/execute/simd_compress/mod.rs @@ -152,6 +152,7 @@ const fn compress_lut( /// The pointer contract of [`filter_slice_by_bitmap`] / [`filter_slice_mut_by_bitmap`] must hold, /// and `bits` must only select elements that are in bounds. #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), not(miri)))] +#[allow(clippy::inline_always)] #[inline(always)] unsafe fn compress_tail( src: *const u8, @@ -180,6 +181,7 @@ unsafe fn compress_tail( /// must not overlap `src` and must have room at `write_pos`; in-place, forward compaction must /// guarantee `write_pos <= word_start`. #[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), not(miri)))] +#[allow(clippy::inline_always)] #[inline(always)] unsafe fn bulk_copy( src: *const u8, diff --git a/vortex-array/src/arrays/filter/execute/slice.rs b/vortex-array/src/arrays/filter/execute/slice.rs index ea428564dcb..52d1328c92e 100644 --- a/vortex-array/src/arrays/filter/execute/slice.rs +++ b/vortex-array/src/arrays/filter/execute/slice.rs @@ -14,6 +14,7 @@ use vortex_mask::MaskValues; /// Invoke `f` with each `(word, word_start, word_len)` of the mask bitmap, where `word` holds /// the mask bits for elements `word_start..word_start + word_len` in its low `word_len` bits. +#[allow(clippy::inline_always)] #[inline(always)] pub(super) fn for_each_mask_word(mask: &MaskValues, mut f: impl FnMut(u64, usize, usize)) { let bits = mask.bit_buffer(); diff --git a/vortex-array/src/arrays/fixed_width/take/avx2/gather.rs b/vortex-array/src/arrays/fixed_width/take/avx2/gather.rs index 5f201a65192..79e0a43358f 100644 --- a/vortex-array/src/arrays/fixed_width/take/avx2/gather.rs +++ b/vortex-array/src/arrays/fixed_width/take/avx2/gather.rs @@ -110,6 +110,7 @@ macro_rules! impl_gather { const STRIDE: usize = $STRIDE; #[allow(unused_unsafe, clippy::cast_possible_truncation)] + #[allow(clippy::inline_always)] #[inline(always)] unsafe fn gather( indices: *const $idx, diff --git a/vortex-array/src/arrays/fixed_width/take/avx2/mod.rs b/vortex-array/src/arrays/fixed_width/take/avx2/mod.rs index 6a0ba762552..cba089f1011 100644 --- a/vortex-array/src/arrays/fixed_width/take/avx2/mod.rs +++ b/vortex-array/src/arrays/fixed_width/take/avx2/mod.rs @@ -94,6 +94,7 @@ const fn i32_gather_can_address(values_len: usize) -> bool { /// `size_of::()`). Each valid lane copies the initialized representation of an existing /// `Out`; invalid lanes are masked and cause a panic before the output buffer is initialized. /// Gather instructions tolerate the source's potentially weaker alignment. +#[allow(clippy::inline_always)] #[inline(always)] fn exec_take(values: &[Out], indices: &[Idx]) -> Buffer where diff --git a/vortex-array/src/arrays/fixed_width/take/scalar.rs b/vortex-array/src/arrays/fixed_width/take/scalar.rs index 7a1d1865be3..5a3510efc2d 100644 --- a/vortex-array/src/arrays/fixed_width/take/scalar.rs +++ b/vortex-array/src/arrays/fixed_width/take/scalar.rs @@ -6,6 +6,7 @@ use vortex_buffer::BufferMut; use crate::dtype::IntegerPType; +#[allow(clippy::inline_always)] #[inline(always)] pub(crate) fn take_values_scalar( values: &[T], diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 9ad44db8442..7f3a436996f 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -32,6 +32,7 @@ impl Display for ScalarFnData { impl ScalarFnData { /// Get the scalar function bound to this array. + #[allow(clippy::inline_always)] #[inline(always)] pub fn scalar_fn(&self) -> &ScalarFnRef { &self.scalar_fn diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index 3fff845b361..1fcd313fa46 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -153,6 +153,7 @@ fn collect_lane_bits( /// /// Offsets at null positions are not validated, so an out-of-bounds or inverted range is /// possible there; such lanes answer `false`, and validity masks them out of the result anyway. +#[allow(clippy::inline_always)] #[inline(always)] fn value_eq(bytes: &[u8], start: usize, end: usize, constant: &[u8]) -> bool { // A lane can only match when its length equals the constant's, so lanes of a different @@ -164,6 +165,7 @@ fn value_eq(bytes: &[u8], start: usize, end: usize, constant: &[u8]) -> bool { /// Order `bytes[start..end]` against `constant`, treating the unvalidated garbage ranges that /// can appear at null positions as empty; validity masks those lanes out of the result anyway. +#[allow(clippy::inline_always)] #[inline(always)] fn value_cmp(bytes: &[u8], start: usize, end: usize, constant: &[u8]) -> Ordering { bytes.get(start..end).unwrap_or_default().cmp(constant) diff --git a/vortex-array/src/arrays/varbinview/compact.rs b/vortex-array/src/arrays/varbinview/compact.rs index 9f1fa8c350f..75ed5d813db 100644 --- a/vortex-array/src/arrays/varbinview/compact.rs +++ b/vortex-array/src/arrays/varbinview/compact.rs @@ -65,6 +65,7 @@ impl VarBinViewArray { /// Iterates over all valid, non-inlined views, calling the provided /// closure for each one. + #[allow(clippy::inline_always)] #[inline(always)] fn iter_valid_views(&self, ctx: &mut ExecutionCtx, mut f: F) -> VortexResult<()> where diff --git a/vortex-array/src/dtype/decimal/precision.rs b/vortex-array/src/dtype/decimal/precision.rs index 79d47c65462..62c00b80e0d 100644 --- a/vortex-array/src/dtype/decimal/precision.rs +++ b/vortex-array/src/dtype/decimal/precision.rs @@ -93,12 +93,14 @@ impl PrecisionScale { } /// The precision is the number of significant figures that the decimal tracks. + #[allow(clippy::inline_always)] #[inline(always)] pub fn precision(&self) -> u8 { self.precision.get() } /// The scale is the maximum number of digits relative to the decimal point. + #[allow(clippy::inline_always)] #[inline(always)] pub fn scale(&self) -> i8 { self.scale diff --git a/vortex-array/src/matcher.rs b/vortex-array/src/matcher.rs index c3f9e542f13..4bda03f420e 100644 --- a/vortex-array/src/matcher.rs +++ b/vortex-array/src/matcher.rs @@ -24,11 +24,13 @@ pub struct AnyArray; impl Matcher for AnyArray { type Match<'a> = &'a ArrayRef; + #[allow(clippy::inline_always)] #[inline(always)] fn matches(_array: &ArrayRef) -> bool { true } + #[allow(clippy::inline_always)] #[inline(always)] fn try_match(array: &ArrayRef) -> Option> { Some(array) diff --git a/vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs b/vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs index 1247358dce1..2ca23f023ed 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs +++ b/vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs @@ -93,6 +93,7 @@ fn compare_primitive_typed( Ok(BoolArray::try_new(bits, validity)?.into_array()) } +#[allow(clippy::inline_always)] #[inline(always)] fn apply_op(lhs: T, rhs: T, op: CompareOperator) -> bool { match op { diff --git a/vortex-array/src/scalar_fn/typed.rs b/vortex-array/src/scalar_fn/typed.rs index 84670b537d7..3ee15367e6c 100644 --- a/vortex-array/src/scalar_fn/typed.rs +++ b/vortex-array/src/scalar_fn/typed.rs @@ -113,11 +113,13 @@ pub(super) trait DynScalarFn: 'static + Send + Sync + super::sealed::Sealed { } impl DynScalarFn for TypedScalarFnInstance { + #[allow(clippy::inline_always)] #[inline(always)] fn as_any(&self) -> &dyn Any { self } + #[allow(clippy::inline_always)] #[inline(always)] fn id(&self) -> ScalarFnId { V::id(&self.vtable) diff --git a/vortex-array/src/test_harness/trace/mod.rs b/vortex-array/src/test_harness/trace/mod.rs index 0890f85f067..1e053cf5a7a 100644 --- a/vortex-array/src/test_harness/trace/mod.rs +++ b/vortex-array/src/test_harness/trace/mod.rs @@ -262,6 +262,7 @@ pub fn trace_op_with( } /// Returns true when the current thread has an active trace recorder. +#[allow(clippy::inline_always)] #[inline(always)] pub(crate) fn is_active() -> bool { if ACTIVE_TRACE_COUNT.load(Ordering::Relaxed) == 0 { @@ -270,6 +271,7 @@ pub(crate) fn is_active() -> bool { TRACE_INTEREST.with(|interest| interest.get().is_active()) } +#[allow(clippy::inline_always)] #[inline(always)] fn attempts_enabled() -> bool { if ATTEMPTS_TRACE_COUNT.load(Ordering::Relaxed) == 0 { diff --git a/vortex-buffer/src/bit/buf.rs b/vortex-buffer/src/bit/buf.rs index 1d7f42b70fb..6bc2d3c3626 100644 --- a/vortex-buffer/src/bit/buf.rs +++ b/vortex-buffer/src/bit/buf.rs @@ -308,12 +308,14 @@ impl BitBuffer { } /// Offset of the start of the buffer in bits. + #[allow(clippy::inline_always)] #[inline(always)] pub fn offset(&self) -> usize { self.offset } /// Get a reference to the underlying buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub fn inner(&self) -> &ByteBuffer { &self.buffer diff --git a/vortex-buffer/src/bit/buf_mut.rs b/vortex-buffer/src/bit/buf_mut.rs index d7dc59e7fe6..99935f627f3 100644 --- a/vortex-buffer/src/bit/buf_mut.rs +++ b/vortex-buffer/src/bit/buf_mut.rs @@ -17,6 +17,7 @@ use crate::bit::unset_bit_unchecked; use crate::buffer_mut; /// Sets all bits in the bit-range `[start_bit, end_bit)` of `slice` to `value`. +#[allow(clippy::inline_always)] #[inline(always)] pub(crate) fn fill_bits(slice: &mut [u8], start_bit: usize, end_bit: usize, value: bool) { if start_bit >= end_bit { @@ -156,6 +157,7 @@ impl BitBufferMut { } /// Create a new empty `BitBufferMut`. + #[allow(clippy::inline_always)] #[inline(always)] pub fn empty() -> Self { Self::with_capacity(0) @@ -242,24 +244,28 @@ impl BitBufferMut { } /// Get the current populated length of the buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub fn len(&self) -> usize { self.len } /// True if the buffer has length 0. + #[allow(clippy::inline_always)] #[inline(always)] pub fn is_empty(&self) -> bool { self.len == 0 } /// Get the current bit offset of the buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub fn offset(&self) -> usize { self.offset } /// Get the value at the requested index. + #[allow(clippy::inline_always)] #[inline(always)] pub fn value(&self, index: usize) -> bool { assert!(index < self.len); @@ -272,12 +278,14 @@ impl BitBufferMut { /// # Safety /// /// The caller must ensure that `index` is less than the length of the buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub unsafe fn value_unchecked(&self, index: usize) -> bool { unsafe { get_bit_unchecked(self.buffer.as_ptr(), self.offset + index) } } /// Get the bit capacity of the buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub fn capacity(&self) -> usize { (self.buffer.capacity() * 8) - self.offset @@ -385,6 +393,7 @@ impl BitBufferMut { /// /// - `new_len` must be less than or equal to [`capacity()`](Self::capacity) /// - The elements at `old_len..new_len` must be initialized + #[allow(clippy::inline_always)] #[inline(always)] pub unsafe fn set_len(&mut self, new_len: usize) { debug_assert!( @@ -502,6 +511,7 @@ impl BitBufferMut { /// /// This operates on an arbitrary range within the existing length of the buffer. /// Panics if `end > self.len` or `start > end`. + #[allow(clippy::inline_always)] #[inline(always)] pub fn fill_range(&mut self, start: usize, end: usize, value: bool) { assert!(end <= self.len, "end {end} exceeds len {}", self.len); @@ -517,6 +527,7 @@ impl BitBufferMut { /// # Safety /// /// The caller must ensure that `start <= end <= self.len`. + #[allow(clippy::inline_always)] #[inline(always)] pub unsafe fn fill_range_unchecked(&mut self, start: usize, end: usize, value: bool) { fill_bits( diff --git a/vortex-buffer/src/bit/meta.rs b/vortex-buffer/src/bit/meta.rs index 621783b8e4a..d7197e912d8 100644 --- a/vortex-buffer/src/bit/meta.rs +++ b/vortex-buffer/src/bit/meta.rs @@ -71,18 +71,21 @@ impl BitBufferMeta { } /// The sub-byte bit offset. Always `< 8`. + #[allow(clippy::inline_always)] #[inline(always)] pub fn offset(&self) -> usize { self.offset } /// The logical length of the bitset in bits. + #[allow(clippy::inline_always)] #[inline(always)] pub fn len(&self) -> usize { self.len } /// Returns `true` if the bitset is empty. + #[allow(clippy::inline_always)] #[inline(always)] pub fn is_empty(&self) -> bool { self.len == 0 diff --git a/vortex-buffer/src/bit/mod.rs b/vortex-buffer/src/bit/mod.rs index 46d1bee89be..c0d82e10b3f 100644 --- a/vortex-buffer/src/bit/mod.rs +++ b/vortex-buffer/src/bit/mod.rs @@ -158,6 +158,7 @@ where /// # Panics /// /// Panics if `index` is not between 0 and length of `buf * 8`. +#[allow(clippy::inline_always)] #[inline(always)] pub fn get_bit(buf: &[u8], index: usize) -> bool { buf[index / 8] & (1 << (index % 8)) != 0 @@ -168,6 +169,7 @@ pub fn get_bit(buf: &[u8], index: usize) -> bool { /// # Safety /// /// `index` must be between 0 and length of `buf * 8`. +#[allow(clippy::inline_always)] #[inline(always)] pub unsafe fn get_bit_unchecked(buf: *const u8, index: usize) -> bool { (unsafe { *buf.add(index / 8) } & (1 << (index % 8))) != 0 @@ -178,6 +180,7 @@ pub unsafe fn get_bit_unchecked(buf: *const u8, index: usize) -> bool { /// # Safety /// /// `index` must be between 0 and length of `buf * 8`. +#[allow(clippy::inline_always)] #[inline(always)] pub unsafe fn set_bit_unchecked(buf: *mut u8, index: usize) { unsafe { *buf.add(index / 8) |= 1 << (index % 8) }; @@ -188,6 +191,7 @@ pub unsafe fn set_bit_unchecked(buf: *mut u8, index: usize) { /// # Safety /// /// `index` must be between 0 and length of `buf * 8`. +#[allow(clippy::inline_always)] #[inline(always)] pub unsafe fn unset_bit_unchecked(buf: *mut u8, index: usize) { unsafe { *buf.add(index / 8) &= !(1 << (index % 8)) }; diff --git a/vortex-buffer/src/bit/pack.rs b/vortex-buffer/src/bit/pack.rs index 565ce1e8345..128e27eb007 100644 --- a/vortex-buffer/src/bit/pack.rs +++ b/vortex-buffer/src/bit/pack.rs @@ -56,6 +56,7 @@ where /// the wider pack saves — and an indirect call per word is worse still (~4x on cheap /// predicates), since an opaque call target blocks fill/pack fusion regardless of how cheap /// the kernel *selection* is. For provably cheap predicates, use [`collect_bool_words_multiversioned`]. +#[allow(clippy::inline_always)] #[inline(always)] pub(crate) fn collect_bool_words_inline(words: &mut [u64], len: usize, f: F) where @@ -150,6 +151,7 @@ where /// /// Marked `#[inline(always)]` so each `#[target_feature]` wrapper gets its own fully-inlined /// copy compiled with that feature set. +#[allow(clippy::inline_always)] #[inline(always)] fn collect_bool_words_with(words: &mut [u64], len: usize, mut f: F, pack: P) where diff --git a/vortex-buffer/src/bit/view.rs b/vortex-buffer/src/bit/view.rs index f4d46c93da7..073df5943bc 100644 --- a/vortex-buffer/src/bit/view.rs +++ b/vortex-buffer/src/bit/view.rs @@ -112,12 +112,14 @@ impl<'a> BitBufferView<'a> { } /// Offset of the start of the view in bits. Always `< 8`. + #[allow(clippy::inline_always)] #[inline(always)] pub fn offset(&self) -> usize { self.offset } /// Get a reference to the underlying byte slice. + #[allow(clippy::inline_always)] #[inline(always)] pub fn inner(&self) -> &'a [u8] { self.buffer @@ -301,6 +303,7 @@ impl<'a> BitBufferMutView<'a> { } /// Offset of the start of the view in bits. Always `< 8`. + #[allow(clippy::inline_always)] #[inline(always)] pub fn offset(&self) -> usize { self.offset @@ -429,6 +432,7 @@ impl<'a> BitBufferMutView<'a> { /// Sets all bits in the range `[start, end)` to `value`. /// /// Panics if `end > self.len()` or `start > end`. + #[allow(clippy::inline_always)] #[inline(always)] pub fn fill_range(&mut self, start: usize, end: usize, value: bool) { assert!(end <= self.len, "end {end} exceeds len {}", self.len); @@ -442,6 +446,7 @@ impl<'a> BitBufferMutView<'a> { /// # Safety /// /// Caller must ensure that `start <= end <= self.len()`. + #[allow(clippy::inline_always)] #[inline(always)] pub unsafe fn fill_range_unchecked(&mut self, start: usize, end: usize, value: bool) { fill_bits(self.buffer, self.offset + start, self.offset + end, value); diff --git a/vortex-buffer/src/buffer.rs b/vortex-buffer/src/buffer.rs index 92215c26c63..8c78b9f8a1c 100644 --- a/vortex-buffer/src/buffer.rs +++ b/vortex-buffer/src/buffer.rs @@ -270,24 +270,28 @@ impl Buffer { } /// Returns the length of the buffer in elements of type T. + #[allow(clippy::inline_always)] #[inline(always)] pub fn len(&self) -> usize { self.length } /// Returns whether the buffer is empty. + #[allow(clippy::inline_always)] #[inline(always)] pub fn is_empty(&self) -> bool { self.length == 0 } /// Returns the alignment of the buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub fn alignment(&self) -> Alignment { self.alignment } /// Returns a slice over the buffer of elements of type T. + #[allow(clippy::inline_always)] #[inline(always)] pub fn as_slice(&self) -> &[T] { // SAFETY: alignment of Buffer is checked on construction @@ -295,6 +299,7 @@ impl Buffer { } /// Return a view over the buffer as an opaque byte slice. + #[allow(clippy::inline_always)] #[inline(always)] pub fn as_bytes(&self) -> &[u8] { self.bytes.as_ref() @@ -313,6 +318,7 @@ impl Buffer { /// /// Requires that `begin <= end` and `end <= self.len()`. /// Also requires that both `begin` and `end` are aligned to the buffer's required alignment. + #[allow(clippy::inline_always)] #[inline(always)] pub fn slice(&self, range: impl RangeBounds) -> Self { self.slice_with_alignment(range, self.alignment) @@ -324,6 +330,7 @@ impl Buffer { /// # Panics /// /// Requires that `begin <= end` and `end <= self.len()`. + #[allow(clippy::inline_always)] #[inline(always)] pub fn slice_unaligned(&self, range: impl RangeBounds) -> Self { self.slice_with_alignment(range, Alignment::of::()) @@ -399,6 +406,7 @@ impl Buffer { /// /// # Panics: /// Requires that the given sub slice is in fact contained within the Bytes buffer; otherwise this function will panic. + #[allow(clippy::inline_always)] #[inline(always)] pub fn slice_ref(&self, subset: &[T]) -> Self { self.slice_ref_with_alignment(subset, Alignment::of::()) diff --git a/vortex-buffer/src/buffer_mut.rs b/vortex-buffer/src/buffer_mut.rs index e5cb03c558b..044b3b3f270 100644 --- a/vortex-buffer/src/buffer_mut.rs +++ b/vortex-buffer/src/buffer_mut.rs @@ -204,12 +204,14 @@ impl BufferMut { } /// Get the alignment of the buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub fn alignment(&self) -> Alignment { self.alignment } /// Returns the length of the buffer. + #[allow(clippy::inline_always)] #[inline(always)] pub fn len(&self) -> usize { debug_assert_eq!(self.length, self.bytes.len() / size_of::()); @@ -217,6 +219,7 @@ impl BufferMut { } /// Returns whether the buffer is empty. + #[allow(clippy::inline_always)] #[inline(always)] pub fn is_empty(&self) -> bool { self.length == 0 diff --git a/vortex-buffer/src/const.rs b/vortex-buffer/src/const.rs index bd696c371bb..d37631c3eea 100644 --- a/vortex-buffer/src/const.rs +++ b/vortex-buffer/src/const.rs @@ -30,6 +30,7 @@ impl ConstBuffer { } /// Returns a slice over the buffer of elements of type T. + #[allow(clippy::inline_always)] #[inline(always)] pub fn as_slice(&self) -> &[T] { self.0.as_slice() diff --git a/vortex-compressor/src/stats/integer.rs b/vortex-compressor/src/stats/integer.rs index 6396660aa51..cded1a76aa4 100644 --- a/vortex-compressor/src/stats/integer.rs +++ b/vortex-compressor/src/stats/integer.rs @@ -479,6 +479,7 @@ struct LoopState { } /// Inner loop for non-null chunks of 64 values. +#[allow(clippy::inline_always)] #[inline(always)] fn inner_loop_nonnull( values: &[T; 64], @@ -500,6 +501,7 @@ fn inner_loop_nonnull( } /// Inner loop for nullable chunks of 64 values. +#[allow(clippy::inline_always)] #[inline(always)] fn inner_loop_nullable( values: &[T; 64], @@ -524,6 +526,7 @@ fn inner_loop_nullable( } /// Fallback inner loop for remainder values. +#[allow(clippy::inline_always)] #[inline(always)] fn inner_loop_naive( values: &[T], diff --git a/vortex-compute/src/lane_kernels/map_in_place.rs b/vortex-compute/src/lane_kernels/map_in_place.rs index 335e1c456df..7ca58ac1609 100644 --- a/vortex-compute/src/lane_kernels/map_in_place.rs +++ b/vortex-compute/src/lane_kernels/map_in_place.rs @@ -37,6 +37,7 @@ pub trait IndexedSinkExt: IndexedSink + Sized { where F: FnMut(Self::Item) -> Self::Write, { + #[allow(clippy::inline_always)] #[inline(always)] fn chunk(values: &mut S, f: &mut F, base: usize, count: usize) where @@ -88,6 +89,7 @@ pub trait IndexedSinkExt: IndexedSink + Sized { Self::Write: Default, F: FnMut(Self::Item) -> Option, { + #[allow(clippy::inline_always)] #[inline(always)] fn chunk(values: &mut S, base: usize, count: usize, f: &mut F) -> Option where @@ -162,6 +164,7 @@ pub trait IndexedSinkExt: IndexedSink + Sized { /// Bit-pack `is_none()` flags per lane, then AND with `src_chunk` post-loop to /// drop null-lane failures. The per-lane attribution work is `OR + shift` /// (no `min`/`csel`), giving LLVM more freedom to vectorize the value pipeline. + #[allow(clippy::inline_always)] #[inline(always)] fn chunk( values: &mut S, diff --git a/vortex-compute/src/lane_kernels/map_into.rs b/vortex-compute/src/lane_kernels/map_into.rs index 5add896c855..ee8c10b1477 100644 --- a/vortex-compute/src/lane_kernels/map_into.rs +++ b/vortex-compute/src/lane_kernels/map_into.rs @@ -57,6 +57,7 @@ pub trait IndexedSourceExt: IndexedSource + Sized { R: Copy + Default, F: Fn(Self::Item) -> Option, { + #[allow(clippy::inline_always)] #[inline(always)] fn chunk( values: &S, @@ -126,6 +127,7 @@ pub trait IndexedSourceExt: IndexedSource + Sized { where F: Fn(Self::Item) -> R, { + #[allow(clippy::inline_always)] #[inline(always)] fn chunk(values: &S, out: &mut [MaybeUninit], f: &F, base: usize, count: usize) where @@ -181,6 +183,7 @@ pub trait IndexedSourceExt: IndexedSource + Sized { where F: Fn(Self::Item) -> bool, { + #[allow(clippy::inline_always)] #[inline(always)] fn chunk(values: &S, f: &F, base: usize, count: usize) -> u64 where @@ -299,6 +302,7 @@ pub trait IndexedSourceExt: IndexedSource + Sized { /// Returns `true` if any lane in `[base, base+count)` failed (OR-reduced); /// the cold attribution path is called at the kernel level so it can be /// inlined separately for full vs remainder. + #[allow(clippy::inline_always)] #[inline(always)] fn chunk( values: &S, diff --git a/vortex-error/src/lib.rs b/vortex-error/src/lib.rs index 261de577f73..13ee1cfb31f 100644 --- a/vortex-error/src/lib.rs +++ b/vortex-error/src/lib.rs @@ -339,6 +339,7 @@ where { type Output = T; + #[allow(clippy::inline_always)] #[inline(always)] fn vortex_expect(self, msg: &'static str) -> Self::Output { self.map_err(|err| err.into()) @@ -349,6 +350,7 @@ where impl VortexExpect for Option { type Output = T; + #[allow(clippy::inline_always)] #[inline(always)] fn vortex_expect(self, msg: &'static str) -> Self::Output { self.unwrap_or_else(|| { diff --git a/vortex-mask/src/lib.rs b/vortex-mask/src/lib.rs index fcb4ee40e7f..ac680289ac1 100644 --- a/vortex-mask/src/lib.rs +++ b/vortex-mask/src/lib.rs @@ -304,6 +304,7 @@ impl Mask { })) } + #[allow(clippy::inline_always)] #[inline(always)] fn check_slices(len: usize, vec: &[(usize, usize)]) { assert!(vec.iter().all(|&(b, e)| b < e && e <= len)); diff --git a/vortex-row/src/codec.rs b/vortex-row/src/codec.rs index 58fbc3ccbbb..3f43bbdf7bd 100644 --- a/vortex-row/src/codec.rs +++ b/vortex-row/src/codec.rs @@ -1152,6 +1152,7 @@ fn encode_non_empty_varlen_body(bytes: &[u8], out: &mut [u8], descending: bool) /// # Safety /// `src` must be valid for 32 reads, `dst` valid for 32 writes, and the regions must not /// overlap. +#[allow(clippy::inline_always)] #[inline(always)] unsafe fn xor_copy_block(src: *const u8, dst: *mut u8) { // Four u64 lanes of 8 bytes each = 32 bytes total.