Add Fastor library
This commit is contained in:
Bassem Girgis
16
noarch/include/Fastor/simd_vector/SIMDVector.h
Normal file
16
noarch/include/Fastor/simd_vector/SIMDVector.h
Normal file
@@ -0,0 +1,16 @@
|
||||
#ifndef SIMDVECTOR_H
|
||||
#define SIMDVECTOR_H
|
||||
|
||||
#include "Fastor/simd_vector/simd_vector_base.h"
|
||||
#include "Fastor/simd_vector/simd_vector_scalar.h"
|
||||
#include "Fastor/simd_vector/simd_vector_float.h"
|
||||
#include "Fastor/simd_vector/simd_vector_double.h"
|
||||
#include "Fastor/simd_vector/simd_vector_int32.h"
|
||||
#include "Fastor/simd_vector/simd_vector_int64.h"
|
||||
#include "Fastor/simd_vector/simd_vector_complex_scalar.h"
|
||||
#include "Fastor/simd_vector/simd_vector_complex_float.h"
|
||||
#include "Fastor/simd_vector/simd_vector_complex_double.h"
|
||||
#include "Fastor/simd_vector/simd_vector_common.h"
|
||||
|
||||
#endif // SIMDVECTOR_H
|
||||
|
||||
1255
noarch/include/Fastor/simd_vector/extintrin.h
Normal file
1255
noarch/include/Fastor/simd_vector/extintrin.h
Normal file
File diff suppressed because it is too large
Load Diff
225
noarch/include/Fastor/simd_vector/simd_vector_abi.h
Normal file
225
noarch/include/Fastor/simd_vector/simd_vector_abi.h
Normal file
@@ -0,0 +1,225 @@
|
||||
#ifndef FASTOR_SIMD_VECTOR_ABI
|
||||
#define FASTOR_SIMD_VECTOR_ABI
|
||||
|
||||
#include "Fastor/meta/meta.h"
|
||||
#include "Fastor/config/config.h"
|
||||
#include <complex>
|
||||
#include <type_traits>
|
||||
|
||||
namespace Fastor {
|
||||
|
||||
namespace simd_abi {
|
||||
|
||||
struct scalar {};
|
||||
struct sse {};
|
||||
struct avx {};
|
||||
struct avx512 {};
|
||||
struct mic {};
|
||||
template<size_t N> struct fixed_size {};
|
||||
|
||||
#ifndef FASTOR_DONT_VECTORISE
|
||||
#if defined(FASTOR_AVX512_IMPL)
|
||||
using native = simd_abi::avx512;
|
||||
#elif defined(FASTOR_AVX_IMPL)
|
||||
using native = simd_abi::avx;
|
||||
#elif defined(FASTOR_SSE2_IMPL)
|
||||
using native = simd_abi::sse;
|
||||
#else
|
||||
using native = simd_abi::scalar;
|
||||
#endif
|
||||
#else
|
||||
using native = simd_abi::scalar;
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
// Definition of DEFAULT_ABI is here
|
||||
//--------------------------------------------------------------------------------------------------------------//
|
||||
#define DEFAULT_ABI simd_abi::native
|
||||
//--------------------------------------------------------------------------------------------------------------//
|
||||
|
||||
|
||||
//--------------------------------------------------------------------------------------------------------------//
|
||||
namespace internal {
|
||||
|
||||
template<class __svec>
|
||||
struct get_simd_vector_size;
|
||||
template<template<typename, typename> class __svec, typename T, typename ABI>
|
||||
struct get_simd_vector_size<__svec<T,ABI>> {
|
||||
static constexpr size_t bitsize = std::is_same<ABI,simd_abi::avx512>::value
|
||||
? FASTOR_AVX512_BITSIZE : (std::is_same<ABI,simd_abi::avx>::value
|
||||
? FASTOR_AVX_BITSIZE : (std::is_same<ABI,simd_abi::sse>::value
|
||||
? FASTOR_SSE_BITSIZE : sizeof(T)*8));
|
||||
|
||||
// Size should be at least 1UL
|
||||
static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL;
|
||||
};
|
||||
// Specialisation for fixed size simd vectors
|
||||
template<template<typename, typename> class __svec, typename T, size_t N>
|
||||
struct get_simd_vector_size<__svec<T,simd_abi::fixed_size<N> > > {
|
||||
static constexpr size_t bitsize = N*8UL;
|
||||
static constexpr size_t value = N;
|
||||
};
|
||||
// Specialisation for complex simd vectors
|
||||
template<template<typename, typename> class __svec, typename ABI>
|
||||
struct get_simd_vector_size<__svec<std::complex<float>,ABI>> {
|
||||
using T = float;
|
||||
static constexpr size_t bitsize = std::is_same<ABI,simd_abi::avx512>::value
|
||||
? FASTOR_AVX512_BITSIZE : (std::is_same<ABI,simd_abi::avx>::value
|
||||
? FASTOR_AVX_BITSIZE : (std::is_same<ABI,simd_abi::sse>::value
|
||||
? FASTOR_SSE_BITSIZE : sizeof(T)*8));
|
||||
|
||||
// Size should be at least 1UL
|
||||
static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL;
|
||||
};
|
||||
template<template<typename, typename> class __svec, typename ABI>
|
||||
struct get_simd_vector_size<__svec<std::complex<double>,ABI>> {
|
||||
using T = double;
|
||||
static constexpr size_t bitsize = std::is_same<ABI,simd_abi::avx512>::value
|
||||
? FASTOR_AVX512_BITSIZE : (std::is_same<ABI,simd_abi::avx>::value
|
||||
? FASTOR_AVX_BITSIZE : (std::is_same<ABI,simd_abi::sse>::value
|
||||
? FASTOR_SSE_BITSIZE : sizeof(T)*8));
|
||||
|
||||
// Size should be at least 1UL
|
||||
static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL;
|
||||
};
|
||||
|
||||
|
||||
template<class __svec>
|
||||
struct get_half_simd_type;
|
||||
template<template<typename, typename> class __svec, typename T, typename ABI>
|
||||
struct get_half_simd_type<__svec<T,ABI>> {
|
||||
// If not a half of simd we give back the actual incoming type
|
||||
using actual_type = __svec<T,ABI>;
|
||||
using type = typename std::conditional< std::is_same<ABI,simd_abi::avx512>::value, __svec<T,simd_abi::avx>,
|
||||
typename std::conditional< std::is_same<ABI,simd_abi::avx>::value, __svec<T,simd_abi::sse>, actual_type>::type
|
||||
>::type;
|
||||
};
|
||||
|
||||
|
||||
template<class __svec>
|
||||
struct get_quarter_simd_type;
|
||||
template<template<typename, typename> class __svec, typename T, typename ABI>
|
||||
struct get_quarter_simd_type<__svec<T,ABI>> {
|
||||
// If not a quarter of simd we give back the actual incoming type
|
||||
using actual_type = __svec<T,ABI>;
|
||||
using type = typename std::conditional< std::is_same<ABI,simd_abi::avx512>::value, __svec<T,simd_abi::sse>, actual_type>::type;
|
||||
};
|
||||
|
||||
|
||||
template<class __svec, size_t N>
|
||||
struct is_exact_multiple_of_smaller_simd;
|
||||
template<template<typename, typename> class __svec, typename T, typename ABI, size_t N>
|
||||
struct is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N> {
|
||||
// If not a fraction of simd we give back the actual incoming type
|
||||
using actual_type = __svec<T,ABI>;
|
||||
// if N is half simd which=2, if it is a 1/4th which=4, else which=1
|
||||
static constexpr int which = get_simd_vector_size<__svec<T,ABI>>::value / N == 2UL ? 2UL
|
||||
: (get_simd_vector_size<__svec<T,ABI>>::value / N == 4UL ? 4UL : 1UL);
|
||||
static constexpr bool value = which != 1UL && !std::is_same<ABI,simd_abi::sse>::value ? true : false;
|
||||
|
||||
static constexpr bool is_half_of_avx512 = std::is_same<ABI,simd_abi::avx512>::value && which==2UL;
|
||||
static constexpr bool is_half_of_avx = std::is_same<ABI,simd_abi::avx>::value && which==2UL;
|
||||
static constexpr bool is_4th_of_avx512 = std::is_same<ABI,simd_abi::avx512>::value && which==4UL;
|
||||
// static constexpr bool is_4th_of_avx = std::is_same<ABI,simd_abi::avx>::value && which==4UL;
|
||||
|
||||
static constexpr bool is_half_type = is_half_of_avx512 || is_half_of_avx;
|
||||
static constexpr bool is_quarter_type = is_4th_of_avx512;
|
||||
|
||||
using half_type = typename std::conditional< is_half_of_avx512, __svec<T,simd_abi::avx>,
|
||||
typename std::conditional< is_half_of_avx, __svec<T,simd_abi::sse>, actual_type>::type
|
||||
>::type;
|
||||
using quarter_type = typename std::conditional< is_4th_of_avx512, __svec<T,simd_abi::sse>, actual_type>::type;
|
||||
|
||||
using type = typename std::conditional<is_half_type, half_type,
|
||||
typename std::conditional<is_quarter_type, quarter_type, actual_type>::type
|
||||
>::type;
|
||||
};
|
||||
|
||||
template<class __svec, size_t N>
|
||||
struct choose_best_simd_type;
|
||||
template<template<typename, typename> class __svec, typename TT, typename ABI, size_t N>
|
||||
struct choose_best_simd_type<__svec<TT,ABI>,N> {
|
||||
using T = remove_cv_ref_t<TT>;
|
||||
using actual_type = __svec<T,ABI>;
|
||||
static constexpr size_t _vec_size = get_simd_vector_size<__svec<T,ABI>>::value;
|
||||
// For exact fractions simd gets proper speed up for instance for matmul
|
||||
using exact_multiple_t = typename is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::type;
|
||||
static constexpr bool is_exact_multiple = is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::value;
|
||||
|
||||
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
|
||||
using size_based_type = typename std::conditional<is_exact_multiple, exact_multiple_t, actual_type>::type;
|
||||
#else
|
||||
using size_based_type = typename std::conditional<is_exact_multiple, exact_multiple_t,
|
||||
typename std::conditional<is_less<N,_vec_size>::value, typename get_half_simd_type<__svec<T,ABI>>::type, actual_type
|
||||
>::type
|
||||
>::type;
|
||||
#endif
|
||||
|
||||
// using size_based_type = typename std::conditional<is_exact_multiple, typename is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::type,
|
||||
// typename std::conditional<is_greater<_vec_size,2UL*N>::value, typename get_quarter_simd_type<__svec<T,ABI>>::type,
|
||||
// typename std::conditional<is_greater<_vec_size,N>::value, typename get_half_simd_type<__svec<T,ABI>>::type, actual_type
|
||||
// >::type
|
||||
// >::type
|
||||
// >::type;
|
||||
|
||||
// // For other fractions masking might be a better idea than, hence this special logic for remainder using is_less.
|
||||
// // For no special logic use the above case
|
||||
// using size_based_type = typename std::conditional<is_exact_multiple, typename is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::type,
|
||||
// typename std::conditional<is_greater<_vec_size,2UL*N>::value && is_greater<_vec_size % N,1UL>::value,
|
||||
// typename get_quarter_simd_type<__svec<T,ABI>>::type,
|
||||
// typename std::conditional<is_greater<_vec_size,N>::value && is_greater<_vec_size % N,1UL>::value,
|
||||
// typename get_half_simd_type<__svec<T,ABI>>::type, actual_type
|
||||
// >::type
|
||||
// >::type
|
||||
// >::type;
|
||||
|
||||
using type = typename std::conditional< std::is_same<T,float>::value ||
|
||||
std::is_same<T,double>::value ||
|
||||
std::is_same<T,std::complex<float>>::value ||
|
||||
std::is_same<T,std::complex<double>>::value ||
|
||||
std::is_same<T,int32_t>::value ||
|
||||
std::is_same<T,int64_t>::value,
|
||||
size_based_type,
|
||||
__svec<T,simd_abi::scalar>
|
||||
>::type;
|
||||
};
|
||||
|
||||
} // end of namesapce internal
|
||||
|
||||
|
||||
// This is specifically for matmul and other backend tensor kernels that need
|
||||
// to choose/switch between best simd types for utmost performance
|
||||
template<class __svec, size_t N>
|
||||
using choose_best_simd_t = typename internal::choose_best_simd_type<__svec,N>::type;
|
||||
//--------------------------------------------------------------------------------------------------------------//
|
||||
|
||||
|
||||
// For switching between complex and non-complex implementation of simd we need different return types
|
||||
//--------------------------------------------------------------------------------------------------------------//
|
||||
template<typename T, typename T2 = void>
|
||||
struct get_simd_cmplx_value_type;
|
||||
template<typename T>
|
||||
struct get_simd_cmplx_value_type<T, enable_if_t_<!is_complex_v_<T> > > {
|
||||
using type = T;
|
||||
};
|
||||
template<typename T>
|
||||
struct get_simd_cmplx_value_type<T, enable_if_t_<is_complex_v_<T> > > {
|
||||
using type = typename T::value_type;
|
||||
};
|
||||
|
||||
template<class VecType>
|
||||
struct simd_cmplx_value_type;
|
||||
template<template<typename, typename> class __svec, typename T, typename ABI>
|
||||
struct simd_cmplx_value_type< __svec<T,ABI> > {
|
||||
using type = typename get_simd_cmplx_value_type<T>::type;
|
||||
};
|
||||
template<class VecType>
|
||||
using simd_cmplx_value_t = typename simd_cmplx_value_type<VecType>::type;
|
||||
//--------------------------------------------------------------------------------------------------------------//
|
||||
|
||||
|
||||
|
||||
} // end of namesapce Fastor
|
||||
|
||||
#endif // FASTOR_SIMD_VECTOR_ABI
|
||||
391
noarch/include/Fastor/simd_vector/simd_vector_base.h
Normal file
391
noarch/include/Fastor/simd_vector/simd_vector_base.h
Normal file
@@ -0,0 +1,391 @@
|
||||
#ifndef SIMD_VECTOR_BASE_H
|
||||
#define SIMD_VECTOR_BASE_H
|
||||
|
||||
#include "Fastor/meta/meta.h"
|
||||
#include "Fastor/config/config.h"
|
||||
#include "Fastor/simd_vector/extintrin.h"
|
||||
#include "Fastor/simd_vector/simd_vector_abi.h"
|
||||
|
||||
#include<cmath>
|
||||
#include<complex>
|
||||
|
||||
|
||||
namespace Fastor {
|
||||
|
||||
/* The default SIMDVector class that falls back to scalar implementation
|
||||
* if SIMD types are not available or if vectorisation is disallowed
|
||||
*/
|
||||
//--------------------------------------------------------------------------------------------------------------------//
|
||||
template <typename CVT, typename ABI = simd_abi::native>
|
||||
struct SIMDVector {
|
||||
using T = remove_cv_ref_t<CVT>;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<T,ABI>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<T,ABI>>::value;}
|
||||
using vector_type = SIMDVector<T,ABI>;
|
||||
using value_type = T[Size];
|
||||
using scalar_value_type = T;
|
||||
using abi_type = ABI;
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value{} {}
|
||||
FASTOR_INLINE SIMDVector(T num) { std::fill(value, value+Size, num); }
|
||||
FASTOR_INLINE SIMDVector(const SIMDVector<T,ABI> &a) { std::copy(a.value,a.value+a.Size,value); }
|
||||
FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) { std::copy(data,data+Size,value); unused(Aligned); }
|
||||
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator=(T num) { std::fill(value, value+Size, num); return *this;}
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator=(const SIMDVector<T,ABI> &a) { std::copy(a.value,a.value+a.Size,value); return *this; };
|
||||
|
||||
FASTOR_INLINE void load(const T *data, bool Aligned=true ) { std::copy(data,data+Size,value); unused(Aligned);}
|
||||
FASTOR_INLINE void store(T *data, bool Aligned=true ) const { std::copy(value,value+Size,data); unused(Aligned);}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const T *data) { std::copy(data,data+Size,value); }
|
||||
FASTOR_INLINE void aligned_store(T *data) const { std::copy(value,value+Size,data);}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
std::fill(value, value+Size, 0);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
}
|
||||
|
||||
FASTOR_INLINE T operator[](FASTOR_INDEX i) const {return value[i];}
|
||||
FASTOR_INLINE T operator()(FASTOR_INDEX i) const {return value[i];}
|
||||
|
||||
// For compatibility with complex simd vector
|
||||
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> real() const {
|
||||
simd_cmplx_value_t<vector_type> arr[Size];
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
arr[i] = value[i].real();
|
||||
}
|
||||
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
|
||||
return out;
|
||||
}
|
||||
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> imag() const {
|
||||
simd_cmplx_value_t<vector_type> arr[Size];
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
arr[i] = value[i].imag();
|
||||
}
|
||||
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void set(T num) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] = num;
|
||||
}
|
||||
template<typename U, typename ... Args>
|
||||
FASTOR_INLINE void set(U first, Args ... args) {
|
||||
static_assert(sizeof...(args)+1==Size,"CANNOT SET VECTOR WITH SPECIFIED NUMBER OF VALUES DUE TO ABI CONSIDERATION");
|
||||
T arr[Size] = {first,args...};
|
||||
std::reverse_copy(arr, arr+Size, value);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(T num0) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] = num0+(T)i;
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(T num) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] += num;
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<T,ABI> &a) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] += a.value[i];
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(T num) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] -= num;
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<T,ABI> &a) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] -= a.value[i];
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(T num) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] *= num;
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<T,ABI> &a) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] *= a.value[i];
|
||||
}
|
||||
FASTOR_INLINE void operator/=(T num) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] /= num;
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<T,ABI> &a) {
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
value[i] /= a.value[i];
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE SIMDVector<T,ABI> shift(FASTOR_INDEX i) {
|
||||
SIMDVector<T,ABI> out;
|
||||
std::fill(out.value,out.value+out.Size,static_cast<T>(0));
|
||||
std::copy(value,value+Size, out.value+i);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE T sum() {
|
||||
T quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
quan += value[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE T product() {
|
||||
//! Don't use prod as that is the name of a meta-function
|
||||
T quan = 1;
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
quan *= value[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<T,ABI> reverse() {
|
||||
SIMDVector<T,ABI> out;
|
||||
std::copy(value,value+Size,out.value);
|
||||
std::reverse(out.value,out.value+Size);
|
||||
return out;
|
||||
}
|
||||
// This is for comatibility with complex simd vectors
|
||||
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> magnitude() {
|
||||
simd_cmplx_value_t<vector_type> arr[Size];
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
arr[i] = std::abs(value[i]);
|
||||
}
|
||||
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
|
||||
return out;
|
||||
}
|
||||
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> norm() {
|
||||
simd_cmplx_value_t<vector_type> arr[Size];
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
arr[i] = std::norm(value[i]);
|
||||
}
|
||||
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE T minimum() {
|
||||
T quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
if (value[i]<quan)
|
||||
quan = value[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE T maximum() {
|
||||
T quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
if (value[i]>quan)
|
||||
quan = value[i];
|
||||
return quan;
|
||||
}
|
||||
|
||||
FASTOR_INLINE T dot(const SIMDVector<T,ABI> &other) {
|
||||
T quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size;++i)
|
||||
quan += value[i]*other.value[i];
|
||||
return quan;
|
||||
}
|
||||
|
||||
template<typename U>
|
||||
FASTOR_INLINE SIMDVector<U,ABI> cast() {
|
||||
SIMDVector<U,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<Size;++i) {
|
||||
out.value[i] = static_cast<U>(value[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
#ifdef FASTOR_ZERO_INITIALISE
|
||||
FASTOR_ARCH_ALIGN T value[Size] = {};
|
||||
#else
|
||||
FASTOR_ARCH_ALIGN T value[Size];
|
||||
#endif
|
||||
};
|
||||
|
||||
template <typename T, typename ABI>
|
||||
constexpr FASTOR_INDEX SIMDVector<T,ABI>::Size;
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<T,ABI> a) {
|
||||
os << "[";
|
||||
for (FASTOR_INDEX i=0; i<a.size(); ++i)
|
||||
os << a.value[i] << ' ';
|
||||
os << "]";
|
||||
return os;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator+(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] + b.value[i];
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator+(const SIMDVector<T,ABI> &a, T b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] + b;
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator+(T a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a + b.value[i];
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator+(const SIMDVector<T,ABI> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator-(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] - b.value[i];
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator-(const SIMDVector<T,ABI> &a, T b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] - b;
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator-(T a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a - b.value[i];
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator-(const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = -b.value[i];
|
||||
return out;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator*(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] * b.value[i];
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator*(const SIMDVector<T,ABI> &a, T b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] * b;
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator*(T a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a * b.value[i];
|
||||
return out;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator/(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] / b.value[i];
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator/(const SIMDVector<T,ABI> &a, T b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a.value[i] / b;
|
||||
return out;
|
||||
}
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> operator/(T a, const SIMDVector<T,ABI> &b) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
|
||||
out.value[i] = a / b.value[i];
|
||||
return out;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> rcp(const SIMDVector<T,ABI> &a) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<a.Size; ++i)
|
||||
out.value[i] = T(1.)/a.value[i];
|
||||
return out;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> sqrt(const SIMDVector<T,ABI> &a) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<a.Size; ++i)
|
||||
out.value[i] = std::sqrt(a.value[i]);
|
||||
return out;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> rsqrt(const SIMDVector<T,ABI> &a) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<a.Size; ++i)
|
||||
out.value[i] = T(1.)/std::sqrt(a.value[i]);
|
||||
return out;
|
||||
}
|
||||
|
||||
template<typename T, typename ABI>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> abs(const SIMDVector<T,ABI> &a) {
|
||||
SIMDVector<T,ABI> out;
|
||||
for (FASTOR_INDEX i=0; i<a.Size; ++i)
|
||||
out.value[i] = std::abs(a.value[i]);
|
||||
return out;
|
||||
}
|
||||
|
||||
// For compatibility with complex simd vectors
|
||||
template<typename T, typename ABI, enable_if_t_<is_complex_v_<T>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> conj(const SIMDVector<T,ABI> &a) {
|
||||
T arr[SIMDVector<T,ABI>::Size];
|
||||
for (FASTOR_INDEX i=0UL; i<SIMDVector<T,ABI>::Size; ++i) {
|
||||
arr[i] = std::conj(a[i]);
|
||||
}
|
||||
return SIMDVector<T,ABI>(arr,false);
|
||||
}
|
||||
template<typename T, typename ABI, enable_if_t_<is_complex_v_<T>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<T,ABI> arg(const SIMDVector<T,ABI> &a) {
|
||||
T arr[SIMDVector<T,ABI>::Size];
|
||||
for (FASTOR_INDEX i=0UL; i<SIMDVector<T,ABI>::Size; ++i) {
|
||||
arr[i] = std::arg(a[i]);
|
||||
}
|
||||
return SIMDVector<T,ABI>(arr,false);
|
||||
}
|
||||
|
||||
} // end of namespace Fastor
|
||||
|
||||
#endif // SIMD_VECTOR_H
|
||||
|
||||
1177
noarch/include/Fastor/simd_vector/simd_vector_common.h
Normal file
1177
noarch/include/Fastor/simd_vector/simd_vector_common.h
Normal file
File diff suppressed because it is too large
Load Diff
1611
noarch/include/Fastor/simd_vector/simd_vector_complex_double.h
Normal file
1611
noarch/include/Fastor/simd_vector/simd_vector_complex_double.h
Normal file
File diff suppressed because it is too large
Load Diff
1619
noarch/include/Fastor/simd_vector/simd_vector_complex_float.h
Normal file
1619
noarch/include/Fastor/simd_vector/simd_vector_complex_float.h
Normal file
File diff suppressed because it is too large
Load Diff
378
noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h
Normal file
378
noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h
Normal file
@@ -0,0 +1,378 @@
|
||||
#ifndef SIMD_VECTOR_COMPLEX_SCALAR
|
||||
#define SIMD_VECTOR_COMPLEX_SCALAR
|
||||
|
||||
#include "Fastor/util/extended_algorithms.h"
|
||||
#include "Fastor/simd_vector/extintrin.h"
|
||||
#include "Fastor/simd_vector/simd_vector_base.h"
|
||||
#include "Fastor/simd_vector/simd_vector_double.h"
|
||||
#include <cmath>
|
||||
#include <complex>
|
||||
|
||||
|
||||
namespace Fastor {
|
||||
|
||||
|
||||
// SCALAR IMPLEMENTATION OF SIMDVECTOR FOR COMPLEX<T>
|
||||
//------------------------------------------------------------------------------------------------------------
|
||||
|
||||
template <typename T>
|
||||
struct SIMDVector<std::complex<T>, simd_abi::scalar> {
|
||||
using vector_type = SIMDVector<std::complex<T>, simd_abi::scalar>;
|
||||
using value_type = T;
|
||||
using scalar_value_type = std::complex<T>;
|
||||
using abi_type = simd_abi::scalar;
|
||||
static constexpr FASTOR_INDEX Size = 1;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value_r(0), value_i(0) {}
|
||||
FASTOR_INLINE SIMDVector(scalar_value_type num) {
|
||||
value_r = num.real();
|
||||
value_i = num.imag();
|
||||
}
|
||||
FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {}
|
||||
FASTOR_INLINE SIMDVector(const scalar_value_type *data, bool Aligned=true) {
|
||||
value_r = (*data).real();
|
||||
value_i = (*data).imag();
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator=(std::complex<T> num) {
|
||||
value_r = num.real();
|
||||
value_i = num.imag();
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const scalar_value_type *data, bool ) {
|
||||
value_r = (*data).real();
|
||||
value_i = (*data).imag();
|
||||
}
|
||||
FASTOR_INLINE void store(scalar_value_type *data, bool ) const {
|
||||
data[0] = scalar_value_type(value_r,value_i);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const T *data) {
|
||||
value_r = (*data).real();
|
||||
value_i = (*data).imag();
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(T *data) const {
|
||||
data[0] = scalar_value_type(value_r,value_i);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool ) {
|
||||
if (mask != 0x0) {
|
||||
value_r = (*data).real();
|
||||
value_i = (*data).imag();
|
||||
}
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool) const {
|
||||
if (mask != 0x0) {
|
||||
data[0] = scalar_value_type(value_r,value_i);
|
||||
}
|
||||
}
|
||||
|
||||
FASTOR_INLINE T operator[](FASTOR_INDEX) const {return scalar_value_type(value_r,value_i);}
|
||||
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> real() const {
|
||||
return value_r;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> imag() const {
|
||||
return value_i;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void set(scalar_value_type num) {
|
||||
value_r = num.real();
|
||||
value_i = num.imag();
|
||||
}
|
||||
|
||||
FASTOR_INLINE void set_sequential(scalar_value_type num) {
|
||||
value_r = num.real();
|
||||
value_i = num.imag();
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE void operator+=(U num) {
|
||||
value_r += num;
|
||||
}
|
||||
FASTOR_INLINE void operator+=(scalar_value_type num) {
|
||||
value_r += num.real();
|
||||
value_i += num.imag();
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const vector_type &a) {
|
||||
value_r += a.value_r;
|
||||
value_i += a.value_i;
|
||||
}
|
||||
|
||||
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE void operator-=(U num) {
|
||||
value_r -= num;
|
||||
}
|
||||
FASTOR_INLINE void operator-=(scalar_value_type num) {
|
||||
value_r -= num.real();
|
||||
value_i -= num.imag();
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const vector_type &a) {
|
||||
value_r -= a.value_r;
|
||||
value_i -= a.value_i;
|
||||
}
|
||||
|
||||
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE void operator*=(U num) {
|
||||
value_r *= num;
|
||||
value_i *= num;
|
||||
}
|
||||
FASTOR_INLINE void operator*=(scalar_value_type num) {
|
||||
scalar_value_type tmp(value_r, value_i);
|
||||
tmp *= num;
|
||||
value_r = tmp.real();
|
||||
value_i = tmp.imag();
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const vector_type &a) {
|
||||
scalar_value_type tmp0(value_r, value_i);
|
||||
scalar_value_type tmp1(a.value_r, a.value_i);
|
||||
tmp0 *= tmp1;
|
||||
value_r = tmp0.real();
|
||||
value_i = tmp0.imag();
|
||||
}
|
||||
|
||||
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE void operator/=(U num) {
|
||||
value_r /= num;
|
||||
value_i /= num;
|
||||
}
|
||||
FASTOR_INLINE void operator/=(scalar_value_type num) {
|
||||
scalar_value_type tmp(value_r, value_i);
|
||||
tmp /= num;
|
||||
value_r = tmp.real();
|
||||
value_i = tmp.imag();
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const vector_type &a) {
|
||||
scalar_value_type tmp0(value_r, value_i);
|
||||
scalar_value_type tmp1(a.value_r, a.value_i);
|
||||
tmp0 /= tmp1;
|
||||
value_r = tmp0.real();
|
||||
value_i = tmp0.imag();
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE scalar_value_type sum() const {return scalar_value_type(value_r, value_i);}
|
||||
FASTOR_INLINE scalar_value_type product() const {return scalar_value_type(value_r, value_i);}
|
||||
FASTOR_INLINE vector_type reverse() const { return scalar_value_type(value_r, value_i); }
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> magnitude() const { return std::abs(scalar_value_type(value_r, value_i));}
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> norm() const { return std::norm(scalar_value_type(value_r, value_i));}
|
||||
FASTOR_INLINE scalar_value_type minimum() const {return scalar_value_type(value_r, value_i);}
|
||||
FASTOR_INLINE scalar_value_type maximum() const {return scalar_value_type(value_r, value_i);}
|
||||
FASTOR_INLINE scalar_value_type dot(const vector_type &other) const {
|
||||
return vector_type(value_r, value_i)*vector_type(other.value_r, other.value_i);
|
||||
}
|
||||
|
||||
value_type value_r;
|
||||
value_type value_i;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<std::complex<T>,simd_abi::scalar> a) {
|
||||
os << "[" << a.value_r << signum_string(a.value_i) << std::abs(a.value_i) << "j]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out += b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out += b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator+(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out += b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out += b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator+(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
|
||||
out += b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out -= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out -= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator-(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out -= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out -= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator-(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
|
||||
out -= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
return SIMDVector<std::complex<T>,simd_abi::scalar>(0,0) - b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator*(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out *= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator*(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out *= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator*(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out *= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator*(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out *= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator*(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
|
||||
out *= b;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator/(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out /= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator/(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out /= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator/(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out /= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator/(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
|
||||
out /= b;
|
||||
return out;
|
||||
}
|
||||
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
operator/(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
|
||||
out /= b;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
rcp(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(1,0));
|
||||
out /= a;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
sqrt(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
|
||||
std::complex<T> out(a.value_r,a.value_i);
|
||||
return std::sqrt(out);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
rsqrt(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
|
||||
return rcp(sqrt(a));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
abs(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
|
||||
SIMDVector<std::complex<T>,simd_abi::scalar> out;
|
||||
out.value_r = a.magnitude().value;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
conj(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
|
||||
std::complex<T> out(a.value_r,a.value_i);
|
||||
return std::conj(out);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
|
||||
arg(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
|
||||
std::complex<T> out(a.value_r,a.value_i);
|
||||
return SIMDVector<std::complex<T>,simd_abi::scalar>(std::arg(out),0);
|
||||
}
|
||||
//------------------------------------------------------------------------------------------------------------
|
||||
|
||||
} // end of namespace Fastor
|
||||
|
||||
#endif // SIMD_VECTOR_COMPLEX_SCALAR
|
||||
914
noarch/include/Fastor/simd_vector/simd_vector_double.h
Normal file
914
noarch/include/Fastor/simd_vector/simd_vector_double.h
Normal file
@@ -0,0 +1,914 @@
|
||||
#ifndef SIMD_VECTOR_DOUBLE_H
|
||||
#define SIMD_VECTOR_DOUBLE_H
|
||||
|
||||
#include "Fastor/simd_vector/simd_vector_base.h"
|
||||
|
||||
namespace Fastor {
|
||||
|
||||
// AVX512 VERSION
|
||||
//--------------------------------------------------------------------------------------------------
|
||||
#ifdef FASTOR_AVX512_IMPL
|
||||
|
||||
template <>
|
||||
struct SIMDVector<double, simd_abi::avx512> {
|
||||
using value_type = __m512d;
|
||||
using scalar_value_type = double;
|
||||
using abi_type = simd_abi::avx512;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx512>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx512>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm512_setzero_pd()) {}
|
||||
FASTOR_INLINE SIMDVector(double num) : value(_mm512_set1_pd(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m512d regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm512_load_pd(data);
|
||||
else
|
||||
value = _mm512_loadu_pd(data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator=(double num) {
|
||||
value = _mm512_set1_pd(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator=(__m512d regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const double *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm512_load_pd(data);
|
||||
else
|
||||
value = _mm512_loadu_pd(data);
|
||||
}
|
||||
FASTOR_INLINE void store(double *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm512_store_pd(data,value);
|
||||
else
|
||||
_mm512_storeu_pd(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const double *data) {
|
||||
value =_mm512_load_pd(data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(double *data) const {
|
||||
_mm512_store_pd(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm512_mask_loadu_pd(value, mask, a);
|
||||
else
|
||||
value = _mm512_mask_load_pd(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm512_setzero_pd();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm512_mask_storeu_pd(a, mask, value);
|
||||
else
|
||||
_mm512_mask_store_pd(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE double operator[](FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
|
||||
FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(double num) {
|
||||
value = _mm512_set1_pd(num);
|
||||
}
|
||||
FASTOR_INLINE void set(double num0, double num1, double num2, double num3, double num4, double num5, double num6, double num7) {
|
||||
value = _mm512_set_pd(num0,num1,num2,num3,num4,num5,num6,num7);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(double num0) {
|
||||
value = _mm512_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0,num0+4.0,num0+5.0,num0+6.0,num0+7.0);
|
||||
}
|
||||
FASTOR_INLINE void broadcast(const double *data) {
|
||||
// value = _mm512_broadcast_sd(data);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(double num) {
|
||||
value = _mm512_add_pd(value,_mm512_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m512d regi) {
|
||||
value = _mm512_add_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
value = _mm512_add_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(double num) {
|
||||
value = _mm512_sub_pd(value,_mm512_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m512d regi) {
|
||||
value = _mm512_sub_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
value = _mm512_sub_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(double num) {
|
||||
value = _mm512_mul_pd(value,_mm512_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m512d regi) {
|
||||
value = _mm512_mul_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
value = _mm512_mul_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(double num) {
|
||||
value = _mm512_div_pd(value,_mm512_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m512d regi) {
|
||||
value = _mm512_div_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
value = _mm512_div_pd(value,a.value);
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
// FASTOR_INLINE SIMDVector<double,simd_abi::avx512> shift(FASTOR_INDEX i) {
|
||||
// SIMDVector<double,simd_abi::avx512> out;
|
||||
// if (i==1)
|
||||
// out.value = _mm512_shift1_pd(value);
|
||||
// else if (i==2)
|
||||
// out.value = _mm512_shift2_pd(value);
|
||||
// else if (i==3)
|
||||
// out.value = _mm512_shift3_pd(value);
|
||||
// return out;
|
||||
// }
|
||||
FASTOR_INLINE double sum() {
|
||||
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
|
||||
return _mm512_reduce_add_pd(value);
|
||||
#else
|
||||
__m256d low = _mm512_castpd512_pd256(value);
|
||||
__m256d high = _mm512_extractf64x4_pd(value,0x1);
|
||||
return _mm256_sum_pd(_mm256_add_pd(low,high));
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE double product() {
|
||||
__m256d low = _mm512_castpd512_pd256(value);
|
||||
__m256d high = _mm512_extractf64x4_pd(value,1);
|
||||
return _mm256_prod_pd(_mm256_mul_pd(low,high));
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> reverse() {
|
||||
return _mm512_reverse_pd(value);
|
||||
}
|
||||
// FASTOR_INLINE double minimum() {return _mm512_hmin_pd(value);}
|
||||
// FASTOR_INLINE double maximum() {return _mm512_hmax_pd(value);}
|
||||
|
||||
FASTOR_INLINE double dot(const SIMDVector<double,simd_abi::avx512> &other) {
|
||||
__m512d res = _mm512_mul_pd(value,other.value);
|
||||
__m256d low = _mm512_castpd512_pd256(res);
|
||||
__m256d high = _mm512_extractf64x4_pd(res,1);
|
||||
return _mm256_sum_pd(_mm256_add_pd(low,high));
|
||||
}
|
||||
|
||||
__m512d value;
|
||||
};
|
||||
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<double,simd_abi::avx512> a) {
|
||||
// ICC crashes without a copy
|
||||
const __m512d v = a.value;
|
||||
const double* value = reinterpret_cast<const double*>(&v);
|
||||
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3]
|
||||
<< " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(
|
||||
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(const SIMDVector<double,simd_abi::avx512> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_pd(a.value,_mm512_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(double a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_pd(_mm512_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(
|
||||
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(const SIMDVector<double,simd_abi::avx512> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_pd(a.value,_mm512_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(double a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_pd(_mm512_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
return _mm512_neg_pd(b.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator*(
|
||||
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_mul_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator*(const SIMDVector<double,simd_abi::avx512> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_mul_pd(a.value,_mm512_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator*(double a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_mul_pd(_mm512_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator/(
|
||||
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_div_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator/(const SIMDVector<double,simd_abi::avx512> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_div_pd(a.value,_mm512_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator/(double a, const SIMDVector<double,simd_abi::avx512> &b) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_div_pd(_mm512_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> rcp(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_rcp14_pd(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> sqrt(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_sqrt_pd(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> rsqrt(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
out.value = _mm512_rsqrt14_pd(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> abs(const SIMDVector<double,simd_abi::avx512> &a) {
|
||||
SIMDVector<double,simd_abi::avx512> out;
|
||||
#ifdef FASTOR_HAS_AVX512_ABS
|
||||
out.value = _mm512_abs_pd(a.value);
|
||||
#else
|
||||
for (FASTOR_INDEX i=0UL; i<8UL; ++i) {
|
||||
((double*)&out.value)[i] = std::abs(((double*)&a.value)[i]);
|
||||
}
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
// AVX VERSION
|
||||
//--------------------------------------------------------------------------------------------------
|
||||
#ifdef FASTOR_AVX_IMPL
|
||||
|
||||
template <>
|
||||
struct SIMDVector<double, simd_abi::avx> {
|
||||
using value_type = __m256d;
|
||||
using scalar_value_type = double;
|
||||
using abi_type = simd_abi::avx;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm256_setzero_pd()) {}
|
||||
FASTOR_INLINE SIMDVector(double num) : value(_mm256_set1_pd(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m256d regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm256_load_pd(data);
|
||||
else
|
||||
value = _mm256_loadu_pd(data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator=(double num) {
|
||||
value = _mm256_set1_pd(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator=(__m256d regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const double *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm256_load_pd(data);
|
||||
else
|
||||
value = _mm256_loadu_pd(data);
|
||||
}
|
||||
FASTOR_INLINE void store(double *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm256_store_pd(data,value);
|
||||
else
|
||||
_mm256_storeu_pd(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const double *data) {
|
||||
value =_mm256_load_pd(data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(double *data) const {
|
||||
_mm256_store_pd(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm256_mask_loadu_pd(value, mask, a);
|
||||
else
|
||||
value = _mm256_mask_load_pd(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm256_setzero_pd();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm256_mask_storeu_pd(a, mask, value);
|
||||
else
|
||||
_mm256_mask_store_pd(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE double operator[](FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
|
||||
FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(double num) {
|
||||
value = _mm256_set1_pd(num);
|
||||
}
|
||||
FASTOR_INLINE void set(double num0, double num1, double num2, double num3) {
|
||||
value = _mm256_set_pd(num0,num1,num2,num3);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(double num0) {
|
||||
value = _mm256_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0);
|
||||
}
|
||||
FASTOR_INLINE void broadcast(const double *data) {
|
||||
value = _mm256_broadcast_sd(data);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(double num) {
|
||||
value = _mm256_add_pd(value,_mm256_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m256d regi) {
|
||||
value = _mm256_add_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
value = _mm256_add_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(double num) {
|
||||
value = _mm256_sub_pd(value,_mm256_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m256d regi) {
|
||||
value = _mm256_sub_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
value = _mm256_sub_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(double num) {
|
||||
value = _mm256_mul_pd(value,_mm256_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m256d regi) {
|
||||
value = _mm256_mul_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
value = _mm256_mul_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(double num) {
|
||||
value = _mm256_div_pd(value,_mm256_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m256d regi) {
|
||||
value = _mm256_div_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
value = _mm256_div_pd(value,a.value);
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> shift(FASTOR_INDEX i) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
if (i==1)
|
||||
out.value = _mm256_shift1_pd(value);
|
||||
else if (i==2)
|
||||
out.value = _mm256_shift2_pd(value);
|
||||
else if (i==3)
|
||||
out.value = _mm256_shift3_pd(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE double sum() {return _mm256_sum_pd(value);}
|
||||
FASTOR_INLINE double product() {return _mm256_prod_pd(value);}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> reverse() {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_reverse_pd(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE double minimum() {return _mm256_hmin_pd(value);}
|
||||
FASTOR_INLINE double maximum() {return _mm256_hmax_pd(value);}
|
||||
|
||||
FASTOR_INLINE double dot(const SIMDVector<double,simd_abi::avx> &other) {
|
||||
return _mm_cvtsd_f64(_mm256_dp_pd(value,other.value));
|
||||
}
|
||||
|
||||
__m256d value;
|
||||
};
|
||||
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<double,simd_abi::avx> a) {
|
||||
// ICC crashes without a copy
|
||||
const __m256d v = a.value;
|
||||
const double* value = reinterpret_cast<const double*>(&v);
|
||||
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_add_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(const SIMDVector<double,simd_abi::avx> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_add_pd(a.value,_mm256_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(double a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_add_pd(_mm256_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(const SIMDVector<double,simd_abi::avx> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(const SIMDVector<double,simd_abi::avx> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_pd(a.value,_mm256_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(double a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_pd(_mm256_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_neg_pd(b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator*(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator*(const SIMDVector<double,simd_abi::avx> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_pd(a.value,_mm256_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator*(double a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_pd(_mm256_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator/(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_div_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator/(const SIMDVector<double,simd_abi::avx> &a, double b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_div_pd(a.value,_mm256_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator/(double a, const SIMDVector<double,simd_abi::avx> &b) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_div_pd(_mm256_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> rcp(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
// This is very inaccurate for double precision
|
||||
out.value = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value)));
|
||||
return out;
|
||||
|
||||
// // For making it more accurate using Newton Raphson use this
|
||||
// __m128d xmm0 = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value)));
|
||||
// xmm0 = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0)));
|
||||
// out.value = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0)));
|
||||
// return out;
|
||||
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> sqrt(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_sqrt_pd(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> rsqrt(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
// This is very inaccurate for double precision
|
||||
out.value = _mm256_cvtps_pd(_mm_rsqrt_ps(_mm256_cvtpd_ps(a.value)));
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::avx> abs(const SIMDVector<double,simd_abi::avx> &a) {
|
||||
SIMDVector<double,simd_abi::avx> out;
|
||||
out.value = _mm256_abs_pd(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// SSE VERSION
|
||||
//------------------------------------------------------------------------------------------------------------
|
||||
|
||||
#ifdef FASTOR_SSE2_IMPL
|
||||
|
||||
template <>
|
||||
struct SIMDVector<double, simd_abi::sse> {
|
||||
using value_type = __m128d;
|
||||
using scalar_value_type = double;
|
||||
using abi_type = simd_abi::sse;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<double,simd_abi::sse>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<double,simd_abi::sse>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm_setzero_pd()) {}
|
||||
FASTOR_INLINE SIMDVector(double num) : value(_mm_set1_pd(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m128d regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm_load_pd(data);
|
||||
else
|
||||
value = _mm_loadu_pd(data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator=(double num) {
|
||||
value = _mm_set1_pd(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator=(__m128d regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const double *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm_load_pd(data);
|
||||
else
|
||||
value = _mm_loadu_pd(data);
|
||||
}
|
||||
FASTOR_INLINE void store(double *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm_store_pd(data,value);
|
||||
else
|
||||
_mm_storeu_pd(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const double *data) {
|
||||
value =_mm_load_pd(data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(double *data) const {
|
||||
_mm_store_pd(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm_mask_loadu_pd(value, mask, a);
|
||||
else
|
||||
value = _mm_mask_load_pd(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm_setzero_pd();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm_mask_storeu_pd(a, mask, value);
|
||||
else
|
||||
_mm_mask_store_pd(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE double operator[](FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
|
||||
FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(double num) {
|
||||
value = _mm_set1_pd(num);
|
||||
}
|
||||
FASTOR_INLINE void set(double num0, double num1) {
|
||||
value = _mm_set_pd(num0,num1);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(double num0) {
|
||||
value = _mm_setr_pd(num0,num0+1.0);
|
||||
}
|
||||
FASTOR_INLINE void broadcast(const double *data) {
|
||||
value = _mm_load1_pd(data);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(double num) {
|
||||
value = _mm_add_pd(value,_mm_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m128d regi) {
|
||||
value = _mm_add_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
value = _mm_add_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(double num) {
|
||||
value = _mm_sub_pd(value,_mm_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m128d regi) {
|
||||
value = _mm_sub_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
value = _mm_sub_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(double num) {
|
||||
value = _mm_mul_pd(value,_mm_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m128d regi) {
|
||||
value = _mm_mul_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
value = _mm_mul_pd(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(double num) {
|
||||
value = _mm_div_pd(value,_mm_set1_pd(num));
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m128d regi) {
|
||||
value = _mm_div_pd(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
value = _mm_div_pd(value,a.value);
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> shift(FASTOR_INDEX i) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
FASTOR_ASSERT(i==1,"INCORRECT SHIFT INDEX");
|
||||
out.value = _mm_shift1_pd(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE double sum() {return _mm_sum_pd(value);}
|
||||
FASTOR_INLINE double product() {return _mm_prod_pd(value);}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> reverse() {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_reverse_pd(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE double minimum() {return _mm_hmin_pd(value);}
|
||||
FASTOR_INLINE double maximum() {return _mm_hmax_pd(value);}
|
||||
|
||||
FASTOR_INLINE double dot(const SIMDVector<double,simd_abi::sse> &other) {
|
||||
#ifdef FASTOR_SSE4_1_IMPL
|
||||
return _mm_cvtsd_f64(_mm_dp_pd(value,other.value,0xff));
|
||||
#else
|
||||
return _mm_sum_pd(_mm_mul_pd(value,other.value));
|
||||
#endif
|
||||
}
|
||||
|
||||
__m128d value;
|
||||
};
|
||||
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<double,simd_abi::sse> a) {
|
||||
// ICC crashes without a copy
|
||||
const __m128d v = a.value;
|
||||
const double* value = reinterpret_cast<const double*>(&v);
|
||||
os << "[" << value[0] << " " << value[1] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_add_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(const SIMDVector<double,simd_abi::sse> &a, double b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_add_pd(a.value,_mm_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(double a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_add_pd(_mm_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(const SIMDVector<double,simd_abi::sse> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_sub_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(const SIMDVector<double,simd_abi::sse> &a, double b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_sub_pd(a.value,_mm_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(double a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_sub_pd(_mm_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_neg_pd(b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator*(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_mul_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator*(const SIMDVector<double,simd_abi::sse> &a, double b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_mul_pd(a.value,_mm_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator*(double a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_mul_pd(_mm_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator/(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_div_pd(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator/(const SIMDVector<double,simd_abi::sse> &a, double b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_div_pd(a.value,_mm_set1_pd(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator/(double a, const SIMDVector<double,simd_abi::sse> &b) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_div_pd(_mm_set1_pd(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> rcp(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
// This is very inaccurate for double precision
|
||||
out.value = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value)));
|
||||
return out;
|
||||
|
||||
/*
|
||||
// For making it more accurate using Newton Raphson use this
|
||||
__m128d xmm0 = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value)));
|
||||
xmm0 = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0)));
|
||||
out.value = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0)));
|
||||
return out;
|
||||
*/
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> sqrt(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_sqrt_pd(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> rsqrt(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
// This is very inaccurate for double precision
|
||||
out.value = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(a.value)));
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<double,simd_abi::sse> abs(const SIMDVector<double,simd_abi::sse> &a) {
|
||||
SIMDVector<double,simd_abi::sse> out;
|
||||
out.value = _mm_abs_pd(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
} // end of namespace Fastor
|
||||
|
||||
#endif // // SIMD_VECTOR_DOUBLE_H
|
||||
918
noarch/include/Fastor/simd_vector/simd_vector_float.h
Normal file
918
noarch/include/Fastor/simd_vector/simd_vector_float.h
Normal file
@@ -0,0 +1,918 @@
|
||||
#ifndef SIMD_VECTOR_FLOAT_H
|
||||
#define SIMD_VECTOR_FLOAT_H
|
||||
|
||||
#include "Fastor/simd_vector/simd_vector_base.h"
|
||||
|
||||
namespace Fastor {
|
||||
|
||||
|
||||
// AVX512 VERSION
|
||||
//--------------------------------------------------------------------------------------------------
|
||||
|
||||
#ifdef FASTOR_AVX512_IMPL
|
||||
|
||||
template <>
|
||||
struct SIMDVector<float,simd_abi::avx512> {
|
||||
using value_type = __m512;
|
||||
using scalar_value_type = float;
|
||||
using abi_type = simd_abi::avx512;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx512>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx512>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm512_setzero_ps()) {}
|
||||
FASTOR_INLINE SIMDVector(float num) : value(_mm512_set1_ps(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m512 regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm512_load_ps(data);
|
||||
else
|
||||
value = _mm512_loadu_ps(data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator=(float num) {
|
||||
value = _mm512_set1_ps(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator=(__m512 regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const float *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm512_load_ps(data);
|
||||
else
|
||||
value = _mm512_loadu_ps(data);
|
||||
}
|
||||
FASTOR_INLINE void store(float *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm512_store_ps(data,value);
|
||||
else
|
||||
_mm512_storeu_ps(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const float *data) {
|
||||
value =_mm512_load_ps(data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(float *data) const {
|
||||
_mm512_store_ps(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint16_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm512_mask_loadu_ps(value, mask, a);
|
||||
else
|
||||
value = _mm512_mask_load_ps(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm512_setzero_ps();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint16_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm512_mask_storeu_ps(a, mask, value);
|
||||
else
|
||||
_mm512_mask_store_ps(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE float operator[](FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
|
||||
FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(float num) {
|
||||
value = _mm512_set1_ps(num);
|
||||
}
|
||||
FASTOR_INLINE void set(float num0, float num1, float num2, float num3,
|
||||
float num4, float num5, float num6, float num7,
|
||||
float num8, float num9, float num10, float num11,
|
||||
float num12, float num13, float num14, float num15) {
|
||||
value = _mm512_set_ps(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(float num0) {
|
||||
value = _mm512_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f,
|
||||
num0+8.f,num0+9.f,num0+10.f,num0+11.f,num0+12.f,num0+13.f,num0+14.f,num0+15.f);
|
||||
}
|
||||
FASTOR_INLINE void broadcast(const float *data) {
|
||||
// value = _mm512_broadcast_ss(data);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(float num) {
|
||||
value = _mm512_add_ps(value,_mm512_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m512 regi) {
|
||||
value = _mm512_add_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
value = _mm512_add_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(float num) {
|
||||
value = _mm512_sub_ps(value,_mm512_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m512 regi) {
|
||||
value = _mm512_sub_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
value = _mm512_sub_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(float num) {
|
||||
value = _mm512_mul_ps(value,_mm512_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m512 regi) {
|
||||
value = _mm512_mul_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
value = _mm512_mul_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(float num) {
|
||||
value = _mm512_div_ps(value,_mm512_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m512 regi) {
|
||||
value = _mm512_div_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
value = _mm512_div_ps(value,a.value);
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE float sum() {
|
||||
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
|
||||
return _mm512_reduce_add_ps(value);
|
||||
#else
|
||||
__m256 low = _mm512_castps512_ps256(value);
|
||||
__m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1));
|
||||
return _mm256_sum_ps(_mm256_add_ps(low,high));
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE float product() {
|
||||
__m256 low = _mm512_castps512_ps256(value);
|
||||
__m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1));
|
||||
return _mm256_prod_ps(_mm256_mul_ps(low,high));
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> reverse() {
|
||||
return _mm512_reverse_ps(value);
|
||||
}
|
||||
// FASTOR_INLINE float minimum() {return _mm512_hmin_ps(value);}
|
||||
// FASTOR_INLINE float maximum() {return _mm512_hmax_ps(value);}
|
||||
|
||||
FASTOR_INLINE float dot(const SIMDVector<float,simd_abi::avx512> &other) {
|
||||
__m512 res = _mm512_mul_ps(value,other.value);
|
||||
__m256 low = _mm512_castps512_ps256(res);
|
||||
__m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(res),1));
|
||||
return _mm256_sum_ps(_mm256_add_ps(low,high));
|
||||
}
|
||||
|
||||
__m512 value;
|
||||
};
|
||||
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<float,simd_abi::avx512> a) {
|
||||
// ICC crashes without a copy
|
||||
const __m512 v = a.value;
|
||||
const float* value = reinterpret_cast<const float*>(&v);
|
||||
os << "["
|
||||
<< value[0] << " " << value[1] << " "
|
||||
<< value[2] << " " << value[3] << " "
|
||||
<< value[4] << " " << value[5] << " "
|
||||
<< value[6] << " " << value[7] << " "
|
||||
<< value[8] << " " << value[9] << " "
|
||||
<< value[10] << " " << value[11] << " "
|
||||
<< value[12] << " " << value[13] << " "
|
||||
<< value[14] << " " << value[15] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(
|
||||
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(const SIMDVector<float,simd_abi::avx512> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_ps(a.value,_mm512_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(float a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_ps(_mm512_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(
|
||||
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(const SIMDVector<float,simd_abi::avx512> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_ps(a.value,_mm512_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(float a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_ps(_mm512_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
return _mm512_neg_ps(b.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator*(
|
||||
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_mul_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator*(const SIMDVector<float,simd_abi::avx512> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_mul_ps(a.value,_mm512_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator*(float a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_mul_ps(_mm512_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator/(
|
||||
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_div_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator/(const SIMDVector<float,simd_abi::avx512> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_div_ps(a.value,_mm512_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator/(float a, const SIMDVector<float,simd_abi::avx512> &b) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_div_ps(_mm512_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> rcp(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_rcp14_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> sqrt(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_sqrt_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> rsqrt(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
out.value = _mm512_rsqrt14_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> abs(const SIMDVector<float,simd_abi::avx512> &a) {
|
||||
SIMDVector<float,simd_abi::avx512> out;
|
||||
#ifdef FASTOR_HAS_AVX512_ABS
|
||||
out.value = _mm512_abs_ps(a.value);
|
||||
#else
|
||||
for (FASTOR_INDEX i=0UL; i<16UL; ++i) {
|
||||
((float*)&out.value)[i] = std::abs(((float*)&a.value)[i]);
|
||||
}
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
// AVX VERSION
|
||||
//--------------------------------------------------------------------------------------------------
|
||||
|
||||
#ifdef FASTOR_AVX_IMPL
|
||||
|
||||
template <>
|
||||
struct SIMDVector<float,simd_abi::avx> {
|
||||
using value_type = __m256;
|
||||
using scalar_value_type = float;
|
||||
using abi_type = simd_abi::avx;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm256_setzero_ps()) {}
|
||||
FASTOR_INLINE SIMDVector(float num) : value(_mm256_set1_ps(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m256 regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm256_load_ps(data);
|
||||
else
|
||||
value = _mm256_loadu_ps(data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator=(float num) {
|
||||
value = _mm256_set1_ps(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator=(__m256 regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const float *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm256_load_ps(data);
|
||||
else
|
||||
value = _mm256_loadu_ps(data);
|
||||
}
|
||||
FASTOR_INLINE void store(float *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm256_store_ps(data,value);
|
||||
else
|
||||
_mm256_storeu_ps(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const float *data) {
|
||||
value =_mm256_load_ps(data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(float *data) const {
|
||||
_mm256_store_ps(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm256_mask_loadu_ps(value, mask, a);
|
||||
else
|
||||
value = _mm256_mask_load_ps(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm256_setzero_ps();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm256_mask_storeu_ps(a, mask, value);
|
||||
else
|
||||
_mm256_mask_store_ps(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE float operator[](FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
|
||||
FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(float num) {
|
||||
value = _mm256_set1_ps(num);
|
||||
}
|
||||
FASTOR_INLINE void set(float num0, float num1, float num2, float num3,
|
||||
float num4, float num5, float num6, float num7) {
|
||||
value = _mm256_set_ps(num0,num1,num2,num3,num4,num5,num6,num7);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(float num0) {
|
||||
value = _mm256_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f);
|
||||
}
|
||||
FASTOR_INLINE void broadcast(const float *data) {
|
||||
value = _mm256_broadcast_ss(data);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(float num) {
|
||||
value = _mm256_add_ps(value,_mm256_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m256 regi) {
|
||||
value = _mm256_add_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
value = _mm256_add_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(float num) {
|
||||
value = _mm256_sub_ps(value,_mm256_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m256 regi) {
|
||||
value = _mm256_sub_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
value = _mm256_sub_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(float num) {
|
||||
value = _mm256_mul_ps(value,_mm256_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m256 regi) {
|
||||
value = _mm256_mul_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
value = _mm256_mul_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(float num) {
|
||||
value = _mm256_div_ps(value,_mm256_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m256 regi) {
|
||||
value = _mm256_div_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
value = _mm256_div_ps(value,a.value);
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> shift(FASTOR_INDEX i) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
if (i==1)
|
||||
out.value = _mm256_shift1_ps(value);
|
||||
else if (i==2)
|
||||
out.value = _mm256_shift2_ps(value);
|
||||
else if (i==3)
|
||||
out.value = _mm256_shift3_ps(value);
|
||||
else if (i==4)
|
||||
out.value = _mm256_shift4_ps(value);
|
||||
else if (i==5)
|
||||
out.value = _mm256_shift5_ps(value);
|
||||
else if (i==6)
|
||||
out.value = _mm256_shift6_ps(value);
|
||||
else if (i==7)
|
||||
out.value = _mm256_shift7_ps(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE float sum() {return _mm256_sum_ps(value);}
|
||||
FASTOR_INLINE float product() {return _mm256_prod_ps(value);}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> reverse() {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_reverse_ps(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE float minimum() {return _mm256_hmin_ps(value);}
|
||||
FASTOR_INLINE float maximum() {return _mm256_hmax_ps(value);}
|
||||
|
||||
FASTOR_INLINE float dot(const SIMDVector<float,simd_abi::avx> &other) {
|
||||
__m256 tmp = _mm256_dp_ps(value,other.value,0xff);
|
||||
return _mm256_get0_ps(tmp)+_mm256_get4_ps(tmp);
|
||||
}
|
||||
|
||||
__m256 value;
|
||||
};
|
||||
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<float,simd_abi::avx> a) {
|
||||
// ICC crashes without a copy
|
||||
const __m256 v = a.value;
|
||||
const float* value = reinterpret_cast<const float*>(&v);
|
||||
os << "[" << value[0] << " " << value[1] << " "
|
||||
<< value[2] << " " << value[3] << " "
|
||||
<< value[4] << " " << value[5] << " "
|
||||
<< value[6] << " " << value[7] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_add_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(const SIMDVector<float,simd_abi::avx> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_add_ps(a.value,_mm256_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(float a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_add_ps(_mm256_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(const SIMDVector<float,simd_abi::avx> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(const SIMDVector<float,simd_abi::avx> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_ps(a.value,_mm256_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(float a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_ps(_mm256_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_neg_ps(b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator*(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator*(const SIMDVector<float,simd_abi::avx> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_ps(a.value,_mm256_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator*(float a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_ps(_mm256_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator/(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_div_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator/(const SIMDVector<float,simd_abi::avx> &a, float b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_div_ps(a.value,_mm256_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator/(float a, const SIMDVector<float,simd_abi::avx> &b) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_div_ps(_mm256_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> rcp(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_rcp_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> sqrt(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_sqrt_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> rsqrt(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_rsqrt_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::avx> abs(const SIMDVector<float,simd_abi::avx> &a) {
|
||||
SIMDVector<float,simd_abi::avx> out;
|
||||
out.value = _mm256_abs_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// SSE VERSION
|
||||
//--------------------------------------------------------------------------------------------------
|
||||
|
||||
#ifdef FASTOR_SSE2_IMPL
|
||||
|
||||
template <>
|
||||
struct SIMDVector<float,simd_abi::sse> {
|
||||
using value_type = __m128;
|
||||
using scalar_value_type = float;
|
||||
using abi_type = simd_abi::sse;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<float,simd_abi::sse>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<float,simd_abi::sse>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm_setzero_ps()) {}
|
||||
FASTOR_INLINE SIMDVector(float num) : value(_mm_set1_ps(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m128 regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm_load_ps(data);
|
||||
else
|
||||
value = _mm_loadu_ps(data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator=(float num) {
|
||||
value = _mm_set1_ps(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator=(__m128 regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const float *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm_load_ps(data);
|
||||
else
|
||||
value = _mm_loadu_ps(data);
|
||||
}
|
||||
FASTOR_INLINE void store(float *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm_store_ps(data,value);
|
||||
else
|
||||
_mm_storeu_ps(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const float *data) {
|
||||
value =_mm_load_ps(data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(float *data) const {
|
||||
_mm_store_ps(data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm_mask_loadu_ps(value, mask, a);
|
||||
else
|
||||
value = _mm_mask_load_ps(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm_setzero_ps();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm_mask_storeu_ps(a, mask, value);
|
||||
else
|
||||
_mm_mask_store_ps(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE float operator[](FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
|
||||
FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(float num) {
|
||||
value = _mm_set1_ps(num);
|
||||
}
|
||||
FASTOR_INLINE void set(float num0, float num1, float num2, float num3) {
|
||||
value = _mm_set_ps(num0,num1,num2,num3);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(float num0) {
|
||||
value = _mm_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f);
|
||||
}
|
||||
FASTOR_INLINE void broadcast(const float *data) {
|
||||
value = _mm_load1_ps(data);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(float num) {
|
||||
value = _mm_add_ps(value,_mm_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m128 regi) {
|
||||
value = _mm_add_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
value = _mm_add_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(float num) {
|
||||
value = _mm_sub_ps(value,_mm_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m128 regi) {
|
||||
value = _mm_sub_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
value = _mm_sub_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(float num) {
|
||||
value = _mm_mul_ps(value,_mm_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m128 regi) {
|
||||
value = _mm_mul_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
value = _mm_mul_ps(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(float num) {
|
||||
value = _mm_div_ps(value,_mm_set1_ps(num));
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m128 regi) {
|
||||
value = _mm_div_ps(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
value = _mm_div_ps(value,a.value);
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> shift(FASTOR_INDEX i) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
if (i==1)
|
||||
out.value = _mm_shift1_ps(value);
|
||||
else if (i==2)
|
||||
out.value = _mm_shift2_ps(value);
|
||||
else if (i==3)
|
||||
out.value = _mm_shift3_ps(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE float sum() {return _mm_sum_ps(value);}
|
||||
FASTOR_INLINE float product() {return _mm_prod_ps(value);}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> reverse() {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_reverse_ps(value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE float minimum() {return _mm_hmin_ps(value);}
|
||||
FASTOR_INLINE float maximum() {return _mm_hmax_ps(value);}
|
||||
|
||||
FASTOR_INLINE float dot(const SIMDVector<float,simd_abi::sse> &other) {
|
||||
#ifdef FASTOR_SSE4_1_IMPL
|
||||
return _mm_cvtss_f32(_mm_dp_ps(value,other.value,0xff));
|
||||
#else
|
||||
return _mm_sum_ps(_mm_mul_ps(value,other.value));
|
||||
#endif
|
||||
}
|
||||
|
||||
__m128 value;
|
||||
};
|
||||
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<float,simd_abi::sse> a) {
|
||||
// ICC crashes without a copy
|
||||
const __m128 v = a.value;
|
||||
const float* value = reinterpret_cast<const float*>(&v);
|
||||
os << "[" << value[0] << " " << value[1] << " "
|
||||
<< value[2] << " " << value[3] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_add_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(const SIMDVector<float,simd_abi::sse> &a, float b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_add_ps(a.value,_mm_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(float a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_add_ps(_mm_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(const SIMDVector<float,simd_abi::sse> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_sub_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(const SIMDVector<float,simd_abi::sse> &a, float b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_sub_ps(a.value,_mm_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(float a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_sub_ps(_mm_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_neg_ps(b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator*(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_mul_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator*(const SIMDVector<float,simd_abi::sse> &a, float b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_mul_ps(a.value,_mm_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator*(float a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_mul_ps(_mm_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator/(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_div_ps(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator/(const SIMDVector<float,simd_abi::sse> &a, float b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_div_ps(a.value,_mm_set1_ps(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator/(float a, const SIMDVector<float,simd_abi::sse> &b) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_div_ps(_mm_set1_ps(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> rcp(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_rcp_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> sqrt(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_sqrt_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> rsqrt(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_rsqrt_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<float,simd_abi::sse> abs(const SIMDVector<float,simd_abi::sse> &a) {
|
||||
SIMDVector<float,simd_abi::sse> out;
|
||||
out.value = _mm_abs_ps(a.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
} // end of namespace Fastor
|
||||
|
||||
|
||||
#endif // SIMD_VECTOR_FLOAT_H
|
||||
988
noarch/include/Fastor/simd_vector/simd_vector_int32.h
Normal file
988
noarch/include/Fastor/simd_vector/simd_vector_int32.h
Normal file
@@ -0,0 +1,988 @@
|
||||
#ifndef SIMD_VECTOR_INT_H
|
||||
#define SIMD_VECTOR_INT_H
|
||||
|
||||
#include "Fastor/simd_vector/simd_vector_base.h"
|
||||
#include <cstdint>
|
||||
|
||||
namespace Fastor {
|
||||
|
||||
|
||||
// AVX512 VERSION
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifdef FASTOR_AVX512F_IMPL
|
||||
|
||||
template<>
|
||||
struct SIMDVector<int32_t,simd_abi::avx512> {
|
||||
using value_type = __m512i;
|
||||
using scalar_value_type = int32_t;
|
||||
using abi_type = simd_abi::avx512;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx512>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx512>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm512_setzero_si512()) {}
|
||||
FASTOR_INLINE SIMDVector(int32_t num) : value(_mm512_set1_epi32(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m512i regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm512_load_si512((__m512i*)data);
|
||||
else
|
||||
value = _mm512_loadu_si512((__m512i*)data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator=(int32_t num) {
|
||||
value = _mm512_set1_epi32(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator=(__m512i regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm512_load_si512((__m512i*)data);
|
||||
else
|
||||
value = _mm512_loadu_si512((__m512i*)data);
|
||||
}
|
||||
FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm512_store_si512((__m512i*)data,value);
|
||||
else
|
||||
_mm512_storeu_si512((__m512i*)data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const int32_t *data) {
|
||||
value =_mm512_load_si512((__m512i*)data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(int32_t *data) const {
|
||||
_mm512_store_si512((__m512i*)data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm512_mask_loadu_epi32(value, mask, a);
|
||||
else
|
||||
value = _mm512_mask_load_epi32(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm512_setzero_si512();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm512_mask_storeu_epi32(a, mask, value);
|
||||
else
|
||||
_mm512_mask_store_epi32(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t operator[](FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
|
||||
FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(int32_t num) {
|
||||
value = _mm512_set1_epi32(num);
|
||||
}
|
||||
FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7,
|
||||
int32_t num8, int32_t num9, int32_t num10, int32_t num11, int32_t num12, int32_t num13, int32_t num14, int32_t num15) {
|
||||
value = _mm512_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(int32_t num0) {
|
||||
value = _mm512_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7,
|
||||
num0+8,num0+9,num0+10,num0+11,num0+12,num0+13,num0+14,num0+15);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(int32_t num) {
|
||||
value = _mm512_add_epi32(value,_mm512_set1_epi32(num));
|
||||
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m512i regi) {
|
||||
value = _mm512_add_epi32(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
|
||||
value = _mm512_add_epi32(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(int32_t num) {
|
||||
value = _mm512_sub_epi32(value,_mm512_set1_epi32(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m512i regi) {
|
||||
value = _mm512_sub_epi32(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
|
||||
value = _mm512_sub_epi32(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(int32_t num) {
|
||||
value = _mm512_mullo_epi32(value,_mm512_set1_epi32(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m512i regi) {
|
||||
value = _mm512_mullo_epi32(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
|
||||
value = _mm512_mullo_epi32(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(int32_t num) {
|
||||
#ifdef FASTOR_INTEL
|
||||
value = _mm512_div_epi32(value,_mm512_set1_epi32(num));
|
||||
#else
|
||||
int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= num;
|
||||
}
|
||||
value = _mm512_loadu_si512((__m512i*)val);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m512i regi) {
|
||||
#ifdef FASTOR_INTEL
|
||||
value = _mm512_div_epi32(value,regi);
|
||||
#else
|
||||
int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value);
|
||||
int32_t val_num[Size]; _mm512_storeu_si512((__m512i*)val_num, regi);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= val_num[i];
|
||||
}
|
||||
value = _mm512_loadu_si512((__m512i*)val);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
|
||||
#ifdef FASTOR_INTEL
|
||||
value = _mm512_div_epi32(value,a.value);
|
||||
#else
|
||||
int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value);
|
||||
int32_t val_a[Size]; _mm512_storeu_si512((__m512i*)val_a, a.value);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= val_a[i];
|
||||
}
|
||||
value = _mm512_loadu_si512((__m512i*)val);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t minimum() {
|
||||
int32_t *vals = (int32_t*)&value;
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
if (vals[i]<quan)
|
||||
quan = vals[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE int32_t maximum() {
|
||||
int32_t *vals = (int32_t*)&value;
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
if (vals[i]>quan)
|
||||
quan = vals[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> reverse() {
|
||||
return _mm512_reverse_epi32(value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t sum() {
|
||||
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
|
||||
return _mm512_reduce_add_epi32(value);
|
||||
#else
|
||||
int32_t vals[Size]; _mm512_storeu_si512((__m512i*)vals, value);
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
quan += vals[i];
|
||||
return quan;
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t product() {
|
||||
int32_t vals[Size]; _mm512_storeu_si512((__m512i*)vals, value);
|
||||
int32_t quan = 1;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
quan *= vals[i];
|
||||
return quan;
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t dot(const SIMDVector<int32_t,simd_abi::avx512> &other) {
|
||||
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
|
||||
return _mm512_reduce_add_epi32(_mm512_mullo_epi32(value,other.value));
|
||||
#else
|
||||
return SIMDVector<int32_t,simd_abi::avx512>(_mm512_mullo_epi32(value,other.value)).sum();
|
||||
#endif
|
||||
}
|
||||
|
||||
__m512i value;
|
||||
};
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<int32_t,simd_abi::avx512> a) {
|
||||
const int32_t *value = (int32_t*) &a.value;
|
||||
os << "["
|
||||
<< value[0] << " " << value[1] << " "
|
||||
<< value[2] << " " << value[3] << " "
|
||||
<< value[4] << " " << value[5] << " "
|
||||
<< value[6] << " " << value[7] << " "
|
||||
<< value[8] << " " << value[9] << " "
|
||||
<< value[10] << " " << value[11] << " "
|
||||
<< value[12] << " " << value[13] << " "
|
||||
<< value[14] << " " << value[15] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_epi32(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_epi32(a.value,_mm512_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_add_epi32(_mm512_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_epi32(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_epi32(a.value,_mm512_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_sub_epi32(_mm512_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
return _mm512_castps_si512(_mm512_neg_ps(_mm512_castsi512_ps(b.value)));
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator*(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_mullo_epi32(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator*(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_mullo_epi32(a.value,_mm512_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator*(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
out.value = _mm512_mullo_epi32(_mm512_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator/(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
#ifdef FASTOR_INTEL
|
||||
out.value = _mm512_div_epi32(a.value,b.value);
|
||||
#else
|
||||
int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value);
|
||||
int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value);
|
||||
int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = val_a[i] / val_b[i];
|
||||
}
|
||||
out.value = _mm512_loadu_si512((__m512i*)val);
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator/(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
#ifdef FASTOR_INTEL
|
||||
out.value = _mm512_div_epi32(a.value,_mm512_set1_epi32(b));
|
||||
#else
|
||||
int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value);
|
||||
int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = val_a[i] / b;
|
||||
}
|
||||
out.value = _mm512_loadu_si512((__m512i*)val);
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator/(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
#ifdef FASTOR_INTEL
|
||||
out.value = _mm512_div_epi32(_mm512_set1_epi32(a),b.value);
|
||||
#else
|
||||
int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value);
|
||||
int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = a / val_b[i];
|
||||
}
|
||||
out.value = _mm512_loadu_si512((__m512i*)val);
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> abs(const SIMDVector<int32_t,simd_abi::avx512> &a) {
|
||||
SIMDVector<int32_t,simd_abi::avx512> out;
|
||||
#ifdef FASTOR_HAS_AVX512_ABS
|
||||
out.value = _mm512_abs_epi32(a.value);
|
||||
#else
|
||||
for (FASTOR_INDEX i=0UL; i<16UL; ++i) {
|
||||
((int32_t*)&out.value)[i] = std::abs(((int32_t*)&a.value)[i]);
|
||||
}
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// AVX VERSION
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifdef FASTOR_AVX2_IMPL
|
||||
|
||||
template<>
|
||||
struct SIMDVector<int32_t,simd_abi::avx> {
|
||||
using value_type = __m256i;
|
||||
using scalar_value_type = int32_t;
|
||||
using abi_type = simd_abi::avx;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm256_setzero_si256()) {}
|
||||
FASTOR_INLINE SIMDVector(int32_t num) : value(_mm256_set1_epi32(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m256i regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm256_load_si256((__m256i*)data);
|
||||
else
|
||||
value = _mm256_loadu_si256((__m256i*)data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator=(int32_t num) {
|
||||
value = _mm256_set1_epi32(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator=(__m256i regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm256_load_si256((__m256i*)data);
|
||||
else
|
||||
value = _mm256_loadu_si256((__m256i*)data);
|
||||
}
|
||||
FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm256_store_si256((__m256i*)data,value);
|
||||
else
|
||||
_mm256_storeu_si256((__m256i*)data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const int32_t *data) {
|
||||
value =_mm256_load_si256((__m256i*)data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(int32_t *data) const {
|
||||
_mm256_store_si256((__m256i*)data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm256_mask_loadu_epi32(value, mask, a);
|
||||
else
|
||||
value = _mm256_mask_load_epi32(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm256_setzero_si256();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm256_mask_storeu_epi32(a, mask, value);
|
||||
else
|
||||
_mm256_mask_store_epi32(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t operator[](FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
|
||||
FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(int32_t num) {
|
||||
value = _mm256_set1_epi32(num);
|
||||
}
|
||||
FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7) {
|
||||
value = _mm256_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(int32_t num0) {
|
||||
value = _mm256_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(int32_t num) {
|
||||
value = _mm256_add_epi32x(value,_mm256_set1_epi32(num));
|
||||
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m256i regi) {
|
||||
value = _mm256_add_epi32x(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<int32_t,simd_abi::avx> &a) {
|
||||
value = _mm256_add_epi32x(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(int32_t num) {
|
||||
value = _mm256_sub_epi32x(value,_mm256_set1_epi32(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m256i regi) {
|
||||
value = _mm256_sub_epi32x(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<int32_t,simd_abi::avx> &a) {
|
||||
value = _mm256_sub_epi32x(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(int32_t num) {
|
||||
value = _mm256_mul_epi32x(value,_mm256_set1_epi32(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m256i regi) {
|
||||
value = _mm256_mul_epi32x(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<int32_t,simd_abi::avx> &a) {
|
||||
value = _mm256_mul_epi32x(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(int32_t num) {
|
||||
int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= num;
|
||||
}
|
||||
value = _mm256_loadu_si256((__m256i*)val);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m256i regi) {
|
||||
int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value);
|
||||
int32_t val_num[Size]; _mm256_storeu_si256((__m256i*)val_num, regi);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= val_num[i];
|
||||
}
|
||||
value = _mm256_loadu_si256((__m256i*)val);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<int32_t,simd_abi::avx> &a) {
|
||||
int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value);
|
||||
int32_t val_a[Size]; _mm256_storeu_si256((__m256i*)val_a, a.value);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= val_a[i];
|
||||
}
|
||||
value = _mm256_loadu_si256((__m256i*)val);
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t minimum() {
|
||||
int32_t *vals = (int32_t*)&value;
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
if (vals[i]<quan)
|
||||
quan = vals[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE int32_t maximum() {
|
||||
int32_t *vals = (int32_t*)&value;
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
if (vals[i]>quan)
|
||||
quan = vals[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> reverse() {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_reverse_epi32(value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t sum() {
|
||||
int32_t vals[Size]; _mm256_storeu_si256((__m256i*)vals, value);
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
quan += vals[i];
|
||||
return quan;
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t dot(const SIMDVector<int32_t,simd_abi::avx> &other) {
|
||||
int32_t vals0[Size]; _mm256_storeu_si256((__m256i*)vals0, value);
|
||||
int32_t vals1[Size]; _mm256_storeu_si256((__m256i*)vals1, other.value);
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
quan += vals0[i]*vals1[i];
|
||||
return quan;
|
||||
}
|
||||
|
||||
__m256i value;
|
||||
};
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<int32_t,simd_abi::avx> a) {
|
||||
const int32_t *value = (int32_t*) &a.value;
|
||||
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3]
|
||||
<< " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_add_epi32x(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_add_epi32x(a.value,_mm256_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_add_epi32x(_mm256_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_epi32x(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_epi32x(a.value,_mm256_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_sub_epi32x(_mm256_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
return _mm256_castps_si256(_mm256_neg_ps(_mm256_castsi256_ps(b.value)));
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator*(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_epi32x(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator*(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_epi32x(a.value,_mm256_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator*(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
out.value = _mm256_mul_epi32x(_mm256_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator/(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value);
|
||||
int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value);
|
||||
int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = val_a[i] / val_b[i];
|
||||
}
|
||||
out.value = _mm256_loadu_si256((__m256i*)val);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator/(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value);
|
||||
int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = val_a[i] / b;
|
||||
}
|
||||
out.value = _mm256_loadu_si256((__m256i*)val);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator/(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value);
|
||||
int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = a / val_b[i];
|
||||
}
|
||||
out.value = _mm256_loadu_si256((__m256i*)val);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> abs(const SIMDVector<int32_t,simd_abi::avx> &a) {
|
||||
SIMDVector<int32_t,simd_abi::avx> out;
|
||||
#ifdef __AVX2__
|
||||
out.value = _mm256_abs_epi32(a.value);
|
||||
#else
|
||||
// THIS IS ALSO AVX2 VERSION!
|
||||
// __m128i lo = _mm_abs_epi32(_mm256_castsi256_si128(a.value));
|
||||
// __m128i hi = _mm_abs_epi32(_mm256_extracti128_si256(a.value,0x1));
|
||||
// out.value = _mm256_castsi128_si256(lo);
|
||||
// out.value = _mm256_insertf128_si256(out.value,hi,0x1);
|
||||
|
||||
int32_t *value = (int32_t*) &a.value;
|
||||
for (int32_t i=0; i<8; ++i) {
|
||||
value[i] = std::abs(value[i]);
|
||||
}
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// SSE VERSION
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifdef FASTOR_SSE2_IMPL
|
||||
|
||||
template<>
|
||||
struct SIMDVector<int32_t,simd_abi::sse> {
|
||||
using value_type = __m128i;
|
||||
using scalar_value_type = int32_t;
|
||||
using abi_type = simd_abi::sse;
|
||||
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::sse>>::value;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::sse>>::value;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(_mm_setzero_si128()) {}
|
||||
FASTOR_INLINE SIMDVector(int32_t num) : value(_mm_set1_epi32(num)) {}
|
||||
FASTOR_INLINE SIMDVector(__m128i regi) : value(regi) {}
|
||||
FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm_load_si128((__m128i*)data);
|
||||
else
|
||||
value = _mm_loadu_si128((__m128i*)data);
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator=(int32_t num) {
|
||||
value = _mm_set1_epi32(num);
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator=(__m128i regi) {
|
||||
value = regi;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) {
|
||||
if (Aligned)
|
||||
value =_mm_load_si128((__m128i*)data);
|
||||
else
|
||||
value = _mm_loadu_si128((__m128i*)data);
|
||||
}
|
||||
FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const {
|
||||
if (Aligned)
|
||||
_mm_store_si128((__m128i*)data,value);
|
||||
else
|
||||
_mm_storeu_si128((__m128i*)data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void aligned_load(const int32_t *data) {
|
||||
value =_mm_load_si128((__m128i*)data);
|
||||
}
|
||||
FASTOR_INLINE void aligned_store(int32_t *data) const {
|
||||
_mm_store_si128((__m128i*)data,value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
value = _mm_mask_loadu_epi32(value, mask, a);
|
||||
else
|
||||
value = _mm_mask_load_epi32(value, mask, a);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
value = _mm_setzero_si128();
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
|
||||
#ifdef FASTOR_HAS_AVX512_MASKS
|
||||
if (!Aligned)
|
||||
_mm_mask_storeu_epi32(a, mask, value);
|
||||
else
|
||||
_mm_mask_store_epi32(a, mask, value);
|
||||
#else
|
||||
// perhaps very inefficient but they never get used
|
||||
int maska[Size];
|
||||
mask_to_array(mask,maska);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
if (maska[i] == -1) {
|
||||
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
|
||||
}
|
||||
else {
|
||||
a[Size - i - 1] = 0;
|
||||
}
|
||||
}
|
||||
unused(Aligned);
|
||||
#endif
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t operator[](FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
|
||||
FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
|
||||
|
||||
FASTOR_INLINE void set(int32_t num) {
|
||||
value = _mm_set1_epi32(num);
|
||||
}
|
||||
FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3) {
|
||||
value = _mm_set_epi32(num0,num1,num2,num3);
|
||||
}
|
||||
FASTOR_INLINE void set_sequential(int32_t num0) {
|
||||
value = _mm_setr_epi32(num0,num0+1,num0+2,num0+3);
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(int32_t num) {
|
||||
value = _mm_add_epi32(value,_mm_set1_epi32(num));
|
||||
|
||||
}
|
||||
FASTOR_INLINE void operator+=(__m128i regi) {
|
||||
value = _mm_add_epi32(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<int32_t,simd_abi::sse> &a) {
|
||||
value = _mm_add_epi32(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(int32_t num) {
|
||||
value = _mm_sub_epi32(value,_mm_set1_epi32(num));
|
||||
}
|
||||
FASTOR_INLINE void operator-=(__m128i regi) {
|
||||
value = _mm_sub_epi32(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<int32_t,simd_abi::sse> &a) {
|
||||
value = _mm_sub_epi32(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(int32_t num) {
|
||||
value = _mm_mul_epi32x(value,_mm_set1_epi32(num));
|
||||
}
|
||||
FASTOR_INLINE void operator*=(__m128i regi) {
|
||||
value = _mm_mul_epi32x(value,regi);
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<int32_t,simd_abi::sse> &a) {
|
||||
value = _mm_mul_epi32x(value,a.value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(int32_t num) {
|
||||
int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= num;
|
||||
}
|
||||
value = _mm_loadu_si128((__m128i*)val);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(__m128i regi) {
|
||||
int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value);
|
||||
int32_t val_num[Size]; _mm_storeu_si128((__m128i*)val_num, regi);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= val_num[i];
|
||||
}
|
||||
value = _mm_loadu_si128((__m128i*)val);
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<int32_t,simd_abi::sse> &a) {
|
||||
int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value);
|
||||
int32_t val_a[Size]; _mm_storeu_si128((__m128i*)val_a, a.value);
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i) {
|
||||
val[i] /= val_a[i];
|
||||
}
|
||||
value = _mm_loadu_si128((__m128i*)val);
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t minimum() {
|
||||
int32_t *vals = (int32_t*)&value;
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
if (vals[i]<quan)
|
||||
quan = vals[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE int32_t maximum() {
|
||||
int32_t *vals = (int32_t*)&value;
|
||||
int32_t quan = 0;
|
||||
for (FASTOR_INDEX i=0; i<Size; ++i)
|
||||
if (vals[i]>quan)
|
||||
quan = vals[i];
|
||||
return quan;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> reverse() {
|
||||
return _mm_reverse_epi32(value);
|
||||
}
|
||||
|
||||
FASTOR_INLINE int32_t sum() {return _mm_sum_epi32(value);}
|
||||
FASTOR_INLINE int32_t product() {return _mm_prod_epi32(value);}
|
||||
|
||||
FASTOR_INLINE int32_t dot(const SIMDVector<int32_t,simd_abi::sse> &other) {
|
||||
return _mm_sum_epi32(_mm_mul_epi32x(value,other.value));
|
||||
}
|
||||
|
||||
__m128i value;
|
||||
};
|
||||
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<int32_t,simd_abi::sse> a) {
|
||||
const int32_t *value = (int32_t*) &a.value;
|
||||
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_add_epi32(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_add_epi32(a.value,_mm_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_add_epi32(_mm_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_sub_epi32(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_sub_epi32(a.value,_mm_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_sub_epi32(_mm_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
return _mm_castps_si128(_mm_neg_ps(_mm_castsi128_ps(b.value)));
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator*(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_mul_epi32x(a.value,b.value);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator*(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_mul_epi32x(a.value,_mm_set1_epi32(b));
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator*(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
out.value = _mm_mul_epi32x(_mm_set1_epi32(a),b.value);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator/(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value);
|
||||
int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value);
|
||||
int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = val_a[i] / val_b[i];
|
||||
}
|
||||
out.value = _mm_loadu_si128((__m128i*)val);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator/(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value);
|
||||
int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = val_a[i] / b;
|
||||
}
|
||||
out.value = _mm_loadu_si128((__m128i*)val);
|
||||
return out;
|
||||
}
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator/(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value);
|
||||
int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value);
|
||||
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
|
||||
val[i] = a / val_b[i];
|
||||
}
|
||||
out.value = _mm_loadu_si128((__m128i*)val);
|
||||
return out;
|
||||
}
|
||||
|
||||
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> abs(const SIMDVector<int32_t,simd_abi::sse> &a) {
|
||||
SIMDVector<int32_t,simd_abi::sse> out;
|
||||
#ifdef FASTOR_SSSE3_IMPL
|
||||
out.value = _mm_abs_epi32(a.value);
|
||||
#else // SSE2
|
||||
__m128i sign = _mm_srai_epi32(a.value, 31);
|
||||
__m128i inv = _mm_xor_si128(a.value, sign);
|
||||
out.value = _mm_sub_epi32(inv, sign);
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
} // end of namespace Fastor
|
||||
|
||||
|
||||
#endif // SIMD_VECTOR_INT_H
|
||||
1040
noarch/include/Fastor/simd_vector/simd_vector_int64.h
Normal file
1040
noarch/include/Fastor/simd_vector/simd_vector_int64.h
Normal file
File diff suppressed because it is too large
Load Diff
216
noarch/include/Fastor/simd_vector/simd_vector_scalar.h
Normal file
216
noarch/include/Fastor/simd_vector/simd_vector_scalar.h
Normal file
@@ -0,0 +1,216 @@
|
||||
#ifndef SIMD_VECTOR_T_SCALAR_H
|
||||
#define SIMD_VECTOR_T_SCALAR_H
|
||||
|
||||
#include "Fastor/simd_vector/simd_vector_base.h"
|
||||
|
||||
namespace Fastor {
|
||||
|
||||
template <typename T>
|
||||
struct SIMDVector<T, simd_abi::scalar> {
|
||||
using value_type = T;
|
||||
using scalar_value_type = T;
|
||||
using abi_type = simd_abi::scalar;
|
||||
static constexpr FASTOR_INDEX Size = 1;
|
||||
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;}
|
||||
|
||||
FASTOR_INLINE SIMDVector() : value(0) {}
|
||||
FASTOR_INLINE SIMDVector(T num) : value(num) {}
|
||||
FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) : value(*data) {}
|
||||
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator=(T num) {
|
||||
value = num;
|
||||
return *this;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void load(const T *data, bool Aligned=true) { value = *data; unused(Aligned); }
|
||||
FASTOR_INLINE void store(T *data, bool Aligned=true) const { data[0] = value; unused(Aligned); }
|
||||
|
||||
FASTOR_INLINE void aligned_load(const T *data) { value = *data; }
|
||||
FASTOR_INLINE void aligned_store(T *data) const { data[0] = value; }
|
||||
|
||||
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool ) {
|
||||
if (mask != 0x0) value = *a;
|
||||
}
|
||||
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool) const {
|
||||
if (mask != 0x0) a[0] = value;
|
||||
}
|
||||
|
||||
FASTOR_INLINE T operator[](FASTOR_INDEX) const {return value;}
|
||||
FASTOR_INLINE T operator()(FASTOR_INDEX) const {return value;}
|
||||
|
||||
FASTOR_INLINE void set(T num) {
|
||||
value = num;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void set_sequential(T num) {
|
||||
value = num;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void broadcast(const T *data) {
|
||||
value = *data;
|
||||
}
|
||||
|
||||
// In-place operators
|
||||
FASTOR_INLINE void operator+=(T num) {
|
||||
value += num;
|
||||
}
|
||||
FASTOR_INLINE void operator+=(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
value += a.value;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator-=(T num) {
|
||||
value -= num;
|
||||
}
|
||||
FASTOR_INLINE void operator-=(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
value -= a.value;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator*=(T num) {
|
||||
value *= num;
|
||||
}
|
||||
FASTOR_INLINE void operator*=(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
value *= a.value;
|
||||
}
|
||||
|
||||
FASTOR_INLINE void operator/=(T num) {
|
||||
value /= num;
|
||||
}
|
||||
FASTOR_INLINE void operator/=(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
value /= a.value;
|
||||
}
|
||||
// end of in-place operators
|
||||
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> shift(FASTOR_INDEX) {
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE T sum() {return value;}
|
||||
FASTOR_INLINE T product() {return value;}
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> reverse() {
|
||||
return *this;
|
||||
}
|
||||
FASTOR_INLINE T minimum() {return value;}
|
||||
FASTOR_INLINE T maximum() {return value;}
|
||||
|
||||
FASTOR_INLINE T dot(const SIMDVector<T,simd_abi::scalar> &other) {
|
||||
return value*other.value;
|
||||
}
|
||||
|
||||
T value;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<T,simd_abi::scalar> a) {
|
||||
os << "[" << a.value << "]\n";
|
||||
return os;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value+b.value;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(const SIMDVector<T,simd_abi::scalar> &a, T b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value+b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(T a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a+b.value;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
return b;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value-b.value;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(const SIMDVector<T,simd_abi::scalar> &a, T b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value-b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(T a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a-b.value;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = -b.value;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator*(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value*b.value;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator*(const SIMDVector<T,simd_abi::scalar> &a, T b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value*b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator*(T a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a*b.value;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator/(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value/b.value;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator/(const SIMDVector<T,simd_abi::scalar> &a, T b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a.value/b;
|
||||
return out;
|
||||
}
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator/(T a, const SIMDVector<T,simd_abi::scalar> &b) {
|
||||
SIMDVector<T,simd_abi::scalar> out;
|
||||
out.value = a/b.value;
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> rcp(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
return T(1) / a.value;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> sqrt(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
return std::sqrt(a.value);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> rsqrt(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
return T(1) / std::sqrt(a.value);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> abs(const SIMDVector<T,simd_abi::scalar> &a) {
|
||||
return std::abs(a.value);
|
||||
}
|
||||
|
||||
} // end of namespace Fastor
|
||||
|
||||
|
||||
#endif // SIMD_VECTOR_T_SCALAR_H
|
||||
Reference in New Issue
Block a user