diff --git a/noarch/include/Fastor/Fastor.h b/noarch/include/Fastor/Fastor.h new file mode 100644 index 00000000..1e066aaa --- /dev/null +++ b/noarch/include/Fastor/Fastor.h @@ -0,0 +1,39 @@ +#ifndef FASTOR_ALL_INCLUDE_H +#define FASTOR_ALL_INCLUDE_H + +#if defined(_MSC_VER) +// Disable MSVC warnings for macros +#pragma warning (disable: 4003) +// Disable MSVC warnings for conversion +#pragma warning (disable: 4244) +// Disable MSVC unreferenced parameters +#pragma warning (disable: 4100) +// Disable MSVC narrowing conversion +#pragma warning (disable: 4267) +#endif + +#include "config/config.h" +#include "util/util.h" +#include "simd_vector/SIMDVector.h" +#include "simd_math/simd_math.h" +#include "tensor/Tensor.h" +#include "tensor/TensorMap.h" +#include "tensor/TensorIO.h" +#include "tensor/TensorFunctions.h" +#include "tensor/AbstractTensorFunctions.h" +#include "tensor_algebra/einsum.h" +#include "tensor_algebra/network_einsum.h" +#include "tensor_algebra/einsum_explicit.h" +#include "tensor_algebra/abstract_contraction.h" +#include "expressions/expressions.h" +#include "backend/voigt.h" + +#if defined(_MSC_VER) +#pragma warning (default: 4003) +#pragma warning (default: 4244) +#pragma warning (default: 4100) +#pragma warning (default: 4267) +#endif + +#endif // FASTOR_ALL_INCLUDE_H + diff --git a/noarch/include/Fastor/backend/adjoint.h b/noarch/include/Fastor/backend/adjoint.h new file mode 100644 index 00000000..8a5dd1eb --- /dev/null +++ b/noarch/include/Fastor/backend/adjoint.h @@ -0,0 +1,105 @@ +#ifndef ADJOINT_H +#define ADJOINT_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/extintrin.h" + +namespace Fastor { + +template, bool> = false> +FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); + +template, bool> = false> +FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { + *dst = *src; +} + +template, bool> = false> +FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T src0 = src[0]; + T src1 = src[1]; + T src2 = src[2]; + T src3 = src[3]; + + /* Compute adjoint: */ + dst[0] = + src3; + dst[1] = - src1; + dst[2] = - src2; + dst[3] = + src0; +} + +template, bool> = false> +FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T src0 = src[0]; + T src1 = src[1]; + T src2 = src[2]; + T src3 = src[3]; + T src4 = src[4]; + T src5 = src[5]; + T src6 = src[6]; + T src7 = src[7]; + T src8 = src[8]; + + /* Compute adjoint: */ + dst[0] = + src4 * src8 - src5 * src7; + dst[1] = - src1 * src8 + src2 * src7; + dst[2] = + src1 * src5 - src2 * src4; + dst[3] = - src3 * src8 + src5 * src6; + dst[4] = + src0 * src8 - src2 * src6; + dst[5] = - src0 * src5 + src2 * src3; + dst[6] = + src3 * src7 - src4 * src6; + dst[7] = - src0 * src7 + src1 * src6; + dst[8] = + src0 * src4 - src1 * src3; +} + +template, bool> = false> +FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + /* Compute adjoint: */ + T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2]; + T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1]; + T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1]; + + dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3; + dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3; + + T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0]; + T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0]; + + dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5; + dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5; + + t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0]; + + dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1; + dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1; + dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1; + dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1; + + t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2]; + t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1]; + t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1]; + + dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3; + dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3; + + t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0]; + t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0]; + + dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5; + dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5; + + t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0]; + + dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1; + dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1; + dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1; + dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1; +} + +} // end of namespace Fastor + +#endif // ADJOINT_H diff --git a/noarch/include/Fastor/backend/backend.h b/noarch/include/Fastor/backend/backend.h new file mode 100644 index 00000000..c232b650 --- /dev/null +++ b/noarch/include/Fastor/backend/backend.h @@ -0,0 +1,24 @@ +#ifndef BACKEND_H +#define BACKEND_H + + +#include "Fastor/backend/adjoint.h" +#include "Fastor/backend/cofactor.h" +#include "Fastor/backend/cyclic_0.h" +#include "Fastor/backend/determinant.h" +#include "Fastor/backend/doublecontract.h" +#include "Fastor/backend/dyadic.h" +#include "Fastor/backend/inner.h" +#include "Fastor/backend/inverse.h" +#include "Fastor/backend/lufact.h" +#include "Fastor/backend/lut_inverse.h" +#include "Fastor/backend/matmul/matmul.h" +#include "Fastor/backend/matmul/tmatmul.h" +#include "Fastor/backend/norm.h" +#include "Fastor/backend/outer.h" +#include "Fastor/backend/tensor_cross.h" +#include "Fastor/backend/trace.h" +#include "Fastor/backend/transpose/transpose.h" + +#endif // BACKEND_H + diff --git a/noarch/include/Fastor/backend/cofactor.h b/noarch/include/Fastor/backend/cofactor.h new file mode 100644 index 00000000..8bece5af --- /dev/null +++ b/noarch/include/Fastor/backend/cofactor.h @@ -0,0 +1,105 @@ +#ifndef COFACTOR_H +#define COFACTOR_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/extintrin.h" + +namespace Fastor { + +template, bool> = false> +FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); + +template, bool> = false> +FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { + *dst = *src; +} + +template, bool> = false> +FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T src0 = src[0]; + T src1 = src[1]; + T src2 = src[2]; + T src3 = src[3]; + + /* Compute cofactor: */ + dst[0] = + src3; + dst[1] = - src2; + dst[2] = - src1; + dst[3] = + src0; +} + +template, bool> = false> +FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T src0 = src[0]; + T src1 = src[1]; + T src2 = src[2]; + T src3 = src[3]; + T src4 = src[4]; + T src5 = src[5]; + T src6 = src[6]; + T src7 = src[7]; + T src8 = src[8]; + + /* Compute cofactor: */ + dst[0] = + src4 * src8 - src5 * src7; + dst[1] = - src3 * src8 + src5 * src6; + dst[2] = + src3 * src7 - src4 * src6; + dst[3] = - src1 * src8 + src2 * src7; + dst[4] = + src0 * src8 - src2 * src6; + dst[5] = - src0 * src7 + src1 * src6; + dst[6] = + src1 * src5 - src2 * src4; + dst[7] = - src0 * src5 + src2 * src3; + dst[8] = + src0 * src4 - src1 * src3; +} + +template, bool> = false> +FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + /* Compute cofactor: */ + T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2]; + T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1]; + T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1]; + + dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3; + dst[4] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3; + + T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0]; + T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0]; + + dst[1] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5; + dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5; + + t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0]; + + dst[2] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1; + dst[6] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1; + dst[3] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1; + dst[7] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1; + + t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2]; + t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1]; + t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1]; + + dst[8] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3; + dst[12] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3; + + t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0]; + t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0]; + + dst[9] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5; + dst[13] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5; + + t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0]; + + dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1; + dst[14] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1; + dst[11] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1; + dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1; +} + +} + +#endif // COFACTOR_H diff --git a/noarch/include/Fastor/backend/cyclic_0.h b/noarch/include/Fastor/backend/cyclic_0.h new file mode 100644 index 00000000..c0c643ff --- /dev/null +++ b/noarch/include/Fastor/backend/cyclic_0.h @@ -0,0 +1,172 @@ +#ifndef CYCLIC_0_H +#define CYCLIC_0_H + +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" + +namespace Fastor { + + +//! Version 0 of cyclic product of two second order tensors i.e. C_ijkl = A_ik * B_jl + +template +FASTOR_HINT_INLINE void _cyclic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + for (size_t i=0; i +FASTOR_HINT_INLINE void _cyclic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + __m256d as = _mm256_load_pd(a); + __m256d bs = _mm256_load_pd(b); + + // First 2x2 block + __m256d c0 = _mm256_mul_pd(as,bs); + _mm_store_sd(out,_mm256_castpd256_pd128(c0)); + _mm_store_sd(out+1,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1)); + __m128d c0_high = _mm256_extractf128_pd(c0,0x1); + _mm_store_sd(out+3,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1)); + _mm_store_sd(out+4,_mm_shuffle_pd(c0_high,c0_high,0x1)); + // c_02 + __m128d a0 = _mm256_castpd256_pd128(as); + __m128d b0 = _mm256_castpd256_pd128(bs); + b0 = _mm_shuffle_pd(b0,b0,0x1); + __m128d c1 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0))); + _mm_store_sd(out+2,c1); + _mm_store_sd(out+6,c1); + // c_12 + __m128d a1 = _mm256_extractf128_pd(as,0x1); + __m128d b1 = _mm256_extractf128_pd(bs,0x1); + b1 = _mm_shuffle_pd(b1,b1,0x1); + __m128d c2 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1))); + _mm_store_sd(out+5,c2); + _mm_store_sd(out+7,c2); + // c_22 + __m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b1))); + _mm_store_sd(out+8,c3); +} + +template<> +FASTOR_HINT_INLINE void _cyclic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + // 34+ OPS + __m256d a_low = _mm256_load_pd(a); + __m256d a_high = _mm256_load_pd(a+4); + __m128d a_end = _mm_load_sd(a+8); + + __m256d b_low = _mm256_load_pd(b); + __m256d b_high = _mm256_load_pd(b+4); + __m128d b_end = _mm_load_sd(b+8); + + // The first 3x3 block + __m256d c0 = _mm256_mul_pd(a_low,b_low); + __m256d c1 = _mm256_mul_pd(a_high,b_high); + __m128d c2 = _mm_mul_sd(a_end,b_end); + + _mm256_store_pd(out,c0); + _mm_store_sd(out+6,_mm_set_sd(_mm256_get1_pd(c0))); + _mm256_storeu_pd(out+7,c1); +// _mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0))); + _mm_store_sd(out+12,_mm256_extractf128_pd(c0,0x1)); + _mm_store_sd(out+13,_mm_set_sd(_mm256_get1_pd(c1))); + _mm_store_sd(out+14,c2); + + // rest + // c_03 + __m128d a0 = _mm256_castpd256_pd128(a_low); + __m128d b0 = _mm256_castpd256_pd128(b_low); + b0 = _mm_shuffle_pd(b0,b0,0x1); + __m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0))); + _mm_store_sd(out+3,c3); + _mm_store_sd(out+18,c3); + // c_04 + __m128d a1 = _mm_set_pd(_mm256_get0_pd(a_low),_mm256_get2_pd(a_low)); + __m128d b1 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get0_pd(b_low)); + __m128d c4 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1))); + _mm_store_sd(out+4,c4); + _mm_store_sd(out+24,c4); + // c_05 + __m128d a2 = _mm_set_pd(_mm256_get1_pd(a_low),_mm256_get2_pd(a_low)); + __m128d b2 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get1_pd(b_low)); + __m128d c5 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b2))); + _mm_store_sd(out+5,c5); + _mm_store_sd(out+30,c5); + // c_13 + __m128d a3 = _mm_setr_pd(_mm256_get3_pd(a_low),_mm256_get0_pd(a_high)); + __m128d b3 = _mm_setr_pd(_mm256_get0_pd(b_high),_mm256_get3_pd(b_low)); + __m128d c6 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b3))); + _mm_store_sd(out+9,c6); + _mm_store_sd(out+19,c6); + // c_14 + __m128d a4 = _mm_set_pd(_mm256_get3_pd(a_low),_mm256_get1_pd(a_high)); + __m128d b4 = _mm_set_pd(_mm256_get1_pd(b_high),_mm256_get3_pd(b_low)); + __m128d c7 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b4))); + _mm_store_sd(out+10,c7); + _mm_store_sd(out+25,c7); + // c_15 + __m128d a5 = _mm256_castpd256_pd128(a_high); + __m128d b5 = _mm256_castpd256_pd128(b_high); + b5 = _mm_shuffle_pd(b5,b5,0x1); + __m128d c8 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,b5))); + _mm_store_sd(out+11,c8); + _mm_store_sd(out+31,c8); + // c_23 + __m128d a6 = _mm256_extractf128_pd(a_high,0x1); + __m128d b6 = _mm256_extractf128_pd(b_high,0x1); + b6 = _mm_shuffle_pd(b6,b6,0x1); + __m128d c9 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a6,b6))); + _mm_store_sd(out+15,c9); + _mm_store_sd(out+20,c9); + // c_24 + __m128d a7 = _mm_set_pd(_mm256_get2_pd(a_high),_mm_get0_pd(a_end)); + __m128d b7 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get2_pd(b_high)); + __m128d c10 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a7,b7))); + _mm_store_sd(out+16,c10); + _mm_store_sd(out+26,c10); + // c_25 + __m128d a8 = _mm_set_pd(_mm256_get3_pd(a_high),_mm_get0_pd(a_end)); + __m128d b8 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get3_pd(b_high)); + __m128d c11 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a8,b8))); + _mm_store_sd(out+17,c11); + _mm_store_sd(out+32,c11); + // c_33 + __m128d c12 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b3))); + _mm_store_sd(out+21,c12); + // c_34 + __m128d c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b4))); + _mm_store_sd(out+22,c13); + // c_43 + c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b6))); + _mm_store_sd(out+27,c13); + // c_35 + __m128d c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,_mm_shuffle_pd(b5,b5,0x1)))); + _mm_store_sd(out+23,c14); + // c_53 + c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b6))); + _mm_store_sd(out+33,c14); + // c_44 + __m128d c15 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b7))); + _mm_store_sd(out+28,c15); + // c_45 + __m128d c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b8))); + _mm_store_sd(out+29,c16); + c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b7))); + _mm_store_sd(out+34,c16); + // c_55 + __m128d c17 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,_mm_shuffle_pd(b8,b8,0x1)))); + _mm_store_sd(out+35,c17); +} + +#endif + +} + +#endif // CYCLIC_0_H + diff --git a/noarch/include/Fastor/backend/determinant.h b/noarch/include/Fastor/backend/determinant.h new file mode 100644 index 00000000..e3cd2f2e --- /dev/null +++ b/noarch/include/Fastor/backend/determinant.h @@ -0,0 +1,116 @@ +#ifndef DETERMINANT_H +#define DETERMINANT_H + +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/meta/tensor_meta.h" + +namespace Fastor { + + +#ifndef FASTOR_AVX_IMPL +template::type=0> +#else +template::value && + !std::is_same::value && M==2 && N==2, bool>::type=0> +#endif +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { + return a[0] * a[3] - a[1] * a[2]; +} + + +#ifndef FASTOR_AVX_IMPL +template::type=0> +#else +template::value && + !std::is_same::value && M==3 && N==3, bool>::type=0> +#endif +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { + return a[0]*a[4]*a[8] + a[1]*a[5]*a[6] + a[2]*a[3]*a[7] - a[2]*a[4]*a[6] - a[1]*a[3]*a[8] - a[0]*a[5]*a[7]; +} + +template::type=0> +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT m) { + return m[12] * m[9] * m[6] * m[3] - m[8] * m[13] * m[6] * m[3] - + m[12] * m[5] * m[10] * m[3] + m[4] * m[13] * m[10] * m[3] + + m[8] * m[5] * m[14] * m[3] - m[4] * m[9] * m[14] * m[3] - + m[12] * m[9] * m[2] * m[7] + m[8] * m[13] * m[2] * m[7] + + m[12] * m[1] * m[10] * m[7] - m[0] * m[13] * m[10] * m[7] - + m[8] * m[1] * m[14] * m[7] + m[0] * m[9] * m[14] * m[7] + + m[12] * m[5] * m[2] * m[11] - m[4] * m[13] * m[2] * m[11] - + m[12] * m[1] * m[6] * m[11] + m[0] * m[13] * m[6] * m[11] + + m[4] * m[1] * m[14] * m[11] - m[0] * m[5] * m[14] * m[11] - + m[8] * m[5] * m[2] * m[15] + m[4] * m[9] * m[2] * m[15] + + m[8] * m[1] * m[6] * m[15] - m[0] * m[9] * m[6] * m[15] - + m[4] * m[1] * m[10] * m[15] + m[0] * m[5] * m[10] * m[15]; +} + +template::value || is_greater::value, bool>::type=0> +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { + static_assert(M==N, "2D TENSOR MUST BE SQUARE"); + assert(false && "2D TENSOR MUST BE SQUARE"); +} + + +#ifdef FASTOR_AVX_IMPL +template::value && + std::is_same::value && M==2 && N==2, bool>::type=0> +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { + // 10 OPS + __m128 a1 = _mm_load_ps(a); + __m128 a2 = _mm_shuffle_ps(a1,a1,_MM_SHUFFLE(0,1,2,3)); + __m128 a3 = _mm_mul_ps(a1,a2); + return _mm_cvtss_f32(_mm_sub_ss(a3,_mm_shuffle_ps(a3,a3,_MM_SHUFFLE(0,0,0,1)))); +} + +template::value && + std::is_same::value && M==3 && N==3, bool>::type=0> +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { + // ?? OPS + __m128 r0 = {a[2],a[1],a[0],0.}; + __m128 r1 = {a[3],a[5],a[4],0.}; + __m128 r2 = {a[7],a[6],a[8],0.}; + + __m128 r3 = {a[6],a[7],a[8],0.}; + __m128 r4 = {a[4],a[5],a[3],0.}; + __m128 r5 = {a[2],a[0],a[1],0.}; + + __m128 out0 = _mm_mul_ps(r2,_mm_mul_ps(r0,r1)); + __m128 out1 = _mm_mul_ps(r3,_mm_mul_ps(r4,r5)); + + return _mm_cvtss_f32(_mm_sub_ss(_add_ps(out0),_add_ps(out1))); +} + +template::value && + !std::is_same::value && M==2 && N==2, bool>::type=0> +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { + // 10 OPS + __m128d a1 = _mm_load_pd(a); + __m128d a2 = _mm_load_pd(a+2); + __m128d a3 = _mm_mul_pd(a1,_mm_shuffle_pd(a2,a2,1)); + return _mm_cvtsd_f64(_mm_sub_pd(a3,_mm_shuffle_pd(a3,a3,0x1))); +} + +template::value && + !std::is_same::value && M==3 && N==3, bool>::type=0> +FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { + // ?? OPS + __m256d r0 = {a[2],a[1],a[0],0.}; + __m256d r1 = {a[3],a[5],a[4],0.}; + __m256d r2 = {a[7],a[6],a[8],0.}; + + __m256d r3 = {a[6],a[7],a[8],0.}; + __m256d r4 = {a[4],a[5],a[3],0.}; + __m256d r5 = {a[2],a[0],a[1],0.}; + + __m256d out0 = _mm256_mul_pd(r2,_mm256_mul_pd(r0,r1)); + __m256d out1 = _mm256_mul_pd(r3,_mm256_mul_pd(r4,r5)); + + return _mm_cvtsd_f64(_mm_sub_sd(_add_pd(out0),_add_pd(out1))); +} +#endif + +} + +#endif // DETERMINANT_H + diff --git a/noarch/include/Fastor/backend/doublecontract.h b/noarch/include/Fastor/backend/doublecontract.h new file mode 100644 index 00000000..a2cf1599 --- /dev/null +++ b/noarch/include/Fastor/backend/doublecontract.h @@ -0,0 +1,128 @@ +#ifndef DOUBLECONTRACT_H +#define DOUBLECONTRACT_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" + +namespace Fastor { + + +template,M*N>::Size, M*N >, bool> = false> +FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { + + constexpr size_t Size = M*N; + using V = choose_best_simd_t,Size>; + + V omm0; + size_t i = 0; + for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) { + const V amm0(&a[i],false); + const V bmm0(&b[i],false); + + omm0 = fmadd(amm0,bmm0,omm0); + } + T scalar = static_cast(0); + for (; i < Size; ++i) { + scalar += a[i]*b[i]; + } + return omm0.sum() + scalar; +} + +template,M*N>::Size, M*N >, bool> = false> +FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { + + constexpr size_t Size = M*N; + using V = choose_best_simd_t,Size>; + + V omm0, omm1, omm2, omm3; + size_t i = 0; + for (; i< ROUND_DOWN(Size,4*V::Size); i+=4*V::Size) { + const V amm0(&a[i],false); + const V amm1(&a[i+V::Size],false); + const V amm2(&a[i+2*V::Size],false); + const V amm3(&a[i+3*V::Size],false); + + const V bmm0(&b[i],false); + const V bmm1(&b[i+V::Size],false); + const V bmm2(&b[i+2*V::Size],false); + const V bmm3(&b[i+3*V::Size],false); + + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm1,bmm1,omm1); + omm2 = fmadd(amm2,bmm2,omm2); + omm3 = fmadd(amm3,bmm3,omm3); + } + for (; i< ROUND_DOWN(Size,2*V::Size); i+=2*V::Size) { + const V amm0(&a[i],false); + const V amm1(&a[i+V::Size],false); + + const V bmm0(&b[i],false); + const V bmm1(&b[i+V::Size],false); + + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm1,bmm1,omm1); + } + for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) { + const V amm0(&a[i],false); + const V bmm0(&b[i],false); + + omm0 = fmadd(amm0,bmm0,omm0); + } + T scalar = static_cast(0); + for (; i < Size; ++i) { + scalar += a[i]*b[i]; + } + return (omm0 + omm1 + omm2 + omm3).sum() + scalar; +} + + +#ifdef FASTOR_AVX_IMPL + +template<> +FASTOR_INLINE float _doublecontract(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) { + return _mm_sum_ps(_mm_mul_ps(_mm_loadu_ps(a),_mm_loadu_ps(b))); +} + +template<> +FASTOR_INLINE float _doublecontract(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) { + float r1 = _mm256_sum_ps(_mm256_mul_ps(_mm256_loadu_ps(a),_mm256_loadu_ps(b))); + float r2 = _mm_sum_ps(_mm_mul_ss(_mm_load_ss(a+8),_mm_load_ss(b+8))); + return r1+r2; +} + +template<> +FASTOR_INLINE double _doublecontract(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) { + return _mm256_sum_pd(_mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b))); +} + +template<> +FASTOR_INLINE double _doublecontract(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) { + __m256d r1 = _mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b)); + __m256d r2 = _mm256_mul_pd(_mm256_loadu_pd(a+4),_mm256_loadu_pd(b+4)); + __m128d r3 = _mm_mul_sd(_mm_load_sd(a+8),_mm_load_sd(b+8)); + __m128d r4 = _add_pd(_mm256_add_pd(r1,r2)); + __m128d summ = _mm_add_pd(_add_pd(r3),r4); + return _mm_cvtsd_f64(summ); +} + + +#endif + +// doublecontract and transpose +template +FASTOR_INLINE T _doublecontract_transpose(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { + T dc = static_cast(0); + for (FASTOR_INDEX i=0; i +FASTOR_INLINE +void _dyadic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + using V = typename internal::choose_best_simd_type,SizeB>::type; + // constexpr size_t unrollOuterloop = 4UL; + // constexpr size_t M0 = SizeA / unrollOuterloop * unrollOuterloop; + // Unrolling the inner loop beyond 4 does not give any benefit neither on AVX + // nor on AVX512 + + size_t i = 0; + for (; i +FASTOR_INLINE +void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + + __m128 vec_a = _mm_load_ps(a); + __m128 vec_b = _mm_load_ps(b); + + __m128 a00 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0)); + __m128 a01 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1)); + __m128 a10 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2)); + __m128 a11 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(3,3,3,3)); + + __m256 a0001 = _mm256_castps128_ps256(a00); + a0001 = _mm256_insertf128_ps(a0001,a01,0x1); + __m256 a1011 = _mm256_castps128_ps256(a10); + a1011 = _mm256_insertf128_ps(a1011,a11,0x1); + + __m256 vec_b2 = _mm256_castps128_ps256(vec_b); + vec_b2 = _mm256_insertf128_ps(vec_b2,vec_b,0x1); + + _mm256_store_ps(out,_mm256_mul_ps(a0001,vec_b2)); + _mm256_store_ps(out+8,_mm256_mul_ps(a1011,vec_b2)); +} + + +// Outer product (2x2) x (2x2) +template<> +FASTOR_INLINE +void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + + __m256d vec_b = _mm256_loadu_pd(b); + + __m256d a00 = _mm256_set1_pd(a[0]); + __m256d a01 = _mm256_set1_pd(a[1]); + __m256d a10 = _mm256_set1_pd(a[2]); + __m256d a11 = _mm256_set1_pd(a[3]); + + _mm256_storeu_pd(out,_mm256_mul_pd(a00,vec_b)); + _mm256_storeu_pd(out+4,_mm256_mul_pd(a01,vec_b)); + _mm256_storeu_pd(out+8,_mm256_mul_pd(a10,vec_b)); + _mm256_storeu_pd(out+12,_mm256_mul_pd(a11,vec_b)); +} + + + +// Outer product (1x2) x (1x2) [for vectors] +template<> +FASTOR_INLINE +void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + // 7 OPS + __m128 vec_a = _mm_loadu_ps(a); + __m128 vec_b = _mm_loadu_ps(b); + + vec_a = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,0,0)); + vec_b = _mm_shuffle_ps(vec_b,vec_b,_MM_SHUFFLE(1,0,1,0)); + + _mm_storeu_ps(out,_mm_mul_ps(vec_a,vec_b)); +} + + +// Outer product (1x2) x (1x2) [for vectors] +template<> +FASTOR_INLINE +void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + // IVY 9 OPS / HW 13 OPS + __m128d vec_a = _mm_loadu_pd(a); + __m128d vec_b = _mm_loadu_pd(b); + + __m128d a0 = _mm_shuffle_pd(vec_a,vec_a,0x0); + __m128d a1 = _mm_shuffle_pd(vec_a,vec_a,0x3); + __m256d as = _mm256_castpd128_pd256(a0); + as = _mm256_insertf128_pd(as,a1,0x1); + + __m256d bs = _mm256_castpd128_pd256(vec_b); + bs = _mm256_insertf128_pd(bs,vec_b,0x1); + + _mm256_storeu_pd(out,_mm256_mul_pd(as,bs)); +} + + +// Outer product (1x3) x (1x3) [for vectors] +template<> +FASTOR_INLINE +void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + // 18 OPS + __m128 vec_a = _mm_loadu_ps(a); + __m128 vec_b = _mm_loadu_ps(b); + + __m128 a0 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0)); + __m128 a1 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1)); + __m128 a2 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2)); + + _mm_storeu_ps(out,_mm_mul_ps(a0,vec_b)); + _mm_storeu_ps(out+3,_mm_mul_ps(a1,vec_b)); + _mm_storeu_ps(out+6,_mm_mul_ps(a2,vec_b)); +} + + +// Outer product (1x3) x (1x3) [for vectors] +template<> +FASTOR_INLINE +void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + // 15 OPS + set OPS + __m256d vec_b = _mm256_loadu_pd(b); + __m256d a0 = _mm256_set1_pd(a[0]); + __m256d a1 = _mm256_set1_pd(a[1]); + __m256d a2 = _mm256_set1_pd(a[2]); + + _mm256_storeu_pd(out,_mm256_mul_pd(a0,vec_b)); + _mm256_storeu_pd(out+3,_mm256_mul_pd(a1,vec_b)); + _mm256_storeu_pd(out+6,_mm256_mul_pd(a2,vec_b)); +} + +#endif +//--------------------------------------------------------------------------------------------------- + + +//--------------------------------------------------------------------------------------------------- +// Outer product of scalars +template<> +FASTOR_INLINE +void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + out[0] = a[0]*b[0]; +} +template<> +FASTOR_INLINE +void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + out[0] = a[0]*b[0]; +} +//--------------------------------------------------------------------------------------------------- + +} + +#endif // DYADIC_H diff --git a/noarch/include/Fastor/backend/inner.h b/noarch/include/Fastor/backend/inner.h new file mode 100644 index 00000000..cb0619b3 --- /dev/null +++ b/noarch/include/Fastor/backend/inner.h @@ -0,0 +1,27 @@ +#ifndef INNER_H_ +#define INNER_H_ + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/doublecontract.h" + +namespace Fastor { + +/* The dependency on doublecontract here is on purpose + as it creates a necessary layer of indirection to avoid + the case where M == 0 +*/ +template, bool> = false> +FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { + return _doublecontract(a,b); +} + +template = false> +FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { + return (*a)*(*b); +} + +} // end of namespace Fastor + +#endif // INNER_H_ diff --git a/noarch/include/Fastor/backend/inverse.h b/noarch/include/Fastor/backend/inverse.h new file mode 100644 index 00000000..48274664 --- /dev/null +++ b/noarch/include/Fastor/backend/inverse.h @@ -0,0 +1,450 @@ +#ifndef INVERSE_H +#define INVERSE_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/extintrin.h" + +namespace Fastor { + +template, bool> = false> +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); + +template, bool> = false> +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { + *dst = T(1) / (*src); +} + +#ifdef FASTOR_SSE2_IMPL +template && !is_same_v_ && !is_same_v_, bool> = false> +#else +template, bool> = false> +#endif +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T det; + + T src0 = src[0]; + T src1 = src[1]; + T src2 = src[2]; + T src3 = src[3]; + + /* Compute adjoint: */ + dst[0] = + src3; + dst[1] = - src1; + dst[2] = - src2; + dst[3] = + src0; + + /* Compute determinant: */ + det = src0 * dst[0] + src1 * dst[2]; + + /* Multiply adjoint with reciprocal of determinant: */ + det = T(1.0) / det; + dst[0] *= det; + dst[1] *= det; + dst[2] *= det; + dst[3] *= det; +} + +#ifdef FASTOR_SSE2_IMPL +template && is_same_v_, bool> = false> +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + // This is much superior to the scalar code as + // gcc/clang can't auto-vectorise the scalar code + + // 6 shuffles + 1 add + 1 mul + 1 div + // Sky 6 + 4 + 4 + 11 = 25 + + __m128 mat = _mm_loadu_ps(src); + // xor to swap off-diagonals sings + __m128 nmat = _mm_neg_ps(mat); + // two shuffles to get adjoint + __m128 adj = _mm_shuffle_ps(mat, nmat, 0x009C ); + adj = _mm_shuffle_ps(adj, adj , 0x39 ); + + // compute determinat + __m128 tmp0 = _mm_shuffle_ps(mat , mat , 0x00D8); + tmp0 = _mm_mul_ps (adj , tmp0 ); + __m128 tmp1 = _mm_shuffle_ps(tmp0, tmp0, 0x1 ); + __m128 det = _mm_div_ss (ONEPS, _mm_add_ss(tmp0,tmp1)); + // broadcast det to all elements of __m128 + det = _mm_shuffle_ps(det, det, 0x0 ); + // divide adjoint by determinant + __m128 inv = _mm_mul_ps (adj, det); + + _mm_storeu_ps(dst, inv); +} + +template && is_same_v_, bool> = false> +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + // This is much superior to the scalar code as + // gcc/clang can't auto-vectorise the scalar code + + // 8 shuffles + 1 add + 3 mul + 1 div + // Sky 8 + 4 + 12 + 14 = 38 + + __m128d row0 = _mm_loadu_pd(src); + __m128d row1 = _mm_loadu_pd(src+2); + + __m128d tmp = row0; + row0 = _mm_shuffle_pd(row0,_mm_neg_pd(row0),0x2); + row1 = _mm_shuffle_pd(_mm_neg_pd(row1),row1,0x2); + // these two registers hold the adjoint + __m128d irow0 = _mm_shuffle_pd(row1,row0,0x3); + __m128d irow1 = _mm_shuffle_pd(row1,row0,0x0); + // dot product to compute determinant + __m128d det = _mm_mul_pd(tmp,_mm_reverse_pd(row1)); + det = _mm_add_pd(det,_mm_reverse_pd(det)); + // one by determinant + __m128d invdet = _mm_div_pd(_mm_set1_pd(1.0),det); + // scale + irow0 = _mm_mul_pd(irow0,invdet); + irow1 = _mm_mul_pd(irow1,invdet); + + _mm_storeu_pd(dst ,irow0); + _mm_storeu_pd(dst+2,irow1); +} +#endif + +template, bool> = false> +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T det; + + T src0 = src[0]; + T src1 = src[1]; + T src2 = src[2]; + T src3 = src[3]; + T src4 = src[4]; + T src5 = src[5]; + T src6 = src[6]; + T src7 = src[7]; + T src8 = src[8]; + + /* Compute adjoint: */ + dst[0] = + src4 * src8 - src5 * src7; + dst[1] = - src1 * src8 + src2 * src7; + dst[2] = + src1 * src5 - src2 * src4; + dst[3] = - src3 * src8 + src5 * src6; + dst[4] = + src0 * src8 - src2 * src6; + dst[5] = - src0 * src5 + src2 * src3; + dst[6] = + src3 * src7 - src4 * src6; + dst[7] = - src0 * src7 + src1 * src6; + dst[8] = + src0 * src4 - src1 * src3; + + /* Compute determinant: */ + det = src0 * dst[0] + src1 * dst[3] + src2 * dst[6]; + + /* Multiply adjoint with reciprocal of determinant: */ + det = T(1.0) / det; + + dst[0] *= det; + dst[1] *= det; + dst[2] *= det; + dst[3] *= det; + dst[4] *= det; + dst[5] *= det; + dst[6] *= det; + dst[7] *= det; + dst[8] *= det; +} + + +#ifdef FASTOR_SSE2_IMPL +template && !is_same_v_ && !is_same_v_, bool> = false> +#else +template, bool> = false> +#endif +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2]; + T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1]; + T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1]; + + dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3; + dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3; + + T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0]; + T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0]; + + dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5; + dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5; + + t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0]; + + dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1; + dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1; + dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1; + dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1; + + t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2]; + t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1]; + t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1]; + + dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3; + dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3; + + t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0]; + t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0]; + + dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5; + dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5; + + t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0]; + + dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1; + dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1; + dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1; + dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1; + + const T __det = src[0]*dst[0] + src[1]*dst[4] + src[2]*dst[8] + src[3]*dst[12]; + const T __invdet = T(1)/__det; + for (int i=0; i<16; ++i) + dst[i] *= __invdet; +} + + +#ifdef FASTOR_SSE2_IMPL +template && is_same_v_, bool> = false> +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + // From Intel's SSE matrix library + // The inverse is calculated using "Divide and Conquer" technique. The + // original matrix is divide into four 2x2 sub-matrices. Since each + // register of the matrix holds two elements, the smaller matrices are + // consisted of two registers. Hence we get a better locality of the + // calculations. + + const __m128 p4f_sign_PNNP = _mm_castsi128_ps(_mm_set_epi32(0x00000000, 0x80000000, 0x80000000, 0x00000000)); + + // Load the full matrix into registers + __m128 _L1 = _mm_loadu_ps(src + 0); + __m128 _L2 = _mm_loadu_ps(src + 4); + __m128 _L3 = _mm_loadu_ps(src + 8); + __m128 _L4 = _mm_loadu_ps(src + 12); + + __m128 A, B, C, D; // the four sub-matrices + + A = _mm_movelh_ps(_L1, _L2); + B = _mm_movehl_ps(_L2, _L1); + C = _mm_movelh_ps(_L3, _L4); + D = _mm_movehl_ps(_L4, _L3); + + // partial inverse of the sub-matrices + __m128 iA, iB, iC, iD, DC, AB; + __m128 dA, dB, dC, dD; // determinant of the sub-matrices + __m128 det, d, d1, d2; + __m128 rd; // reciprocal of the determinant + + // AB = A# * B + AB = _mm_mul_ps(_mm_shuffle_ps(A,A,0x0F), B); + AB = _mm_sub_ps(AB,_mm_mul_ps(_mm_shuffle_ps(A,A,0xA5), _mm_shuffle_ps(B,B,0x4E))); + // DC = D# * C + DC = _mm_mul_ps(_mm_shuffle_ps(D,D,0x0F), C); + DC = _mm_sub_ps(DC,_mm_mul_ps(_mm_shuffle_ps(D,D,0xA5), _mm_shuffle_ps(C,C,0x4E))); + + // dA = |A| + dA = _mm_mul_ps(_mm_shuffle_ps(A, A, 0x5F),A); + dA = _mm_sub_ss(dA, _mm_movehl_ps(dA,dA)); + // dB = |B| + dB = _mm_mul_ps(_mm_shuffle_ps(B, B, 0x5F),B); + dB = _mm_sub_ss(dB, _mm_movehl_ps(dB,dB)); + + // dC = |C| + dC = _mm_mul_ps(_mm_shuffle_ps(C, C, 0x5F),C); + dC = _mm_sub_ss(dC, _mm_movehl_ps(dC,dC)); + // dD = |D| + dD = _mm_mul_ps(_mm_shuffle_ps(D, D, 0x5F),D); + dD = _mm_sub_ss(dD, _mm_movehl_ps(dD,dD)); + + // d = trace(AB*DC) = trace(A#*B*D#*C) + d = _mm_mul_ps(_mm_shuffle_ps(DC,DC,0xD8),AB); + + // iD = C*A#*B + iD = _mm_mul_ps(_mm_shuffle_ps(C,C,0xA0), _mm_movelh_ps(AB,AB)); + iD = _mm_add_ps(iD,_mm_mul_ps(_mm_shuffle_ps(C,C,0xF5), _mm_movehl_ps(AB,AB))); + // iA = B*D#*C + iA = _mm_mul_ps(_mm_shuffle_ps(B,B,0xA0), _mm_movelh_ps(DC,DC)); + iA = _mm_add_ps(iA,_mm_mul_ps(_mm_shuffle_ps(B,B,0xF5), _mm_movehl_ps(DC,DC))); + + // d = trace(AB*DC) = trace(A#*B*D#*C) [continue] + d = _mm_add_ps(d, _mm_movehl_ps(d, d)); + d = _mm_add_ss(d, _mm_shuffle_ps(d, d, 1)); + d1 = _mm_mul_ss(dA,dD); + d2 = _mm_mul_ss(dB,dC); + + // iD = D*|A| - C*A#*B + iD = _mm_sub_ps(_mm_mul_ps(D,_mm_shuffle_ps(dA,dA,0)), iD); + + // iA = A*|D| - B*D#*C; + iA = _mm_sub_ps(_mm_mul_ps(A,_mm_shuffle_ps(dD,dD,0)), iA); + + // det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C) + det = _mm_sub_ss(_mm_add_ss(d1,d2),d); + rd = _mm_div_ss(_mm_set_ss(1.0f), det); + + // iB = D * (A#B)# = D*B#*A + iB = _mm_mul_ps(D, _mm_shuffle_ps(AB,AB,0x33)); + iB = _mm_sub_ps(iB, _mm_mul_ps(_mm_shuffle_ps(D,D,0xB1), _mm_shuffle_ps(AB,AB,0x66))); + // iC = A * (D#C)# = A*C#*D + iC = _mm_mul_ps(A, _mm_shuffle_ps(DC,DC,0x33)); + iC = _mm_sub_ps(iC, _mm_mul_ps(_mm_shuffle_ps(A,A,0xB1), _mm_shuffle_ps(DC,DC,0x66))); + + rd = _mm_shuffle_ps(rd,rd,0); + rd = _mm_xor_ps(rd, p4f_sign_PNNP); + + // iB = C*|B| - D*B#*A + iB = _mm_sub_ps(_mm_mul_ps(C,_mm_shuffle_ps(dB,dB,0)), iB); + + // iC = B*|C| - A*C#*D; + iC = _mm_sub_ps(_mm_mul_ps(B,_mm_shuffle_ps(dC,dC,0)), iC); + + // iX = iX / det + iA = _mm_mul_ps(rd,iA); + iB = _mm_mul_ps(rd,iB); + iC = _mm_mul_ps(rd,iC); + iD = _mm_mul_ps(rd,iD); + + _mm_storeu_ps(dst+0, _mm_shuffle_ps(iA,iB,0x77)); + _mm_storeu_ps(dst+4, _mm_shuffle_ps(iA,iB,0x22)); + _mm_storeu_ps(dst+8, _mm_shuffle_ps(iC,iD,0x77)); + _mm_storeu_ps(dst+12, _mm_shuffle_ps(iC,iD,0x22)); +} + + +template && is_same_v_, bool> = false> +FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + // From Intel's SSE matrix library + // The inverse is calculated using "Divide and Conquer" technique. The + // original matrix is divide into four 2x2 sub-matrices. Since each + // register of the matrix holds two elements, the smaller matrices are + // consisted of two registers. Hence we get a better locality of the + // calculations. + + const __m128d _Sign_NP = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0)); + const __m128d _Sign_PN = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0)); + + // the four sub-matrices + __m128d A1, A2, B1, B2, C1, C2, D1, D2; + + A1 = _mm_loadu_pd(src + 0); B1 = _mm_loadu_pd(src + 2); + A2 = _mm_loadu_pd(src + 4); B2 = _mm_loadu_pd(src + 6); + C1 = _mm_loadu_pd(src + 8); D1 = _mm_loadu_pd(src +10); + C2 = _mm_loadu_pd(src +12); D2 = _mm_loadu_pd(src +14); + + // partial inverse of the sub-matrices + __m128d iA1, iA2, iB1, iB2, iC1, iC2, iD1, iD2, DC1, DC2, AB1, AB2; + __m128d dA, dB, dC, dD; // determinant of the sub-matrices + __m128d det, d1, d2, rd; + + // dA = |A| + dA = _mm_shuffle_pd(A2, A2, 1); + dA = _mm_mul_pd(A1, dA); + dA = _mm_sub_sd(dA, _mm_shuffle_pd(dA,dA,3)); + // dB = |B| + dB = _mm_shuffle_pd(B2, B2, 1); + dB = _mm_mul_pd(B1, dB); + dB = _mm_sub_sd(dB, _mm_shuffle_pd(dB,dB,3)); + + // AB = A# * B + AB1 = _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,3)); + AB2 = _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,0)); + AB1 = _mm_sub_pd(AB1, _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,3))); + AB2 = _mm_sub_pd(AB2, _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,0))); + + // dC = |C| + dC = _mm_shuffle_pd(C2, C2, 1); + dC = _mm_mul_pd(C1, dC); + dC = _mm_sub_sd(dC, _mm_shuffle_pd(dC,dC,3)); + // dD = |D| + dD = _mm_shuffle_pd(D2, D2, 1); + dD = _mm_mul_pd(D1, dD); + dD = _mm_sub_sd(dD, _mm_shuffle_pd(dD,dD,3)); + + // DC = D# * C + DC1 = _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,3)); + DC2 = _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,0)); + DC1 = _mm_sub_pd(DC1, _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,3))); + DC2 = _mm_sub_pd(DC2, _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,0))); + + // rd = trace(AB*DC) = trace(A#*B*D#*C) + d1 = _mm_mul_pd(AB1, _mm_shuffle_pd(DC1, DC2, 0)); + d2 = _mm_mul_pd(AB2, _mm_shuffle_pd(DC1, DC2, 3)); + rd = _mm_add_pd(d1, d2); + rd = _mm_add_sd(rd, _mm_shuffle_pd(rd, rd,3)); + + // iD = C*A#*B + iD1 = _mm_mul_pd(AB1, _mm_shuffle_pd(C1,C1,0)); + iD2 = _mm_mul_pd(AB1, _mm_shuffle_pd(C2,C2,0)); + iD1 = _mm_add_pd(iD1, _mm_mul_pd(AB2, _mm_shuffle_pd(C1,C1,3))); + iD2 = _mm_add_pd(iD2, _mm_mul_pd(AB2, _mm_shuffle_pd(C2,C2,3))); + + // iA = B*D#*C + iA1 = _mm_mul_pd(DC1, _mm_shuffle_pd(B1,B1,0)); + iA2 = _mm_mul_pd(DC1, _mm_shuffle_pd(B2,B2,0)); + iA1 = _mm_add_pd(iA1, _mm_mul_pd(DC2, _mm_shuffle_pd(B1,B1,3))); + iA2 = _mm_add_pd(iA2, _mm_mul_pd(DC2, _mm_shuffle_pd(B2,B2,3))); + + // iD = D*|A| - C*A#*B + dA = _mm_shuffle_pd(dA,dA,0); + iD1 = _mm_sub_pd(_mm_mul_pd(D1, dA), iD1); + iD2 = _mm_sub_pd(_mm_mul_pd(D2, dA), iD2); + + // iA = A*|D| - B*D#*C; + dD = _mm_shuffle_pd(dD,dD,0); + iA1 = _mm_sub_pd(_mm_mul_pd(A1, dD), iA1); + iA2 = _mm_sub_pd(_mm_mul_pd(A2, dD), iA2); + + d1 = _mm_mul_sd(dA, dD); + d2 = _mm_mul_sd(dB, dC); + + // iB = D * (A#B)# = D*B#*A + iB1 = _mm_mul_pd(D1, _mm_shuffle_pd(AB2,AB1,1)); + iB2 = _mm_mul_pd(D2, _mm_shuffle_pd(AB2,AB1,1)); + iB1 = _mm_sub_pd(iB1, _mm_mul_pd(_mm_shuffle_pd(D1,D1,1), _mm_shuffle_pd(AB2,AB1,2))); + iB2 = _mm_sub_pd(iB2, _mm_mul_pd(_mm_shuffle_pd(D2,D2,1), _mm_shuffle_pd(AB2,AB1,2))); + + // det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C) + det = _mm_add_sd(d1, d2); + det = _mm_sub_sd(det, rd); + + // iC = A * (D#C)# = A*C#*D + iC1 = _mm_mul_pd(A1, _mm_shuffle_pd(DC2,DC1,1)); + iC2 = _mm_mul_pd(A2, _mm_shuffle_pd(DC2,DC1,1)); + iC1 = _mm_sub_pd(iC1, _mm_mul_pd(_mm_shuffle_pd(A1,A1,1), _mm_shuffle_pd(DC2,DC1,2))); + iC2 = _mm_sub_pd(iC2, _mm_mul_pd(_mm_shuffle_pd(A2,A2,1), _mm_shuffle_pd(DC2,DC1,2))); + + rd = _mm_div_sd(_mm_set_sd(1.0), det); + rd = _mm_shuffle_pd(rd,rd,0); + + // iB = C*|B| - D*B#*A + dB = _mm_shuffle_pd(dB,dB,0); + iB1 = _mm_sub_pd(_mm_mul_pd(C1, dB), iB1); + iB2 = _mm_sub_pd(_mm_mul_pd(C2, dB), iB2); + + d1 = _mm_xor_pd(rd, _Sign_PN); + d2 = _mm_xor_pd(rd, _Sign_NP); + + // iC = B*|C| - A*C#*D; + dC = _mm_shuffle_pd(dC,dC,0); + iC1 = _mm_sub_pd(_mm_mul_pd(B1, dC), iC1); + iC2 = _mm_sub_pd(_mm_mul_pd(B2, dC), iC2); + + _mm_storeu_pd(dst+0, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 3), d1)); + _mm_storeu_pd(dst+4, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 0), d2)); + _mm_storeu_pd(dst+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 3), d1)); + _mm_storeu_pd(dst+4+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 0), d2)); + _mm_storeu_pd(dst+2*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 3), d1)); + _mm_storeu_pd(dst+3*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 0), d2)); + _mm_storeu_pd(dst+2*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 3), d1)); + _mm_storeu_pd(dst+3*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 0), d2)); +} +#endif + + +} // end of namespace Fastor + +#endif // INVERSE_H diff --git a/noarch/include/Fastor/backend/lufact.h b/noarch/include/Fastor/backend/lufact.h new file mode 100644 index 00000000..1d8422c4 --- /dev/null +++ b/noarch/include/Fastor/backend/lufact.h @@ -0,0 +1,753 @@ +#ifndef LUFACT_H +#define LUFACT_H + +#include "Fastor/meta/meta.h" +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" + +namespace Fastor { + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u); + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u) { + *l = 1; + *u = *a; +} + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { + + // [a11 a12] [1 0] [u11 u12] + // [a21 a22] [l21 1] [0 u22] + + const T L21 = A[2]/A[0]; + + L[0] = 1; + L[1] = 0; + L[2] = L21; + L[3] = 1; + + + U[0] = A[0]; + U[1] = A[1]; + U[2] = 0; + U[3] = A[3] - L21 * A[1]; +} + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { + + const T A00 = A[0]; + const T L21 = A[3]/A00; + const T L31 = A[6]/A00; + + const T U22 = A[4] - L21 * A[1]; + const T U23 = A[5] - L21 * A[2]; + + const T L32 = (A[7] - L31 * A[1]) / U22; + + const T U33 = A[8] - L31 * A[2] - L32 * U23; + + L[0] = 1; + L[1] = 0; + L[2] = 0; + L[3] = L21; + L[4] = 1; + L[5] = 0; + L[6] = L31; + L[7] = L32; + L[8] = 1; + + U[0] = A00; + U[1] = A[1]; + U[2] = A[2]; + U[3] = 0; + U[4] = U22; + U[5] = U23; + U[6] = 0; + U[7] = 0; + U[8] = U33; +} + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { + + const T A00 = A[0]; + const T L21 = A[4] /A00; + const T L31 = A[8] /A00; + const T L41 = A[12]/A00; + + const T U22 = A[5] - L21 * A[1]; + const T U23 = A[6] - L21 * A[2]; + const T U24 = A[7] - L21 * A[3]; + + const T L32 = (A[9 ] - L31 * A[1]) / U22; + const T L42 = (A[13] - L41 * A[1]) / U22; + + const T U33 = A[10] - L31 * A[2] - L32 * U23; + const T U34 = A[11] - L31 * A[3] - L32 * U24; + + const T L43 = (A[14] - L41 * A[2] - L42 * U23) / U33; + + const T U44 = A[15] - L41 * A[3] - L42 * U24 - L43 * U34; + + L[0] = 1; + L[1] = 0; + L[2] = 0; + L[3] = 0; + L[4] = L21; + L[5] = 1; + L[6] = 0; + L[7] = 0; + L[8] = L31; + L[9] = L32; + L[10] = 1; + L[11] = 0; + L[12] = L41; + L[13] = L42; + L[14] = L43; + L[15] = 1; + + U[0] = A00; + U[1] = A[1]; + U[2] = A[2]; + U[3] = A[3]; + U[4] = 0; + U[5] = U22; + U[6] = U23; + U[7] = U24; + U[8] = 0; + U[9] = 0; + U[10] = U33; + U[11] = U34; + U[12] = 0; + U[13] = 0; + U[14] = 0; + U[15] = U44; +} + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { + + const T A00 = A[0*N]; + const T L21 = A[1*N] /A00; + const T L31 = A[2*N] /A00; + const T L41 = A[3*N] /A00; + const T L51 = A[4*N] /A00; + + const T U22 = A[1*N+1] - L21 * A[1]; + const T U23 = A[1*N+2] - L21 * A[2]; + const T U24 = A[1*N+3] - L21 * A[3]; + const T U25 = A[1*N+4] - L21 * A[4]; + + const T L32 = (A[2*N+1] - L31 * A[1]) / U22; + const T L42 = (A[3*N+1] - L41 * A[1]) / U22; + const T L52 = (A[4*N+1] - L51 * A[1]) / U22; + + const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; + const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; + const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; + + const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; + const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; + + const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; + const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; + + const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; + + const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; + + // L + L[0*N+0] = 1; + L[0*N+1] = 0; + L[0*N+2] = 0; + L[0*N+3] = 0; + L[0*N+4] = 0; + + L[1*N+0] = L21; + L[1*N+1] = 1; + L[1*N+2] = 0; + L[1*N+3] = 0; + L[1*N+4] = 0; + + L[2*N+0] = L31; + L[2*N+1] = L32; + L[2*N+2] = 1; + L[2*N+3] = 0; + L[2*N+4] = 0; + + L[3*N+0] = L41; + L[3*N+1] = L42; + L[3*N+2] = L43; + L[3*N+3] = 1; + L[3*N+4] = 0; + + L[4*N+0] = L51; + L[4*N+1] = L52; + L[4*N+2] = L53; + L[4*N+3] = L54; + L[4*N+4] = 1; + + // U + U[0*N+0] = A00; + U[0*N+1] = A[1]; + U[0*N+2] = A[2]; + U[0*N+3] = A[3]; + U[0*N+4] = A[4]; + + U[1*N+0] = 0; + U[1*N+1] = U22; + U[1*N+2] = U23; + U[1*N+3] = U24; + U[1*N+4] = U25; + + U[2*N+0] = 0; + U[2*N+1] = 0; + U[2*N+2] = U33; + U[2*N+3] = U34; + U[2*N+4] = U35; + + U[3*N+0] = 0; + U[3*N+1] = 0; + U[3*N+2] = 0; + U[3*N+3] = U44; + U[3*N+4] = U45; + + U[4*N+0] = 0; + U[4*N+1] = 0; + U[4*N+2] = 0; + U[4*N+3] = 0; + U[4*N+4] = U55; +} + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { + + const T A00 = A[0*N]; + const T L21 = A[1*N] /A00; + const T L31 = A[2*N] /A00; + const T L41 = A[3*N] /A00; + const T L51 = A[4*N] /A00; + const T L61 = A[5*N] /A00; + + const T U22 = A[1*N+1] - L21 * A[1]; + const T U23 = A[1*N+2] - L21 * A[2]; + const T U24 = A[1*N+3] - L21 * A[3]; + const T U25 = A[1*N+4] - L21 * A[4]; + const T U26 = A[1*N+5] - L21 * A[5]; + + const T L32 = (A[2*N+1] - L31 * A[1]) / U22; + const T L42 = (A[3*N+1] - L41 * A[1]) / U22; + const T L52 = (A[4*N+1] - L51 * A[1]) / U22; + const T L62 = (A[5*N+1] - L61 * A[1]) / U22; + + const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; + const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; + const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; + const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26; + + const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; + const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; + const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33; + + const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; + const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; + const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36; + + const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; + const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44; + + const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; + const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46; + + const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55; + + const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56; + + // L + L[0*N+0] = 1; + L[0*N+1] = 0; + L[0*N+2] = 0; + L[0*N+3] = 0; + L[0*N+4] = 0; + L[0*N+5] = 0; + + L[1*N+0] = L21; + L[1*N+1] = 1; + L[1*N+2] = 0; + L[1*N+3] = 0; + L[1*N+4] = 0; + L[1*N+5] = 0; + + L[2*N+0] = L31; + L[2*N+1] = L32; + L[2*N+2] = 1; + L[2*N+3] = 0; + L[2*N+4] = 0; + L[2*N+5] = 0; + + L[3*N+0] = L41; + L[3*N+1] = L42; + L[3*N+2] = L43; + L[3*N+3] = 1; + L[3*N+4] = 0; + L[3*N+5] = 0; + + L[4*N+0] = L51; + L[4*N+1] = L52; + L[4*N+2] = L53; + L[4*N+3] = L54; + L[4*N+4] = 1; + L[4*N+5] = 0; + + L[5*N+0] = L61; + L[5*N+1] = L62; + L[5*N+2] = L63; + L[5*N+3] = L64; + L[5*N+4] = L65; + L[5*N+5] = 1; + + // U + U[0*N+0] = A00; + U[0*N+1] = A[1]; + U[0*N+2] = A[2]; + U[0*N+3] = A[3]; + U[0*N+4] = A[4]; + U[0*N+5] = A[5]; + + U[1*N+0] = 0; + U[1*N+1] = U22; + U[1*N+2] = U23; + U[1*N+3] = U24; + U[1*N+4] = U25; + U[1*N+5] = U26; + + U[2*N+0] = 0; + U[2*N+1] = 0; + U[2*N+2] = U33; + U[2*N+3] = U34; + U[2*N+4] = U35; + U[2*N+5] = U36; + + U[3*N+0] = 0; + U[3*N+1] = 0; + U[3*N+2] = 0; + U[3*N+3] = U44; + U[3*N+4] = U45; + U[3*N+5] = U46; + + U[4*N+0] = 0; + U[4*N+1] = 0; + U[4*N+2] = 0; + U[4*N+3] = 0; + U[4*N+4] = U55; + U[4*N+5] = U56; + + U[5*N+0] = 0; + U[5*N+1] = 0; + U[5*N+2] = 0; + U[5*N+3] = 0; + U[5*N+4] = 0; + U[5*N+5] = U66; +} + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { + + const T A00 = A[0*N]; + const T L21 = A[1*N] /A00; + const T L31 = A[2*N] /A00; + const T L41 = A[3*N] /A00; + const T L51 = A[4*N] /A00; + const T L61 = A[5*N] /A00; + const T L71 = A[6*N] /A00; + + const T U22 = A[1*N+1] - L21 * A[1]; + const T U23 = A[1*N+2] - L21 * A[2]; + const T U24 = A[1*N+3] - L21 * A[3]; + const T U25 = A[1*N+4] - L21 * A[4]; + const T U26 = A[1*N+5] - L21 * A[5]; + const T U27 = A[1*N+6] - L21 * A[6]; + + const T L32 = (A[2*N+1] - L31 * A[1]) / U22; + const T L42 = (A[3*N+1] - L41 * A[1]) / U22; + const T L52 = (A[4*N+1] - L51 * A[1]) / U22; + const T L62 = (A[5*N+1] - L61 * A[1]) / U22; + const T L72 = (A[6*N+1] - L71 * A[1]) / U22; + + const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; + const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; + const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; + const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26; + const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27; + + const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; + const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; + const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33; + const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33; + + const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; + const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; + const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36; + const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37; + + const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; + const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44; + const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44; + + const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; + const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46; + const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47; + + const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55; + const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55; + + const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56; + const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57; + + const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66; + + const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67; + + // L + L[0*N+0] = 1; + L[0*N+1] = 0; + L[0*N+2] = 0; + L[0*N+3] = 0; + L[0*N+4] = 0; + L[0*N+5] = 0; + L[0*N+6] = 0; + + L[1*N+0] = L21; + L[1*N+1] = 1; + L[1*N+2] = 0; + L[1*N+3] = 0; + L[1*N+4] = 0; + L[1*N+5] = 0; + L[1*N+6] = 0; + + L[2*N+0] = L31; + L[2*N+1] = L32; + L[2*N+2] = 1; + L[2*N+3] = 0; + L[2*N+4] = 0; + L[2*N+5] = 0; + L[2*N+6] = 0; + + L[3*N+0] = L41; + L[3*N+1] = L42; + L[3*N+2] = L43; + L[3*N+3] = 1; + L[3*N+4] = 0; + L[3*N+5] = 0; + L[3*N+6] = 0; + + L[4*N+0] = L51; + L[4*N+1] = L52; + L[4*N+2] = L53; + L[4*N+3] = L54; + L[4*N+4] = 1; + L[4*N+5] = 0; + L[4*N+6] = 0; + + L[5*N+0] = L61; + L[5*N+1] = L62; + L[5*N+2] = L63; + L[5*N+3] = L64; + L[5*N+4] = L65; + L[5*N+5] = 1; + L[5*N+6] = 0; + + L[6*N+0] = L71; + L[6*N+1] = L72; + L[6*N+2] = L73; + L[6*N+3] = L74; + L[6*N+4] = L75; + L[6*N+5] = L76; + L[6*N+6] = 1; + + // U + U[0*N+0] = A00; + U[0*N+1] = A[1]; + U[0*N+2] = A[2]; + U[0*N+3] = A[3]; + U[0*N+4] = A[4]; + U[0*N+5] = A[5]; + U[0*N+6] = A[6]; + + U[1*N+0] = 0; + U[1*N+1] = U22; + U[1*N+2] = U23; + U[1*N+3] = U24; + U[1*N+4] = U25; + U[1*N+5] = U26; + U[1*N+6] = U27; + + U[2*N+0] = 0; + U[2*N+1] = 0; + U[2*N+2] = U33; + U[2*N+3] = U34; + U[2*N+4] = U35; + U[2*N+5] = U36; + U[2*N+6] = U37; + + U[3*N+0] = 0; + U[3*N+1] = 0; + U[3*N+2] = 0; + U[3*N+3] = U44; + U[3*N+4] = U45; + U[3*N+5] = U46; + U[3*N+6] = U47; + + U[4*N+0] = 0; + U[4*N+1] = 0; + U[4*N+2] = 0; + U[4*N+3] = 0; + U[4*N+4] = U55; + U[4*N+5] = U56; + U[4*N+6] = U57; + + U[5*N+0] = 0; + U[5*N+1] = 0; + U[5*N+2] = 0; + U[5*N+3] = 0; + U[5*N+4] = 0; + U[5*N+5] = U66; + U[5*N+6] = U67; + + U[6*N+0] = 0; + U[6*N+1] = 0; + U[6*N+2] = 0; + U[6*N+3] = 0; + U[6*N+4] = 0; + U[6*N+5] = 0; + U[6*N+6] = U77; +} + +template, bool> = false> +FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { + + const T A00 = A[0*N]; + const T L21 = A[1*N] /A00; + const T L31 = A[2*N] /A00; + const T L41 = A[3*N] /A00; + const T L51 = A[4*N] /A00; + const T L61 = A[5*N] /A00; + const T L71 = A[6*N] /A00; + const T L81 = A[7*N] /A00; + + const T U22 = A[1*N+1] - L21 * A[1]; + const T U23 = A[1*N+2] - L21 * A[2]; + const T U24 = A[1*N+3] - L21 * A[3]; + const T U25 = A[1*N+4] - L21 * A[4]; + const T U26 = A[1*N+5] - L21 * A[5]; + const T U27 = A[1*N+6] - L21 * A[6]; + const T U28 = A[1*N+7] - L21 * A[7]; + + const T L32 = (A[2*N+1] - L31 * A[1]) / U22; + const T L42 = (A[3*N+1] - L41 * A[1]) / U22; + const T L52 = (A[4*N+1] - L51 * A[1]) / U22; + const T L62 = (A[5*N+1] - L61 * A[1]) / U22; + const T L72 = (A[6*N+1] - L71 * A[1]) / U22; + const T L82 = (A[7*N+1] - L81 * A[1]) / U22; + + const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; + const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; + const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; + const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26; + const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27; + const T U38 = A[2*N+7] - L31 * A[7] - L32 * U28; + + const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; + const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; + const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33; + const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33; + const T L83 = (A[7*N+2] - L81 * A[2] - L82 * U23) / U33; + + const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; + const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; + const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36; + const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37; + const T U48 = A[3*N+7] - L41 * A[7] - L42 * U28 - L43 * U38; + + const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; + const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44; + const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44; + const T L84 = (A[7*N+3] - L81 * A[3] - L82 * U24 - L83 * U34) / U44; + + const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; + const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46; + const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47; + const T U58 = A[4*N+7] - L51 * A[7] - L52 * U28 - L53 * U38 - L54 * U48; + + const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55; + const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55; + const T L85 = (A[7*N+4] - L81 * A[4] - L82 * U25 - L83 * U35 - L84 * U45) / U55; + + const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56; + const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57; + const T U68 = A[5*N+7] - L61 * A[7] - L62 * U28 - L63 * U38 - L64 * U48 - L65 * U58; + + const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66; + const T L86 = (A[7*N+5] - L81 * A[5] - L82 * U26 - L83 * U36 - L84 * U46 - L85 * U56) / U66; + + const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67; + const T U78 = A[6*N+7] - L71 * A[7] - L72 * U28 - L73 * U38 - L74 * U48 - L75 * U58 - L76 * U68; + + const T L87 = (A[7*N+6] - L81 * A[6] - L82 * U27 - L83 * U37 - L84 * U47 - L85 * U57 - L86 * U67) / U77; + + const T U88 = A[7*N+7] - L81 * A[7] - L82 * U28 - L83 * U38 - L84 * U48 - L85 * U58 - L86 * U68 - L87 * U78; + + // L + L[0*N+0] = 1; + L[0*N+1] = 0; + L[0*N+2] = 0; + L[0*N+3] = 0; + L[0*N+4] = 0; + L[0*N+5] = 0; + L[0*N+6] = 0; + L[0*N+7] = 0; + + L[1*N+0] = L21; + L[1*N+1] = 1; + L[1*N+2] = 0; + L[1*N+3] = 0; + L[1*N+4] = 0; + L[1*N+5] = 0; + L[1*N+6] = 0; + L[1*N+7] = 0; + + L[2*N+0] = L31; + L[2*N+1] = L32; + L[2*N+2] = 1; + L[2*N+3] = 0; + L[2*N+4] = 0; + L[2*N+5] = 0; + L[2*N+6] = 0; + L[2*N+7] = 0; + + L[3*N+0] = L41; + L[3*N+1] = L42; + L[3*N+2] = L43; + L[3*N+3] = 1; + L[3*N+4] = 0; + L[3*N+5] = 0; + L[3*N+6] = 0; + L[3*N+7] = 0; + + L[4*N+0] = L51; + L[4*N+1] = L52; + L[4*N+2] = L53; + L[4*N+3] = L54; + L[4*N+4] = 1; + L[4*N+5] = 0; + L[4*N+6] = 0; + L[4*N+7] = 0; + + L[5*N+0] = L61; + L[5*N+1] = L62; + L[5*N+2] = L63; + L[5*N+3] = L64; + L[5*N+4] = L65; + L[5*N+5] = 1; + L[5*N+6] = 0; + L[5*N+7] = 0; + + L[6*N+0] = L71; + L[6*N+1] = L72; + L[6*N+2] = L73; + L[6*N+3] = L74; + L[6*N+4] = L75; + L[6*N+5] = L76; + L[6*N+6] = 1; + L[6*N+7] = 0; + + L[7*N+0] = L81; + L[7*N+1] = L82; + L[7*N+2] = L83; + L[7*N+3] = L84; + L[7*N+4] = L85; + L[7*N+5] = L86; + L[7*N+6] = L87; + L[7*N+7] = 1; + + // U + U[0*N+0] = A00; + U[0*N+1] = A[1]; + U[0*N+2] = A[2]; + U[0*N+3] = A[3]; + U[0*N+4] = A[4]; + U[0*N+5] = A[5]; + U[0*N+6] = A[6]; + U[0*N+7] = A[7]; + + U[1*N+0] = 0; + U[1*N+1] = U22; + U[1*N+2] = U23; + U[1*N+3] = U24; + U[1*N+4] = U25; + U[1*N+5] = U26; + U[1*N+6] = U27; + U[1*N+7] = U28; + + U[2*N+0] = 0; + U[2*N+1] = 0; + U[2*N+2] = U33; + U[2*N+3] = U34; + U[2*N+4] = U35; + U[2*N+5] = U36; + U[2*N+6] = U37; + U[2*N+7] = U38; + + U[3*N+0] = 0; + U[3*N+1] = 0; + U[3*N+2] = 0; + U[3*N+3] = U44; + U[3*N+4] = U45; + U[3*N+5] = U46; + U[3*N+6] = U47; + U[3*N+7] = U48; + + U[4*N+0] = 0; + U[4*N+1] = 0; + U[4*N+2] = 0; + U[4*N+3] = 0; + U[4*N+4] = U55; + U[4*N+5] = U56; + U[4*N+6] = U57; + U[4*N+7] = U58; + + U[5*N+0] = 0; + U[5*N+1] = 0; + U[5*N+2] = 0; + U[5*N+3] = 0; + U[5*N+4] = 0; + U[5*N+5] = U66; + U[5*N+6] = U67; + U[5*N+7] = U68; + + U[6*N+0] = 0; + U[6*N+1] = 0; + U[6*N+2] = 0; + U[6*N+3] = 0; + U[6*N+4] = 0; + U[6*N+5] = 0; + U[6*N+6] = U77; + U[6*N+7] = U78; + + U[7*N+0] = 0; + U[7*N+1] = 0; + U[7*N+2] = 0; + U[7*N+3] = 0; + U[7*N+4] = 0; + U[7*N+5] = 0; + U[7*N+6] = 0; + U[7*N+7] = U88; +} + +} // end of namespace Fastor + +#endif // LUFACT_H diff --git a/noarch/include/Fastor/backend/lut_inverse.h b/noarch/include/Fastor/backend/lut_inverse.h new file mode 100644 index 00000000..5debb56d --- /dev/null +++ b/noarch/include/Fastor/backend/lut_inverse.h @@ -0,0 +1,80 @@ +#ifndef LOWUNITRI_INVERSE_H +#define LOWUNITRI_INVERSE_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" + +namespace Fastor { + +template, bool> = false> +FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); + +template, bool> = false> +FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { + *dst = T(1); +} + +template, bool> = false> +FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + /* Compute adjoint: */ + dst[0] = 1; + dst[1] = 0; + dst[2] = - src[2]; + dst[3] = 1; +} + +template, bool> = false> +FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + T src3 = src[3]; + T src6 = src[6]; + T src7 = src[7]; + T src8 = src[8]; + + /* Compute adjoint: */ + dst[0] = 1; + dst[1] = 0; + dst[2] = 0; + dst[3] = - src3 * src8; + dst[4] = 1; + dst[5] = 0; + dst[6] = + src3 * src7 - src6; + dst[7] = - src7; + dst[8] = 1; +} + + +template, bool> = false> +FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) +{ + dst[0] = 1; + dst[1] = 0; + dst[2] = 0; + dst[3] = 0; + + dst[4] = - src[1*4+0]; + dst[5] = 1; + dst[6] = 0; + dst[7] = 0; + + T t2 = src[2*4+1]; + T t3 = src[2*4+1]*src[3*4+2] - src[3*4+1]; + + T t4 = src[2*4+0]; + T t5 = src[2*4+0]*src[3*4+2] - src[3*4+0]; + + dst[8] = src[1*4+0]*t2 - t4; + dst[9] = - t2; + dst[10] = 1; + dst[11] = 0; + + dst[12] = t5 - src[1*4+0]*t3; + dst[13] = t3; + dst[14] = - src[3*4+2]; + dst[15] = 1; +} + +} // end of namespace Fastor + +#endif // LOWUNITRI_INVERSE_H diff --git a/noarch/include/Fastor/backend/matmul/libxsmm_backend.h b/noarch/include/Fastor/backend/matmul/libxsmm_backend.h new file mode 100644 index 00000000..fe79c560 --- /dev/null +++ b/noarch/include/Fastor/backend/matmul/libxsmm_backend.h @@ -0,0 +1,209 @@ +#ifndef LIBXSMM_BACKEND_H +#define LIBXSMM_BACKEND_H + +#include + +#ifdef FASTOR_USE_LIBXSMM + +#include + +namespace Fastor { +namespace blas { + +// single +template +FASTOR_INLINE +void matmulNN_libxsmm( + const float * FASTOR_RESTRICT a_data, + const float * FASTOR_RESTRICT b_data, + float * FASTOR_RESTRICT out_data) { + + constexpr int MM= M; + constexpr int KK= K; + constexpr int NN= N; + constexpr float alpha = 1.0; + constexpr float beta = 0.0; + + constexpr char transa = 'N'; + constexpr char transb = 'N'; + + libxsmm_sgemm( + &transa /*transa*/, + &transb /*transb*/, + &NN /*required*/, + &MM /*required*/, + &KK /*required*/, + &alpha /*alpha*/, + b_data /*required*/, + &NN /*lda*/, + a_data /*required*/, + &KK /*ldb*/, + &beta /*beta*/, + out_data /*required*/, + &NN /*ldc*/ + ); +} + +// double +template +FASTOR_INLINE +void matmulNN_libxsmm( + const double * FASTOR_RESTRICT a_data, + const double * FASTOR_RESTRICT b_data, + double * FASTOR_RESTRICT out_data) { + + constexpr int MM= M; + constexpr int KK= K; + constexpr int NN= N; + constexpr double alpha = 1.0; + constexpr double beta = 0.0; + + constexpr char transa = 'N'; + constexpr char transb = 'N'; + + libxsmm_dgemm( + &transa /*transa*/, + &transb /*transb*/, + &NN /*required*/, + &MM /*required*/, + &KK /*required*/, + &alpha /*alpha*/, + b_data /*required*/, + &NN /*lda*/, + a_data /*required*/, + &KK /*ldb*/, + &beta /*beta*/, + out_data /*required*/, + &NN /*ldc*/ + ); +} + + +template +FASTOR_INLINE +void matmulTN_libxsmm( + const double * FASTOR_RESTRICT a_data, + const double * FASTOR_RESTRICT b_data, + double * FASTOR_RESTRICT out_data) { + + constexpr int MM= M; + constexpr int KK= K; + constexpr int NN= N; + constexpr double alpha = 1.0; + constexpr double beta = 0.0; + + constexpr char transa = 'N'; + constexpr char transb = 'Y'; + + libxsmm_dgemm( + &transa /*transa*/, + &transb /*transb*/, + &NN /*required*/, + &MM /*required*/, + &KK /*required*/, + &alpha /*alpha*/, + b_data /*required*/, + &NN /*lda*/, + a_data /*required*/, + &MM /*ldb*/, + &beta /*beta*/, + out_data /*required*/, + &NN /*ldc*/ + ); +} + + +template +FASTOR_INLINE +void matmulNT_libxsmm( + const double * FASTOR_RESTRICT a_data, + const double * FASTOR_RESTRICT b_data, + double * FASTOR_RESTRICT out_data) { + + constexpr int MM= M; + constexpr int KK= K; + constexpr int NN= N; + constexpr double alpha = 1.0; + constexpr double beta = 0.0; + + constexpr char transa = 'Y'; + constexpr char transb = 'N'; + + libxsmm_dgemm( + &transa /*transa*/, + &transb /*transb*/, + &NN /*required*/, + &MM /*required*/, + &KK /*required*/, + &alpha /*alpha*/, + b_data /*required*/, + &KK /*lda*/, + a_data /*required*/, + &KK /*ldb*/, + &beta /*beta*/, + out_data /*required*/, + &NN /*ldc*/ + ); +} + + +template +FASTOR_INLINE +void matmulTT_libxsmm( + const double * FASTOR_RESTRICT a_data, + const double * FASTOR_RESTRICT b_data, + double * FASTOR_RESTRICT out_data) { + + constexpr int MM= M; + constexpr int KK= K; + constexpr int NN= N; + constexpr double alpha = 1.0; + constexpr double beta = 0.0; + + constexpr char transa = 'Y'; + constexpr char transb = 'Y'; + + libxsmm_dgemm( + &transa /*transa*/, + &transb /*transb*/, + &NN /*required*/, + &MM /*required*/, + &KK /*required*/, + &alpha /*alpha*/, + b_data /*required*/, + &KK /*lda*/, + a_data /*required*/, + &MM /*ldb*/, + &beta /*beta*/, + out_data /*required*/, + &NN /*ldc*/ + ); +} + + +template::value,bool>::type=0> +FASTOR_INLINE +void matmul_libxsmm( + const T * FASTOR_RESTRICT a_data, + const T * FASTOR_RESTRICT b_data, + T * FASTOR_RESTRICT out_data) { + matmulNN_libxsmm(a_data,b_data,out_data); +} + +template::value,bool>::type=0> +FASTOR_INLINE +void matmul_libxsmm( + const T * FASTOR_RESTRICT a_data, + const T * FASTOR_RESTRICT b_data, + T * FASTOR_RESTRICT out_data) { + matmulNN_libxsmm(a_data,b_data,out_data); +} + +} // end of namespace blas +} // end of namespace Fastor + +#endif // FASTOR_USE_LIBXSMM + +#endif // LIBXSMM_BACKEND_H diff --git a/noarch/include/Fastor/backend/matmul/matmul.h b/noarch/include/Fastor/backend/matmul/matmul.h new file mode 100644 index 00000000..9526b171 --- /dev/null +++ b/noarch/include/Fastor/backend/matmul/matmul.h @@ -0,0 +1,150 @@ +#ifndef MATMUL_H +#define MATMUL_H + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/matmul/matmul_kernels.h" + +#ifdef FASTOR_USE_LIBXSMM +#include "Fastor/backend/matmul/libxsmm_backend.h" +#endif +#ifdef FASTOR_USE_MKL +#include "Fastor/backend/matmul/mkl_backend.h" +#endif + +namespace Fastor { + + + +// Forward declare +//----------------------------------------------------------------------------------------------------------- +namespace internal { +template +FASTOR_INLINE +void _matvecmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out); +} // internal +//----------------------------------------------------------------------------------------------------------- + + +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +#if !defined(FASTOR_USE_LIBXSMM) && !defined(FASTOR_USE_MKL) +template || is_same_v_) ),bool> = 0> +#else +template || is_same_v_) ) + && is_less_equal::value,1>::value, + bool> = 0> +#endif +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + // Non-primitive types + FASTOR_IF_CONSTEXPR (!is_primitive_v_) { + internal::_matmul_base_non_primitive(a,b,out); + return; + } + + // Matrix-vector specialisation + FASTOR_IF_CONSTEXPR (N==1UL) { + internal::_matvecmul(a,b,out); + return; + } + + using nativeV = SIMDVector; + using V = choose_best_simd_t; + + // Use specialised kernels + FASTOR_IF_CONSTEXPR((N==V::Size || N==2*V::Size || N==3*V::Size || N==4*V::Size || N==5*V::Size) && V::Size!=1UL) { + internal::_matmul_mk_smalln(a,b,out); + return; + } + +#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) + FASTOR_IF_CONSTEXPR((N<5*V::Size && N!=1UL)) { + internal::_matmul_mk_smalln(a,b,out); + return; + } +#endif + +#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) + FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size <= 1UL) { + internal::_matmul_base(a,b,out); + return; + } + else FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size > 1UL) { + internal::_matmul_base_masked(a,b,out); + return; + } +#else + FASTOR_IF_CONSTEXPR( M*N*K > 27UL ) { + internal::_matmul_base(a,b,out); + return; + } +#endif + else + { + // For all other cases where M,N,K is too small + // this simple version is sufficient + constexpr int ROUND_ = ROUND_DOWN(N,V::Size); + for (size_t j=0; j || is_same_v_) ) + && is_greater::value,1>::value, + bool> = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + blas::matmul_libxsmm(a,b,c); +} +#endif + +#if !defined(FASTOR_USE_LIBXSMM) && defined(FASTOR_USE_MKL) +template || is_same_v_) ) + && is_greater::value,1>::value, + bool> = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + blas::matmul_mkl(a,b,c); +} +#endif +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- + + +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +#include "Fastor/backend/matmul/matmul_specialisations_kernels.h" +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- + + +} // end of namespace + +#endif // MATMUL_H diff --git a/noarch/include/Fastor/backend/matmul/matmul_kernels.h b/noarch/include/Fastor/backend/matmul/matmul_kernels.h new file mode 100644 index 00000000..da10cefd --- /dev/null +++ b/noarch/include/Fastor/backend/matmul/matmul_kernels.h @@ -0,0 +1,823 @@ +#ifndef MATMUL_KERNELS_H +#define MATMUL_KERNELS_H + + +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/meta/tensor_meta.h" + + +namespace Fastor { + +namespace internal { + + +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- + +// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel) +// unroll the inner-most loop (on unrollOuterloop) +//----------------------------------------------------------------------------------------------------------- +template::type = false> +FASTOR_INLINE +void interior_block_matmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + + const V bmm0(&b[k*N+j],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + } + } +} + +template::type = false> +FASTOR_INLINE +void interior_block_matmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_matmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + const V bmm2(&b[k*N+j+2*V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_matmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + const V bmm2(&b[k*N+j+2*V::Size],false); + const V bmm3(&b[k*N+j+3*V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); + c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); + c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_matmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + const V bmm2(&b[k*N+j+2*V::Size],false); + const V bmm3(&b[k*N+j+3*V::Size],false); + const V bmm4(&b[k*N+j+4*V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); + c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); + c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); + c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); + c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_matmul_scalar_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + T c_ij[unrollOuterloop*numSIMDCols] = {}; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + const T bmm0(b[k*N+j]); + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] += amm0*bmm0; + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n]; + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_matmul_mask_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j, const int (&maska)[V::Size]) { + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + + const V bmm0(maskload(&b[k*N+j],maska)); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_matmul_mask_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j, const MaskT mask) { + + V bmm0; + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = 0; k < K; ++k) { + + bmm0.mask_load(&b[k*N+j],mask,false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false); + } + } +} +//----------------------------------------------------------------------------------------------------------- + + + +//----------------------------------------------------------------------------------------------------------- +// This is the base implementation of matrix-matrix multiplication for all 2D tensors and +// higher order tensor products that can be expressed as gemm +// The function uses two level unrolling one based on block sizes and one based on register widths +// with any remainder left treated in a scalar fashion +template +FASTOR_INLINE +void _matmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + + using V = typename internal::choose_best_simd_type,N>::type; + + // This parameter can be adjusted and does not need to be 4UL/8UL etc + // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; + constexpr size_t unrollOuterloop = 4UL; + +#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE + // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] + constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); +#else + constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; +#endif +#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE + // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] + constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; +#else + constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; +#endif + + // The goal is to get 10 parallel independent chains of accumulators + // to saturate the pipeline by having a completely unrolled block of + // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of + // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains + // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. + // However, most recent X86/64 architectures can do 2 FMAs per load so + // so unrolling with numSIMDCols > 2 is not beneficial + + constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; + // Number of rows of c (M) that can be safely unrolled with this block size. + constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; + + constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; + // Number of columns of c (N) that can be safely unrolled with this block size + constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + size_t i = 0; + for (; i < M0; i += unrollOuterBlock) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N1 columns + for (; j < N; ++j) { + interior_block_matmul_scalar_impl(a,b,c,i,j); + } + + } + + // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. + // This is necessary as for small sizes the earlier block loop may not be + // triggered if the size of the block is bigger than the number of rows of + // (a and c) i.e. M + constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); + for (; i < M1; i += unrollOuterloop) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + V c_ij[unrollOuterloop]; + for (size_t k = 0; k < K; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i + n)*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; ++j) { + T c_ij[unrollOuterloop] = {}; + for (size_t k = 0; k < K; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] += a[(i + n)*K+k] * b[k*N+j]; + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c[(i + n)*N+j] = c_ij[n]; + } + } + } + + // Now treat the remaining M-M1 rows + FASTOR_IF_CONSTEXPR (M-M1 > 0) { + // Hack to get around zero length array issue + constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + // If MM1==0 the function never gets invoked anyway + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + V c_ij[MM1]; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); + c_ij[n-M1].store(&c[n*N+j],false); + } + } + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1].store(&c[n*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; ++j) { + T c_ij[MM1] = {}; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1] += a[n*K+k] * b[k*N+j]; + c[n*N+j] = c_ij[n-M1]; + } + } + for (size_t n = M1; n < M; ++n) { + c[n*N+j] = c_ij[n-M1]; + } + } + } +} +//----------------------------------------------------------------------------------------------------------- + + + +//----------------------------------------------------------------------------------------------------------- +// This is the base implementation of matrix-matrix multiplication for all 2D tensors and +// higher order tensor products that can be expressed as gemm +// The function uses two level unrolling one based on block sizes and one based on register widths +// with any remainder left treated in vector mode with masked and conditional load/stores. +// Note that conditional load/store requires at least AVX intrinsics +template +FASTOR_INLINE +void _matmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + + using V = typename internal::choose_best_simd_type,N>::type; + + // This parameter can be adjusted and does not need to be 4UL/8UL etc + // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; + constexpr size_t unrollOuterloop = 4UL; + +#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE + // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] + constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); +#else + constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; +#endif +#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE + // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] + constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; +#else + constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; +#endif + + // The goal is to get 10 parallel independent chains of accumulators + // to saturate the pipeline by having a completely unrolled block of + // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of + // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains + // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. + // However, most recent X86/64 architectures can do 2 FMAs per load so + // so unrolling with numSIMDCols > 2 is not beneficial + + constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; + // Number of rows of c (M) that can be safely unrolled with this block size. + constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; + + constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; + // Number of columns of c (N) that can be safely unrolled with this block size + constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + int maska[V::Size]; + std::fill(maska,&maska[V::Size], -1); + for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; +#ifdef FASTOR_HAS_AVX512_MASKS + const auto mask = array_to_mask(maska); +#endif + + size_t i = 0; + for (; i < M0; i += unrollOuterBlock) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N1 columns + for (; j < N; j+= N-N1) { +#ifdef FASTOR_HAS_AVX512_MASKS + interior_block_matmul_mask_impl(a,b,c,i,j,mask); +#else + interior_block_matmul_mask_impl(a,b,c,i,j,maska); +#endif + } + } + + // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. + // This is necessary as for small sizes the earlier block loop may not be + // triggered if the size of the block is bigger than the number of rows of + // (a and c) i.e. M + constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); + for (; i < M1; i += unrollOuterloop) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + V c_ij[unrollOuterloop]; + for (size_t k = 0; k < K; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i + n)*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; j+=N-N1) { + V c_ij[unrollOuterloop]; + for (size_t k = 0; k < K; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; bmm0.mask_load(&b[k*N+j],mask); +#else + const V bmm0(maskload(&b[k*N+j],maska)); +#endif + const V amm0 = a[(i + n)*K+k]; + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + c_ij[n].mask_store(&c[(i+n)*N+j],mask,false); +#else + maskstore(&c[(i+n)*N+j],maska,c_ij[n]); +#endif + } + } + } + + // Now treat the remaining M-M1 rows + FASTOR_IF_CONSTEXPR (M-M1 > 0) { + // Hack to get around zero length array issue + constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + // If MM1==0 the function never gets invoked anyway + interior_block_matmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + V c_ij[MM1]; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); + c_ij[n-M1].store(&c[n*N+j],false); + } + } + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1].store(&c[n*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; j+=N-N1) { + V c_ij[MM1] = {}; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; bmm0.mask_load(&b[k*N+j],mask); +#else + const V bmm0(maskload(&b[k*N+j],maska)); +#endif + const V amm0 = a[n*K+k]; + c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]); + } + } + for (size_t n = M1; n < M; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + c_ij[n-M1].mask_store(&c[n*N+j],mask,false); +#else + maskstore(&c[n*N+j],maska,c_ij[n-M1]); +#endif + } + } + } +} +//----------------------------------------------------------------------------------------------------------- + + + + +//----------------------------------------------------------------------------------------------------------- +// matmul kernel for non-fundamental types +// The assumption here is that non-fundamental types are not SIMD vectorisable for instance +// Tensor,3,3> or Tensor,...> plus they cannot fuse [do fused-add-multiply] +// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in +// the inner most loops of matmul +template +FASTOR_INLINE +void _matmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + // There is no SIMD here as V::Size == 1 anyway + // No outer loop unrolling otherwise the innermost loop + // will create unnecessary temporaries + for (size_t i=0; i +FASTOR_INLINE +void _matmul_mkn_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + + using V = typename internal::choose_best_simd_type,N>::type; + + // Get 10 parallel independent chains of accumulators for bigger matrices + constexpr size_t unrollOuterloop = M >= 64 ? 10UL : (M % 8 == 0 ? 8UL : V::Size); + + // The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore + // the last rows may need special treatment if M is not a multiple of unrollOuterloop. + // M0 is the number of rows that can safely be iterated with a stride of + // unrollOuterloop. + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + for (size_t i = 0; i < M0; i += unrollOuterloop) { + // The iteration over the column index of b and c uses a stride of V::Size. This + // enables row-vector loads (from b) and stores (to c). The matrix storage is + // padded accordingly, ensuring correct bounds and alignment. + for (size_t j = 0; j < N; j += V::Size) { + // This temporary variables are used to accumulate the results of the products + // producing the new values for the c matrix. This variable is necessary + // because we need a V object for data-parallel accumulation. Storing to c + // directly stores to scalar objects and thus would drop the ability for + // data-parallel (SIMD) addition. + V c_ij[unrollOuterloop]; + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] = a[(i + n)*K]*V(&b[j]); + } + for (size_t k = 1; k < K - 1; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j]); + c_ij[n].store(&c[(i + n)*N+j]); + } + } + } +} + +// This is the same implementation as the above case but does not unroll on block sizes and does not require +// the registers to be zeroed out but K must be !=1 +template +FASTOR_INLINE +void _matmul_mkn_non_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + + // This variant strictly cannot deal outer-product i.e. with K==1 + + using V = typename internal::choose_best_simd_type,N>::type; + + // Get 10 parallel independent chains of accumulators for bigger matrices + constexpr size_t unrollOuterloop = M < V::Size ? 1UL : + (( M >= 64 && K > 10 && N > V::Size ) ? 10UL : (M % 8 == 0 && N > V::Size ? 8UL : V::Size)); + constexpr bool isPadded = N % V::Size == 0; + + // The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore + // the last rows may need special treatment if M is not a multiple of unrollOuterloop. + // M0 is the number of rows that can safely be iterated with a stride of + // unrollOuterloop. + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + constexpr size_t N0 = N / V::Size * V::Size; + + for (size_t i = 0; i < M0; i += unrollOuterloop) { + // The iteration over the column index of b and c uses a stride of V::size(). This + // enables row-vector loads (from b) and stores (to c). The matrix storage is + // padded accordingly, ensuring correct bounds and alignment. + size_t j = 0; + for (; j < N0; j += V::Size) { + // This temporary variables are used to accumulate the results of the products + // producing the new values for the c matrix. This variable is necessary + // because we need a V object for data-parallel accumulation. Storing to c + // directly stores to scalar objects and thus would drop the ability for + // data-parallel (SIMD) addition. + V c_ij[unrollOuterloop]; + for (size_t n = 0; n < unrollOuterloop; ++n) { // correct + c_ij[n] = a[(i + n)*K]*V(&b[j], isPadded); + } + for (size_t k = 1; k < K - 1; ++k) { // correct + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j], false); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { // correct + c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j], false); + c_ij[n].store(&c[(i + n)*N+j], isPadded); + } + } + + // Remainder N - N0 columns + for (; j < N; ++j) { + T c_ij[unrollOuterloop]; + for (size_t n = 0; n < unrollOuterloop; ++n) { // correct + c_ij[n] = a[(i + n)*K]*b[j]; + } + for (size_t k = 1; k < K - 1; ++k) { // correct + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] += a[(i + n)*K+k] * b[k*N+j]; + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { // correct + c_ij[n] += a[(i + n)*K+(K - 1)] * b[(K - 1)*N+j]; + c[(i + n)*N+j] = c_ij[n]; + } + } + } + + // This final loop treats the remaining M - M0 rows. + size_t j = 0; + for (; j < N0; j += V::Size) { + V c_ij[M-M0]; + for (size_t n = M0; n < M; ++n) { // correct + c_ij[n - M0] = a[n*K] * V(&b[j], isPadded); + } + for (size_t k = 1; k < K - 1; ++k) { // correct + for (size_t n = M0; n < M; ++n) { // correct + c_ij[n - M0] += a[n*K+k] * V(&b[k*N+j], false); + } + } + for (size_t n = M0; n < M; ++n) { // correct + c_ij[n - M0] += a[n*K+(K - 1)] * V(&b[(K - 1)*N+j], false); + c_ij[n - M0].store(&c[n*N+j], isPadded); + } + } + + for (; j < N; ++j) { + T c_ij[M-M0]; + for (size_t n = M0; n < M; ++n) { // correct + c_ij[n - M0] = a[n*K] * b[j]; + } + for (size_t k = 1; k < K - 1; ++k) { // correct + for (size_t n = M0; n < M; ++n) { // correct + c_ij[n - M0] += a[n*K+k] * b[k*N+j]; + } + } + for (size_t n = M0; n < M; ++n) { // correct + c_ij[n - M0] += a[n*K+(K - 1)] * b[(K - 1)*N+j]; + c[n*N+j] = c_ij[n - M0]; + } + } + +} +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- + +} // end of namespace internal + +} // end of namespace Fastor + +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +#include "Fastor/backend/matmul/matmul_mk_smalln.h" +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- + + +#endif // MATMUL_KERNELS_H diff --git a/noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h b/noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h new file mode 100644 index 00000000..c6810c98 --- /dev/null +++ b/noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h @@ -0,0 +1,3271 @@ +#ifndef MATMUL_MK_SMALLODDN_H +#define MATMUL_MK_SMALLODDN_H + + +#include "Fastor/simd_vector/simd_vector_abi.h" +#include "Fastor/simd_vector/SIMDVector.h" + +namespace Fastor { + +namespace internal { + + + +// This implementation covers all matrix-matrix multiplications with any M and K and +// and N<=5*SIMDVector::Size. Given that it uses choose_best_simd_type it can switch +// between SSE, AVX and AVX512 to cover all ranges of N + + +//----------------------------------------------------------------------------------------------------------- +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + const V amm6(a[(j+6)*K]); + const V amm7(a[(j+7)*K]); + const V amm8(a[(j+8)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + // row 6 + V omm6(amm6*bmm0); + // row 7 + V omm7(amm7*bmm0); + // row 8 + V omm8(amm8*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + const V amm4(a[(j+4)*K+i]); + const V amm5(a[(j+5)*K+i]); + const V amm6(a[(j+6)*K+i]); + const V amm7(a[(j+7)*K+i]); + + const V amm8(a[(j+8)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + // row 3 + omm3 = fmadd(amm3,bmm0,omm3); + // row 4 + omm4 = fmadd(amm4,bmm0,omm4); + // row 5 + omm5 = fmadd(amm5,bmm0,omm5); + // row 6 + omm6 = fmadd(amm6,bmm0,omm6); + // row 7 + omm7 = fmadd(amm7,bmm0,omm7); + // row 8 + omm8 = fmadd(amm8,bmm0,omm8); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); + omm3.mask_store(&out[(j+3)*N],mask,false); + omm4.mask_store(&out[(j+4)*N],mask,false); + omm5.mask_store(&out[(j+5)*N],mask,false); + omm6.mask_store(&out[(j+6)*N],mask,false); + omm7.mask_store(&out[(j+7)*N],mask,false); + omm8.mask_store(&out[(j+8)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); + maskstore(&out[(j+3)*N],maska,omm3); + maskstore(&out[(j+4)*N],maska,omm4); + maskstore(&out[(j+5)*N],maska,omm5); + maskstore(&out[(j+6)*N],maska,omm6); + maskstore(&out[(j+7)*N],maska,omm7); + maskstore(&out[(j+8)*N],maska,omm8); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + const V amm6(a[(j+6)*K]); + const V amm7(a[(j+7)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + // row 6 + V omm6(amm6*bmm0); + // row 7 + V omm7(amm7*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + const V amm4(a[(j+4)*K+i]); + const V amm5(a[(j+5)*K+i]); + const V amm6(a[(j+6)*K+i]); + const V amm7(a[(j+7)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + // row 3 + omm3 = fmadd(amm3,bmm0,omm3); + // row 4 + omm4 = fmadd(amm4,bmm0,omm4); + // row 5 + omm5 = fmadd(amm5,bmm0,omm5); + // row 6 + omm6 = fmadd(amm6,bmm0,omm6); + // row 7 + omm7 = fmadd(amm7,bmm0,omm7); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); + omm3.mask_store(&out[(j+3)*N],mask,false); + omm4.mask_store(&out[(j+4)*N],mask,false); + omm5.mask_store(&out[(j+5)*N],mask,false); + omm6.mask_store(&out[(j+6)*N],mask,false); + omm7.mask_store(&out[(j+7)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); + maskstore(&out[(j+3)*N],maska,omm3); + maskstore(&out[(j+4)*N],maska,omm4); + maskstore(&out[(j+5)*N],maska,omm5); + maskstore(&out[(j+6)*N],maska,omm6); + maskstore(&out[(j+7)*N],maska,omm7); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + const V amm6(a[(j+6)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + // row 6 + V omm6(amm6*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + const V amm4(a[(j+4)*K+i]); + const V amm5(a[(j+5)*K+i]); + const V amm6(a[(j+6)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + // row 3 + omm3 = fmadd(amm3,bmm0,omm3); + // row 4 + omm4 = fmadd(amm4,bmm0,omm4); + // row 5 + omm5 = fmadd(amm5,bmm0,omm5); + // row 6 + omm6 = fmadd(amm6,bmm0,omm6); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); + omm3.mask_store(&out[(j+3)*N],mask,false); + omm4.mask_store(&out[(j+4)*N],mask,false); + omm5.mask_store(&out[(j+5)*N],mask,false); + omm6.mask_store(&out[(j+6)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); + maskstore(&out[(j+3)*N],maska,omm3); + maskstore(&out[(j+4)*N],maska,omm4); + maskstore(&out[(j+5)*N],maska,omm5); + maskstore(&out[(j+6)*N],maska,omm6); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + const V amm4(a[(j+4)*K+i]); + const V amm5(a[(j+5)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + // row 3 + omm3 = fmadd(amm3,bmm0,omm3); + // row 4 + omm4 = fmadd(amm4,bmm0,omm4); + // row 5 + omm5 = fmadd(amm5,bmm0,omm5); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); + omm3.mask_store(&out[(j+3)*N],mask,false); + omm4.mask_store(&out[(j+4)*N],mask,false); + omm5.mask_store(&out[(j+5)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); + maskstore(&out[(j+3)*N],maska,omm3); + maskstore(&out[(j+4)*N],maska,omm4); + maskstore(&out[(j+5)*N],maska,omm5); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + const V amm4(a[(j+4)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + // row 3 + omm3 = fmadd(amm3,bmm0,omm3); + // row 4 + omm4 = fmadd(amm4,bmm0,omm4); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); + omm3.mask_store(&out[(j+3)*N],mask,false); + omm4.mask_store(&out[(j+4)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); + maskstore(&out[(j+3)*N],maska,omm3); + maskstore(&out[(j+4)*N],maska,omm4); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + // row 3 + omm3 = fmadd(amm3,bmm0,omm3); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); + omm3.mask_store(&out[(j+3)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); + maskstore(&out[(j+3)*N],maska,omm3); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); +#endif + return; +} + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; + bmm0.mask_load(&b[0],mask,false); +#else + const V bmm0(maskload(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + + // row 0 + V omm0(amm0*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + } + +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); +#endif + return; +} + + + +#ifdef FASTOR_HAS_AVX512_MASKS +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#else +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { +#endif + return; +} + + +template,N>::type::Size>::value,bool> = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + using V = typename choose_best_simd_type,N>::type; + // using V = SIMDVector; + // Unroll a by 10 + constexpr size_t unrollOuterloop = 10UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + int maska[V::Size]; + std::fill(maska,&maska[V::Size], -1); + for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; +#ifdef FASTOR_HAS_AVX512_MASKS + const auto mask = array_to_mask(maska); + V bmm0; +#endif + + size_t j=0; + for (; j(&b[0],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + const V amm6(a[(j+6)*K]); + const V amm7(a[(j+7)*K]); + const V amm8(a[(j+8)*K]); + const V amm9(a[(j+9)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + // row 6 + V omm6(amm6*bmm0); + // row 7 + V omm7(amm7*bmm0); + // row 8 + V omm8(amm8*bmm0); + // row 9 + V omm9(amm9*bmm0); + + for (size_t i=1; i(&b[i*N],maska)); +#endif + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + const V amm4(a[(j+4)*K+i]); + const V amm5(a[(j+5)*K+i]); + const V amm6(a[(j+6)*K+i]); + const V amm7(a[(j+7)*K+i]); + const V amm8(a[(j+8)*K+i]); + const V amm9(a[(j+9)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + // row 1 + omm1 = fmadd(amm1,bmm0,omm1); + // row 2 + omm2 = fmadd(amm2,bmm0,omm2); + // row 3 + omm3 = fmadd(amm3,bmm0,omm3); + // row 4 + omm4 = fmadd(amm4,bmm0,omm4); + // row 5 + omm5 = fmadd(amm5,bmm0,omm5); + // row 6 + omm6 = fmadd(amm6,bmm0,omm6); + // row 7 + omm7 = fmadd(amm7,bmm0,omm7); + // row 8 + omm8 = fmadd(amm8,bmm0,omm8); + // row 9 + omm9 = fmadd(amm9,bmm0,omm9); + } + + // Not all aside from the last vector + // needs to be mask stored, however clang + // just does not like +#ifdef FASTOR_HAS_AVX512_MASKS + omm0.mask_store(&out[(j )*N],mask,false); + omm1.mask_store(&out[(j+1)*N],mask,false); + omm2.mask_store(&out[(j+2)*N],mask,false); + omm3.mask_store(&out[(j+3)*N],mask,false); + omm4.mask_store(&out[(j+4)*N],mask,false); + omm5.mask_store(&out[(j+5)*N],mask,false); + omm6.mask_store(&out[(j+6)*N],mask,false); + omm7.mask_store(&out[(j+7)*N],mask,false); + omm8.mask_store(&out[(j+8)*N],mask,false); + omm9.mask_store(&out[(j+9)*N],mask,false); +#else + maskstore(&out[(j )*N],maska,omm0); + maskstore(&out[(j+1)*N],maska,omm1); + maskstore(&out[(j+2)*N],maska,omm2); + maskstore(&out[(j+3)*N],maska,omm3); + maskstore(&out[(j+4)*N],maska,omm4); + maskstore(&out[(j+5)*N],maska,omm5); + maskstore(&out[(j+6)*N],maska,omm6); + maskstore(&out[(j+7)*N],maska,omm7); + maskstore(&out[(j+8)*N],maska,omm8); + maskstore(&out[(j+9)*N],maska,omm9); +#endif + } + +#ifdef FASTOR_HAS_AVX512_MASKS + matmul_mk_uptosimd_remainder_kernel(j,mask,a,b,out); +#else + matmul_mk_uptosimd_remainder_kernel(j,maska,a,b,out); +#endif +} +//----------------------------------------------------------------------------------------------------------- + + + + + + + + + + + + + + +// Take care of N==V::Size +//----------------------------------------------------------------------------------------------------------- +template = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + const V amm6(a[(j+6)*K]); + const V amm7(a[(j+7)*K]); + const V amm8(a[(j+8)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + // row 6 + V omm6(amm6*bmm0); + // row 7 + V omm7(amm7*bmm0); + // row 8 + V omm8(amm8*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + const V amm6(a[(j+6)*K]); + const V amm7(a[(j+7)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + // row 6 + V omm6(amm6*bmm0); + // row 7 + V omm7(amm7*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + const V amm6(a[(j+6)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + // row 6 + V omm6(amm6*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + const V amm5(a[(j+5)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + // row 5 + V omm5(amm5*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + // row 4 + V omm4(amm4*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + // row 3 + V omm3(amm3*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + // row 2 + V omm2(amm2*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + + // row 0 + V omm0(amm0*bmm0); + // row 1 + V omm1(amm1*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + + const V bmm0(&b[0],false); + + const V amm0(a[(j )*K]); + + // row 0 + V omm0(amm0*bmm0); + + for (size_t i=1; i = false> +FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, + const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { + return; +} + + +template,N>::type::Size,bool> = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + using V = typename choose_best_simd_type,N>::type; + + // Unroll a by 10 + constexpr size_t unrollOuterloop = 10UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + + size_t j=0; + for (; j(j,a,b,out); +} +//----------------------------------------------------------------------------------------------------------- + + + + +// Take care of [V::Size < N < 2*V::Size] +// The function implements standard loop unrolling over M. It uses conditional +// loads and store using masks and requires at least AVX. The efficiency of the method comes from +// the fact that it attempts to achieve exact two FMA per load. Both GCC and Clang emit excellent +// code for this at O3 +// A recursive implementation of this using compile time unrolling is available at: +// https://gist.github.com/romeric/a176e28127a8348c3c37c5a369051451 +//----------------------------------------------------------------------------------------------------------- +template,N>::type::Size>::value && + is_greater,N>::type::Size>::value),bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + using V = typename internal::choose_best_simd_type,N>::type; + // We unroll a by 5 and load 2 simd wide columns of b to get two FMA per load + // Unrolling by 5 does not hurt as the fall back cases 4,3,2,1 are also explicitly + // unrolled + constexpr size_t unrollOuterloop = 5UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + int maska[V::Size]; + std::fill(maska,&maska[V::Size], -1); + for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; +#ifdef FASTOR_HAS_AVX512_MASKS + const auto mask = array_to_mask(maska); + V bmm1; +#endif + + size_t j=0; + for (; j(&b[V::Size],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + const V amm4(a[(j+4)*K]); + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + // row 1 + V omm2(amm1*bmm0); + V omm3(amm1*bmm1); + // row 2 + V omm4(amm2*bmm0); + V omm5(amm2*bmm1); + // row 3 + V omm6(amm3*bmm0); + V omm7(amm3*bmm1); + // row 4 + V omm8(amm4*bmm0); + V omm9(amm4*bmm1); + + for (size_t i=1; i(&b[i*N+V::Size],maska)); +#endif + + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + const V amm4(a[(j+4)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + // row 1 + omm2 = fmadd(amm1,bmm0,omm2); + omm3 = fmadd(amm1,bmm1,omm3); + // row 2 + omm4 = fmadd(amm2,bmm0,omm4); + omm5 = fmadd(amm2,bmm1,omm5); + // row 3 + omm6 = fmadd(amm3,bmm0,omm6); + omm7 = fmadd(amm3,bmm1,omm7); + // row 4 + omm8 = fmadd(amm4,bmm0,omm8); + omm9 = fmadd(amm4,bmm1,omm9); + } + + omm0.store(&out[(j )*N],false); + omm1.store(&out[(j )*N+V::Size],false); + omm2.store(&out[(j+1)*N],false); + omm3.store(&out[(j+1)*N+V::Size],false); + omm4.store(&out[(j+2)*N],false); + omm5.store(&out[(j+2)*N+V::Size],false); + omm6.store(&out[(j+3)*N],false); + omm7.store(&out[(j+3)*N+V::Size],false); + omm8.store(&out[(j+4)*N],false); +#ifdef FASTOR_HAS_AVX512_MASKS + omm9.mask_store(&out[(j+4)*N+V::Size],mask,false); +#else + maskstore(&out[(j+4)*N+V::Size],maska,omm9); +#endif + } + + // Remainder M-M0 rows + // Explicitly unroll remaining loops, there is going to be atmost 4 + FASTOR_IF_CONSTEXPR (M-M0==4) { + const V bmm0(&b[0], false); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm1.mask_load(&b[V::Size],mask,false); +#else + const V bmm1(maskload(&b[V::Size],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + const V amm3(a[(j+3)*K]); + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + // row 1 + V omm2(amm1*bmm0); + V omm3(amm1*bmm1); + // row 2 + V omm4(amm2*bmm0); + V omm5(amm2*bmm1); + // row 3 + V omm6(amm3*bmm0); + V omm7(amm3*bmm1); + + for (size_t i=1; i(&b[i*N+V::Size],maska)); +#endif + + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + // row 1 + omm2 = fmadd(amm1,bmm0,omm2); + omm3 = fmadd(amm1,bmm1,omm3); + // row 2 + omm4 = fmadd(amm2,bmm0,omm4); + omm5 = fmadd(amm2,bmm1,omm5); + // row 3 + omm6 = fmadd(amm3,bmm0,omm6); + omm7 = fmadd(amm3,bmm1,omm7); + } + + omm0.store(&out[(j )*N],false); + omm1.store(&out[(j )*N+V::Size],false); + omm2.store(&out[(j+1)*N],false); + omm3.store(&out[(j+1)*N+V::Size],false); + omm4.store(&out[(j+2)*N],false); + omm5.store(&out[(j+2)*N+V::Size],false); + omm6.store(&out[(j+3)*N],false); +#ifdef FASTOR_HAS_AVX512_MASKS + omm7.mask_store(&out[(j+3)*N+V::Size],mask,false); +#else + maskstore(&out[(j+3)*N+V::Size],maska,omm7); +#endif + } + + else FASTOR_IF_CONSTEXPR (M-M0==3) { + const V bmm0(&b[0], false); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm1.mask_load(&b[V::Size],mask,false); +#else + const V bmm1(maskload(&b[V::Size],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + // row 1 + V omm2(amm1*bmm0); + V omm3(amm1*bmm1); + // row 2 + V omm4(amm2*bmm0); + V omm5(amm2*bmm1); + + for (size_t i=1; i(&b[i*N+V::Size],maska)); +#endif + + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + // row 1 + omm2 = fmadd(amm1,bmm0,omm2); + omm3 = fmadd(amm1,bmm1,omm3); + // row 2 + omm4 = fmadd(amm2,bmm0,omm4); + omm5 = fmadd(amm2,bmm1,omm5); + } + + omm0.store(&out[(j )*N],false); + omm1.store(&out[(j )*N+V::Size],false); + omm2.store(&out[(j+1)*N],false); + omm3.store(&out[(j+1)*N+V::Size],false); + omm4.store(&out[(j+2)*N],false); +#ifdef FASTOR_HAS_AVX512_MASKS + omm5.mask_store(&out[(j+2)*N+V::Size],mask,false); +#else + maskstore(&out[(j+2)*N+V::Size],maska,omm5); +#endif + } + + else FASTOR_IF_CONSTEXPR (M-M0==2) { + const V bmm0(&b[0], false); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm1.mask_load(&b[V::Size],mask,false); +#else + const V bmm1(maskload(&b[V::Size],maska)); +#endif + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + // row 1 + V omm2(amm1*bmm0); + V omm3(amm1*bmm1); + + for (size_t i=1; i(&b[i*N+V::Size],maska)); +#endif + + const V amm0(a[(j )*K+i]); + const V amm1(a[(j+1)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + // row 1 + omm2 = fmadd(amm1,bmm0,omm2); + omm3 = fmadd(amm1,bmm1,omm3); + } + + omm0.store(&out[(j )*N],false); + omm1.store(&out[(j )*N+V::Size],false); + omm2.store(&out[(j+1)*N],false); +#ifdef FASTOR_HAS_AVX512_MASKS + omm3.mask_store(&out[(j+1)*N+V::Size],mask,false); +#else + maskstore(&out[(j+1)*N+V::Size],maska,omm3); +#endif + } + + else FASTOR_IF_CONSTEXPR (M-M0==1) { + const V bmm0(&b[0], false); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm1.mask_load(&b[V::Size],mask,false); +#else + const V bmm1(maskload(&b[V::Size],maska)); +#endif + + const V amm0(a[(j )*K]); + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + + for (size_t i=1; i(&b[i*N+V::Size],maska)); +#endif + + const V amm0(a[(j )*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + } + + omm0.store(&out[(j )*N],false); +#ifdef FASTOR_HAS_AVX512_MASKS + omm1.mask_store(&out[(j)*N+V::Size],mask,false); +#else + maskstore(&out[(j)*N+V::Size],maska,omm1); +#endif + } +} +//----------------------------------------------------------------------------------------------------------- + + + +// Take care of 2*V::Size cases +//----------------------------------------------------------------------------------------------------------- +template,N>::type::Size,bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + using V = typename internal::choose_best_simd_type,N>::type; + // We unroll a by 5 and load 2 simd wide columns of b to get two FMA per load + // Unrolling by 5 does not hurt as the fall back cases 4,3,2,1 are also explicitly + // unrolled + constexpr size_t unrollOuterloop = 5UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + + size_t j=0; + for (; j,N>::type::Size>::value && + is_less,N>::type::Size>::value),bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + // Unrolling by 4 to get 12 independent fma + using V = typename internal::choose_best_simd_type,N>::type; + constexpr size_t unrollOuterloop = 4UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + // constexpr size_t remainder = M < unrollOuterloop ? 0 : M0-unrollOuterloop; + constexpr bool isBAligned = false; + constexpr bool isCAligned = false; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + int maska[V::Size]; + std::fill(maska,&maska[V::Size], -1); + for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; +#ifdef FASTOR_HAS_AVX512_MASKS + const auto mask = array_to_mask(maska); + V bmm2; +#endif + + size_t j=0; + for (; j(&b[2*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + // row 1 + V omm3(amm1*bmm0); + V omm4(amm1*bmm1); + V omm5(amm1*bmm2); + // row 2 + V omm6(amm2*bmm0); + V omm7(amm2*bmm1); + V omm8(amm2*bmm2); + // row 3 + V omm9 (amm3*bmm0); + V omm10(amm3*bmm1); + V omm11(amm3*bmm2); + + for (size_t i=1; i(&b[i*N+2*V::Size],maska)); +#endif + + const V amm0(a[j*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + const V amm3(a[(j+3)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + // row 1 + omm3 = fmadd(amm1,bmm0,omm3); + omm4 = fmadd(amm1,bmm1,omm4); + omm5 = fmadd(amm1,bmm2,omm5); + // row 2 + omm6 = fmadd(amm2,bmm0,omm6); + omm7 = fmadd(amm2,bmm1,omm7); + omm8 = fmadd(amm2,bmm2,omm8); + // row 3 + omm9 = fmadd(amm3,bmm0,omm9); + omm10 = fmadd(amm3,bmm1,omm10); + omm11 = fmadd(amm3,bmm2,omm11); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); + + omm3.store(&out[(j+1)*N],isCAligned); + omm4.store(&out[(j+1)*N+V::Size],isCAligned); + omm5.store(&out[(j+1)*N+2*V::Size],isCAligned); + + omm6.store(&out[(j+2)*N],isCAligned); + omm7.store(&out[(j+2)*N+V::Size],isCAligned); + omm8.store(&out[(j+2)*N+2*V::Size],isCAligned); + + omm9.store(&out[(j+3)*N],isCAligned); + omm10.store(&out[(j+3)*N+V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm11.mask_store(&out[(j+3)*N+2*V::Size],mask,false); +#else + maskstore(&out[(j+3)*N+2*V::Size],maska,omm11); +#endif + } + + FASTOR_IF_CONSTEXPR (M-M0==3) { + const V amm0(a[j*K]); + const V amm1(a[(j+1)*K]); + const V amm2(a[(j+2)*K]); + + const V bmm0(&b[0], isBAligned); + const V bmm1((&b[V::Size]),isBAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm2.mask_load(&b[2*V::Size],mask,false); +#else + const V bmm2(maskload(&b[2*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + // row 1 + V omm3(amm1*bmm0); + V omm4(amm1*bmm1); + V omm5(amm1*bmm2); + // row 2 + V omm6(amm2*bmm0); + V omm7(amm2*bmm1); + V omm8(amm2*bmm2); + + for (size_t i=1; i(&b[i*N+2*V::Size],maska)); +#endif + + const V amm0(a[j*K+i]); + const V amm1(a[(j+1)*K+i]); + const V amm2(a[(j+2)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + // row 1 + omm3 = fmadd(amm1,bmm0,omm3); + omm4 = fmadd(amm1,bmm1,omm4); + omm5 = fmadd(amm1,bmm2,omm5); + // row 2 + omm6 = fmadd(amm2,bmm0,omm6); + omm7 = fmadd(amm2,bmm1,omm7); + omm8 = fmadd(amm2,bmm2,omm8); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); + + omm3.store(&out[(j+1)*N],isCAligned); + omm4.store(&out[(j+1)*N+V::Size],isCAligned); + omm5.store(&out[(j+1)*N+2*V::Size],isCAligned); + + omm6.store(&out[(j+2)*N],isCAligned); + omm7.store(&out[(j+2)*N+V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm8.mask_store(&out[(j+2)*N+2*V::Size],mask,false); +#else + maskstore(&out[(j+2)*N+2*V::Size],maska,omm8); +#endif + } + + else FASTOR_IF_CONSTEXPR (M-M0==2) { + + const V amm0(a[j*K]); + const V amm1(a[(j+1)*K]); + + const V bmm0(&b[0], isBAligned); + const V bmm1((&b[V::Size]),isBAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm2.mask_load(&b[2*V::Size],mask,false); +#else + const V bmm2(maskload(&b[2*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + // row 1 + V omm3(amm1*bmm0); + V omm4(amm1*bmm1); + V omm5(amm1*bmm2); + + for (size_t i=1; i(&b[i*N+2*V::Size],maska)); +#endif + + const V amm0(a[j*K+i]); + const V amm1(a[(j+1)*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + // row 1 + omm3 = fmadd(amm1,bmm0,omm3); + omm4 = fmadd(amm1,bmm1,omm4); + omm5 = fmadd(amm1,bmm2,omm5); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); + + omm3.store(&out[(j+1)*N],isCAligned); + omm4.store(&out[(j+1)*N+V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm5.mask_store(&out[(j+1)*N+2*V::Size],mask,false); +#else + maskstore(&out[(j+1)*N+2*V::Size],maska,omm5); +#endif + } + + FASTOR_IF_CONSTEXPR (M-M0==1) { + + const V amm0(a[j*K]); + + const V bmm0(&b[0], isBAligned); + const V bmm1((&b[V::Size]),isBAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm2.mask_load(&b[2*V::Size],mask,false); +#else + const V bmm2(maskload(&b[2*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + + for (size_t i=1; i(&b[i*N+2*V::Size],maska)); +#endif + + const V amm0(a[j*K+i]); + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + +#ifdef FASTOR_HAS_AVX512_MASKS + omm2.mask_store(&out[j*N+2*V::Size],mask,false); +#else + maskstore(&out[j*N+2*V::Size],maska,omm2); +#endif + } +} +//----------------------------------------------------------------------------------------------------------- + + + +// Take care of 3*V::Size cases +// Note that you get the exact same performance by tuning _matmul_base parameters as +// [unrollOuterloop = 2, nSIMDRows=1 and nSIMDCols=3] however these parameters affect +// the universal behaviour of that method specially for big matrices +// This method unrolls M by 2 only as when N==3*V::Size unrolling the entire column of c +// and doing 2 rows at a time is the most benificial. Unrolling M any further hurts the +// performance really bad +//----------------------------------------------------------------------------------------------------------- +template,N>::type::Size,bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + // Unrolling by 4 to get 12 independent fma + using V = typename internal::choose_best_simd_type,N>::type; + constexpr size_t unrollOuterloop = 4UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + constexpr bool isBAligned = false; + constexpr bool isCAligned = false; + + size_t j=0; + for (; j,N>::type::Size>::value && + is_less,N>::type::Size>::value),bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + using V = typename internal::choose_best_simd_type,N>::type; + constexpr size_t unrollOuterloop = 3UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + constexpr bool isBAligned = false; + constexpr bool isCAligned = false; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + int maska[V::Size]; + std::fill(maska,&maska[V::Size], -1); + for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; +#ifdef FASTOR_HAS_AVX512_MASKS + const auto mask = array_to_mask(maska); + V bmm3; +#endif + + size_t j=0; + for (; j(&b[3*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + V omm3(amm0*bmm3); + // row 1 + V omm4(amm1*bmm0); + V omm5(amm1*bmm1); + V omm6(amm1*bmm2); + V omm7(amm1*bmm3); + // row 2 + V omm8 (amm2*bmm0); + V omm9 (amm2*bmm1); + V omm10(amm2*bmm2); + V omm11(amm2*bmm3); + + for (size_t i=1; i(&b[i*N+3*V::Size],maska)); +#endif + + const V amm0 = a[(j )*K+i]; + const V amm1 = a[(j+1)*K+i]; + const V amm2 = a[(j+2)*K+i]; + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + omm3 = fmadd(amm0,bmm3,omm3); + // row 1 + omm4 = fmadd(amm1,bmm0,omm4); + omm5 = fmadd(amm1,bmm1,omm5); + omm6 = fmadd(amm1,bmm2,omm6); + omm7 = fmadd(amm1,bmm3,omm7); + // row 2 + omm8 = fmadd(amm2,bmm0,omm8); + omm9 = fmadd(amm2,bmm1,omm9); + omm10 = fmadd(amm2,bmm2,omm10); + omm11 = fmadd(amm2,bmm3,omm11); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); + omm3.store(&out[j*N+3*V::Size],isCAligned); + + omm4.store(&out[(j+1)*N],isCAligned); + omm5.store(&out[(j+1)*N+V::Size],isCAligned); + omm6.store(&out[(j+1)*N+2*V::Size],isCAligned); + omm7.store(&out[(j+1)*N+3*V::Size],isCAligned); + + omm8.store (&out[(j+2)*N],isCAligned); + omm9.store(&out[(j+2)*N+V::Size],isCAligned); + omm10.store(&out[(j+2)*N+2*V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm11.mask_store(&out[(j+2)*N+3*V::Size],mask,false); +#else + maskstore(&out[(j+2)*N+3*V::Size],maska,omm11); +#endif + } + + FASTOR_IF_CONSTEXPR (M-M0==2) { + + const V amm0(a[(j )*K]); + const V amm1(a[(j+1)*K]); + + const V bmm0(&b[0], isBAligned); + const V bmm1(&b[V::Size],isBAligned); + const V bmm2(&b[2*V::Size],isBAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm3.mask_load(&b[3*V::Size],mask,false); +#else + const V bmm3(maskload(&b[3*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + V omm3(amm0*bmm3); + // row 1 + V omm4(amm1*bmm0); + V omm5(amm1*bmm1); + V omm6(amm1*bmm2); + V omm7(amm1*bmm3); + + for (size_t i=1; i(&b[i*N+3*V::Size],maska)); +#endif + + const V amm0 = a[(j )*K+i]; + const V amm1 = a[(j+1)*K+i]; + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + omm3 = fmadd(amm0,bmm3,omm3); + // row 1 + omm4 = fmadd(amm1,bmm0,omm4); + omm5 = fmadd(amm1,bmm1,omm5); + omm6 = fmadd(amm1,bmm2,omm6); + omm7 = fmadd(amm1,bmm3,omm7); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); + omm3.store(&out[j*N+3*V::Size],isCAligned); + + omm4.store(&out[(j+1)*N],isCAligned); + omm5.store(&out[(j+1)*N+V::Size],isCAligned); + omm6.store(&out[(j+1)*N+2*V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm7.mask_store(&out[(j+1)*N+3*V::Size],mask,false); +#else + maskstore(&out[(j+1)*N+3*V::Size],maska,omm7); +#endif + } + + else FASTOR_IF_CONSTEXPR (M-M0==1) { + + const V amm0(a[(j )*K]); + + const V bmm0(&b[0], isBAligned); + const V bmm1(&b[V::Size],isBAligned); + const V bmm2(&b[2*V::Size],isBAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm3.mask_load(&b[3*V::Size],mask,false); +#else + const V bmm3(maskload(&b[3*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + V omm3(amm0*bmm3); + + for (size_t i=1; i(&b[i*N+3*V::Size],maska)); +#endif + + const V amm0 = a[(j )*K+i]; + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + omm3 = fmadd(amm0,bmm3,omm3); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm3.mask_store(&out[(j)*N+3*V::Size],mask,false); +#else + maskstore(&out[(j)*N+3*V::Size],maska,omm3); +#endif + } +} +//----------------------------------------------------------------------------------------------------------- + + + + +// Take care of 4*V::Size cases +// Note that you get the exact same performance by tuning _matmul_base parameters as +// [unrollOuterloop = 2, nSIMDRows=1 and nSIMDCols=4] however these parameters affect +// the universal behaviour of that method specially for big matrices +// This method unrolls M by 2 only as when N==4*V::Size unrolling the entire column of c +// and doing 2 rows at a time is the most benificial. Unrolling M any further hurts the +// performance really bad +//----------------------------------------------------------------------------------------------------------- +template,N>::type::Size,bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + + using V = typename internal::choose_best_simd_type,N>::type; + constexpr size_t unrollOuterloop = 3UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + constexpr bool isBAligned = false; + constexpr bool isCAligned = false; + + size_t j=0; + for (; j,N>::type::Size>::value && + is_less,N>::type::Size>::value),bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + + using V = typename internal::choose_best_simd_type,N>::type; + constexpr size_t unrollOuterloop = 2UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + // constexpr size_t remainder = M < unrollOuterloop ? 0 : M0-unrollOuterloop; + constexpr bool isBAligned = false; + constexpr bool isCAligned = false; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + int maska[V::Size]; + std::fill(maska,&maska[V::Size], -1); + for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; +#ifdef FASTOR_HAS_AVX512_MASKS + const auto mask = array_to_mask(maska); + V bmm4; +#endif + + size_t j=0; + for (; j(&b[4*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + V omm3(amm0*bmm3); + V omm4(amm0*bmm4); + // row 1 + V omm5(amm1*bmm0); + V omm6(amm1*bmm1); + V omm7(amm1*bmm2); + V omm8(amm1*bmm3); + V omm9(amm1*bmm4); + + + for (size_t i=1; i(&b[i*N+4*V::Size],maska)); +#endif + + const V amm0 = a[j*K+i]; + const V amm1 = a[(j+1)*K+i]; + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + omm3 = fmadd(amm0,bmm3,omm3); + omm4 = fmadd(amm0,bmm4,omm4); + // row 1 + omm5 = fmadd(amm1,bmm0,omm5); + omm6 = fmadd(amm1,bmm1,omm6); + omm7 = fmadd(amm1,bmm2,omm7); + omm8 = fmadd(amm1,bmm3,omm8); + omm9 = fmadd(amm1,bmm4,omm9); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); + omm3.store(&out[j*N+3*V::Size],isCAligned); + omm4.store(&out[j*N+4*V::Size],isCAligned); + + omm5.store(&out[(j+1)*N],isCAligned); + omm6.store(&out[(j+1)*N+V::Size],isCAligned); + omm7.store(&out[(j+1)*N+2*V::Size],isCAligned); + omm8.store(&out[(j+1)*N+3*V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm9.mask_store(&out[(j+1)*N+4*V::Size],mask,false); +#else + maskstore(&out[(j+1)*N+4*V::Size],maska,omm9); +#endif + } + + FASTOR_IF_CONSTEXPR (M-M0==1) { + V amm0(a[j*K]); + + const V bmm0(&b[0], isBAligned); + const V bmm1((&b[V::Size]),isBAligned); + const V bmm2((&b[2*V::Size]),isBAligned); + const V bmm3((&b[3*V::Size]),isBAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + bmm4.mask_load(&b[4*V::Size],mask,false); +#else + const V bmm4(maskload(&b[4*V::Size],maska)); +#endif + + // row 0 + V omm0(amm0*bmm0); + V omm1(amm0*bmm1); + V omm2(amm0*bmm2); + V omm3(amm0*bmm3); + V omm4(amm0*bmm4); + + for (size_t i=1; i(&b[i*N+4*V::Size],maska)); +#endif + + const V amm0 = a[j*K+i]; + + // row 0 + omm0 = fmadd(amm0,bmm0,omm0); + omm1 = fmadd(amm0,bmm1,omm1); + omm2 = fmadd(amm0,bmm2,omm2); + omm3 = fmadd(amm0,bmm3,omm3); + omm4 = fmadd(amm0,bmm4,omm4); + } + + omm0.store(&out[j*N],isCAligned); + omm1.store(&out[j*N+V::Size],isCAligned); + omm2.store(&out[j*N+2*V::Size],isCAligned); + omm3.store(&out[j*N+3*V::Size],isCAligned); +#ifdef FASTOR_HAS_AVX512_MASKS + omm4.mask_store(&out[j*N+4*V::Size],mask,false); +#else + maskstore(&out[j*N+4*V::Size],maska,omm4); +#endif + } +} +//----------------------------------------------------------------------------------------------------------- + + + +// N==5*V::Size case +//----------------------------------------------------------------------------------------------------------- +template,N>::type::Size,bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + + using V = typename internal::choose_best_simd_type,N>::type; + constexpr size_t unrollOuterloop = 2UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + // constexpr size_t remainder = M < unrollOuterloop ? 0 : M0-unrollOuterloop; + constexpr bool isBAligned = false; + constexpr bool isCAligned = false; + + size_t j=0; + for (; j,N>::type::Size>::value,bool>::type = 0> +FASTOR_INLINE +void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + _matmul_base_masked(a,b,out); +} + + + +} // internal + +} // Fastor + + +#endif // MATMUL_MK_SMALLODDN_H diff --git a/noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h b/noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h new file mode 100644 index 00000000..fb916f41 --- /dev/null +++ b/noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h @@ -0,0 +1,1187 @@ +#ifndef MATMUL_SPECIALISATIONS_KERNELS_H +#define MATMUL_SPECIALISATIONS_KERNELS_H + + +// Forward declare +//----------------------------------------------------------------------------------------------------------- +namespace internal { +template::value,bool>::type = 0 > +FASTOR_INLINE +void _matmul8k8_float(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out); + +template::value,bool>::type = 0 > +FASTOR_INLINE +void _matmul8k8_double(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out); + +} // internal +//----------------------------------------------------------------------------------------------------------- + + + +#ifdef FASTOR_SSE2_IMPL +// (2xk) x (kx2) matrices +template::value),bool>::type = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m128d out_row0 = ZEROPD; + __m128d out_row1 = ZEROPD; + + for (size_t i=0; i::value),bool>::type = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m128 out_row0 = ZEROPS; + __m128 out_row1 = ZEROPS; + + __m128 brow = ZEROPS; + for (size_t i=0; i::type = 0> +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + internal::_matmul_base(a,b,out); +} +#endif + +#ifdef FASTOR_AVX_IMPL +// (3xk) x (kx3) matrices +template::value),bool>::type = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m256d out_row0 = VZEROPD; + __m256d out_row1 = VZEROPD; + __m256d out_row2 = VZEROPD; + + for (size_t i=0; i::value,bool>::type = 0> +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + internal::_matmul_base(a,b,out); +} +#endif + +#ifdef FASTOR_SSE2_IMPL +// (3xk) x (kx3) matrices +template::value),bool>::type = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m128 out_row0 = ZEROPS; + __m128 out_row1 = ZEROPS; + __m128 out_row2 = ZEROPS; + + for (size_t i=0; i::value),bool>::type = 0> +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + internal::_matmul_base(a,b,out); +} +#endif + +#ifdef FASTOR_AVX_IMPL +// (4xk) x (kx4) matrices +template::value),bool>::type = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m256d out_row0 = VZEROPD; + __m256d out_row1 = VZEROPD; + __m256d out_row2 = VZEROPD; + __m256d out_row3 = VZEROPD; + + for (size_t i=0; i::value,bool>::type = 0> +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + internal::_matmul_base(a,b,out); +} +#endif + +#ifdef FASTOR_SSE2_IMPL +// (4xk) x (kx4) matrices +template::value),bool>::type = 0> +FASTOR_INLINE +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m128 out_row0 = ZEROPS; + __m128 out_row1 = ZEROPS; + __m128 out_row2 = ZEROPS; + __m128 out_row3 = ZEROPS; + + for (size_t i=0; i::value,bool>::type = 0> +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + internal::_matmul_base(a,b,out); +} +#endif + + +// (8xk) x (kx8) matrices +template::value,bool>::type = 0> +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { +#ifdef FASTOR_AVX512_IMPL + FASTOR_IF_CONSTEXPR(K<=64) + internal::_matmul8k8_double(a,b,out); + else + internal::_matmul_mk_smalln(a,b,out); +#else + internal::_matmul_mk_smalln(a,b,out); +#endif +} + +template::value,bool>::type = 0> +void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { +#ifdef FASTOR_AVX_IMPL + internal::_matmul8k8_float(a,b,out); +#else + internal::_matmul_base(a,b,out); +#endif +} + + + + + +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE +void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + + // Ivy 17 - Sky 16 [12 if fused] OPS + __m128 ar = _mm_loadu_ps(a); + __m128 br = _mm_loadu_ps(b); + __m128 ar0 = _mm_shuffle_ps(ar,ar,_MM_SHUFFLE(2,2,0,0)); + __m128 ar1 = _mm_shuffle_ps(ar,ar,_MM_SHUFFLE(3,3,1,1)); + __m128 br0 = _mm_shuffle_ps(br,br,_MM_SHUFFLE(1,0,1,0)); + __m128 br1 = _mm_shuffle_ps(br,br,_MM_SHUFFLE(3,2,3,2)); + __m128 res = _mm_add_ps(_mm_mul_ps(ar0,br0),_mm_mul_ps(ar1,br1)); + _mm_storeu_ps(out,res); +} + +template<> +FASTOR_INLINE +void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + + // Ivy 63 - HW 45 - Sky 36 OPS + // Fully vectorised approach that reduces + // (27 scalar mul + 18 scalar add) to + // (9 SSE mul + 6 SEE add) [non-FMA] + // (3 SSE mul + 6 SEE FMA) [ FMA] + + __m128 bmm0 = _mm_loadu_ps (b ); + __m128 bmm1 = _mm_loadu_ps (b+3); + __m128 bmm2 = _mm_loadul3_ps(b+6); + + __m128 omm0(_mm_mul_ps(_mm_set1_ps(a[0]),bmm0)); + __m128 omm1(_mm_mul_ps(_mm_set1_ps(a[3]),bmm0)); + __m128 omm2(_mm_mul_ps(_mm_set1_ps(a[6]),bmm0)); +#ifndef FASTOR_FMA_IMPL + omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[1]),bmm1)); + omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[4]),bmm1)); + omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[7]),bmm1)); + + omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[2]),bmm2)); + omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[5]),bmm2)); + omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[8]),bmm2)); +#else + omm0 = _mm_fmadd_ps(_mm_set1_ps(a[1]),bmm1,omm0); + omm1 = _mm_fmadd_ps(_mm_set1_ps(a[4]),bmm1,omm1); + omm2 = _mm_fmadd_ps(_mm_set1_ps(a[7]),bmm1,omm2); + + omm0 = _mm_fmadd_ps(_mm_set1_ps(a[2]),bmm2,omm0); + omm1 = _mm_fmadd_ps(_mm_set1_ps(a[5]),bmm2,omm1); + omm2 = _mm_fmadd_ps(_mm_set1_ps(a[8]),bmm2,omm2); +#endif + + _mm_storeu_ps (out , omm0); + _mm_storeu_ps (out+3, omm1); + _mm_storeul3_ps(out+6, omm2); +} + +template<> +FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + + // Ivy 116 - HW 80 - Sky 64 OPS + // Fully vectorised approach that reduces + // (64 scalar mul + 48 scalar add) to + // (16 SSE mul + 12 SEE add) [non-FMA] + // (4 SSE mul + 12 SEE FMA) [ FMA] + + __m128 bmm0 = _mm_loadu_ps (b ); + __m128 bmm1 = _mm_loadu_ps (b+4 ); + __m128 bmm2 = _mm_loadu_ps (b+8 ); + __m128 bmm3 = _mm_loadu_ps (b+12); + + __m128 omm0(_mm_mul_ps(_mm_set1_ps(a[0 ]),bmm0)); + __m128 omm1(_mm_mul_ps(_mm_set1_ps(a[4 ]),bmm0)); + __m128 omm2(_mm_mul_ps(_mm_set1_ps(a[8 ]),bmm0)); + __m128 omm3(_mm_mul_ps(_mm_set1_ps(a[12]),bmm0)); +#ifndef FASTOR_FMA_IMPL + omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[1 ]),bmm1)); + omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[5 ]),bmm1)); + omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[9 ]),bmm1)); + omm3 = _mm_add_ps(omm3,_mm_mul_ps(_mm_set1_ps(a[13]),bmm1)); + + omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[2 ]),bmm2)); + omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[6 ]),bmm2)); + omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[10]),bmm2)); + omm3 = _mm_add_ps(omm3,_mm_mul_ps(_mm_set1_ps(a[14]),bmm2)); + + omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[3 ]),bmm3)); + omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[7 ]),bmm3)); + omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[11]),bmm3)); + omm3 = _mm_add_ps(omm3,_mm_mul_ps(_mm_set1_ps(a[15]),bmm3)); +#else + omm0 = _mm_fmadd_ps(_mm_set1_ps(a[1 ]),bmm1,omm0); + omm1 = _mm_fmadd_ps(_mm_set1_ps(a[5 ]),bmm1,omm1); + omm2 = _mm_fmadd_ps(_mm_set1_ps(a[9 ]),bmm1,omm2); + omm3 = _mm_fmadd_ps(_mm_set1_ps(a[13]),bmm1,omm3); + + omm0 = _mm_fmadd_ps(_mm_set1_ps(a[2 ]),bmm2,omm0); + omm1 = _mm_fmadd_ps(_mm_set1_ps(a[6 ]),bmm2,omm1); + omm2 = _mm_fmadd_ps(_mm_set1_ps(a[10]),bmm2,omm2); + omm3 = _mm_fmadd_ps(_mm_set1_ps(a[14]),bmm2,omm3); + + omm0 = _mm_fmadd_ps(_mm_set1_ps(a[3 ]),bmm3,omm0); + omm1 = _mm_fmadd_ps(_mm_set1_ps(a[7 ]),bmm3,omm1); + omm2 = _mm_fmadd_ps(_mm_set1_ps(a[11]),bmm3,omm2); + omm3 = _mm_fmadd_ps(_mm_set1_ps(a[15]),bmm3,omm3); +#endif + + _mm_storeu_ps (out , omm0); + _mm_storeu_ps (out+4 , omm1); + _mm_storeu_ps (out+8 , omm2); + _mm_storeu_ps (out+12, omm3); +} +#endif + +#ifdef FASTOR_AVX_IMPL +namespace internal { +// This is the common interface for 8k8 matmul and not only for 888 so do not +// make it specific to 888 floats +template::value,bool>::type> +FASTOR_INLINE +void _matmul8k8_float(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m256 out_row0 = VZEROPS; + __m256 out_row1 = VZEROPS; + __m256 out_row2 = VZEROPS; + __m256 out_row3 = VZEROPS; + __m256 out_row4 = VZEROPS; + __m256 out_row5 = VZEROPS; + __m256 out_row6 = VZEROPS; + __m256 out_row7 = VZEROPS; + + for (size_t i=0; i +FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + internal::_matmul8k8_float(a,b,out); + return; +} +#endif + + +#ifdef FASTOR_AVX512_IMPL +namespace internal { +template::value,bool>::type> +FASTOR_INLINE +void _matmul8k8_double(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + __m512d out_row0 = _mm512_setzero_pd(); + __m512d out_row1 = _mm512_setzero_pd(); + __m512d out_row2 = _mm512_setzero_pd(); + __m512d out_row3 = _mm512_setzero_pd(); + __m512d out_row4 = _mm512_setzero_pd(); + __m512d out_row5 = _mm512_setzero_pd(); + __m512d out_row6 = _mm512_setzero_pd(); + __m512d out_row7 = _mm512_setzero_pd(); + + for (size_t i=0; i +FASTOR_INLINE void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + internal::_matmul8k8_double(a,b,out); + return; +} +#endif + + +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE +void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + + // Ivy 26 - HW 20 - Sky 16 OPS + + __m128d bmm0 = _mm_loadu_pd (b ); + __m128d bmm1 = _mm_loadu_pd (b+2); + + __m128d omm0(_mm_mul_pd(_mm_set1_pd(a[0]),bmm0)); + __m128d omm1(_mm_mul_pd(_mm_set1_pd(a[2]),bmm0)); +#ifndef FASTOR_FMA_IMPL + omm0 = _mm_add_pd(omm0,_mm_mul_pd(_mm_set1_pd(a[1]),bmm1)); + omm1 = _mm_add_pd(omm1,_mm_mul_pd(_mm_set1_pd(a[3]),bmm1)); +#else + omm0 = _mm_fmadd_pd(_mm_set1_pd(a[1]),bmm1,omm0); + omm1 = _mm_fmadd_pd(_mm_set1_pd(a[3]),bmm1,omm1); +#endif + _mm_storeu_pd (out , omm0); + _mm_storeu_pd (out+2, omm1); +} +#endif + +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE +void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + + // Ivy 63 - HW 45 - Sky 36 OPS + // Fully vectorised approach that reduces + // (27 scalar mul + 18 scalar add) to + // (9 AVX mul + 6 AVX add) [non-FMA] + // (3 AVX mul + 6 AVX FMA) [ FMA] + + __m256d bmm0 = _mm256_loadu_pd (b ); + __m256d bmm1 = _mm256_loadu_pd (b+3); + __m256d bmm2 = _mm256_loadul3_pd(b+6); + + __m256d omm0(_mm256_mul_pd(_mm256_set1_pd(a[0]),bmm0)); + __m256d omm1(_mm256_mul_pd(_mm256_set1_pd(a[3]),bmm0)); + __m256d omm2(_mm256_mul_pd(_mm256_set1_pd(a[6]),bmm0)); +#ifndef FASTOR_FMA_IMPL + omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[1]),bmm1)); + omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[4]),bmm1)); + omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[7]),bmm1)); + + omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[2]),bmm2)); + omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[5]),bmm2)); + omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[8]),bmm2)); +#else + omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[1]),bmm1,omm0); + omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[4]),bmm1,omm1); + omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[7]),bmm1,omm2); + + omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[2]),bmm2,omm0); + omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[5]),bmm2,omm1); + omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[8]),bmm2,omm2); +#endif + + _mm256_storeu_pd (out , omm0); + _mm256_storeu_pd (out+3, omm1); + _mm256_storeul3_pd(out+6, omm2); +} + +template<> +FASTOR_INLINE +void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + + // Ivy 116 - HW 80 - Sky 64 OPS + // Fully vectorised approach that reduces + // (64 scalar mul + 48 scalar add) to + // (16 SSE mul + 12 SEE add) [non-FMA] + // (4 SSE mul + 12 SEE FMA) [ FMA] + + __m256d bmm0 = _mm256_loadu_pd (b ); + __m256d bmm1 = _mm256_loadu_pd (b+4 ); + __m256d bmm2 = _mm256_loadu_pd (b+8 ); + __m256d bmm3 = _mm256_loadu_pd (b+12); + + __m256d omm0(_mm256_mul_pd(_mm256_set1_pd(a[0 ]),bmm0)); + __m256d omm1(_mm256_mul_pd(_mm256_set1_pd(a[4 ]),bmm0)); + __m256d omm2(_mm256_mul_pd(_mm256_set1_pd(a[8 ]),bmm0)); + __m256d omm3(_mm256_mul_pd(_mm256_set1_pd(a[12]),bmm0)); +#ifndef FASTOR_FMA_IMPL + omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[1 ]),bmm1)); + omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[5 ]),bmm1)); + omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[9 ]),bmm1)); + omm3 = _mm256_add_pd(omm3,_mm256_mul_pd(_mm256_set1_pd(a[13]),bmm1)); + + omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[2 ]),bmm2)); + omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[6 ]),bmm2)); + omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[10]),bmm2)); + omm3 = _mm256_add_pd(omm3,_mm256_mul_pd(_mm256_set1_pd(a[14]),bmm2)); + + omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[3 ]),bmm3)); + omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[7 ]),bmm3)); + omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[11]),bmm3)); + omm3 = _mm256_add_pd(omm3,_mm256_mul_pd(_mm256_set1_pd(a[15]),bmm3)); +#else + omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[1 ]),bmm1,omm0); + omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[5 ]),bmm1,omm1); + omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[9 ]),bmm1,omm2); + omm3 = _mm256_fmadd_pd(_mm256_set1_pd(a[13]),bmm1,omm3); + + omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[2 ]),bmm2,omm0); + omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[6 ]),bmm2,omm1); + omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[10]),bmm2,omm2); + omm3 = _mm256_fmadd_pd(_mm256_set1_pd(a[14]),bmm2,omm3); + + omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[3 ]),bmm3,omm0); + omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[7 ]),bmm3,omm1); + omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[11]),bmm3,omm2); + omm3 = _mm256_fmadd_pd(_mm256_set1_pd(a[15]),bmm3,omm3); +#endif + + _mm256_storeu_pd (out , omm0); + _mm256_storeu_pd (out+4 , omm1); + _mm256_storeu_pd (out+8 , omm2); + _mm256_storeu_pd (out+12, omm3); +} +#endif + + + + + +//! Matrix-vector multiplication +//----------------------------------------------------------------------------------------------------------- +// Don't call this function directly as it's name is unconventional +// It gets called from within matmul anyway so always call matmul +namespace internal { + +template +FASTOR_INLINE +void _matvecmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + using V = typename choose_best_simd_type,N>::type; + constexpr size_t unrollOuterloop = 8UL; + constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + constexpr bool isAAligned = false; + constexpr bool isBAligned = false; + + size_t i=0; + FASTOR_IF_CONSTEXPR(N < V::Size) { + + for (; i +FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + // 11 OPS + __m128 amm = _mm_loadu_ps(a); + __m128 bmm = ZEROPS; + bmm = _mm_loadl_pi(bmm, (__m64*)b); + + __m128 res = _mm_mul_ps(amm,_mm_movelh_ps(bmm,bmm)); + __m128 res2 = _mm_shuffle_ps(res ,res ,_MM_SHUFFLE(2,3,0,1)); + res2 = _mm_add_ps(res,res2); + __m128 res3 = _mm_shuffle_ps(res2,res2,_MM_SHUFFLE(3,1,2,0)); + _mm_storel_pi((__m64*) out,res3); +} + +template<> +FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + // IVY/HW 47 OPS + // Mask loads on AVX and AVX512 otherwise 3 single loads each + __m128 amm0 = _mm_loadul3_ps(a); + __m128 amm1 = _mm_loadul3_ps(&a[3]); + __m128 amm2 = _mm_loadul3_ps(&a[6]); + __m128 bmm = _mm_loadul3_ps(b); + + out[0] = _mm_sum_ps(_mm_mul_ps(amm0,bmm)); + out[1] = _mm_sum_ps(_mm_mul_ps(amm1,bmm)); + out[2] = _mm_sum_ps(_mm_mul_ps(amm2,bmm)); +} + +template<> +FASTOR_INLINE void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + // IVY/HW/SKY 19 OPS + __m128d amm0 = _mm_loadu_pd(a ); + __m128d amm1 = _mm_loadu_pd(a+2); + __m128d bmm0 = _mm_loadu_pd(b ); + + __m128d omm0 = _mm_mul_pd(amm0,bmm0); + __m128d omm1 = _mm_mul_pd(amm1,bmm0); + omm0 = _mm_add_pd(omm0, _mm_reverse_pd(omm0)); + omm1 = _mm_add_pd(omm1, _mm_reverse_pd(omm1)); + + _mm_storeu_pd(out,_mm_shuffle_pd(omm0,omm1,0x1)); +} +#endif + + +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + // SKY 49 - ICE 87 OPS + // Mask loads on AVX and AVX512 + __m256d amm0 = _mm256_loadul3_pd(a); + __m256d amm1 = _mm256_loadul3_pd(&a[3]); + __m256d amm2 = _mm256_loadul3_pd(&a[6]); + __m256d bmm = _mm256_loadul3_pd(b); + + out[0] = _mm256_sum_pd(_mm256_mul_pd(amm0,bmm)); + out[1] = _mm256_sum_pd(_mm256_mul_pd(amm1,bmm)); + out[2] = _mm256_sum_pd(_mm256_mul_pd(amm2,bmm)); +} +#endif + +#endif // MATMUL_SPECIALISATIONS_KERNELS_H diff --git a/noarch/include/Fastor/backend/matmul/mkl_backend.h b/noarch/include/Fastor/backend/matmul/mkl_backend.h new file mode 100644 index 00000000..e2eb88fe --- /dev/null +++ b/noarch/include/Fastor/backend/matmul/mkl_backend.h @@ -0,0 +1,67 @@ +#ifndef MKL_BACKEND_H +#define MKL_BACKEND_H + +#include + +#ifdef FASTOR_USE_MKL + +// Explicitly activate the jit +#ifndef MKL_DIRECT_CALL_SEQ_JIT +#define MKL_DIRECT_CALL_SEQ_JIT +#endif + +#include + +namespace Fastor { +namespace blas { + +// single +template,bool> = false> +void matmul_mkl( + const float * FASTOR_RESTRICT a_data, + const float * FASTOR_RESTRICT b_data, + float * FASTOR_RESTRICT out_data) { + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N); + +} + +// double +template,bool> = false> +void matmul_mkl( + const double * FASTOR_RESTRICT a_data, + const double * FASTOR_RESTRICT b_data, + double * FASTOR_RESTRICT out_data) { + + cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N); + +} + + +#if 0 +// dedicated jit api, but the jit kernel has to be created before hand +template +Tensor matmul_mkl_jit_api(const Tensor &a, const Tensor &b) { + + Tensor out; + void* jitter; + mkl_jit_status_t status = mkl_jit_create_dgemm(&jitter, MKL_ROW_MAJOR, MKL_NOTRANS, MKL_NOTRANS, M, N, K, 1.0, K, N, 0.0, N); + + dgemm_jit_kernel_t _dgemm_kernel = mkl_jit_get_dgemm_ptr(jitter); + + _dgemm_kernel(jitter, a.data(), b.data(), out.data()); + + mkl_jit_destroy(jitter); + + return out; +} +#endif + +} // end of namespace blas +} // end of namespace Fastor + +#endif // FASTOR_USE_MKL + +#endif // MKL_BACKEND_H diff --git a/noarch/include/Fastor/backend/matmul/tmatmul.h b/noarch/include/Fastor/backend/matmul/tmatmul.h new file mode 100644 index 00000000..96dd6072 --- /dev/null +++ b/noarch/include/Fastor/backend/matmul/tmatmul.h @@ -0,0 +1,803 @@ +#ifndef MATMUL_KERNELS2_H +#define MATMUL_KERNELS2_H + + +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/meta/meta.h" +#include "Fastor/meta/tensor_meta.h" + + +namespace Fastor { + +namespace internal { + +// TRMM implementation of Fastor - matrix-matrix multiplication when either or both operands are +// lower or upper triangular. The matrices do not need to be square and trapezoidal cases are also +// covered. For big matrices the speed-up is 2X or even better over matmul for when one operand is +// is triangular and nearly 4X for when both operands are triangular. +// For small matrices due to aggressive unrolling for SIMD the matrices cannot be exactly traversed +// within their triangular part(s) and a bit of the non-triangular part(s) need(s) to be loaded as well +// hence, the performance may not be exactly 2X/4X over the general matmul case + + +// The functions here are exact replica of those in matmul_kernels.h and will be eventually +// merged together as these variants have no associated overhead for the general case of matmul +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +/* + +For triangular matmul only the iteration span of K is modified +using the following logic [lt = Lower, ut=Upper] + +// if lhs == lt +const size_t kfirst = 0; +const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K); + +// if lhs == ut +const size_t kfirst = i; +const size_t klast = K; + +// if rhs == lt +const size_t kfirst = j; +const size_t klast = K; + +// if rhs == ut +const size_t kfirst = 0; +const size_t klast = min(j+1,K); // or min(j+unrollOuterloop,K); + + +// both lower +const size_t kfirst = j; +const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K); + +// if lhs == lt && rhs == ut +const size_t kfirst = 0; +const size_t klast = min(min(i+1,j+1),K); // or min(min(i+unrollOuterloop,j+unrollInnerloop),K); + +// if lhs == ut && rhs == lt +const size_t kfirst = max(i,j); +const size_t klast = K; + +// if both upper +const size_t kfirst = i; +const size_t klast = min(j+1,K); // or min(j+unrollInnerloop,K); + +*/ + +template +constexpr FASTOR_INLINE T find_kfirst(const T i, const T j) { + return is_same_v_ || is_same_v_ ? + ( is_same_v_ ? j : 0UL ) : + (is_same_v_ ? ( is_same_v_ ? internal::max_(i,j) : i ) : 0UL ); +} +template +constexpr FASTOR_INLINE T find_klast(const T i, const T j) { + return is_same_v_ ? + ( is_same_v_ ? internal::min_(internal::min_(i+unrollOuterloop,j+unrollInnerloop),K) : internal::min_(i+unrollOuterloop,K) ) : + (is_same_v_ || (is_same_v_) ? + ( is_same_v_ ? internal::min_(j+unrollInnerloop,K) : K ) : K ); +} + + +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- + +// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel) +// unroll the inner-most loop (on unrollOuterloop) +//----------------------------------------------------------------------------------------------------------- +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + + const V bmm0(&b[k*N+j],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + } + } +} + +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + const V bmm2(&b[k*N+j+2*V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + const V bmm2(&b[k*N+j+2*V::Size],false); + const V bmm3(&b[k*N+j+3*V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); + c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); + c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + + const V bmm0(&b[k*N+j],false); + const V bmm1(&b[k*N+j+V::Size],false); + const V bmm2(&b[k*N+j+2*V::Size],false); + const V bmm3(&b[k*N+j+3*V::Size],false); + const V bmm4(&b[k*N+j+4*V::Size],false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); + c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); + c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); + c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); + c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); + c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); + c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); + c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_scalar_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + T c_ij[unrollOuterloop*numSIMDCols] = {}; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + const T bmm0(b[k*N+j]); + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] += amm0*bmm0; + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n]; + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_mask_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j, const int (&maska)[V::Size]) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + + const V bmm0(maskload(&b[k*N+j],maska)); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]); + } + } +} + + +template::type = false> +FASTOR_INLINE +void interior_block_tmatmul_mask_impl( + const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, + const size_t i, const size_t j, const MaskT mask) { + + V bmm0; + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + for (size_t ii = 0; ii < numSIMDRows; ++ii) { + + V c_ij[unrollOuterloop*numSIMDCols]; + // Loop over columns of a (rows of b) + for (size_t k = kfirst; k < klast; ++k) { + + bmm0.mask_load(&b[k*N+j],mask,false); + + for (size_t n = 0; n < unrollOuterloop; ++n) { + + const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; + + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false); + } + } +} +//----------------------------------------------------------------------------------------------------------- + + + +//----------------------------------------------------------------------------------------------------------- +// This is the base implementation of triangular matrix-matrix multiplication for all 2D tensors and +// higher order tensor products that can be expressed as trmm +// The function uses two level unrolling one based on block sizes and one based on register widths +// with any remainder left treated in a scalar fashion +template +FASTOR_INLINE +void _tmatmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + + using V = choose_best_simd_t,N>; + + // This parameter can be adjusted and does not need to be 4UL/8UL etc + // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; + constexpr size_t unrollOuterloop = 4UL; + +#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE + // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] + constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); +#else + constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; +#endif +#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE + // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] + constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; +#else + constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; +#endif + + // The goal is to get 10 parallel independent chains of accumulators + // to saturate the pipeline by having a completely unrolled block of + // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of + // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains + // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. + // However, most recent X86/64 architectures can do 2 FMAs per load so + // so unrolling with numSIMDCols > 2 is not beneficial + + constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; + // Number of rows of c (M) that can be safely unrolled with this block size. + constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; + + constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; + // Number of columns of c (N) that can be safely unrolled with this block size + constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + size_t i = 0; + for (; i < M0; i += unrollOuterBlock) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N1 columns + for (; j < N; ++j) { + interior_block_tmatmul_scalar_impl(a,b,c,i,j); + } + + } + + // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. + // This is necessary as for small sizes the earlier block loop may not be + // triggered if the size of the block is bigger than the number of rows of + // (a and c) i.e. M + constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); + for (; i < M1; i += unrollOuterloop) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + V c_ij[unrollOuterloop]; + for (size_t k = kfirst; k < klast; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i + n)*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; ++j) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + T c_ij[unrollOuterloop] = {}; + for (size_t k = kfirst; k < klast; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] += a[(i + n)*K+k] * b[k*N+j]; + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c[(i + n)*N+j] = c_ij[n]; + } + } + } + + // Now treat the remaining M-M1 rows - here the klast - kfirst range is not used + // so the implementation is exactly the same as matmul_base + FASTOR_IF_CONSTEXPR (M-M1 > 0) { + // Hack to get around zero length array issue + constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + // If MM1==0 the function never gets invoked anyway + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + + V c_ij[MM1]; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); + c_ij[n-M1].store(&c[n*N+j],false); + } + } + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1].store(&c[n*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; ++j) { + + T c_ij[MM1] = {}; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1] += a[n*K+k] * b[k*N+j]; + c[n*N+j] = c_ij[n-M1]; + } + } + for (size_t n = M1; n < M; ++n) { + c[n*N+j] = c_ij[n-M1]; + } + } + } +} +//----------------------------------------------------------------------------------------------------------- + + + + +//----------------------------------------------------------------------------------------------------------- +// This the base implementation of triangular matrix-matrix multiplication for all 2D tensors and +// higher order tensor products that can be expressed as trmm +// The function uses two level unrolling one based on block sizes and one based on register widths +// with any remainder left treated in vector mode with masked and conditional load/stores. +// Note that conditional load/store requires at least AVX intrinsics +template +FASTOR_INLINE +void _tmatmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + + using V = typename internal::choose_best_simd_type,N>::type; + + // This parameter can be adjusted and does not need to be 4UL/8UL etc + // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; + constexpr size_t unrollOuterloop = 4UL; + +#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE + // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] + constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); +#else + constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; +#endif +#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE + // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] + constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; +#else + constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; +#endif + + // The goal is to get 10 parallel independent chains of accumulators + // to saturate the pipeline by having a completely unrolled block of + // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of + // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains + // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. + // However, most recent X86/64 architectures can do 2 FMAs per load so + // so unrolling with numSIMDCols > 2 is not beneficial + + constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; + // Number of rows of c (M) that can be safely unrolled with this block size. + constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; + + constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; + // Number of columns of c (N) that can be safely unrolled with this block size + constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; + + // Number of columns of c (N) that can be safely unrolled with V::Size + constexpr size_t N1 = N / V::Size * V::Size; + + int maska[V::Size]; + std::fill(maska,&maska[V::Size], -1); + for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; +#ifdef FASTOR_HAS_AVX512_MASKS + const auto mask = array_to_mask(maska); +#endif + + size_t i = 0; + for (; i < M0; i += unrollOuterBlock) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N1 columns + for (; j < N; j+= N-N1) { +#ifdef FASTOR_HAS_AVX512_MASKS + interior_block_matmul_mask_impl(a,b,c,i,j,mask); +#else + interior_block_matmul_mask_impl(a,b,c,i,j,maska); +#endif + } + } + + // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. + // This is necessary as for small sizes the earlier block loop may not be + // triggered if the size of the block is bigger than the number of rows of + // (a and c) i.e. M + constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); + for (; i < M1; i += unrollOuterloop) { + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + V c_ij[unrollOuterloop]; + for (size_t k = kfirst; k < klast; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { + c_ij[n].store(&c[(i + n)*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; j+=N-N1) { + + const size_t kfirst = find_kfirst(i,j); + const size_t klast = find_klast (i,j); + + V c_ij[unrollOuterloop]; + for (size_t k = kfirst; k < klast; ++k) { + for (size_t n = 0; n < unrollOuterloop; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; bmm0.mask_load(&b[k*N+j],mask); +#else + const V bmm0(maskload(&b[k*N+j],maska)); +#endif + const V amm0 = a[(i + n)*K+k]; + c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); + } + } + for (size_t n = 0; n < unrollOuterloop; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + c_ij[n].mask_store(&c[(i+n)*N+j],mask,false); +#else + maskstore(&c[(i+n)*N+j],maska,c_ij[n]); +#endif + } + } + } + + // Now treat the remaining M-M1 rows - here the klast - kfirst range is not used + // so the implementation is exactly the same as matmul_base + FASTOR_IF_CONSTEXPR (M-M1 > 0) { + // Hack to get around zero length array issue + constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; + size_t j = 0; + for (; j < N0; j += unrollInnerBlock) { + // If MM1==0 the function never gets invoked anyway + interior_block_tmatmul_impl(a,b,c,i,j); + } + + // Remaining N - N0 columns + for (; j < N1; j += V::Size) { + V c_ij[MM1]; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); + c_ij[n-M1].store(&c[n*N+j],false); + } + } + for (size_t n = M1; n < M; ++n) { + c_ij[n-M1].store(&c[n*N+j],false); + } + } + + // Remaining N - N1 columns + for (; j < N; j+=N-N1) { + V c_ij[MM1] = {}; + for (size_t k = 0; k < K; ++k) { + for (size_t n = M1; n < M; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + V bmm0; bmm0.mask_load(&b[k*N+j],mask); +#else + const V bmm0(maskload(&b[k*N+j],maska)); +#endif + const V amm0 = a[n*K+k]; + c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]); + } + } + for (size_t n = M1; n < M; ++n) { +#ifdef FASTOR_HAS_AVX512_MASKS + c_ij[n-M1].mask_store(&c[n*N+j],mask,false); +#else + maskstore(&c[n*N+j],maska,c_ij[n-M1]); +#endif + } + } + } +} +//----------------------------------------------------------------------------------------------------------- + + + + +//----------------------------------------------------------------------------------------------------------- +// tmatmul kernel for non-fundamental types +// The assumption here is that non-fundamental types are not SIMD vectorisable for instance +// Tensor,3,3> or Tensor,...> plus they cannot fuse [do fused-add-multiply] +// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in +// the inner most loops of matmul +template +FASTOR_INLINE +void _tmatmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + // There is no SIMD here as V::Size == 1 anyway + // No outer loop unrolling otherwise the innermost loop + // will create unnecessary temporaries + for (size_t i=0; i(i,j); + const size_t klast = find_klast (i,j); + // This could potentially cost as opposed to directly writing in to c + T tmp {}; + for (size_t k=kfirst; k +FASTOR_INLINE +void _tmatmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + + // Non-primitive types + FASTOR_IF_CONSTEXPR (!is_primitive_v_) { + internal::_tmatmul_base_non_primitive(a,b,out); + return; + } + + // Use specialised kernels +#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) + + using nativeV = SIMDVector; + using V = choose_best_simd_t; + + FASTOR_IF_CONSTEXPR(N % V::Size <= 1UL) { + internal::_tmatmul_base(a,b,out); + return; + } + else { + internal::_tmatmul_base_masked(a,b,out); + return; + } +#else + internal::_tmatmul_base(a,b,out); + return; +#endif +} +//----------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------------------------------------- +} // end of namespace Fastor + + +#endif // MATMUL_KERNELS_H diff --git a/noarch/include/Fastor/backend/norm.h b/noarch/include/Fastor/backend/norm.h new file mode 100644 index 00000000..3381177e --- /dev/null +++ b/noarch/include/Fastor/backend/norm.h @@ -0,0 +1,149 @@ +#ifndef NORM_H +#define NORM_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/SIMDVector.h" + +namespace Fastor { + +template,N>::type::Size, N >, bool> = false> +#else + enable_if_t_,N>::type::Size, N >, bool> = false> +#endif +FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) { + + using V = typename internal::choose_best_simd_type,N>::type; + T _scal = 0; +#ifdef FASTOR_AVX512_IMPL + V omm0, omm1, omm2, omm3, omm4, omm5, omm6, omm7; +#else + V omm0, omm1, omm2, omm3; +#endif + FASTOR_INDEX i = 0; + + // With AVX utilises all the 16 registers but hurts the performance + // due to spill if eval has created temporary registers so only + // activated for AVX512 +#ifdef FASTOR_AVX512_IMPL + for (; i < ROUND_DOWN(N,8*V::Size); i+=8*V::Size) { + const V smm0(&a[i] , false); + const V smm1(&a[i+V::Size] , false); + const V smm2(&a[i+2*V::Size] , false); + const V smm3(&a[i+3*V::Size] , false); + const V smm4(&a[i+4*V::Size] , false); + const V smm5(&a[i+5*V::Size] , false); + const V smm6(&a[i+6*V::Size] , false); + const V smm7(&a[i+7*V::Size] , false); + omm0 = fmadd(smm0,smm0,omm0); + omm1 = fmadd(smm1,smm1,omm1); + omm2 = fmadd(smm2,smm2,omm2); + omm3 = fmadd(smm3,smm3,omm3); + omm4 = fmadd(smm4,smm4,omm4); + omm5 = fmadd(smm5,smm5,omm5); + omm6 = fmadd(smm6,smm6,omm6); + omm7 = fmadd(smm7,smm7,omm7); + } +#endif + for (; i < ROUND_DOWN(N,4*V::Size); i+=4*V::Size) { + const V smm0(&a[i] , false); + const V smm1(&a[i+V::Size] , false); + const V smm2(&a[i+2*V::Size] , false); + const V smm3(&a[i+3*V::Size] , false); + omm0 = fmadd(smm0,smm0,omm0); + omm1 = fmadd(smm1,smm1,omm1); + omm2 = fmadd(smm2,smm2,omm2); + omm3 = fmadd(smm3,smm3,omm3); + } + for (; i < ROUND_DOWN(N,2*V::Size); i+=2*V::Size) { + const V smm0(&a[i] , false); + const V smm1(&a[i+V::Size] , false); + omm0 = fmadd(smm0,smm0,omm0); + omm1 = fmadd(smm1,smm1,omm1); + } + for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) { + const V smm0(&a[i] , false); + omm0 = fmadd(smm0,smm0,omm0); + } + for (; i < N; ++i) { + const auto smm0(a[i]); + _scal += smm0*smm0; + } +#ifdef FASTOR_AVX512_IMPL + return sqrts( (omm0 + omm1 + omm2 + omm3 + omm4 + omm5 + omm6 + omm7).sum() + _scal); +#else + return sqrts( (omm0 + omm1 + omm2 + omm3).sum() + _scal); +#endif +} + +template,N>::type::Size, N >, bool> = false> +#else + enable_if_t_,N>::type::Size, N >, bool> = false> +#endif +FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) { + + using V = typename internal::choose_best_simd_type,N>::type; + T _scal = 0; + V omm0; + FASTOR_INDEX i = 0; + for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) { + const V smm0(&a[i] , false); + omm0 = fmadd(smm0,smm0,omm0); + } + for (; i < N; ++i) { + const auto smm0(a[i]); + _scal += smm0*smm0; + } + return sqrts( omm0.sum() + _scal); +} + +#ifdef FASTOR_SSE4_2_IMPL +template<> +FASTOR_INLINE float _norm(const float * FASTOR_RESTRICT a) { + // IVY 33 OPS / HW 31 OPS + __m128 a_reg = _mm_load_ps(a); + return _mm_cvtss_f32(_mm_sqrt_ps(_add_ps(_mm_mul_ps(a_reg,a_reg)))); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE float _norm(const float * FASTOR_RESTRICT a) { + // IVY & HW 61 OPS + __m256 a_reg = _mm256_loadu_ps(a); + __m128 a_end = _mm_load_ss(a+8); + __m128 a0 = _add_ps(_mm256_mul_ps(a_reg,a_reg)); + __m128 a1 = _add_ps(_mm_mul_ps(a_end,a_end)); + return _mm_cvtss_f32(_mm_sqrt_ps(_mm_add_ss(a0,a1))); +} + + +template<> +FASTOR_INLINE double _norm(const double * FASTOR_RESTRICT a) { + // IVY 34 OPS / HW 36 OPS + __m256d a_reg = _mm256_loadu_pd(a); + return _mm_cvtsd_f64(_mm_sqrt_pd(_add_pd(_mm256_mul_pd(a_reg,a_reg)))); +} + +template<> +FASTOR_INLINE double _norm(const double * FASTOR_RESTRICT a) { + // IVY 63 OPS / HW 67 OPS + __m256d a_low = _mm256_loadu_pd(a); + __m256d a_high = _mm256_loadu_pd(a+4); + __m128d a_end = _mm_load_sd(a+8); + __m128d a0 = _add_pd(_mm256_mul_pd(a_low,a_low)); + __m128d a1 = _add_pd(_mm256_mul_pd(a_high,a_high)); + __m128d a2 = _add_pd(_mm_mul_pd(a_end,a_end)); + return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_add_sd(a2,(_mm_add_sd(a0,a1))))); +} +#endif + + +} + +#endif // NORM_H + diff --git a/noarch/include/Fastor/backend/outer.h b/noarch/include/Fastor/backend/outer.h new file mode 100644 index 00000000..5952059d --- /dev/null +++ b/noarch/include/Fastor/backend/outer.h @@ -0,0 +1,292 @@ +#ifndef OUTER_H +#define OUTER_H + +#include "Fastor/config/config.h" + +namespace Fastor { + +template +FASTOR_HINT_INLINE void _outer(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { + for (size_t i=0; i +FASTOR_HINT_INLINE void _outer(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + // 31 OPS + // Fetch a to L1-cache +// _mm_prefetch(a,_MM_HINT_T0); + __m128 a0 = _mm_set1_ps(a[0]); + __m128 a1 = _mm_set1_ps(a[1]); +// __m128 a2 = _mm_set1_ps(a[2]); + __m128 a3 = _mm_set1_ps(a[3]); + __m128 bs = _mm_load_ps(b); + + __m128 r0 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(1,2,3,0)); + __m128 r1 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(3,1,2,0)); + __m128 r2 = _mm_mul_ps(HALFPS,_mm_add_ps(r1,bs)); + r2 = _mm_shuffle_ps(r2,r2,_MM_SHUFFLE(3,0,2,1)); +// __m128 r3 = _mm_shift1_ps(_mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0))); + __m128 r3 = _mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0)); + + // row0 + __m128 row0 = _mm_mul_ps(a0,r3); + _mm_store_ps(out, row0); + // row1 + __m128 row1 = _mm_mul_ps(a3,r3); + row1 = _mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,3,2,1)); + _mm_store_ss(out+3,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,1))); +// _mm_storeu_ps(out+4,row1); + _mm_store_ps(out+4,row1); + // row2 + __m128 row2 = _mm_mul_ps(a1,r3); + row2 = _mm_shuffle_ps(row2,row2,_MM_SHUFFLE(1,0,3,2)); + _mm_store_ss(out+6,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,2))); + _mm_store_ss(out+7,_mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,0,0,1))); + _mm_store_ss(out+8, row2); +} + +template<> +FASTOR_HINT_INLINE void _outer(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { + // 81 OPS + // Fetch a to L1-cache +// _mm_prefetch(a,_MM_HINT_T0); + + __m128 a0 = _mm_set1_ps(a[0]); + __m128 a1 = _mm_set1_ps(a[1]); + __m128 a2 = _mm_set1_ps(a[2]); +// __m128 a3 = _mm_set1_ps(a[3]); + __m128 a4 = _mm_set1_ps(a[4]); + __m128 a5 = _mm_set1_ps(a[5]); +// __m128 a6 = _mm_set1_ps(a[6]); +// __m128 a7 = _mm_set1_ps(a[7]); + __m128 a8 = _mm_set1_ps(a[8]); + + __m128 b_low = _mm_load_ps(b); + __m128 b_high =_mm_load_ps(b+4); + __m128 b_end = _mm_load_ss(b+8); + + __m128 b_diag = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(1,0,1,0)); + b_diag = _mm_shuffle_ps(b_diag,b_end,_MM_SHUFFLE(1,0,2,0)); + __m128 ofdiag_str = _mm_shift1_ps(_mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,1,2,1))); + __m128 ofdiag_rev = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,2,2,3)); + ofdiag_rev = _mm_shift1_ps(_mm_shuffle_ps(ofdiag_rev,ofdiag_rev,_MM_SHUFFLE(3,3,2,0))); + + // Compute this only once + __m128 half_add_diag = _mm_mul_ps(HALFPS,_mm_add_ps(ofdiag_str,ofdiag_rev)); + + // row0 + __m128 c0_d = _mm_mul_ps(b_diag,a0); + __m128 c0_off = _mm_mul_ps(a0,half_add_diag); + c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1)); + _mm_store_ps(out,c0_d); + _mm_storeu_ps(out+3,c0_off); + // row1 + __m128 c1_d = _mm_mul_ps(b_diag,a4); + __m128 c1_off = _mm_mul_ps(a4,half_add_diag); + c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1)); + c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1)); + _mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1))); + _mm_storeu_ps(out+7,c1_d); + _mm_storeu_ps(out+9,c1_off); + // row2 + __m128 c2_d = _mm_mul_ps(b_diag,a8); + __m128 c2_off = _mm_mul_ps(a8,half_add_diag); + c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1)); + _mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2))); + _mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1))); + _mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2))); + _mm_storeu_ps(out+15,c2_off); + // row3 + __m128 c3_off = _mm_mul_ps(a1,half_add_diag); + c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1)); + _mm_store_ss(out+18,c0_off); + _mm_store_ss(out+19,c1_off); + _mm_store_ss(out+20,c2_off); + _mm_storeu_ps(out+21,c3_off); + // row4 + __m128 c4_off = _mm_mul_ps(a2,half_add_diag); + c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1)); + _mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1))); + _mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1))); + _mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1))); + _mm_storeu_ps(out+27,c4_off); + // row5 + __m128 c5_off = _mm_mul_ps(a5,half_add_diag); + c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1)); + _mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2))); + _mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2))); + _mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2))); + _mm_storeu_ps(out+33,c5_off); + + // row0 +// __m128 c0_d = _mm_mul_ps(b_diag,a0); +// __m128 c_os = _mm_mul_ps(ofdiag_str,a0); +// __m128 c_or = _mm_mul_ps(ofdiag_rev,a0); +// __m128 c0_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); +// c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1)); +// _mm_store_ps(out,c0_d); +// _mm_storeu_ps(out+3,c0_off); + +// // row1 +// __m128 c1_d = _mm_mul_ps(b_diag,a4); +// c_os = _mm_mul_ps(ofdiag_str,a4); +// c_or = _mm_mul_ps(ofdiag_rev,a4); +// __m128 c1_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); +// c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1)); +// c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1)); +// _mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1))); +// _mm_storeu_ps(out+7,c1_d); +// _mm_storeu_ps(out+9,c1_off); + +// // row2 +// __m128 c2_d = _mm_mul_ps(b_diag,a8); +// c_os = _mm_mul_ps(ofdiag_str,a8); +// c_or = _mm_mul_ps(ofdiag_rev,a8); +// __m128 c2_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); +// c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1)); +// _mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2))); +// _mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1))); +// _mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2))); +// _mm_storeu_ps(out+15,c2_off); + +// // row3 +// c_os = _mm_mul_ps(ofdiag_str,a1); +// c_or = _mm_mul_ps(ofdiag_rev,a1); +// __m128 c3_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); +// c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1)); +// _mm_store_ss(out+18,c0_off); +// _mm_store_ss(out+19,c1_off); +// _mm_store_ss(out+20,c2_off); +// _mm_storeu_ps(out+21,c3_off); + +// // row4 +// c_os = _mm_mul_ps(ofdiag_str,a2); +// c_or = _mm_mul_ps(ofdiag_rev,a2); +// __m128 c4_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); +// c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1)); +// _mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1))); +// _mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1))); +// _mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1))); +// _mm_storeu_ps(out+27,c4_off); + +// // row5 +// c_os = _mm_mul_ps(ofdiag_str,a5); +// c_or = _mm_mul_ps(ofdiag_rev,a5); +// __m128 c5_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); +// c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1)); +// _mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2))); +// _mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2))); +// _mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2))); +// _mm_storeu_ps(out+33,c5_off); + +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_HINT_INLINE void _outer(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + // Fetch a to L1-cache +// _mm_prefetch(a,_MM_HINT_T0); + __m256d a0 = _mm256_set1_pd(a[0]); + __m256d a1 = _mm256_set1_pd(a[1]); + __m256d a3 = _mm256_set1_pd(a[3]); + __m256d bs = _mm256_load_pd(b); + + __m128d r0 = _mm_setr_pd(_mm256_get2_pd(bs),_mm256_get1_pd(bs)); + __m128d r1 = _mm_setr_pd(_mm256_get1_pd(bs),_mm256_get2_pd(bs)); + __m128d r2 = _mm_mul_pd(HALFPD,_mm_add_pd(r0,r1)); + __m256d r3 = _mm256_setr_pd(_mm256_get0_pd(bs),_mm256_get3_pd(bs),_mm_get0_pd(r2),0.0); + + // row0 + __m256d row0 = _mm256_mul_pd(a0,r3); + _mm256_store_pd(out, row0); + // row1 + __m256d row1 = _mm256_mul_pd(a3,r3); + _mm_store_sd(out+3,_mm_set_sd(_mm256_get1_pd(row0))); + _mm_store_pd(out+4,_mm_setr_pd(_mm256_get1_pd(row1),_mm256_get2_pd(row1))); + // row2 + __m256d row2 = _mm256_mul_pd(a1,r3); + _mm_store_sd(out+6,_mm256_extractf128_pd(row0,0x1)); + _mm_store_sd(out+7,_mm256_extractf128_pd(row1,0x1)); + _mm_store_sd(out+8,_mm256_extractf128_pd(row2,0x1)); +} + +template<> +FASTOR_HINT_INLINE void _outer(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { + // OPS +// _mm_prefetch(a,_MM_HINT_T0); + + __m256d a0 = _mm256_set1_pd(a[0]); + __m256d a1 = _mm256_set1_pd(a[1]); + __m256d a2 = _mm256_set1_pd(a[2]); + __m256d a4 = _mm256_set1_pd(a[4]); + __m256d a5 = _mm256_set1_pd(a[5]); + __m256d a8 = _mm256_set1_pd(a[8]); + + __m256d b_low = _mm256_load_pd(b); + __m256d b_high = _mm256_load_pd(b+4); + __m128d b_end = _mm_load_sd(b+8); + + __m256d b_diag = _mm256_setr_pd(_mm_cvtsd_f64(_mm256_castpd256_pd128(b_low)), + _mm_cvtsd_f64(_mm256_castpd256_pd128(b_high)), + _mm_cvtsd_f64(b_end),0.0); + + __m256d ofdiag_str = _mm256_setr_pd(b[1],b[2],b[5],0.0); + __m256d ofdiag_rev = _mm256_setr_pd(b[3],b[6],b[7],0.0); + // Compute this only once + __m256d half_add_diag = _mm256_mul_pd(VHALFPD,_mm256_add_pd(ofdiag_str,ofdiag_rev)); + + // row0 + __m256d c0_d = _mm256_mul_pd(b_diag,a0); + __m256d c0_off = _mm256_mul_pd(a0,half_add_diag); + _mm256_store_pd(out,c0_d); + _mm256_storeu_pd(out+3,c0_off); + // row1 + __m256d c1_d = _mm256_mul_pd(b_diag,a4); + __m256d c1_off = _mm256_mul_pd(a4,half_add_diag); + _mm256_storeu_pd(out+6,c1_d); + _mm_store_sd(out+6,_mm256_castpd256_pd128(_mm256_shuffle_pd(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1)))); + _mm256_storeu_pd(out+9,c1_off); + // row2 + __m256d c2_d = _mm256_mul_pd(b_diag,a8); + __m256d c2_off = _mm256_mul_pd(a8,half_add_diag); + _mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0_d))); + _mm_store_sd(out+13,_mm_set_sd(_mm256_get2_pd(c1_d))); + _mm_store_sd(out+14,_mm_set_sd(_mm256_get2_pd(c2_d))); + _mm256_storeu_pd(out+15,c2_off); + // row3 + __m256d c3_off = _mm256_mul_pd(a1,half_add_diag); + _mm_store_sd(out+18,_mm256_castpd256_pd128(c0_off)); + _mm_store_sd(out+19,_mm256_castpd256_pd128(c1_off)); + _mm_store_sd(out+20,_mm256_castpd256_pd128(c2_off)); + _mm256_storeu_pd(out+21,c3_off); + // row4 + __m256d c4_off = _mm256_mul_pd(a2,half_add_diag); + _mm_store_sd(out+24,_mm_set_sd(_mm256_get1_pd(c0_off))); + _mm_store_sd(out+25,_mm_set_sd(_mm256_get1_pd(c1_off))); + _mm_store_sd(out+26,_mm_set_sd(_mm256_get1_pd(c2_off))); + _mm256_storeu_pd(out+27,c4_off); + // row4 + __m256d c5_off = _mm256_mul_pd(a5,half_add_diag); + _mm_store_sd(out+30,_mm_set_sd(_mm256_get2_pd(c0_off))); + _mm_store_sd(out+31,_mm_set_sd(_mm256_get2_pd(c1_off))); + _mm_store_sd(out+32,_mm_set_sd(_mm256_get2_pd(c2_off))); + _mm256_storeu_pd(out+33,c5_off); +} +#endif + +} + +#endif // OUTER_H + diff --git a/noarch/include/Fastor/backend/tensor_cross.h b/noarch/include/Fastor/backend/tensor_cross.h new file mode 100644 index 00000000..d14a20e3 --- /dev/null +++ b/noarch/include/Fastor/backend/tensor_cross.h @@ -0,0 +1,1843 @@ +#ifndef TENSOR_CROSS_H +#define TENSOR_CROSS_H + +#include "Fastor/meta/meta.h" +#include "Fastor/config/config.h" + +namespace Fastor { + +// classic vector cross product +//--------------------------------------------------------------------------------------------------------// +template = false> +FASTOR_INLINE void _vector_crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + c[0] = a[1] * b[2] - a[2] * b[1]; + c[1] = a[2] * b[0] - a[0] * b[2]; + c[2] = a[0] * b[1] - a[1] * b[0]; +} + +template = false> +FASTOR_INLINE void _vector_crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + c[0] = 0; + c[1] = 0; + c[2] = a[0] * b[1] - a[1] * b[0]; +} +//--------------------------------------------------------------------------------------------------------// + + +// tensor cross products +//--------------------------------------------------------------------------------------------------------// +template && is_equal_v_ && is_equal_v_,bool> = false> +FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + T ___A00 = a[0]; + T ___A01 = a[1]; + T ___A02 = a[2]; + T ___A10 = a[3]; + T ___A11 = a[4]; + T ___A12 = a[5]; + T ___A20 = a[6]; + T ___A21 = a[7]; + T ___A22 = a[8]; + + T ___B00 = b[0]; + T ___B01 = b[1]; + T ___B02 = b[2]; + T ___B10 = b[3]; + T ___B11 = b[4]; + T ___B12 = b[5]; + T ___B20 = b[6]; + T ___B21 = b[7]; + T ___B22 = b[8]; + + c[0] = ___A11*___B22 - ___A12*___B21 - ___A21*___B12 + ___A22*___B11; + c[1] = ___A12*___B20 - ___A10*___B22 + ___A20*___B12 - ___A22*___B10; + c[2] = ___A10*___B21 - ___A11*___B20 - ___A20*___B11 + ___A21*___B10; + c[3] = ___A02*___B21 - ___A01*___B22 + ___A21*___B02 - ___A22*___B01; + c[4] = ___A00*___B22 - ___A02*___B20 - ___A20*___B02 + ___A22*___B00; + c[5] = ___A01*___B20 - ___A00*___B21 + ___A20*___B01 - ___A21*___B00; + c[6] = ___A01*___B12 - ___A02*___B11 - ___A11*___B02 + ___A12*___B01; + c[7] = ___A02*___B10 - ___A00*___B12 + ___A10*___B02 - ___A12*___B00; + c[8] = ___A00*___B11 - ___A01*___B10 - ___A10*___B01 + ___A11*___B00; +} + +template && is_equal_v_ && is_equal_v_,bool> = false> +FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + T ___A00 = a[0]; + T ___A01 = a[1]; + T ___A10 = a[3]; + T ___A11 = a[4]; + + T ___B00 = b[0]; + T ___B01 = b[1]; + T ___B10 = b[3]; + T ___B11 = b[4]; + + c[0] = 0; + c[1] = 0; + c[2] = 0; + c[3] = 0; + c[4] = 0; + c[5] = 0; + c[6] = 0; + c[7] = 0; + c[8] = ___A00*___B11 - ___A01*___B10 - ___A10*___B01 + ___A11*___B00; +} + +// vectors and 2nd order tensors +//--------------------------------------------------------------------------------------------------------// +template && is_equal_v_ && is_equal_v_,bool> = false> +FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + // vector-tensor cross product - regitster based + T ___A1 = a[0]; + T ___A2 = a[1]; + T ___A3 = a[2]; + + T B1_1 = b[0]; + T B1_2 = b[1]; + T B1_3 = b[2]; + T B2_1 = b[3]; + T B2_2 = b[4]; + T B2_3 = b[5]; + T B3_1 = b[6]; + T B3_2 = b[7]; + T B3_3 = b[8]; + + c[0] = ___A2*B3_1 - ___A3*B2_1; + c[1] = ___A2*B3_2 - ___A3*B2_2; + c[2] = ___A2*B3_3 - ___A3*B2_3; + c[3] = ___A3*B1_1 - ___A1*B3_1; + c[4] = ___A3*B1_2 - ___A1*B3_2; + c[5] = ___A3*B1_3 - ___A1*B3_3; + c[6] = ___A1*B2_1 - ___A2*B1_1; + c[7] = ___A1*B2_2 - ___A2*B1_2; + c[8] = ___A1*B2_3 - ___A2*B1_3; + +// [ A2*B3_1 - A3*B2_1, A2*B3_2 - A3*B2_2, A2*B3_3 - A3*B2_3] +// [ A3*B1_1 - A1*B3_1, A3*B1_2 - A1*B3_2, A3*B1_3 - A1*B3_3] +// [ A1*B2_1 - A2*B1_1, A1*B2_2 - A2*B1_2, A1*B2_3 - A2*B1_3] +} + +template && is_equal_v_ && is_equal_v_,bool> = false> +FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + // vector-tensor cross product - regitster based + T ___A1 = a[0]; + T ___A2 = a[1]; + + T B1_1 = b[0]; + T B1_2 = b[1]; + T B2_1 = b[3]; + T B2_2 = b[4]; + + c[0] = 0; + c[1] = 0; + c[2] = 0; + c[3] = 0; + c[4] = 0; + c[5] = 0; + c[6] = ___A1*B2_1 - ___A2*B1_1; + c[7] = ___A1*B2_2 - ___A2*B1_2; + c[8] = 0; +} + +template && is_equal_v_ && is_equal_v_,bool> = false> +FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + // tensor-vector cross product - regitster based + T ___B1 = b[0]; + T ___B2 = b[1]; + T ___B3 = b[2]; + + T A1_1 = a[0]; + T A1_2 = a[1]; + T A1_3 = a[2]; + T A2_1 = a[3]; + T A2_2 = a[4]; + T A2_3 = a[5]; + T A3_1 = a[6]; + T A3_2 = a[7]; + T A3_3 = a[8]; + + c[0] = A1_2*___B3 - A1_3*___B2; + c[1] = A1_3*___B1 - A1_1*___B3; + c[2] = A1_1*___B2 - A1_2*___B1; + c[3] = A2_2*___B3 - A2_3*___B2; + c[4] = A2_3*___B1 - A2_1*___B3; + c[5] = A2_1*___B2 - A2_2*___B1; + c[6] = A3_2*___B3 - A3_3*___B2; + c[7] = A3_3*___B1 - A3_1*___B3; + c[8] = A3_1*___B2 - A3_2*___B1; + +// [ A1_2*B3 - A1_3*B2, A1_3*B1 - A1_1*B3, A1_1*B2 - A1_2*B1] +// [ A2_2*B3 - A2_3*B2, A2_3*B1 - A2_1*B3, A2_1*B2 - A2_2*B1] +// [ A3_2*B3 - A3_3*B2, A3_3*B1 - A3_1*B3, A3_1*B2 - A3_2*B1] +} + +template && is_equal_v_ && is_equal_v_,bool> = false> +FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + // tensor-vector cross product - regitster based + T ___B1 = b[0]; + T ___B2 = b[1]; + + T A1_1 = a[0]; + T A1_2 = a[1]; + T A2_1 = a[3]; + T A2_2 = a[4]; + + c[0] = 0; + c[1] = 0; + c[2] = A1_1*___B2 - A1_2*___B1; + c[3] = 0; + c[4] = 0; + c[5] = A2_1*___B2 - A2_2*___B1; + c[6] = 0; + c[7] = 0; + c[8] = 0; +} +//--------------------------------------------------------------------------------------------------------// + + + +//--------------------------------------------------------------------------------------------------------// +// for plane strain problems +template +FASTOR_INLINE +void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { + // For plane strain problems a and b need to be 3D with last element a[8]=b[8]=1 + // Note that in ultimate case you might not need the last row/column of the output matrix hence computing c22 + // and storing it is un-necessary, hence the (c) array should really be 2x2=4 in length + // Load a data + T ___A00 = a[0]; + T ___A11 = a[4]; + T ___A22 = a[8]; + T ___A01 = a[1]; + T ___A02 = a[2]; + T ___A12 = a[5]; + T ___A10 = a[3]; + T ___A20 = a[6]; + T ___A21 = a[7]; + + T ___B00 = b[0]; + T ___B11 = b[4]; + T ___B22 = b[8]; + T ___B01 = b[1]; + T ___B02 = b[2]; + T ___B12 = b[5]; + T ___B10 = b[3]; + T ___B20 = b[6]; + T ___B21 = b[7]; + + c[0] = ___A11*___B22 - ___A12*___B21 - ___A21*___B12 + ___A22*___B11; + c[1] = ___A12*___B20 - ___A10*___B22 + ___A20*___B12 - ___A22*___B10; + c[0] = 0; + c[3] = ___A02*___B21 - ___A01*___B22 + ___A21*___B02 - ___A22*___B01; + c[4] = ___A00*___B22 - ___A02*___B20 - ___A20*___B02 + ___A22*___B00; + c[5] = 0; + c[6] = 0; + c[7] = 0; + // c[8] = A00*B11 - A01*B10 - A10*B01 + A11*B00; + c[8] = 0; // 1 ? +} +//--------------------------------------------------------------------------------------------------------// + + + +//--------------------------------------------------------------------------------------------------------// +//(AxB)_{PiIQ} = E_{ijk}E_{IJK}A_{PjJ}B_{kKQ} +// tensor cross product of 3rd order tensors +template +FASTOR_HINT_INLINE void _crossproduct(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { + +T _A000 = A[0]; + T _A001 = A[1]; + T _A002 = A[2]; + T _A010 = A[3]; + T _A011 = A[4]; + T _A012 = A[5]; + T _A020 = A[6]; + T _A021 = A[7]; + T _A022 = A[8]; + T _A100 = A[9]; + T _A101 = A[10]; + T _A102 = A[11]; + T _A110 = A[12]; + T _A111 = A[13]; + T _A112 = A[14]; + T _A120 = A[15]; + T _A121 = A[16]; + T _A122 = A[17]; + T _A200 = A[18]; + T _A201 = A[19]; + T _A202 = A[20]; + T _A210 = A[21]; + T _A211 = A[22]; + T _A212 = A[23]; + T _A220 = A[24]; + T _A221 = A[25]; + T _A222 = A[26]; + + + T _B000 = B[0]; + T _B001 = B[1]; + T _B002 = B[2]; + T _B010 = B[3]; + T _B011 = B[4]; + T _B012 = B[5]; + T _B020 = B[6]; + T _B021 = B[7]; + T _B022 = B[8]; + T _B100 = B[9]; + T _B101 = B[10]; + T _B102 = B[11]; + T _B110 = B[12]; + T _B111 = B[13]; + T _B112 = B[14]; + T _B120 = B[15]; + T _B121 = B[16]; + T _B122 = B[17]; + T _B200 = B[18]; + T _B201 = B[19]; + T _B202 = B[20]; + T _B210 = B[21]; + T _B211 = B[22]; + T _B212 = B[23]; + T _B220 = B[24]; + T _B221 = B[25]; + T _B222 = B[26]; + + + C[0] = _A022*_B110 - _A021*_B120 - _A012*_B210 + _A011*_B220; + C[1] = _A022*_B111 - _A021*_B121 - _A012*_B211 + _A011*_B221; + C[2] = _A022*_B112 - _A021*_B122 - _A012*_B212 + _A011*_B222; + C[3] = _A020*_B120 - _A022*_B100 + _A012*_B200 - _A010*_B220; + C[4] = _A020*_B121 - _A022*_B101 + _A012*_B201 - _A010*_B221; + C[5] = _A020*_B122 - _A022*_B102 + _A012*_B202 - _A010*_B222; + C[6] = _A021*_B100 - _A020*_B110 - _A011*_B200 + _A010*_B210; + C[7] = _A021*_B101 - _A020*_B111 - _A011*_B201 + _A010*_B211; + C[8] = _A021*_B102 - _A020*_B112 - _A011*_B202 + _A010*_B212; + C[9] = _A021*_B020 - _A022*_B010 + _A002*_B210 - _A001*_B220; + C[10] = _A021*_B021 - _A022*_B011 + _A002*_B211 - _A001*_B221; + C[11] = _A021*_B022 - _A022*_B012 + _A002*_B212 - _A001*_B222; + C[12] = _A022*_B000 - _A020*_B020 - _A002*_B200 + _A000*_B220; + C[13] = _A022*_B001 - _A020*_B021 - _A002*_B201 + _A000*_B221; + C[14] = _A022*_B002 - _A020*_B022 - _A002*_B202 + _A000*_B222; + C[15] = _A020*_B010 - _A021*_B000 + _A001*_B200 - _A000*_B210; + C[16] = _A020*_B011 - _A021*_B001 + _A001*_B201 - _A000*_B211; + C[17] = _A020*_B012 - _A021*_B002 + _A001*_B202 - _A000*_B212; + C[18] = _A012*_B010 - _A011*_B020 - _A002*_B110 + _A001*_B120; + C[19] = _A012*_B011 - _A011*_B021 - _A002*_B111 + _A001*_B121; + C[20] = _A012*_B012 - _A011*_B022 - _A002*_B112 + _A001*_B122; + C[21] = _A010*_B020 - _A012*_B000 + _A002*_B100 - _A000*_B120; + C[22] = _A010*_B021 - _A012*_B001 + _A002*_B101 - _A000*_B121; + C[23] = _A010*_B022 - _A012*_B002 + _A002*_B102 - _A000*_B122; + C[24] = _A011*_B000 - _A010*_B010 - _A001*_B100 + _A000*_B110; + C[25] = _A011*_B001 - _A010*_B011 - _A001*_B101 + _A000*_B111; + C[26] = _A011*_B002 - _A010*_B012 - _A001*_B102 + _A000*_B112; + C[27] = _A122*_B110 - _A121*_B120 - _A112*_B210 + _A111*_B220; + C[28] = _A122*_B111 - _A121*_B121 - _A112*_B211 + _A111*_B221; + C[29] = _A122*_B112 - _A121*_B122 - _A112*_B212 + _A111*_B222; + C[30] = _A120*_B120 - _A122*_B100 + _A112*_B200 - _A110*_B220; + C[31] = _A120*_B121 - _A122*_B101 + _A112*_B201 - _A110*_B221; + C[32] = _A120*_B122 - _A122*_B102 + _A112*_B202 - _A110*_B222; + C[33] = _A121*_B100 - _A120*_B110 - _A111*_B200 + _A110*_B210; + C[34] = _A121*_B101 - _A120*_B111 - _A111*_B201 + _A110*_B211; + C[35] = _A121*_B102 - _A120*_B112 - _A111*_B202 + _A110*_B212; + C[36] = _A121*_B020 - _A122*_B010 + _A102*_B210 - _A101*_B220; + C[37] = _A121*_B021 - _A122*_B011 + _A102*_B211 - _A101*_B221; + C[38] = _A121*_B022 - _A122*_B012 + _A102*_B212 - _A101*_B222; + C[39] = _A122*_B000 - _A120*_B020 - _A102*_B200 + _A100*_B220; + C[40] = _A122*_B001 - _A120*_B021 - _A102*_B201 + _A100*_B221; + C[41] = _A122*_B002 - _A120*_B022 - _A102*_B202 + _A100*_B222; + C[42] = _A120*_B010 - _A121*_B000 + _A101*_B200 - _A100*_B210; + C[43] = _A120*_B011 - _A121*_B001 + _A101*_B201 - _A100*_B211; + C[44] = _A120*_B012 - _A121*_B002 + _A101*_B202 - _A100*_B212; + C[45] = _A112*_B010 - _A111*_B020 - _A102*_B110 + _A101*_B120; + C[46] = _A112*_B011 - _A111*_B021 - _A102*_B111 + _A101*_B121; + C[47] = _A112*_B012 - _A111*_B022 - _A102*_B112 + _A101*_B122; + C[48] = _A110*_B020 - _A112*_B000 + _A102*_B100 - _A100*_B120; + C[49] = _A110*_B021 - _A112*_B001 + _A102*_B101 - _A100*_B121; + C[50] = _A110*_B022 - _A112*_B002 + _A102*_B102 - _A100*_B122; + C[51] = _A111*_B000 - _A110*_B010 - _A101*_B100 + _A100*_B110; + C[52] = _A111*_B001 - _A110*_B011 - _A101*_B101 + _A100*_B111; + C[53] = _A111*_B002 - _A110*_B012 - _A101*_B102 + _A100*_B112; + C[54] = _A222*_B110 - _A221*_B120 - _A212*_B210 + _A211*_B220; + C[55] = _A222*_B111 - _A221*_B121 - _A212*_B211 + _A211*_B221; + C[56] = _A222*_B112 - _A221*_B122 - _A212*_B212 + _A211*_B222; + C[57] = _A220*_B120 - _A222*_B100 + _A212*_B200 - _A210*_B220; + C[58] = _A220*_B121 - _A222*_B101 + _A212*_B201 - _A210*_B221; + C[59] = _A220*_B122 - _A222*_B102 + _A212*_B202 - _A210*_B222; + C[60] = _A221*_B100 - _A220*_B110 - _A211*_B200 + _A210*_B210; + C[61] = _A221*_B101 - _A220*_B111 - _A211*_B201 + _A210*_B211; + C[62] = _A221*_B102 - _A220*_B112 - _A211*_B202 + _A210*_B212; + C[63] = _A221*_B020 - _A222*_B010 + _A202*_B210 - _A201*_B220; + C[64] = _A221*_B021 - _A222*_B011 + _A202*_B211 - _A201*_B221; + C[65] = _A221*_B022 - _A222*_B012 + _A202*_B212 - _A201*_B222; + C[66] = _A222*_B000 - _A220*_B020 - _A202*_B200 + _A200*_B220; + C[67] = _A222*_B001 - _A220*_B021 - _A202*_B201 + _A200*_B221; + C[68] = _A222*_B002 - _A220*_B022 - _A202*_B202 + _A200*_B222; + C[69] = _A220*_B010 - _A221*_B000 + _A201*_B200 - _A200*_B210; + C[70] = _A220*_B011 - _A221*_B001 + _A201*_B201 - _A200*_B211; + C[71] = _A220*_B012 - _A221*_B002 + _A201*_B202 - _A200*_B212; + C[72] = _A212*_B010 - _A211*_B020 - _A202*_B110 + _A201*_B120; + C[73] = _A212*_B011 - _A211*_B021 - _A202*_B111 + _A201*_B121; + C[74] = _A212*_B012 - _A211*_B022 - _A202*_B112 + _A201*_B122; + C[75] = _A210*_B020 - _A212*_B000 + _A202*_B100 - _A200*_B120; + C[76] = _A210*_B021 - _A212*_B001 + _A202*_B101 - _A200*_B121; + C[77] = _A210*_B022 - _A212*_B002 + _A202*_B102 - _A200*_B122; + C[78] = _A211*_B000 - _A210*_B010 - _A201*_B100 + _A200*_B110; + C[79] = _A211*_B001 - _A210*_B011 - _A201*_B101 + _A200*_B111; + C[80] = _A211*_B002 - _A210*_B012 - _A201*_B102 + _A200*_B112; +} +//--------------------------------------------------------------------------------------------------------// + + + +//--------------------------------------------------------------------------------------------------------// +//(AxB)_{pPiIqQ} = E_{ijk}E_{IJK}A_{pPjJ}B_{kKqQ} +// tensor cross product of 4th order tensors +template +FASTOR_HINT_INLINE void _crossproduct(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { + + T _A0000 = A[0]; + T _A0001 = A[1]; + T _A0002 = A[2]; + T _A0010 = A[3]; + T _A0011 = A[4]; + T _A0012 = A[5]; + T _A0020 = A[6]; + T _A0021 = A[7]; + T _A0022 = A[8]; + T _A0100 = A[9]; + T _A0101 = A[10]; + T _A0102 = A[11]; + T _A0110 = A[12]; + T _A0111 = A[13]; + T _A0112 = A[14]; + T _A0120 = A[15]; + T _A0121 = A[16]; + T _A0122 = A[17]; + T _A0200 = A[18]; + T _A0201 = A[19]; + T _A0202 = A[20]; + T _A0210 = A[21]; + T _A0211 = A[22]; + T _A0212 = A[23]; + T _A0220 = A[24]; + T _A0221 = A[25]; + T _A0222 = A[26]; + T _A1000 = A[27]; + T _A1001 = A[28]; + T _A1002 = A[29]; + T _A1010 = A[30]; + T _A1011 = A[31]; + T _A1012 = A[32]; + T _A1020 = A[33]; + T _A1021 = A[34]; + T _A1022 = A[35]; + T _A1100 = A[36]; + T _A1101 = A[37]; + T _A1102 = A[38]; + T _A1110 = A[39]; + T _A1111 = A[40]; + T _A1112 = A[41]; + T _A1120 = A[42]; + T _A1121 = A[43]; + T _A1122 = A[44]; + T _A1200 = A[45]; + T _A1201 = A[46]; + T _A1202 = A[47]; + T _A1210 = A[48]; + T _A1211 = A[49]; + T _A1212 = A[50]; + T _A1220 = A[51]; + T _A1221 = A[52]; + T _A1222 = A[53]; + T _A2000 = A[54]; + T _A2001 = A[55]; + T _A2002 = A[56]; + T _A2010 = A[57]; + T _A2011 = A[58]; + T _A2012 = A[59]; + T _A2020 = A[60]; + T _A2021 = A[61]; + T _A2022 = A[62]; + T _A2100 = A[63]; + T _A2101 = A[64]; + T _A2102 = A[65]; + T _A2110 = A[66]; + T _A2111 = A[67]; + T _A2112 = A[68]; + T _A2120 = A[69]; + T _A2121 = A[70]; + T _A2122 = A[71]; + T _A2200 = A[72]; + T _A2201 = A[73]; + T _A2202 = A[74]; + T _A2210 = A[75]; + T _A2211 = A[76]; + T _A2212 = A[77]; + T _A2220 = A[78]; + T _A2221 = A[79]; + T _A2222 = A[80]; + + + T _B0000 = B[0]; + T _B0001 = B[1]; + T _B0002 = B[2]; + T _B0010 = B[3]; + T _B0011 = B[4]; + T _B0012 = B[5]; + T _B0020 = B[6]; + T _B0021 = B[7]; + T _B0022 = B[8]; + T _B0100 = B[9]; + T _B0101 = B[10]; + T _B0102 = B[11]; + T _B0110 = B[12]; + T _B0111 = B[13]; + T _B0112 = B[14]; + T _B0120 = B[15]; + T _B0121 = B[16]; + T _B0122 = B[17]; + T _B0200 = B[18]; + T _B0201 = B[19]; + T _B0202 = B[20]; + T _B0210 = B[21]; + T _B0211 = B[22]; + T _B0212 = B[23]; + T _B0220 = B[24]; + T _B0221 = B[25]; + T _B0222 = B[26]; + T _B1000 = B[27]; + T _B1001 = B[28]; + T _B1002 = B[29]; + T _B1010 = B[30]; + T _B1011 = B[31]; + T _B1012 = B[32]; + T _B1020 = B[33]; + T _B1021 = B[34]; + T _B1022 = B[35]; + T _B1100 = B[36]; + T _B1101 = B[37]; + T _B1102 = B[38]; + T _B1110 = B[39]; + T _B1111 = B[40]; + T _B1112 = B[41]; + T _B1120 = B[42]; + T _B1121 = B[43]; + T _B1122 = B[44]; + T _B1200 = B[45]; + T _B1201 = B[46]; + T _B1202 = B[47]; + T _B1210 = B[48]; + T _B1211 = B[49]; + T _B1212 = B[50]; + T _B1220 = B[51]; + T _B1221 = B[52]; + T _B1222 = B[53]; + T _B2000 = B[54]; + T _B2001 = B[55]; + T _B2002 = B[56]; + T _B2010 = B[57]; + T _B2011 = B[58]; + T _B2012 = B[59]; + T _B2020 = B[60]; + T _B2021 = B[61]; + T _B2022 = B[62]; + T _B2100 = B[63]; + T _B2101 = B[64]; + T _B2102 = B[65]; + T _B2110 = B[66]; + T _B2111 = B[67]; + T _B2112 = B[68]; + T _B2120 = B[69]; + T _B2121 = B[70]; + T _B2122 = B[71]; + T _B2200 = B[72]; + T _B2201 = B[73]; + T _B2202 = B[74]; + T _B2210 = B[75]; + T _B2211 = B[76]; + T _B2212 = B[77]; + T _B2220 = B[78]; + T _B2221 = B[79]; + T _B2222 = B[80]; + + + C[0] = _A0022*_B1100 - _A0021*_B1200 - _A0012*_B2100 + _A0011*_B2200; + C[1] = _A0022*_B1101 - _A0021*_B1201 - _A0012*_B2101 + _A0011*_B2201; + C[2] = _A0022*_B1102 - _A0021*_B1202 - _A0012*_B2102 + _A0011*_B2202; + C[3] = _A0022*_B1110 - _A0021*_B1210 - _A0012*_B2110 + _A0011*_B2210; + C[4] = _A0022*_B1111 - _A0021*_B1211 - _A0012*_B2111 + _A0011*_B2211; + C[5] = _A0022*_B1112 - _A0021*_B1212 - _A0012*_B2112 + _A0011*_B2212; + C[6] = _A0022*_B1120 - _A0021*_B1220 - _A0012*_B2120 + _A0011*_B2220; + C[7] = _A0022*_B1121 - _A0021*_B1221 - _A0012*_B2121 + _A0011*_B2221; + C[8] = _A0022*_B1122 - _A0021*_B1222 - _A0012*_B2122 + _A0011*_B2222; + C[9] = _A0020*_B1200 - _A0022*_B1000 + _A0012*_B2000 - _A0010*_B2200; + C[10] = _A0020*_B1201 - _A0022*_B1001 + _A0012*_B2001 - _A0010*_B2201; + C[11] = _A0020*_B1202 - _A0022*_B1002 + _A0012*_B2002 - _A0010*_B2202; + C[12] = _A0020*_B1210 - _A0022*_B1010 + _A0012*_B2010 - _A0010*_B2210; + C[13] = _A0020*_B1211 - _A0022*_B1011 + _A0012*_B2011 - _A0010*_B2211; + C[14] = _A0020*_B1212 - _A0022*_B1012 + _A0012*_B2012 - _A0010*_B2212; + C[15] = _A0020*_B1220 - _A0022*_B1020 + _A0012*_B2020 - _A0010*_B2220; + C[16] = _A0020*_B1221 - _A0022*_B1021 + _A0012*_B2021 - _A0010*_B2221; + C[17] = _A0020*_B1222 - _A0022*_B1022 + _A0012*_B2022 - _A0010*_B2222; + C[18] = _A0021*_B1000 - _A0020*_B1100 - _A0011*_B2000 + _A0010*_B2100; + C[19] = _A0021*_B1001 - _A0020*_B1101 - _A0011*_B2001 + _A0010*_B2101; + C[20] = _A0021*_B1002 - _A0020*_B1102 - _A0011*_B2002 + _A0010*_B2102; + C[21] = _A0021*_B1010 - _A0020*_B1110 - _A0011*_B2010 + _A0010*_B2110; + C[22] = _A0021*_B1011 - _A0020*_B1111 - _A0011*_B2011 + _A0010*_B2111; + C[23] = _A0021*_B1012 - _A0020*_B1112 - _A0011*_B2012 + _A0010*_B2112; + C[24] = _A0021*_B1020 - _A0020*_B1120 - _A0011*_B2020 + _A0010*_B2120; + C[25] = _A0021*_B1021 - _A0020*_B1121 - _A0011*_B2021 + _A0010*_B2121; + C[26] = _A0021*_B1022 - _A0020*_B1122 - _A0011*_B2022 + _A0010*_B2122; + C[27] = _A0021*_B0200 - _A0022*_B0100 + _A0002*_B2100 - _A0001*_B2200; + C[28] = _A0021*_B0201 - _A0022*_B0101 + _A0002*_B2101 - _A0001*_B2201; + C[29] = _A0021*_B0202 - _A0022*_B0102 + _A0002*_B2102 - _A0001*_B2202; + C[30] = _A0021*_B0210 - _A0022*_B0110 + _A0002*_B2110 - _A0001*_B2210; + C[31] = _A0021*_B0211 - _A0022*_B0111 + _A0002*_B2111 - _A0001*_B2211; + C[32] = _A0021*_B0212 - _A0022*_B0112 + _A0002*_B2112 - _A0001*_B2212; + C[33] = _A0021*_B0220 - _A0022*_B0120 + _A0002*_B2120 - _A0001*_B2220; + C[34] = _A0021*_B0221 - _A0022*_B0121 + _A0002*_B2121 - _A0001*_B2221; + C[35] = _A0021*_B0222 - _A0022*_B0122 + _A0002*_B2122 - _A0001*_B2222; + C[36] = _A0022*_B0000 - _A0020*_B0200 - _A0002*_B2000 + _A0000*_B2200; + C[37] = _A0022*_B0001 - _A0020*_B0201 - _A0002*_B2001 + _A0000*_B2201; + C[38] = _A0022*_B0002 - _A0020*_B0202 - _A0002*_B2002 + _A0000*_B2202; + C[39] = _A0022*_B0010 - _A0020*_B0210 - _A0002*_B2010 + _A0000*_B2210; + C[40] = _A0022*_B0011 - _A0020*_B0211 - _A0002*_B2011 + _A0000*_B2211; + C[41] = _A0022*_B0012 - _A0020*_B0212 - _A0002*_B2012 + _A0000*_B2212; + C[42] = _A0022*_B0020 - _A0020*_B0220 - _A0002*_B2020 + _A0000*_B2220; + C[43] = _A0022*_B0021 - _A0020*_B0221 - _A0002*_B2021 + _A0000*_B2221; + C[44] = _A0022*_B0022 - _A0020*_B0222 - _A0002*_B2022 + _A0000*_B2222; + C[45] = _A0020*_B0100 - _A0021*_B0000 + _A0001*_B2000 - _A0000*_B2100; + C[46] = _A0020*_B0101 - _A0021*_B0001 + _A0001*_B2001 - _A0000*_B2101; + C[47] = _A0020*_B0102 - _A0021*_B0002 + _A0001*_B2002 - _A0000*_B2102; + C[48] = _A0020*_B0110 - _A0021*_B0010 + _A0001*_B2010 - _A0000*_B2110; + C[49] = _A0020*_B0111 - _A0021*_B0011 + _A0001*_B2011 - _A0000*_B2111; + C[50] = _A0020*_B0112 - _A0021*_B0012 + _A0001*_B2012 - _A0000*_B2112; + C[51] = _A0020*_B0120 - _A0021*_B0020 + _A0001*_B2020 - _A0000*_B2120; + C[52] = _A0020*_B0121 - _A0021*_B0021 + _A0001*_B2021 - _A0000*_B2121; + C[53] = _A0020*_B0122 - _A0021*_B0022 + _A0001*_B2022 - _A0000*_B2122; + C[54] = _A0012*_B0100 - _A0011*_B0200 - _A0002*_B1100 + _A0001*_B1200; + C[55] = _A0012*_B0101 - _A0011*_B0201 - _A0002*_B1101 + _A0001*_B1201; + C[56] = _A0012*_B0102 - _A0011*_B0202 - _A0002*_B1102 + _A0001*_B1202; + C[57] = _A0012*_B0110 - _A0011*_B0210 - _A0002*_B1110 + _A0001*_B1210; + C[58] = _A0012*_B0111 - _A0011*_B0211 - _A0002*_B1111 + _A0001*_B1211; + C[59] = _A0012*_B0112 - _A0011*_B0212 - _A0002*_B1112 + _A0001*_B1212; + C[60] = _A0012*_B0120 - _A0011*_B0220 - _A0002*_B1120 + _A0001*_B1220; + C[61] = _A0012*_B0121 - _A0011*_B0221 - _A0002*_B1121 + _A0001*_B1221; + C[62] = _A0012*_B0122 - _A0011*_B0222 - _A0002*_B1122 + _A0001*_B1222; + C[63] = _A0010*_B0200 - _A0012*_B0000 + _A0002*_B1000 - _A0000*_B1200; + C[64] = _A0010*_B0201 - _A0012*_B0001 + _A0002*_B1001 - _A0000*_B1201; + C[65] = _A0010*_B0202 - _A0012*_B0002 + _A0002*_B1002 - _A0000*_B1202; + C[66] = _A0010*_B0210 - _A0012*_B0010 + _A0002*_B1010 - _A0000*_B1210; + C[67] = _A0010*_B0211 - _A0012*_B0011 + _A0002*_B1011 - _A0000*_B1211; + C[68] = _A0010*_B0212 - _A0012*_B0012 + _A0002*_B1012 - _A0000*_B1212; + C[69] = _A0010*_B0220 - _A0012*_B0020 + _A0002*_B1020 - _A0000*_B1220; + C[70] = _A0010*_B0221 - _A0012*_B0021 + _A0002*_B1021 - _A0000*_B1221; + C[71] = _A0010*_B0222 - _A0012*_B0022 + _A0002*_B1022 - _A0000*_B1222; + C[72] = _A0011*_B0000 - _A0010*_B0100 - _A0001*_B1000 + _A0000*_B1100; + C[73] = _A0011*_B0001 - _A0010*_B0101 - _A0001*_B1001 + _A0000*_B1101; + C[74] = _A0011*_B0002 - _A0010*_B0102 - _A0001*_B1002 + _A0000*_B1102; + C[75] = _A0011*_B0010 - _A0010*_B0110 - _A0001*_B1010 + _A0000*_B1110; + C[76] = _A0011*_B0011 - _A0010*_B0111 - _A0001*_B1011 + _A0000*_B1111; + C[77] = _A0011*_B0012 - _A0010*_B0112 - _A0001*_B1012 + _A0000*_B1112; + C[78] = _A0011*_B0020 - _A0010*_B0120 - _A0001*_B1020 + _A0000*_B1120; + C[79] = _A0011*_B0021 - _A0010*_B0121 - _A0001*_B1021 + _A0000*_B1121; + C[80] = _A0011*_B0022 - _A0010*_B0122 - _A0001*_B1022 + _A0000*_B1122; + C[81] = _A0122*_B1100 - _A0121*_B1200 - _A0112*_B2100 + _A0111*_B2200; + C[82] = _A0122*_B1101 - _A0121*_B1201 - _A0112*_B2101 + _A0111*_B2201; + C[83] = _A0122*_B1102 - _A0121*_B1202 - _A0112*_B2102 + _A0111*_B2202; + C[84] = _A0122*_B1110 - _A0121*_B1210 - _A0112*_B2110 + _A0111*_B2210; + C[85] = _A0122*_B1111 - _A0121*_B1211 - _A0112*_B2111 + _A0111*_B2211; + C[86] = _A0122*_B1112 - _A0121*_B1212 - _A0112*_B2112 + _A0111*_B2212; + C[87] = _A0122*_B1120 - _A0121*_B1220 - _A0112*_B2120 + _A0111*_B2220; + C[88] = _A0122*_B1121 - _A0121*_B1221 - _A0112*_B2121 + _A0111*_B2221; + C[89] = _A0122*_B1122 - _A0121*_B1222 - _A0112*_B2122 + _A0111*_B2222; + C[90] = _A0120*_B1200 - _A0122*_B1000 + _A0112*_B2000 - _A0110*_B2200; + C[91] = _A0120*_B1201 - _A0122*_B1001 + _A0112*_B2001 - _A0110*_B2201; + C[92] = _A0120*_B1202 - _A0122*_B1002 + _A0112*_B2002 - _A0110*_B2202; + C[93] = _A0120*_B1210 - _A0122*_B1010 + _A0112*_B2010 - _A0110*_B2210; + C[94] = _A0120*_B1211 - _A0122*_B1011 + _A0112*_B2011 - _A0110*_B2211; + C[95] = _A0120*_B1212 - _A0122*_B1012 + _A0112*_B2012 - _A0110*_B2212; + C[96] = _A0120*_B1220 - _A0122*_B1020 + _A0112*_B2020 - _A0110*_B2220; + C[97] = _A0120*_B1221 - _A0122*_B1021 + _A0112*_B2021 - _A0110*_B2221; + C[98] = _A0120*_B1222 - _A0122*_B1022 + _A0112*_B2022 - _A0110*_B2222; + C[99] = _A0121*_B1000 - _A0120*_B1100 - _A0111*_B2000 + _A0110*_B2100; + C[100] = _A0121*_B1001 - _A0120*_B1101 - _A0111*_B2001 + _A0110*_B2101; + C[101] = _A0121*_B1002 - _A0120*_B1102 - _A0111*_B2002 + _A0110*_B2102; + C[102] = _A0121*_B1010 - _A0120*_B1110 - _A0111*_B2010 + _A0110*_B2110; + C[103] = _A0121*_B1011 - _A0120*_B1111 - _A0111*_B2011 + _A0110*_B2111; + C[104] = _A0121*_B1012 - _A0120*_B1112 - _A0111*_B2012 + _A0110*_B2112; + C[105] = _A0121*_B1020 - _A0120*_B1120 - _A0111*_B2020 + _A0110*_B2120; + C[106] = _A0121*_B1021 - _A0120*_B1121 - _A0111*_B2021 + _A0110*_B2121; + C[107] = _A0121*_B1022 - _A0120*_B1122 - _A0111*_B2022 + _A0110*_B2122; + C[108] = _A0121*_B0200 - _A0122*_B0100 + _A0102*_B2100 - _A0101*_B2200; + C[109] = _A0121*_B0201 - _A0122*_B0101 + _A0102*_B2101 - _A0101*_B2201; + C[110] = _A0121*_B0202 - _A0122*_B0102 + _A0102*_B2102 - _A0101*_B2202; + C[111] = _A0121*_B0210 - _A0122*_B0110 + _A0102*_B2110 - _A0101*_B2210; + C[112] = _A0121*_B0211 - _A0122*_B0111 + _A0102*_B2111 - _A0101*_B2211; + C[113] = _A0121*_B0212 - _A0122*_B0112 + _A0102*_B2112 - _A0101*_B2212; + C[114] = _A0121*_B0220 - _A0122*_B0120 + _A0102*_B2120 - _A0101*_B2220; + C[115] = _A0121*_B0221 - _A0122*_B0121 + _A0102*_B2121 - _A0101*_B2221; + C[116] = _A0121*_B0222 - _A0122*_B0122 + _A0102*_B2122 - _A0101*_B2222; + C[117] = _A0122*_B0000 - _A0120*_B0200 - _A0102*_B2000 + _A0100*_B2200; + C[118] = _A0122*_B0001 - _A0120*_B0201 - _A0102*_B2001 + _A0100*_B2201; + C[119] = _A0122*_B0002 - _A0120*_B0202 - _A0102*_B2002 + _A0100*_B2202; + C[120] = _A0122*_B0010 - _A0120*_B0210 - _A0102*_B2010 + _A0100*_B2210; + C[121] = _A0122*_B0011 - _A0120*_B0211 - _A0102*_B2011 + _A0100*_B2211; + C[122] = _A0122*_B0012 - _A0120*_B0212 - _A0102*_B2012 + _A0100*_B2212; + C[123] = _A0122*_B0020 - _A0120*_B0220 - _A0102*_B2020 + _A0100*_B2220; + C[124] = _A0122*_B0021 - _A0120*_B0221 - _A0102*_B2021 + _A0100*_B2221; + C[125] = _A0122*_B0022 - _A0120*_B0222 - _A0102*_B2022 + _A0100*_B2222; + C[126] = _A0120*_B0100 - _A0121*_B0000 + _A0101*_B2000 - _A0100*_B2100; + C[127] = _A0120*_B0101 - _A0121*_B0001 + _A0101*_B2001 - _A0100*_B2101; + C[128] = _A0120*_B0102 - _A0121*_B0002 + _A0101*_B2002 - _A0100*_B2102; + C[129] = _A0120*_B0110 - _A0121*_B0010 + _A0101*_B2010 - _A0100*_B2110; + C[130] = _A0120*_B0111 - _A0121*_B0011 + _A0101*_B2011 - _A0100*_B2111; + C[131] = _A0120*_B0112 - _A0121*_B0012 + _A0101*_B2012 - _A0100*_B2112; + C[132] = _A0120*_B0120 - _A0121*_B0020 + _A0101*_B2020 - _A0100*_B2120; + C[133] = _A0120*_B0121 - _A0121*_B0021 + _A0101*_B2021 - _A0100*_B2121; + C[134] = _A0120*_B0122 - _A0121*_B0022 + _A0101*_B2022 - _A0100*_B2122; + C[135] = _A0112*_B0100 - _A0111*_B0200 - _A0102*_B1100 + _A0101*_B1200; + C[136] = _A0112*_B0101 - _A0111*_B0201 - _A0102*_B1101 + _A0101*_B1201; + C[137] = _A0112*_B0102 - _A0111*_B0202 - _A0102*_B1102 + _A0101*_B1202; + C[138] = _A0112*_B0110 - _A0111*_B0210 - _A0102*_B1110 + _A0101*_B1210; + C[139] = _A0112*_B0111 - _A0111*_B0211 - _A0102*_B1111 + _A0101*_B1211; + C[140] = _A0112*_B0112 - _A0111*_B0212 - _A0102*_B1112 + _A0101*_B1212; + C[141] = _A0112*_B0120 - _A0111*_B0220 - _A0102*_B1120 + _A0101*_B1220; + C[142] = _A0112*_B0121 - _A0111*_B0221 - _A0102*_B1121 + _A0101*_B1221; + C[143] = _A0112*_B0122 - _A0111*_B0222 - _A0102*_B1122 + _A0101*_B1222; + C[144] = _A0110*_B0200 - _A0112*_B0000 + _A0102*_B1000 - _A0100*_B1200; + C[145] = _A0110*_B0201 - _A0112*_B0001 + _A0102*_B1001 - _A0100*_B1201; + C[146] = _A0110*_B0202 - _A0112*_B0002 + _A0102*_B1002 - _A0100*_B1202; + C[147] = _A0110*_B0210 - _A0112*_B0010 + _A0102*_B1010 - _A0100*_B1210; + C[148] = _A0110*_B0211 - _A0112*_B0011 + _A0102*_B1011 - _A0100*_B1211; + C[149] = _A0110*_B0212 - _A0112*_B0012 + _A0102*_B1012 - _A0100*_B1212; + C[150] = _A0110*_B0220 - _A0112*_B0020 + _A0102*_B1020 - _A0100*_B1220; + C[151] = _A0110*_B0221 - _A0112*_B0021 + _A0102*_B1021 - _A0100*_B1221; + C[152] = _A0110*_B0222 - _A0112*_B0022 + _A0102*_B1022 - _A0100*_B1222; + C[153] = _A0111*_B0000 - _A0110*_B0100 - _A0101*_B1000 + _A0100*_B1100; + C[154] = _A0111*_B0001 - _A0110*_B0101 - _A0101*_B1001 + _A0100*_B1101; + C[155] = _A0111*_B0002 - _A0110*_B0102 - _A0101*_B1002 + _A0100*_B1102; + C[156] = _A0111*_B0010 - _A0110*_B0110 - _A0101*_B1010 + _A0100*_B1110; + C[157] = _A0111*_B0011 - _A0110*_B0111 - _A0101*_B1011 + _A0100*_B1111; + C[158] = _A0111*_B0012 - _A0110*_B0112 - _A0101*_B1012 + _A0100*_B1112; + C[159] = _A0111*_B0020 - _A0110*_B0120 - _A0101*_B1020 + _A0100*_B1120; + C[160] = _A0111*_B0021 - _A0110*_B0121 - _A0101*_B1021 + _A0100*_B1121; + C[161] = _A0111*_B0022 - _A0110*_B0122 - _A0101*_B1022 + _A0100*_B1122; + C[162] = _A0222*_B1100 - _A0221*_B1200 - _A0212*_B2100 + _A0211*_B2200; + C[163] = _A0222*_B1101 - _A0221*_B1201 - _A0212*_B2101 + _A0211*_B2201; + C[164] = _A0222*_B1102 - _A0221*_B1202 - _A0212*_B2102 + _A0211*_B2202; + C[165] = _A0222*_B1110 - _A0221*_B1210 - _A0212*_B2110 + _A0211*_B2210; + C[166] = _A0222*_B1111 - _A0221*_B1211 - _A0212*_B2111 + _A0211*_B2211; + C[167] = _A0222*_B1112 - _A0221*_B1212 - _A0212*_B2112 + _A0211*_B2212; + C[168] = _A0222*_B1120 - _A0221*_B1220 - _A0212*_B2120 + _A0211*_B2220; + C[169] = _A0222*_B1121 - _A0221*_B1221 - _A0212*_B2121 + _A0211*_B2221; + C[170] = _A0222*_B1122 - _A0221*_B1222 - _A0212*_B2122 + _A0211*_B2222; + C[171] = _A0220*_B1200 - _A0222*_B1000 + _A0212*_B2000 - _A0210*_B2200; + C[172] = _A0220*_B1201 - _A0222*_B1001 + _A0212*_B2001 - _A0210*_B2201; + C[173] = _A0220*_B1202 - _A0222*_B1002 + _A0212*_B2002 - _A0210*_B2202; + C[174] = _A0220*_B1210 - _A0222*_B1010 + _A0212*_B2010 - _A0210*_B2210; + C[175] = _A0220*_B1211 - _A0222*_B1011 + _A0212*_B2011 - _A0210*_B2211; + C[176] = _A0220*_B1212 - _A0222*_B1012 + _A0212*_B2012 - _A0210*_B2212; + C[177] = _A0220*_B1220 - _A0222*_B1020 + _A0212*_B2020 - _A0210*_B2220; + C[178] = _A0220*_B1221 - _A0222*_B1021 + _A0212*_B2021 - _A0210*_B2221; + C[179] = _A0220*_B1222 - _A0222*_B1022 + _A0212*_B2022 - _A0210*_B2222; + C[180] = _A0221*_B1000 - _A0220*_B1100 - _A0211*_B2000 + _A0210*_B2100; + C[181] = _A0221*_B1001 - _A0220*_B1101 - _A0211*_B2001 + _A0210*_B2101; + C[182] = _A0221*_B1002 - _A0220*_B1102 - _A0211*_B2002 + _A0210*_B2102; + C[183] = _A0221*_B1010 - _A0220*_B1110 - _A0211*_B2010 + _A0210*_B2110; + C[184] = _A0221*_B1011 - _A0220*_B1111 - _A0211*_B2011 + _A0210*_B2111; + C[185] = _A0221*_B1012 - _A0220*_B1112 - _A0211*_B2012 + _A0210*_B2112; + C[186] = _A0221*_B1020 - _A0220*_B1120 - _A0211*_B2020 + _A0210*_B2120; + C[187] = _A0221*_B1021 - _A0220*_B1121 - _A0211*_B2021 + _A0210*_B2121; + C[188] = _A0221*_B1022 - _A0220*_B1122 - _A0211*_B2022 + _A0210*_B2122; + C[189] = _A0221*_B0200 - _A0222*_B0100 + _A0202*_B2100 - _A0201*_B2200; + C[190] = _A0221*_B0201 - _A0222*_B0101 + _A0202*_B2101 - _A0201*_B2201; + C[191] = _A0221*_B0202 - _A0222*_B0102 + _A0202*_B2102 - _A0201*_B2202; + C[192] = _A0221*_B0210 - _A0222*_B0110 + _A0202*_B2110 - _A0201*_B2210; + C[193] = _A0221*_B0211 - _A0222*_B0111 + _A0202*_B2111 - _A0201*_B2211; + C[194] = _A0221*_B0212 - _A0222*_B0112 + _A0202*_B2112 - _A0201*_B2212; + C[195] = _A0221*_B0220 - _A0222*_B0120 + _A0202*_B2120 - _A0201*_B2220; + C[196] = _A0221*_B0221 - _A0222*_B0121 + _A0202*_B2121 - _A0201*_B2221; + C[197] = _A0221*_B0222 - _A0222*_B0122 + _A0202*_B2122 - _A0201*_B2222; + C[198] = _A0222*_B0000 - _A0220*_B0200 - _A0202*_B2000 + _A0200*_B2200; + C[199] = _A0222*_B0001 - _A0220*_B0201 - _A0202*_B2001 + _A0200*_B2201; + C[200] = _A0222*_B0002 - _A0220*_B0202 - _A0202*_B2002 + _A0200*_B2202; + C[201] = _A0222*_B0010 - _A0220*_B0210 - _A0202*_B2010 + _A0200*_B2210; + C[202] = _A0222*_B0011 - _A0220*_B0211 - _A0202*_B2011 + _A0200*_B2211; + C[203] = _A0222*_B0012 - _A0220*_B0212 - _A0202*_B2012 + _A0200*_B2212; + C[204] = _A0222*_B0020 - _A0220*_B0220 - _A0202*_B2020 + _A0200*_B2220; + C[205] = _A0222*_B0021 - _A0220*_B0221 - _A0202*_B2021 + _A0200*_B2221; + C[206] = _A0222*_B0022 - _A0220*_B0222 - _A0202*_B2022 + _A0200*_B2222; + C[207] = _A0220*_B0100 - _A0221*_B0000 + _A0201*_B2000 - _A0200*_B2100; + C[208] = _A0220*_B0101 - _A0221*_B0001 + _A0201*_B2001 - _A0200*_B2101; + C[209] = _A0220*_B0102 - _A0221*_B0002 + _A0201*_B2002 - _A0200*_B2102; + C[210] = _A0220*_B0110 - _A0221*_B0010 + _A0201*_B2010 - _A0200*_B2110; + C[211] = _A0220*_B0111 - _A0221*_B0011 + _A0201*_B2011 - _A0200*_B2111; + C[212] = _A0220*_B0112 - _A0221*_B0012 + _A0201*_B2012 - _A0200*_B2112; + C[213] = _A0220*_B0120 - _A0221*_B0020 + _A0201*_B2020 - _A0200*_B2120; + C[214] = _A0220*_B0121 - _A0221*_B0021 + _A0201*_B2021 - _A0200*_B2121; + C[215] = _A0220*_B0122 - _A0221*_B0022 + _A0201*_B2022 - _A0200*_B2122; + C[216] = _A0212*_B0100 - _A0211*_B0200 - _A0202*_B1100 + _A0201*_B1200; + C[217] = _A0212*_B0101 - _A0211*_B0201 - _A0202*_B1101 + _A0201*_B1201; + C[218] = _A0212*_B0102 - _A0211*_B0202 - _A0202*_B1102 + _A0201*_B1202; + C[219] = _A0212*_B0110 - _A0211*_B0210 - _A0202*_B1110 + _A0201*_B1210; + C[220] = _A0212*_B0111 - _A0211*_B0211 - _A0202*_B1111 + _A0201*_B1211; + C[221] = _A0212*_B0112 - _A0211*_B0212 - _A0202*_B1112 + _A0201*_B1212; + C[222] = _A0212*_B0120 - _A0211*_B0220 - _A0202*_B1120 + _A0201*_B1220; + C[223] = _A0212*_B0121 - _A0211*_B0221 - _A0202*_B1121 + _A0201*_B1221; + C[224] = _A0212*_B0122 - _A0211*_B0222 - _A0202*_B1122 + _A0201*_B1222; + C[225] = _A0210*_B0200 - _A0212*_B0000 + _A0202*_B1000 - _A0200*_B1200; + C[226] = _A0210*_B0201 - _A0212*_B0001 + _A0202*_B1001 - _A0200*_B1201; + C[227] = _A0210*_B0202 - _A0212*_B0002 + _A0202*_B1002 - _A0200*_B1202; + C[228] = _A0210*_B0210 - _A0212*_B0010 + _A0202*_B1010 - _A0200*_B1210; + C[229] = _A0210*_B0211 - _A0212*_B0011 + _A0202*_B1011 - _A0200*_B1211; + C[230] = _A0210*_B0212 - _A0212*_B0012 + _A0202*_B1012 - _A0200*_B1212; + C[231] = _A0210*_B0220 - _A0212*_B0020 + _A0202*_B1020 - _A0200*_B1220; + C[232] = _A0210*_B0221 - _A0212*_B0021 + _A0202*_B1021 - _A0200*_B1221; + C[233] = _A0210*_B0222 - _A0212*_B0022 + _A0202*_B1022 - _A0200*_B1222; + C[234] = _A0211*_B0000 - _A0210*_B0100 - _A0201*_B1000 + _A0200*_B1100; + C[235] = _A0211*_B0001 - _A0210*_B0101 - _A0201*_B1001 + _A0200*_B1101; + C[236] = _A0211*_B0002 - _A0210*_B0102 - _A0201*_B1002 + _A0200*_B1102; + C[237] = _A0211*_B0010 - _A0210*_B0110 - _A0201*_B1010 + _A0200*_B1110; + C[238] = _A0211*_B0011 - _A0210*_B0111 - _A0201*_B1011 + _A0200*_B1111; + C[239] = _A0211*_B0012 - _A0210*_B0112 - _A0201*_B1012 + _A0200*_B1112; + C[240] = _A0211*_B0020 - _A0210*_B0120 - _A0201*_B1020 + _A0200*_B1120; + C[241] = _A0211*_B0021 - _A0210*_B0121 - _A0201*_B1021 + _A0200*_B1121; + C[242] = _A0211*_B0022 - _A0210*_B0122 - _A0201*_B1022 + _A0200*_B1122; + C[243] = _A1022*_B1100 - _A1021*_B1200 - _A1012*_B2100 + _A1011*_B2200; + C[244] = _A1022*_B1101 - _A1021*_B1201 - _A1012*_B2101 + _A1011*_B2201; + C[245] = _A1022*_B1102 - _A1021*_B1202 - _A1012*_B2102 + _A1011*_B2202; + C[246] = _A1022*_B1110 - _A1021*_B1210 - _A1012*_B2110 + _A1011*_B2210; + C[247] = _A1022*_B1111 - _A1021*_B1211 - _A1012*_B2111 + _A1011*_B2211; + C[248] = _A1022*_B1112 - _A1021*_B1212 - _A1012*_B2112 + _A1011*_B2212; + C[249] = _A1022*_B1120 - _A1021*_B1220 - _A1012*_B2120 + _A1011*_B2220; + C[250] = _A1022*_B1121 - _A1021*_B1221 - _A1012*_B2121 + _A1011*_B2221; + C[251] = _A1022*_B1122 - _A1021*_B1222 - _A1012*_B2122 + _A1011*_B2222; + C[252] = _A1020*_B1200 - _A1022*_B1000 + _A1012*_B2000 - _A1010*_B2200; + C[253] = _A1020*_B1201 - _A1022*_B1001 + _A1012*_B2001 - _A1010*_B2201; + C[254] = _A1020*_B1202 - _A1022*_B1002 + _A1012*_B2002 - _A1010*_B2202; + C[255] = _A1020*_B1210 - _A1022*_B1010 + _A1012*_B2010 - _A1010*_B2210; + C[256] = _A1020*_B1211 - _A1022*_B1011 + _A1012*_B2011 - _A1010*_B2211; + C[257] = _A1020*_B1212 - _A1022*_B1012 + _A1012*_B2012 - _A1010*_B2212; + C[258] = _A1020*_B1220 - _A1022*_B1020 + _A1012*_B2020 - _A1010*_B2220; + C[259] = _A1020*_B1221 - _A1022*_B1021 + _A1012*_B2021 - _A1010*_B2221; + C[260] = _A1020*_B1222 - _A1022*_B1022 + _A1012*_B2022 - _A1010*_B2222; + C[261] = _A1021*_B1000 - _A1020*_B1100 - _A1011*_B2000 + _A1010*_B2100; + C[262] = _A1021*_B1001 - _A1020*_B1101 - _A1011*_B2001 + _A1010*_B2101; + C[263] = _A1021*_B1002 - _A1020*_B1102 - _A1011*_B2002 + _A1010*_B2102; + C[264] = _A1021*_B1010 - _A1020*_B1110 - _A1011*_B2010 + _A1010*_B2110; + C[265] = _A1021*_B1011 - _A1020*_B1111 - _A1011*_B2011 + _A1010*_B2111; + C[266] = _A1021*_B1012 - _A1020*_B1112 - _A1011*_B2012 + _A1010*_B2112; + C[267] = _A1021*_B1020 - _A1020*_B1120 - _A1011*_B2020 + _A1010*_B2120; + C[268] = _A1021*_B1021 - _A1020*_B1121 - _A1011*_B2021 + _A1010*_B2121; + C[269] = _A1021*_B1022 - _A1020*_B1122 - _A1011*_B2022 + _A1010*_B2122; + C[270] = _A1021*_B0200 - _A1022*_B0100 + _A1002*_B2100 - _A1001*_B2200; + C[271] = _A1021*_B0201 - _A1022*_B0101 + _A1002*_B2101 - _A1001*_B2201; + C[272] = _A1021*_B0202 - _A1022*_B0102 + _A1002*_B2102 - _A1001*_B2202; + C[273] = _A1021*_B0210 - _A1022*_B0110 + _A1002*_B2110 - _A1001*_B2210; + C[274] = _A1021*_B0211 - _A1022*_B0111 + _A1002*_B2111 - _A1001*_B2211; + C[275] = _A1021*_B0212 - _A1022*_B0112 + _A1002*_B2112 - _A1001*_B2212; + C[276] = _A1021*_B0220 - _A1022*_B0120 + _A1002*_B2120 - _A1001*_B2220; + C[277] = _A1021*_B0221 - _A1022*_B0121 + _A1002*_B2121 - _A1001*_B2221; + C[278] = _A1021*_B0222 - _A1022*_B0122 + _A1002*_B2122 - _A1001*_B2222; + C[279] = _A1022*_B0000 - _A1020*_B0200 - _A1002*_B2000 + _A1000*_B2200; + C[280] = _A1022*_B0001 - _A1020*_B0201 - _A1002*_B2001 + _A1000*_B2201; + C[281] = _A1022*_B0002 - _A1020*_B0202 - _A1002*_B2002 + _A1000*_B2202; + C[282] = _A1022*_B0010 - _A1020*_B0210 - _A1002*_B2010 + _A1000*_B2210; + C[283] = _A1022*_B0011 - _A1020*_B0211 - _A1002*_B2011 + _A1000*_B2211; + C[284] = _A1022*_B0012 - _A1020*_B0212 - _A1002*_B2012 + _A1000*_B2212; + C[285] = _A1022*_B0020 - _A1020*_B0220 - _A1002*_B2020 + _A1000*_B2220; + C[286] = _A1022*_B0021 - _A1020*_B0221 - _A1002*_B2021 + _A1000*_B2221; + C[287] = _A1022*_B0022 - _A1020*_B0222 - _A1002*_B2022 + _A1000*_B2222; + C[288] = _A1020*_B0100 - _A1021*_B0000 + _A1001*_B2000 - _A1000*_B2100; + C[289] = _A1020*_B0101 - _A1021*_B0001 + _A1001*_B2001 - _A1000*_B2101; + C[290] = _A1020*_B0102 - _A1021*_B0002 + _A1001*_B2002 - _A1000*_B2102; + C[291] = _A1020*_B0110 - _A1021*_B0010 + _A1001*_B2010 - _A1000*_B2110; + C[292] = _A1020*_B0111 - _A1021*_B0011 + _A1001*_B2011 - _A1000*_B2111; + C[293] = _A1020*_B0112 - _A1021*_B0012 + _A1001*_B2012 - _A1000*_B2112; + C[294] = _A1020*_B0120 - _A1021*_B0020 + _A1001*_B2020 - _A1000*_B2120; + C[295] = _A1020*_B0121 - _A1021*_B0021 + _A1001*_B2021 - _A1000*_B2121; + C[296] = _A1020*_B0122 - _A1021*_B0022 + _A1001*_B2022 - _A1000*_B2122; + C[297] = _A1012*_B0100 - _A1011*_B0200 - _A1002*_B1100 + _A1001*_B1200; + C[298] = _A1012*_B0101 - _A1011*_B0201 - _A1002*_B1101 + _A1001*_B1201; + C[299] = _A1012*_B0102 - _A1011*_B0202 - _A1002*_B1102 + _A1001*_B1202; + C[300] = _A1012*_B0110 - _A1011*_B0210 - _A1002*_B1110 + _A1001*_B1210; + C[301] = _A1012*_B0111 - _A1011*_B0211 - _A1002*_B1111 + _A1001*_B1211; + C[302] = _A1012*_B0112 - _A1011*_B0212 - _A1002*_B1112 + _A1001*_B1212; + C[303] = _A1012*_B0120 - _A1011*_B0220 - _A1002*_B1120 + _A1001*_B1220; + C[304] = _A1012*_B0121 - _A1011*_B0221 - _A1002*_B1121 + _A1001*_B1221; + C[305] = _A1012*_B0122 - _A1011*_B0222 - _A1002*_B1122 + _A1001*_B1222; + C[306] = _A1010*_B0200 - _A1012*_B0000 + _A1002*_B1000 - _A1000*_B1200; + C[307] = _A1010*_B0201 - _A1012*_B0001 + _A1002*_B1001 - _A1000*_B1201; + C[308] = _A1010*_B0202 - _A1012*_B0002 + _A1002*_B1002 - _A1000*_B1202; + C[309] = _A1010*_B0210 - _A1012*_B0010 + _A1002*_B1010 - _A1000*_B1210; + C[310] = _A1010*_B0211 - _A1012*_B0011 + _A1002*_B1011 - _A1000*_B1211; + C[311] = _A1010*_B0212 - _A1012*_B0012 + _A1002*_B1012 - _A1000*_B1212; + C[312] = _A1010*_B0220 - _A1012*_B0020 + _A1002*_B1020 - _A1000*_B1220; + C[313] = _A1010*_B0221 - _A1012*_B0021 + _A1002*_B1021 - _A1000*_B1221; + C[314] = _A1010*_B0222 - _A1012*_B0022 + _A1002*_B1022 - _A1000*_B1222; + C[315] = _A1011*_B0000 - _A1010*_B0100 - _A1001*_B1000 + _A1000*_B1100; + C[316] = _A1011*_B0001 - _A1010*_B0101 - _A1001*_B1001 + _A1000*_B1101; + C[317] = _A1011*_B0002 - _A1010*_B0102 - _A1001*_B1002 + _A1000*_B1102; + C[318] = _A1011*_B0010 - _A1010*_B0110 - _A1001*_B1010 + _A1000*_B1110; + C[319] = _A1011*_B0011 - _A1010*_B0111 - _A1001*_B1011 + _A1000*_B1111; + C[320] = _A1011*_B0012 - _A1010*_B0112 - _A1001*_B1012 + _A1000*_B1112; + C[321] = _A1011*_B0020 - _A1010*_B0120 - _A1001*_B1020 + _A1000*_B1120; + C[322] = _A1011*_B0021 - _A1010*_B0121 - _A1001*_B1021 + _A1000*_B1121; + C[323] = _A1011*_B0022 - _A1010*_B0122 - _A1001*_B1022 + _A1000*_B1122; + C[324] = _A1122*_B1100 - _A1121*_B1200 - _A1112*_B2100 + _A1111*_B2200; + C[325] = _A1122*_B1101 - _A1121*_B1201 - _A1112*_B2101 + _A1111*_B2201; + C[326] = _A1122*_B1102 - _A1121*_B1202 - _A1112*_B2102 + _A1111*_B2202; + C[327] = _A1122*_B1110 - _A1121*_B1210 - _A1112*_B2110 + _A1111*_B2210; + C[328] = _A1122*_B1111 - _A1121*_B1211 - _A1112*_B2111 + _A1111*_B2211; + C[329] = _A1122*_B1112 - _A1121*_B1212 - _A1112*_B2112 + _A1111*_B2212; + C[330] = _A1122*_B1120 - _A1121*_B1220 - _A1112*_B2120 + _A1111*_B2220; + C[331] = _A1122*_B1121 - _A1121*_B1221 - _A1112*_B2121 + _A1111*_B2221; + C[332] = _A1122*_B1122 - _A1121*_B1222 - _A1112*_B2122 + _A1111*_B2222; + C[333] = _A1120*_B1200 - _A1122*_B1000 + _A1112*_B2000 - _A1110*_B2200; + C[334] = _A1120*_B1201 - _A1122*_B1001 + _A1112*_B2001 - _A1110*_B2201; + C[335] = _A1120*_B1202 - _A1122*_B1002 + _A1112*_B2002 - _A1110*_B2202; + C[336] = _A1120*_B1210 - _A1122*_B1010 + _A1112*_B2010 - _A1110*_B2210; + C[337] = _A1120*_B1211 - _A1122*_B1011 + _A1112*_B2011 - _A1110*_B2211; + C[338] = _A1120*_B1212 - _A1122*_B1012 + _A1112*_B2012 - _A1110*_B2212; + C[339] = _A1120*_B1220 - _A1122*_B1020 + _A1112*_B2020 - _A1110*_B2220; + C[340] = _A1120*_B1221 - _A1122*_B1021 + _A1112*_B2021 - _A1110*_B2221; + C[341] = _A1120*_B1222 - _A1122*_B1022 + _A1112*_B2022 - _A1110*_B2222; + C[342] = _A1121*_B1000 - _A1120*_B1100 - _A1111*_B2000 + _A1110*_B2100; + C[343] = _A1121*_B1001 - _A1120*_B1101 - _A1111*_B2001 + _A1110*_B2101; + C[344] = _A1121*_B1002 - _A1120*_B1102 - _A1111*_B2002 + _A1110*_B2102; + C[345] = _A1121*_B1010 - _A1120*_B1110 - _A1111*_B2010 + _A1110*_B2110; + C[346] = _A1121*_B1011 - _A1120*_B1111 - _A1111*_B2011 + _A1110*_B2111; + C[347] = _A1121*_B1012 - _A1120*_B1112 - _A1111*_B2012 + _A1110*_B2112; + C[348] = _A1121*_B1020 - _A1120*_B1120 - _A1111*_B2020 + _A1110*_B2120; + C[349] = _A1121*_B1021 - _A1120*_B1121 - _A1111*_B2021 + _A1110*_B2121; + C[350] = _A1121*_B1022 - _A1120*_B1122 - _A1111*_B2022 + _A1110*_B2122; + C[351] = _A1121*_B0200 - _A1122*_B0100 + _A1102*_B2100 - _A1101*_B2200; + C[352] = _A1121*_B0201 - _A1122*_B0101 + _A1102*_B2101 - _A1101*_B2201; + C[353] = _A1121*_B0202 - _A1122*_B0102 + _A1102*_B2102 - _A1101*_B2202; + C[354] = _A1121*_B0210 - _A1122*_B0110 + _A1102*_B2110 - _A1101*_B2210; + C[355] = _A1121*_B0211 - _A1122*_B0111 + _A1102*_B2111 - _A1101*_B2211; + C[356] = _A1121*_B0212 - _A1122*_B0112 + _A1102*_B2112 - _A1101*_B2212; + C[357] = _A1121*_B0220 - _A1122*_B0120 + _A1102*_B2120 - _A1101*_B2220; + C[358] = _A1121*_B0221 - _A1122*_B0121 + _A1102*_B2121 - _A1101*_B2221; + C[359] = _A1121*_B0222 - _A1122*_B0122 + _A1102*_B2122 - _A1101*_B2222; + C[360] = _A1122*_B0000 - _A1120*_B0200 - _A1102*_B2000 + _A1100*_B2200; + C[361] = _A1122*_B0001 - _A1120*_B0201 - _A1102*_B2001 + _A1100*_B2201; + C[362] = _A1122*_B0002 - _A1120*_B0202 - _A1102*_B2002 + _A1100*_B2202; + C[363] = _A1122*_B0010 - _A1120*_B0210 - _A1102*_B2010 + _A1100*_B2210; + C[364] = _A1122*_B0011 - _A1120*_B0211 - _A1102*_B2011 + _A1100*_B2211; + C[365] = _A1122*_B0012 - _A1120*_B0212 - _A1102*_B2012 + _A1100*_B2212; + C[366] = _A1122*_B0020 - _A1120*_B0220 - _A1102*_B2020 + _A1100*_B2220; + C[367] = _A1122*_B0021 - _A1120*_B0221 - _A1102*_B2021 + _A1100*_B2221; + C[368] = _A1122*_B0022 - _A1120*_B0222 - _A1102*_B2022 + _A1100*_B2222; + C[369] = _A1120*_B0100 - _A1121*_B0000 + _A1101*_B2000 - _A1100*_B2100; + C[370] = _A1120*_B0101 - _A1121*_B0001 + _A1101*_B2001 - _A1100*_B2101; + C[371] = _A1120*_B0102 - _A1121*_B0002 + _A1101*_B2002 - _A1100*_B2102; + C[372] = _A1120*_B0110 - _A1121*_B0010 + _A1101*_B2010 - _A1100*_B2110; + C[373] = _A1120*_B0111 - _A1121*_B0011 + _A1101*_B2011 - _A1100*_B2111; + C[374] = _A1120*_B0112 - _A1121*_B0012 + _A1101*_B2012 - _A1100*_B2112; + C[375] = _A1120*_B0120 - _A1121*_B0020 + _A1101*_B2020 - _A1100*_B2120; + C[376] = _A1120*_B0121 - _A1121*_B0021 + _A1101*_B2021 - _A1100*_B2121; + C[377] = _A1120*_B0122 - _A1121*_B0022 + _A1101*_B2022 - _A1100*_B2122; + C[378] = _A1112*_B0100 - _A1111*_B0200 - _A1102*_B1100 + _A1101*_B1200; + C[379] = _A1112*_B0101 - _A1111*_B0201 - _A1102*_B1101 + _A1101*_B1201; + C[380] = _A1112*_B0102 - _A1111*_B0202 - _A1102*_B1102 + _A1101*_B1202; + C[381] = _A1112*_B0110 - _A1111*_B0210 - _A1102*_B1110 + _A1101*_B1210; + C[382] = _A1112*_B0111 - _A1111*_B0211 - _A1102*_B1111 + _A1101*_B1211; + C[383] = _A1112*_B0112 - _A1111*_B0212 - _A1102*_B1112 + _A1101*_B1212; + C[384] = _A1112*_B0120 - _A1111*_B0220 - _A1102*_B1120 + _A1101*_B1220; + C[385] = _A1112*_B0121 - _A1111*_B0221 - _A1102*_B1121 + _A1101*_B1221; + C[386] = _A1112*_B0122 - _A1111*_B0222 - _A1102*_B1122 + _A1101*_B1222; + C[387] = _A1110*_B0200 - _A1112*_B0000 + _A1102*_B1000 - _A1100*_B1200; + C[388] = _A1110*_B0201 - _A1112*_B0001 + _A1102*_B1001 - _A1100*_B1201; + C[389] = _A1110*_B0202 - _A1112*_B0002 + _A1102*_B1002 - _A1100*_B1202; + C[390] = _A1110*_B0210 - _A1112*_B0010 + _A1102*_B1010 - _A1100*_B1210; + C[391] = _A1110*_B0211 - _A1112*_B0011 + _A1102*_B1011 - _A1100*_B1211; + C[392] = _A1110*_B0212 - _A1112*_B0012 + _A1102*_B1012 - _A1100*_B1212; + C[393] = _A1110*_B0220 - _A1112*_B0020 + _A1102*_B1020 - _A1100*_B1220; + C[394] = _A1110*_B0221 - _A1112*_B0021 + _A1102*_B1021 - _A1100*_B1221; + C[395] = _A1110*_B0222 - _A1112*_B0022 + _A1102*_B1022 - _A1100*_B1222; + C[396] = _A1111*_B0000 - _A1110*_B0100 - _A1101*_B1000 + _A1100*_B1100; + C[397] = _A1111*_B0001 - _A1110*_B0101 - _A1101*_B1001 + _A1100*_B1101; + C[398] = _A1111*_B0002 - _A1110*_B0102 - _A1101*_B1002 + _A1100*_B1102; + C[399] = _A1111*_B0010 - _A1110*_B0110 - _A1101*_B1010 + _A1100*_B1110; + C[400] = _A1111*_B0011 - _A1110*_B0111 - _A1101*_B1011 + _A1100*_B1111; + C[401] = _A1111*_B0012 - _A1110*_B0112 - _A1101*_B1012 + _A1100*_B1112; + C[402] = _A1111*_B0020 - _A1110*_B0120 - _A1101*_B1020 + _A1100*_B1120; + C[403] = _A1111*_B0021 - _A1110*_B0121 - _A1101*_B1021 + _A1100*_B1121; + C[404] = _A1111*_B0022 - _A1110*_B0122 - _A1101*_B1022 + _A1100*_B1122; + C[405] = _A1222*_B1100 - _A1221*_B1200 - _A1212*_B2100 + _A1211*_B2200; + C[406] = _A1222*_B1101 - _A1221*_B1201 - _A1212*_B2101 + _A1211*_B2201; + C[407] = _A1222*_B1102 - _A1221*_B1202 - _A1212*_B2102 + _A1211*_B2202; + C[408] = _A1222*_B1110 - _A1221*_B1210 - _A1212*_B2110 + _A1211*_B2210; + C[409] = _A1222*_B1111 - _A1221*_B1211 - _A1212*_B2111 + _A1211*_B2211; + C[410] = _A1222*_B1112 - _A1221*_B1212 - _A1212*_B2112 + _A1211*_B2212; + C[411] = _A1222*_B1120 - _A1221*_B1220 - _A1212*_B2120 + _A1211*_B2220; + C[412] = _A1222*_B1121 - _A1221*_B1221 - _A1212*_B2121 + _A1211*_B2221; + C[413] = _A1222*_B1122 - _A1221*_B1222 - _A1212*_B2122 + _A1211*_B2222; + C[414] = _A1220*_B1200 - _A1222*_B1000 + _A1212*_B2000 - _A1210*_B2200; + C[415] = _A1220*_B1201 - _A1222*_B1001 + _A1212*_B2001 - _A1210*_B2201; + C[416] = _A1220*_B1202 - _A1222*_B1002 + _A1212*_B2002 - _A1210*_B2202; + C[417] = _A1220*_B1210 - _A1222*_B1010 + _A1212*_B2010 - _A1210*_B2210; + C[418] = _A1220*_B1211 - _A1222*_B1011 + _A1212*_B2011 - _A1210*_B2211; + C[419] = _A1220*_B1212 - _A1222*_B1012 + _A1212*_B2012 - _A1210*_B2212; + C[420] = _A1220*_B1220 - _A1222*_B1020 + _A1212*_B2020 - _A1210*_B2220; + C[421] = _A1220*_B1221 - _A1222*_B1021 + _A1212*_B2021 - _A1210*_B2221; + C[422] = _A1220*_B1222 - _A1222*_B1022 + _A1212*_B2022 - _A1210*_B2222; + C[423] = _A1221*_B1000 - _A1220*_B1100 - _A1211*_B2000 + _A1210*_B2100; + C[424] = _A1221*_B1001 - _A1220*_B1101 - _A1211*_B2001 + _A1210*_B2101; + C[425] = _A1221*_B1002 - _A1220*_B1102 - _A1211*_B2002 + _A1210*_B2102; + C[426] = _A1221*_B1010 - _A1220*_B1110 - _A1211*_B2010 + _A1210*_B2110; + C[427] = _A1221*_B1011 - _A1220*_B1111 - _A1211*_B2011 + _A1210*_B2111; + C[428] = _A1221*_B1012 - _A1220*_B1112 - _A1211*_B2012 + _A1210*_B2112; + C[429] = _A1221*_B1020 - _A1220*_B1120 - _A1211*_B2020 + _A1210*_B2120; + C[430] = _A1221*_B1021 - _A1220*_B1121 - _A1211*_B2021 + _A1210*_B2121; + C[431] = _A1221*_B1022 - _A1220*_B1122 - _A1211*_B2022 + _A1210*_B2122; + C[432] = _A1221*_B0200 - _A1222*_B0100 + _A1202*_B2100 - _A1201*_B2200; + C[433] = _A1221*_B0201 - _A1222*_B0101 + _A1202*_B2101 - _A1201*_B2201; + C[434] = _A1221*_B0202 - _A1222*_B0102 + _A1202*_B2102 - _A1201*_B2202; + C[435] = _A1221*_B0210 - _A1222*_B0110 + _A1202*_B2110 - _A1201*_B2210; + C[436] = _A1221*_B0211 - _A1222*_B0111 + _A1202*_B2111 - _A1201*_B2211; + C[437] = _A1221*_B0212 - _A1222*_B0112 + _A1202*_B2112 - _A1201*_B2212; + C[438] = _A1221*_B0220 - _A1222*_B0120 + _A1202*_B2120 - _A1201*_B2220; + C[439] = _A1221*_B0221 - _A1222*_B0121 + _A1202*_B2121 - _A1201*_B2221; + C[440] = _A1221*_B0222 - _A1222*_B0122 + _A1202*_B2122 - _A1201*_B2222; + C[441] = _A1222*_B0000 - _A1220*_B0200 - _A1202*_B2000 + _A1200*_B2200; + C[442] = _A1222*_B0001 - _A1220*_B0201 - _A1202*_B2001 + _A1200*_B2201; + C[443] = _A1222*_B0002 - _A1220*_B0202 - _A1202*_B2002 + _A1200*_B2202; + C[444] = _A1222*_B0010 - _A1220*_B0210 - _A1202*_B2010 + _A1200*_B2210; + C[445] = _A1222*_B0011 - _A1220*_B0211 - _A1202*_B2011 + _A1200*_B2211; + C[446] = _A1222*_B0012 - _A1220*_B0212 - _A1202*_B2012 + _A1200*_B2212; + C[447] = _A1222*_B0020 - _A1220*_B0220 - _A1202*_B2020 + _A1200*_B2220; + C[448] = _A1222*_B0021 - _A1220*_B0221 - _A1202*_B2021 + _A1200*_B2221; + C[449] = _A1222*_B0022 - _A1220*_B0222 - _A1202*_B2022 + _A1200*_B2222; + C[450] = _A1220*_B0100 - _A1221*_B0000 + _A1201*_B2000 - _A1200*_B2100; + C[451] = _A1220*_B0101 - _A1221*_B0001 + _A1201*_B2001 - _A1200*_B2101; + C[452] = _A1220*_B0102 - _A1221*_B0002 + _A1201*_B2002 - _A1200*_B2102; + C[453] = _A1220*_B0110 - _A1221*_B0010 + _A1201*_B2010 - _A1200*_B2110; + C[454] = _A1220*_B0111 - _A1221*_B0011 + _A1201*_B2011 - _A1200*_B2111; + C[455] = _A1220*_B0112 - _A1221*_B0012 + _A1201*_B2012 - _A1200*_B2112; + C[456] = _A1220*_B0120 - _A1221*_B0020 + _A1201*_B2020 - _A1200*_B2120; + C[457] = _A1220*_B0121 - _A1221*_B0021 + _A1201*_B2021 - _A1200*_B2121; + C[458] = _A1220*_B0122 - _A1221*_B0022 + _A1201*_B2022 - _A1200*_B2122; + C[459] = _A1212*_B0100 - _A1211*_B0200 - _A1202*_B1100 + _A1201*_B1200; + C[460] = _A1212*_B0101 - _A1211*_B0201 - _A1202*_B1101 + _A1201*_B1201; + C[461] = _A1212*_B0102 - _A1211*_B0202 - _A1202*_B1102 + _A1201*_B1202; + C[462] = _A1212*_B0110 - _A1211*_B0210 - _A1202*_B1110 + _A1201*_B1210; + C[463] = _A1212*_B0111 - _A1211*_B0211 - _A1202*_B1111 + _A1201*_B1211; + C[464] = _A1212*_B0112 - _A1211*_B0212 - _A1202*_B1112 + _A1201*_B1212; + C[465] = _A1212*_B0120 - _A1211*_B0220 - _A1202*_B1120 + _A1201*_B1220; + C[466] = _A1212*_B0121 - _A1211*_B0221 - _A1202*_B1121 + _A1201*_B1221; + C[467] = _A1212*_B0122 - _A1211*_B0222 - _A1202*_B1122 + _A1201*_B1222; + C[468] = _A1210*_B0200 - _A1212*_B0000 + _A1202*_B1000 - _A1200*_B1200; + C[469] = _A1210*_B0201 - _A1212*_B0001 + _A1202*_B1001 - _A1200*_B1201; + C[470] = _A1210*_B0202 - _A1212*_B0002 + _A1202*_B1002 - _A1200*_B1202; + C[471] = _A1210*_B0210 - _A1212*_B0010 + _A1202*_B1010 - _A1200*_B1210; + C[472] = _A1210*_B0211 - _A1212*_B0011 + _A1202*_B1011 - _A1200*_B1211; + C[473] = _A1210*_B0212 - _A1212*_B0012 + _A1202*_B1012 - _A1200*_B1212; + C[474] = _A1210*_B0220 - _A1212*_B0020 + _A1202*_B1020 - _A1200*_B1220; + C[475] = _A1210*_B0221 - _A1212*_B0021 + _A1202*_B1021 - _A1200*_B1221; + C[476] = _A1210*_B0222 - _A1212*_B0022 + _A1202*_B1022 - _A1200*_B1222; + C[477] = _A1211*_B0000 - _A1210*_B0100 - _A1201*_B1000 + _A1200*_B1100; + C[478] = _A1211*_B0001 - _A1210*_B0101 - _A1201*_B1001 + _A1200*_B1101; + C[479] = _A1211*_B0002 - _A1210*_B0102 - _A1201*_B1002 + _A1200*_B1102; + C[480] = _A1211*_B0010 - _A1210*_B0110 - _A1201*_B1010 + _A1200*_B1110; + C[481] = _A1211*_B0011 - _A1210*_B0111 - _A1201*_B1011 + _A1200*_B1111; + C[482] = _A1211*_B0012 - _A1210*_B0112 - _A1201*_B1012 + _A1200*_B1112; + C[483] = _A1211*_B0020 - _A1210*_B0120 - _A1201*_B1020 + _A1200*_B1120; + C[484] = _A1211*_B0021 - _A1210*_B0121 - _A1201*_B1021 + _A1200*_B1121; + C[485] = _A1211*_B0022 - _A1210*_B0122 - _A1201*_B1022 + _A1200*_B1122; + C[486] = _A2022*_B1100 - _A2021*_B1200 - _A2012*_B2100 + _A2011*_B2200; + C[487] = _A2022*_B1101 - _A2021*_B1201 - _A2012*_B2101 + _A2011*_B2201; + C[488] = _A2022*_B1102 - _A2021*_B1202 - _A2012*_B2102 + _A2011*_B2202; + C[489] = _A2022*_B1110 - _A2021*_B1210 - _A2012*_B2110 + _A2011*_B2210; + C[490] = _A2022*_B1111 - _A2021*_B1211 - _A2012*_B2111 + _A2011*_B2211; + C[491] = _A2022*_B1112 - _A2021*_B1212 - _A2012*_B2112 + _A2011*_B2212; + C[492] = _A2022*_B1120 - _A2021*_B1220 - _A2012*_B2120 + _A2011*_B2220; + C[493] = _A2022*_B1121 - _A2021*_B1221 - _A2012*_B2121 + _A2011*_B2221; + C[494] = _A2022*_B1122 - _A2021*_B1222 - _A2012*_B2122 + _A2011*_B2222; + C[495] = _A2020*_B1200 - _A2022*_B1000 + _A2012*_B2000 - _A2010*_B2200; + C[496] = _A2020*_B1201 - _A2022*_B1001 + _A2012*_B2001 - _A2010*_B2201; + C[497] = _A2020*_B1202 - _A2022*_B1002 + _A2012*_B2002 - _A2010*_B2202; + C[498] = _A2020*_B1210 - _A2022*_B1010 + _A2012*_B2010 - _A2010*_B2210; + C[499] = _A2020*_B1211 - _A2022*_B1011 + _A2012*_B2011 - _A2010*_B2211; + C[500] = _A2020*_B1212 - _A2022*_B1012 + _A2012*_B2012 - _A2010*_B2212; + C[501] = _A2020*_B1220 - _A2022*_B1020 + _A2012*_B2020 - _A2010*_B2220; + C[502] = _A2020*_B1221 - _A2022*_B1021 + _A2012*_B2021 - _A2010*_B2221; + C[503] = _A2020*_B1222 - _A2022*_B1022 + _A2012*_B2022 - _A2010*_B2222; + C[504] = _A2021*_B1000 - _A2020*_B1100 - _A2011*_B2000 + _A2010*_B2100; + C[505] = _A2021*_B1001 - _A2020*_B1101 - _A2011*_B2001 + _A2010*_B2101; + C[506] = _A2021*_B1002 - _A2020*_B1102 - _A2011*_B2002 + _A2010*_B2102; + C[507] = _A2021*_B1010 - _A2020*_B1110 - _A2011*_B2010 + _A2010*_B2110; + C[508] = _A2021*_B1011 - _A2020*_B1111 - _A2011*_B2011 + _A2010*_B2111; + C[509] = _A2021*_B1012 - _A2020*_B1112 - _A2011*_B2012 + _A2010*_B2112; + C[510] = _A2021*_B1020 - _A2020*_B1120 - _A2011*_B2020 + _A2010*_B2120; + C[511] = _A2021*_B1021 - _A2020*_B1121 - _A2011*_B2021 + _A2010*_B2121; + C[512] = _A2021*_B1022 - _A2020*_B1122 - _A2011*_B2022 + _A2010*_B2122; + C[513] = _A2021*_B0200 - _A2022*_B0100 + _A2002*_B2100 - _A2001*_B2200; + C[514] = _A2021*_B0201 - _A2022*_B0101 + _A2002*_B2101 - _A2001*_B2201; + C[515] = _A2021*_B0202 - _A2022*_B0102 + _A2002*_B2102 - _A2001*_B2202; + C[516] = _A2021*_B0210 - _A2022*_B0110 + _A2002*_B2110 - _A2001*_B2210; + C[517] = _A2021*_B0211 - _A2022*_B0111 + _A2002*_B2111 - _A2001*_B2211; + C[518] = _A2021*_B0212 - _A2022*_B0112 + _A2002*_B2112 - _A2001*_B2212; + C[519] = _A2021*_B0220 - _A2022*_B0120 + _A2002*_B2120 - _A2001*_B2220; + C[520] = _A2021*_B0221 - _A2022*_B0121 + _A2002*_B2121 - _A2001*_B2221; + C[521] = _A2021*_B0222 - _A2022*_B0122 + _A2002*_B2122 - _A2001*_B2222; + C[522] = _A2022*_B0000 - _A2020*_B0200 - _A2002*_B2000 + _A2000*_B2200; + C[523] = _A2022*_B0001 - _A2020*_B0201 - _A2002*_B2001 + _A2000*_B2201; + C[524] = _A2022*_B0002 - _A2020*_B0202 - _A2002*_B2002 + _A2000*_B2202; + C[525] = _A2022*_B0010 - _A2020*_B0210 - _A2002*_B2010 + _A2000*_B2210; + C[526] = _A2022*_B0011 - _A2020*_B0211 - _A2002*_B2011 + _A2000*_B2211; + C[527] = _A2022*_B0012 - _A2020*_B0212 - _A2002*_B2012 + _A2000*_B2212; + C[528] = _A2022*_B0020 - _A2020*_B0220 - _A2002*_B2020 + _A2000*_B2220; + C[529] = _A2022*_B0021 - _A2020*_B0221 - _A2002*_B2021 + _A2000*_B2221; + C[530] = _A2022*_B0022 - _A2020*_B0222 - _A2002*_B2022 + _A2000*_B2222; + C[531] = _A2020*_B0100 - _A2021*_B0000 + _A2001*_B2000 - _A2000*_B2100; + C[532] = _A2020*_B0101 - _A2021*_B0001 + _A2001*_B2001 - _A2000*_B2101; + C[533] = _A2020*_B0102 - _A2021*_B0002 + _A2001*_B2002 - _A2000*_B2102; + C[534] = _A2020*_B0110 - _A2021*_B0010 + _A2001*_B2010 - _A2000*_B2110; + C[535] = _A2020*_B0111 - _A2021*_B0011 + _A2001*_B2011 - _A2000*_B2111; + C[536] = _A2020*_B0112 - _A2021*_B0012 + _A2001*_B2012 - _A2000*_B2112; + C[537] = _A2020*_B0120 - _A2021*_B0020 + _A2001*_B2020 - _A2000*_B2120; + C[538] = _A2020*_B0121 - _A2021*_B0021 + _A2001*_B2021 - _A2000*_B2121; + C[539] = _A2020*_B0122 - _A2021*_B0022 + _A2001*_B2022 - _A2000*_B2122; + C[540] = _A2012*_B0100 - _A2011*_B0200 - _A2002*_B1100 + _A2001*_B1200; + C[541] = _A2012*_B0101 - _A2011*_B0201 - _A2002*_B1101 + _A2001*_B1201; + C[542] = _A2012*_B0102 - _A2011*_B0202 - _A2002*_B1102 + _A2001*_B1202; + C[543] = _A2012*_B0110 - _A2011*_B0210 - _A2002*_B1110 + _A2001*_B1210; + C[544] = _A2012*_B0111 - _A2011*_B0211 - _A2002*_B1111 + _A2001*_B1211; + C[545] = _A2012*_B0112 - _A2011*_B0212 - _A2002*_B1112 + _A2001*_B1212; + C[546] = _A2012*_B0120 - _A2011*_B0220 - _A2002*_B1120 + _A2001*_B1220; + C[547] = _A2012*_B0121 - _A2011*_B0221 - _A2002*_B1121 + _A2001*_B1221; + C[548] = _A2012*_B0122 - _A2011*_B0222 - _A2002*_B1122 + _A2001*_B1222; + C[549] = _A2010*_B0200 - _A2012*_B0000 + _A2002*_B1000 - _A2000*_B1200; + C[550] = _A2010*_B0201 - _A2012*_B0001 + _A2002*_B1001 - _A2000*_B1201; + C[551] = _A2010*_B0202 - _A2012*_B0002 + _A2002*_B1002 - _A2000*_B1202; + C[552] = _A2010*_B0210 - _A2012*_B0010 + _A2002*_B1010 - _A2000*_B1210; + C[553] = _A2010*_B0211 - _A2012*_B0011 + _A2002*_B1011 - _A2000*_B1211; + C[554] = _A2010*_B0212 - _A2012*_B0012 + _A2002*_B1012 - _A2000*_B1212; + C[555] = _A2010*_B0220 - _A2012*_B0020 + _A2002*_B1020 - _A2000*_B1220; + C[556] = _A2010*_B0221 - _A2012*_B0021 + _A2002*_B1021 - _A2000*_B1221; + C[557] = _A2010*_B0222 - _A2012*_B0022 + _A2002*_B1022 - _A2000*_B1222; + C[558] = _A2011*_B0000 - _A2010*_B0100 - _A2001*_B1000 + _A2000*_B1100; + C[559] = _A2011*_B0001 - _A2010*_B0101 - _A2001*_B1001 + _A2000*_B1101; + C[560] = _A2011*_B0002 - _A2010*_B0102 - _A2001*_B1002 + _A2000*_B1102; + C[561] = _A2011*_B0010 - _A2010*_B0110 - _A2001*_B1010 + _A2000*_B1110; + C[562] = _A2011*_B0011 - _A2010*_B0111 - _A2001*_B1011 + _A2000*_B1111; + C[563] = _A2011*_B0012 - _A2010*_B0112 - _A2001*_B1012 + _A2000*_B1112; + C[564] = _A2011*_B0020 - _A2010*_B0120 - _A2001*_B1020 + _A2000*_B1120; + C[565] = _A2011*_B0021 - _A2010*_B0121 - _A2001*_B1021 + _A2000*_B1121; + C[566] = _A2011*_B0022 - _A2010*_B0122 - _A2001*_B1022 + _A2000*_B1122; + C[567] = _A2122*_B1100 - _A2121*_B1200 - _A2112*_B2100 + _A2111*_B2200; + C[568] = _A2122*_B1101 - _A2121*_B1201 - _A2112*_B2101 + _A2111*_B2201; + C[569] = _A2122*_B1102 - _A2121*_B1202 - _A2112*_B2102 + _A2111*_B2202; + C[570] = _A2122*_B1110 - _A2121*_B1210 - _A2112*_B2110 + _A2111*_B2210; + C[571] = _A2122*_B1111 - _A2121*_B1211 - _A2112*_B2111 + _A2111*_B2211; + C[572] = _A2122*_B1112 - _A2121*_B1212 - _A2112*_B2112 + _A2111*_B2212; + C[573] = _A2122*_B1120 - _A2121*_B1220 - _A2112*_B2120 + _A2111*_B2220; + C[574] = _A2122*_B1121 - _A2121*_B1221 - _A2112*_B2121 + _A2111*_B2221; + C[575] = _A2122*_B1122 - _A2121*_B1222 - _A2112*_B2122 + _A2111*_B2222; + C[576] = _A2120*_B1200 - _A2122*_B1000 + _A2112*_B2000 - _A2110*_B2200; + C[577] = _A2120*_B1201 - _A2122*_B1001 + _A2112*_B2001 - _A2110*_B2201; + C[578] = _A2120*_B1202 - _A2122*_B1002 + _A2112*_B2002 - _A2110*_B2202; + C[579] = _A2120*_B1210 - _A2122*_B1010 + _A2112*_B2010 - _A2110*_B2210; + C[580] = _A2120*_B1211 - _A2122*_B1011 + _A2112*_B2011 - _A2110*_B2211; + C[581] = _A2120*_B1212 - _A2122*_B1012 + _A2112*_B2012 - _A2110*_B2212; + C[582] = _A2120*_B1220 - _A2122*_B1020 + _A2112*_B2020 - _A2110*_B2220; + C[583] = _A2120*_B1221 - _A2122*_B1021 + _A2112*_B2021 - _A2110*_B2221; + C[584] = _A2120*_B1222 - _A2122*_B1022 + _A2112*_B2022 - _A2110*_B2222; + C[585] = _A2121*_B1000 - _A2120*_B1100 - _A2111*_B2000 + _A2110*_B2100; + C[586] = _A2121*_B1001 - _A2120*_B1101 - _A2111*_B2001 + _A2110*_B2101; + C[587] = _A2121*_B1002 - _A2120*_B1102 - _A2111*_B2002 + _A2110*_B2102; + C[588] = _A2121*_B1010 - _A2120*_B1110 - _A2111*_B2010 + _A2110*_B2110; + C[589] = _A2121*_B1011 - _A2120*_B1111 - _A2111*_B2011 + _A2110*_B2111; + C[590] = _A2121*_B1012 - _A2120*_B1112 - _A2111*_B2012 + _A2110*_B2112; + C[591] = _A2121*_B1020 - _A2120*_B1120 - _A2111*_B2020 + _A2110*_B2120; + C[592] = _A2121*_B1021 - _A2120*_B1121 - _A2111*_B2021 + _A2110*_B2121; + C[593] = _A2121*_B1022 - _A2120*_B1122 - _A2111*_B2022 + _A2110*_B2122; + C[594] = _A2121*_B0200 - _A2122*_B0100 + _A2102*_B2100 - _A2101*_B2200; + C[595] = _A2121*_B0201 - _A2122*_B0101 + _A2102*_B2101 - _A2101*_B2201; + C[596] = _A2121*_B0202 - _A2122*_B0102 + _A2102*_B2102 - _A2101*_B2202; + C[597] = _A2121*_B0210 - _A2122*_B0110 + _A2102*_B2110 - _A2101*_B2210; + C[598] = _A2121*_B0211 - _A2122*_B0111 + _A2102*_B2111 - _A2101*_B2211; + C[599] = _A2121*_B0212 - _A2122*_B0112 + _A2102*_B2112 - _A2101*_B2212; + C[600] = _A2121*_B0220 - _A2122*_B0120 + _A2102*_B2120 - _A2101*_B2220; + C[601] = _A2121*_B0221 - _A2122*_B0121 + _A2102*_B2121 - _A2101*_B2221; + C[602] = _A2121*_B0222 - _A2122*_B0122 + _A2102*_B2122 - _A2101*_B2222; + C[603] = _A2122*_B0000 - _A2120*_B0200 - _A2102*_B2000 + _A2100*_B2200; + C[604] = _A2122*_B0001 - _A2120*_B0201 - _A2102*_B2001 + _A2100*_B2201; + C[605] = _A2122*_B0002 - _A2120*_B0202 - _A2102*_B2002 + _A2100*_B2202; + C[606] = _A2122*_B0010 - _A2120*_B0210 - _A2102*_B2010 + _A2100*_B2210; + C[607] = _A2122*_B0011 - _A2120*_B0211 - _A2102*_B2011 + _A2100*_B2211; + C[608] = _A2122*_B0012 - _A2120*_B0212 - _A2102*_B2012 + _A2100*_B2212; + C[609] = _A2122*_B0020 - _A2120*_B0220 - _A2102*_B2020 + _A2100*_B2220; + C[610] = _A2122*_B0021 - _A2120*_B0221 - _A2102*_B2021 + _A2100*_B2221; + C[611] = _A2122*_B0022 - _A2120*_B0222 - _A2102*_B2022 + _A2100*_B2222; + C[612] = _A2120*_B0100 - _A2121*_B0000 + _A2101*_B2000 - _A2100*_B2100; + C[613] = _A2120*_B0101 - _A2121*_B0001 + _A2101*_B2001 - _A2100*_B2101; + C[614] = _A2120*_B0102 - _A2121*_B0002 + _A2101*_B2002 - _A2100*_B2102; + C[615] = _A2120*_B0110 - _A2121*_B0010 + _A2101*_B2010 - _A2100*_B2110; + C[616] = _A2120*_B0111 - _A2121*_B0011 + _A2101*_B2011 - _A2100*_B2111; + C[617] = _A2120*_B0112 - _A2121*_B0012 + _A2101*_B2012 - _A2100*_B2112; + C[618] = _A2120*_B0120 - _A2121*_B0020 + _A2101*_B2020 - _A2100*_B2120; + C[619] = _A2120*_B0121 - _A2121*_B0021 + _A2101*_B2021 - _A2100*_B2121; + C[620] = _A2120*_B0122 - _A2121*_B0022 + _A2101*_B2022 - _A2100*_B2122; + C[621] = _A2112*_B0100 - _A2111*_B0200 - _A2102*_B1100 + _A2101*_B1200; + C[622] = _A2112*_B0101 - _A2111*_B0201 - _A2102*_B1101 + _A2101*_B1201; + C[623] = _A2112*_B0102 - _A2111*_B0202 - _A2102*_B1102 + _A2101*_B1202; + C[624] = _A2112*_B0110 - _A2111*_B0210 - _A2102*_B1110 + _A2101*_B1210; + C[625] = _A2112*_B0111 - _A2111*_B0211 - _A2102*_B1111 + _A2101*_B1211; + C[626] = _A2112*_B0112 - _A2111*_B0212 - _A2102*_B1112 + _A2101*_B1212; + C[627] = _A2112*_B0120 - _A2111*_B0220 - _A2102*_B1120 + _A2101*_B1220; + C[628] = _A2112*_B0121 - _A2111*_B0221 - _A2102*_B1121 + _A2101*_B1221; + C[629] = _A2112*_B0122 - _A2111*_B0222 - _A2102*_B1122 + _A2101*_B1222; + C[630] = _A2110*_B0200 - _A2112*_B0000 + _A2102*_B1000 - _A2100*_B1200; + C[631] = _A2110*_B0201 - _A2112*_B0001 + _A2102*_B1001 - _A2100*_B1201; + C[632] = _A2110*_B0202 - _A2112*_B0002 + _A2102*_B1002 - _A2100*_B1202; + C[633] = _A2110*_B0210 - _A2112*_B0010 + _A2102*_B1010 - _A2100*_B1210; + C[634] = _A2110*_B0211 - _A2112*_B0011 + _A2102*_B1011 - _A2100*_B1211; + C[635] = _A2110*_B0212 - _A2112*_B0012 + _A2102*_B1012 - _A2100*_B1212; + C[636] = _A2110*_B0220 - _A2112*_B0020 + _A2102*_B1020 - _A2100*_B1220; + C[637] = _A2110*_B0221 - _A2112*_B0021 + _A2102*_B1021 - _A2100*_B1221; + C[638] = _A2110*_B0222 - _A2112*_B0022 + _A2102*_B1022 - _A2100*_B1222; + C[639] = _A2111*_B0000 - _A2110*_B0100 - _A2101*_B1000 + _A2100*_B1100; + C[640] = _A2111*_B0001 - _A2110*_B0101 - _A2101*_B1001 + _A2100*_B1101; + C[641] = _A2111*_B0002 - _A2110*_B0102 - _A2101*_B1002 + _A2100*_B1102; + C[642] = _A2111*_B0010 - _A2110*_B0110 - _A2101*_B1010 + _A2100*_B1110; + C[643] = _A2111*_B0011 - _A2110*_B0111 - _A2101*_B1011 + _A2100*_B1111; + C[644] = _A2111*_B0012 - _A2110*_B0112 - _A2101*_B1012 + _A2100*_B1112; + C[645] = _A2111*_B0020 - _A2110*_B0120 - _A2101*_B1020 + _A2100*_B1120; + C[646] = _A2111*_B0021 - _A2110*_B0121 - _A2101*_B1021 + _A2100*_B1121; + C[647] = _A2111*_B0022 - _A2110*_B0122 - _A2101*_B1022 + _A2100*_B1122; + C[648] = _A2222*_B1100 - _A2221*_B1200 - _A2212*_B2100 + _A2211*_B2200; + C[649] = _A2222*_B1101 - _A2221*_B1201 - _A2212*_B2101 + _A2211*_B2201; + C[650] = _A2222*_B1102 - _A2221*_B1202 - _A2212*_B2102 + _A2211*_B2202; + C[651] = _A2222*_B1110 - _A2221*_B1210 - _A2212*_B2110 + _A2211*_B2210; + C[652] = _A2222*_B1111 - _A2221*_B1211 - _A2212*_B2111 + _A2211*_B2211; + C[653] = _A2222*_B1112 - _A2221*_B1212 - _A2212*_B2112 + _A2211*_B2212; + C[654] = _A2222*_B1120 - _A2221*_B1220 - _A2212*_B2120 + _A2211*_B2220; + C[655] = _A2222*_B1121 - _A2221*_B1221 - _A2212*_B2121 + _A2211*_B2221; + C[656] = _A2222*_B1122 - _A2221*_B1222 - _A2212*_B2122 + _A2211*_B2222; + C[657] = _A2220*_B1200 - _A2222*_B1000 + _A2212*_B2000 - _A2210*_B2200; + C[658] = _A2220*_B1201 - _A2222*_B1001 + _A2212*_B2001 - _A2210*_B2201; + C[659] = _A2220*_B1202 - _A2222*_B1002 + _A2212*_B2002 - _A2210*_B2202; + C[660] = _A2220*_B1210 - _A2222*_B1010 + _A2212*_B2010 - _A2210*_B2210; + C[661] = _A2220*_B1211 - _A2222*_B1011 + _A2212*_B2011 - _A2210*_B2211; + C[662] = _A2220*_B1212 - _A2222*_B1012 + _A2212*_B2012 - _A2210*_B2212; + C[663] = _A2220*_B1220 - _A2222*_B1020 + _A2212*_B2020 - _A2210*_B2220; + C[664] = _A2220*_B1221 - _A2222*_B1021 + _A2212*_B2021 - _A2210*_B2221; + C[665] = _A2220*_B1222 - _A2222*_B1022 + _A2212*_B2022 - _A2210*_B2222; + C[666] = _A2221*_B1000 - _A2220*_B1100 - _A2211*_B2000 + _A2210*_B2100; + C[667] = _A2221*_B1001 - _A2220*_B1101 - _A2211*_B2001 + _A2210*_B2101; + C[668] = _A2221*_B1002 - _A2220*_B1102 - _A2211*_B2002 + _A2210*_B2102; + C[669] = _A2221*_B1010 - _A2220*_B1110 - _A2211*_B2010 + _A2210*_B2110; + C[670] = _A2221*_B1011 - _A2220*_B1111 - _A2211*_B2011 + _A2210*_B2111; + C[671] = _A2221*_B1012 - _A2220*_B1112 - _A2211*_B2012 + _A2210*_B2112; + C[672] = _A2221*_B1020 - _A2220*_B1120 - _A2211*_B2020 + _A2210*_B2120; + C[673] = _A2221*_B1021 - _A2220*_B1121 - _A2211*_B2021 + _A2210*_B2121; + C[674] = _A2221*_B1022 - _A2220*_B1122 - _A2211*_B2022 + _A2210*_B2122; + C[675] = _A2221*_B0200 - _A2222*_B0100 + _A2202*_B2100 - _A2201*_B2200; + C[676] = _A2221*_B0201 - _A2222*_B0101 + _A2202*_B2101 - _A2201*_B2201; + C[677] = _A2221*_B0202 - _A2222*_B0102 + _A2202*_B2102 - _A2201*_B2202; + C[678] = _A2221*_B0210 - _A2222*_B0110 + _A2202*_B2110 - _A2201*_B2210; + C[679] = _A2221*_B0211 - _A2222*_B0111 + _A2202*_B2111 - _A2201*_B2211; + C[680] = _A2221*_B0212 - _A2222*_B0112 + _A2202*_B2112 - _A2201*_B2212; + C[681] = _A2221*_B0220 - _A2222*_B0120 + _A2202*_B2120 - _A2201*_B2220; + C[682] = _A2221*_B0221 - _A2222*_B0121 + _A2202*_B2121 - _A2201*_B2221; + C[683] = _A2221*_B0222 - _A2222*_B0122 + _A2202*_B2122 - _A2201*_B2222; + C[684] = _A2222*_B0000 - _A2220*_B0200 - _A2202*_B2000 + _A2200*_B2200; + C[685] = _A2222*_B0001 - _A2220*_B0201 - _A2202*_B2001 + _A2200*_B2201; + C[686] = _A2222*_B0002 - _A2220*_B0202 - _A2202*_B2002 + _A2200*_B2202; + C[687] = _A2222*_B0010 - _A2220*_B0210 - _A2202*_B2010 + _A2200*_B2210; + C[688] = _A2222*_B0011 - _A2220*_B0211 - _A2202*_B2011 + _A2200*_B2211; + C[689] = _A2222*_B0012 - _A2220*_B0212 - _A2202*_B2012 + _A2200*_B2212; + C[690] = _A2222*_B0020 - _A2220*_B0220 - _A2202*_B2020 + _A2200*_B2220; + C[691] = _A2222*_B0021 - _A2220*_B0221 - _A2202*_B2021 + _A2200*_B2221; + C[692] = _A2222*_B0022 - _A2220*_B0222 - _A2202*_B2022 + _A2200*_B2222; + C[693] = _A2220*_B0100 - _A2221*_B0000 + _A2201*_B2000 - _A2200*_B2100; + C[694] = _A2220*_B0101 - _A2221*_B0001 + _A2201*_B2001 - _A2200*_B2101; + C[695] = _A2220*_B0102 - _A2221*_B0002 + _A2201*_B2002 - _A2200*_B2102; + C[696] = _A2220*_B0110 - _A2221*_B0010 + _A2201*_B2010 - _A2200*_B2110; + C[697] = _A2220*_B0111 - _A2221*_B0011 + _A2201*_B2011 - _A2200*_B2111; + C[698] = _A2220*_B0112 - _A2221*_B0012 + _A2201*_B2012 - _A2200*_B2112; + C[699] = _A2220*_B0120 - _A2221*_B0020 + _A2201*_B2020 - _A2200*_B2120; + C[700] = _A2220*_B0121 - _A2221*_B0021 + _A2201*_B2021 - _A2200*_B2121; + C[701] = _A2220*_B0122 - _A2221*_B0022 + _A2201*_B2022 - _A2200*_B2122; + C[702] = _A2212*_B0100 - _A2211*_B0200 - _A2202*_B1100 + _A2201*_B1200; + C[703] = _A2212*_B0101 - _A2211*_B0201 - _A2202*_B1101 + _A2201*_B1201; + C[704] = _A2212*_B0102 - _A2211*_B0202 - _A2202*_B1102 + _A2201*_B1202; + C[705] = _A2212*_B0110 - _A2211*_B0210 - _A2202*_B1110 + _A2201*_B1210; + C[706] = _A2212*_B0111 - _A2211*_B0211 - _A2202*_B1111 + _A2201*_B1211; + C[707] = _A2212*_B0112 - _A2211*_B0212 - _A2202*_B1112 + _A2201*_B1212; + C[708] = _A2212*_B0120 - _A2211*_B0220 - _A2202*_B1120 + _A2201*_B1220; + C[709] = _A2212*_B0121 - _A2211*_B0221 - _A2202*_B1121 + _A2201*_B1221; + C[710] = _A2212*_B0122 - _A2211*_B0222 - _A2202*_B1122 + _A2201*_B1222; + C[711] = _A2210*_B0200 - _A2212*_B0000 + _A2202*_B1000 - _A2200*_B1200; + C[712] = _A2210*_B0201 - _A2212*_B0001 + _A2202*_B1001 - _A2200*_B1201; + C[713] = _A2210*_B0202 - _A2212*_B0002 + _A2202*_B1002 - _A2200*_B1202; + C[714] = _A2210*_B0210 - _A2212*_B0010 + _A2202*_B1010 - _A2200*_B1210; + C[715] = _A2210*_B0211 - _A2212*_B0011 + _A2202*_B1011 - _A2200*_B1211; + C[716] = _A2210*_B0212 - _A2212*_B0012 + _A2202*_B1012 - _A2200*_B1212; + C[717] = _A2210*_B0220 - _A2212*_B0020 + _A2202*_B1020 - _A2200*_B1220; + C[718] = _A2210*_B0221 - _A2212*_B0021 + _A2202*_B1021 - _A2200*_B1221; + C[719] = _A2210*_B0222 - _A2212*_B0022 + _A2202*_B1022 - _A2200*_B1222; + C[720] = _A2211*_B0000 - _A2210*_B0100 - _A2201*_B1000 + _A2200*_B1100; + C[721] = _A2211*_B0001 - _A2210*_B0101 - _A2201*_B1001 + _A2200*_B1101; + C[722] = _A2211*_B0002 - _A2210*_B0102 - _A2201*_B1002 + _A2200*_B1102; + C[723] = _A2211*_B0010 - _A2210*_B0110 - _A2201*_B1010 + _A2200*_B1110; + C[724] = _A2211*_B0011 - _A2210*_B0111 - _A2201*_B1011 + _A2200*_B1111; + C[725] = _A2211*_B0012 - _A2210*_B0112 - _A2201*_B1012 + _A2200*_B1112; + C[726] = _A2211*_B0020 - _A2210*_B0120 - _A2201*_B1020 + _A2200*_B1120; + C[727] = _A2211*_B0021 - _A2210*_B0121 - _A2201*_B1021 + _A2200*_B1121; + C[728] = _A2211*_B0022 - _A2210*_B0122 - _A2201*_B1022 + _A2200*_B1122; + +} +//--------------------------------------------------------------------------------------------------------// + + + + + + + + + + +//--------------------------------------------------------------------------------------------------------// +//(AxB)_{IjJ} = E_{jpq}E_{JPQ}A_{IpP}B_{qQ} +// tensor cross product of 3rd and 2nd order tensors +template +FASTOR_HINT_INLINE void _crossproduct32(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { + + T _A000 = A[0]; + T _A001 = A[1]; + T _A002 = A[2]; + T _A010 = A[3]; + T _A011 = A[4]; + T _A012 = A[5]; + T _A020 = A[6]; + T _A021 = A[7]; + T _A022 = A[8]; + T _A100 = A[9]; + T _A101 = A[10]; + T _A102 = A[11]; + T _A110 = A[12]; + T _A111 = A[13]; + T _A112 = A[14]; + T _A120 = A[15]; + T _A121 = A[16]; + T _A122 = A[17]; + T _A200 = A[18]; + T _A201 = A[19]; + T _A202 = A[20]; + T _A210 = A[21]; + T _A211 = A[22]; + T _A212 = A[23]; + T _A220 = A[24]; + T _A221 = A[25]; + T _A222 = A[26]; + + + T _B00 = B[0]; + T _B01 = B[1]; + T _B02 = B[2]; + T _B10 = B[3]; + T _B11 = B[4]; + T _B12 = B[5]; + T _B20 = B[6]; + T _B21 = B[7]; + T _B22 = B[8]; + + + C[0] = _A011*_B22 - _A012*_B21 - _A021*_B12 + _A022*_B11; + C[1] = _A012*_B20 - _A010*_B22 + _A020*_B12 - _A022*_B10; + C[2] = _A010*_B21 - _A011*_B20 - _A020*_B11 + _A021*_B10; + C[3] = _A002*_B21 - _A001*_B22 + _A021*_B02 - _A022*_B01; + C[4] = _A000*_B22 - _A002*_B20 - _A020*_B02 + _A022*_B00; + C[5] = _A001*_B20 - _A000*_B21 + _A020*_B01 - _A021*_B00; + C[6] = _A001*_B12 - _A002*_B11 - _A011*_B02 + _A012*_B01; + C[7] = _A002*_B10 - _A000*_B12 + _A010*_B02 - _A012*_B00; + C[8] = _A000*_B11 - _A001*_B10 - _A010*_B01 + _A011*_B00; + C[9] = _A111*_B22 - _A112*_B21 - _A121*_B12 + _A122*_B11; + C[10] = _A112*_B20 - _A110*_B22 + _A120*_B12 - _A122*_B10; + C[11] = _A110*_B21 - _A111*_B20 - _A120*_B11 + _A121*_B10; + C[12] = _A102*_B21 - _A101*_B22 + _A121*_B02 - _A122*_B01; + C[13] = _A100*_B22 - _A102*_B20 - _A120*_B02 + _A122*_B00; + C[14] = _A101*_B20 - _A100*_B21 + _A120*_B01 - _A121*_B00; + C[15] = _A101*_B12 - _A102*_B11 - _A111*_B02 + _A112*_B01; + C[16] = _A102*_B10 - _A100*_B12 + _A110*_B02 - _A112*_B00; + C[17] = _A100*_B11 - _A101*_B10 - _A110*_B01 + _A111*_B00; + C[18] = _A211*_B22 - _A212*_B21 - _A221*_B12 + _A222*_B11; + C[19] = _A212*_B20 - _A210*_B22 + _A220*_B12 - _A222*_B10; + C[20] = _A210*_B21 - _A211*_B20 - _A220*_B11 + _A221*_B10; + C[21] = _A202*_B21 - _A201*_B22 + _A221*_B02 - _A222*_B01; + C[22] = _A200*_B22 - _A202*_B20 - _A220*_B02 + _A222*_B00; + C[23] = _A201*_B20 - _A200*_B21 + _A220*_B01 - _A221*_B00; + C[24] = _A201*_B12 - _A202*_B11 - _A211*_B02 + _A212*_B01; + C[25] = _A202*_B10 - _A200*_B12 + _A210*_B02 - _A212*_B00; + C[26] = _A200*_B11 - _A201*_B10 - _A210*_B01 + _A211*_B00; +} +//--------------------------------------------------------------------------------------------------------// + + + +//--------------------------------------------------------------------------------------------------------// +//(AxB)_{iIJ} = E_{ipq}E_{IPQ}A_{qQJ}B_{pP} +// tensor cross product of 2nd and 3rd order tensors +template +FASTOR_HINT_INLINE void _crossproduct23(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { + + T _A000 = A[0]; + T _A001 = A[1]; + T _A002 = A[2]; + T _A010 = A[3]; + T _A011 = A[4]; + T _A012 = A[5]; + T _A020 = A[6]; + T _A021 = A[7]; + T _A022 = A[8]; + T _A100 = A[9]; + T _A101 = A[10]; + T _A102 = A[11]; + T _A110 = A[12]; + T _A111 = A[13]; + T _A112 = A[14]; + T _A120 = A[15]; + T _A121 = A[16]; + T _A122 = A[17]; + T _A200 = A[18]; + T _A201 = A[19]; + T _A202 = A[20]; + T _A210 = A[21]; + T _A211 = A[22]; + T _A212 = A[23]; + T _A220 = A[24]; + T _A221 = A[25]; + T _A222 = A[26]; + + + T _B00 = B[0]; + T _B01 = B[1]; + T _B02 = B[2]; + T _B10 = B[3]; + T _B11 = B[4]; + T _B12 = B[5]; + T _B20 = B[6]; + T _B21 = B[7]; + T _B22 = B[8]; + + + C[0] = _A110*_B22 - _A120*_B21 - _A210*_B12 + _A220*_B11; + C[1] = _A111*_B22 - _A121*_B21 - _A211*_B12 + _A221*_B11; + C[2] = _A112*_B22 - _A122*_B21 - _A212*_B12 + _A222*_B11; + C[3] = _A120*_B20 - _A100*_B22 + _A200*_B12 - _A220*_B10; + C[4] = _A121*_B20 - _A101*_B22 + _A201*_B12 - _A221*_B10; + C[5] = _A122*_B20 - _A102*_B22 + _A202*_B12 - _A222*_B10; + C[6] = _A100*_B21 - _A110*_B20 - _A200*_B11 + _A210*_B10; + C[7] = _A101*_B21 - _A111*_B20 - _A201*_B11 + _A211*_B10; + C[8] = _A102*_B21 - _A112*_B20 - _A202*_B11 + _A212*_B10; + C[9] = _A020*_B21 - _A010*_B22 + _A210*_B02 - _A220*_B01; + C[10] = _A021*_B21 - _A011*_B22 + _A211*_B02 - _A221*_B01; + C[11] = _A022*_B21 - _A012*_B22 + _A212*_B02 - _A222*_B01; + C[12] = _A000*_B22 - _A020*_B20 - _A200*_B02 + _A220*_B00; + C[13] = _A001*_B22 - _A021*_B20 - _A201*_B02 + _A221*_B00; + C[14] = _A002*_B22 - _A022*_B20 - _A202*_B02 + _A222*_B00; + C[15] = _A010*_B20 - _A000*_B21 + _A200*_B01 - _A210*_B00; + C[16] = _A011*_B20 - _A001*_B21 + _A201*_B01 - _A211*_B00; + C[17] = _A012*_B20 - _A002*_B21 + _A202*_B01 - _A212*_B00; + C[18] = _A010*_B12 - _A020*_B11 - _A110*_B02 + _A120*_B01; + C[19] = _A011*_B12 - _A021*_B11 - _A111*_B02 + _A121*_B01; + C[20] = _A012*_B12 - _A022*_B11 - _A112*_B02 + _A122*_B01; + C[21] = _A020*_B10 - _A000*_B12 + _A100*_B02 - _A120*_B00; + C[22] = _A021*_B10 - _A001*_B12 + _A101*_B02 - _A121*_B00; + C[23] = _A022*_B10 - _A002*_B12 + _A102*_B02 - _A122*_B00; + C[24] = _A000*_B11 - _A010*_B10 - _A100*_B01 + _A110*_B00; + C[25] = _A001*_B11 - _A011*_B10 - _A101*_B01 + _A111*_B00; + C[26] = _A002*_B11 - _A012*_B10 - _A102*_B01 + _A112*_B00; +} +//--------------------------------------------------------------------------------------------------------// + + + + +//--------------------------------------------------------------------------------------------------------// +//(AxB)_{iIjJ} = E_{jpq}E_{JPQ}A_{iIpP}B_{qQ} +// tensor cross product of 4th and 2nd order tensors +template +FASTOR_HINT_INLINE void _crossproduct42(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { + + T _A0000 = A[0]; + T _A0001 = A[1]; + T _A0002 = A[2]; + T _A0010 = A[3]; + T _A0011 = A[4]; + T _A0012 = A[5]; + T _A0020 = A[6]; + T _A0021 = A[7]; + T _A0022 = A[8]; + T _A0100 = A[9]; + T _A0101 = A[10]; + T _A0102 = A[11]; + T _A0110 = A[12]; + T _A0111 = A[13]; + T _A0112 = A[14]; + T _A0120 = A[15]; + T _A0121 = A[16]; + T _A0122 = A[17]; + T _A0200 = A[18]; + T _A0201 = A[19]; + T _A0202 = A[20]; + T _A0210 = A[21]; + T _A0211 = A[22]; + T _A0212 = A[23]; + T _A0220 = A[24]; + T _A0221 = A[25]; + T _A0222 = A[26]; + T _A1000 = A[27]; + T _A1001 = A[28]; + T _A1002 = A[29]; + T _A1010 = A[30]; + T _A1011 = A[31]; + T _A1012 = A[32]; + T _A1020 = A[33]; + T _A1021 = A[34]; + T _A1022 = A[35]; + T _A1100 = A[36]; + T _A1101 = A[37]; + T _A1102 = A[38]; + T _A1110 = A[39]; + T _A1111 = A[40]; + T _A1112 = A[41]; + T _A1120 = A[42]; + T _A1121 = A[43]; + T _A1122 = A[44]; + T _A1200 = A[45]; + T _A1201 = A[46]; + T _A1202 = A[47]; + T _A1210 = A[48]; + T _A1211 = A[49]; + T _A1212 = A[50]; + T _A1220 = A[51]; + T _A1221 = A[52]; + T _A1222 = A[53]; + T _A2000 = A[54]; + T _A2001 = A[55]; + T _A2002 = A[56]; + T _A2010 = A[57]; + T _A2011 = A[58]; + T _A2012 = A[59]; + T _A2020 = A[60]; + T _A2021 = A[61]; + T _A2022 = A[62]; + T _A2100 = A[63]; + T _A2101 = A[64]; + T _A2102 = A[65]; + T _A2110 = A[66]; + T _A2111 = A[67]; + T _A2112 = A[68]; + T _A2120 = A[69]; + T _A2121 = A[70]; + T _A2122 = A[71]; + T _A2200 = A[72]; + T _A2201 = A[73]; + T _A2202 = A[74]; + T _A2210 = A[75]; + T _A2211 = A[76]; + T _A2212 = A[77]; + T _A2220 = A[78]; + T _A2221 = A[79]; + T _A2222 = A[80]; + + + T _B00 = B[0]; + T _B01 = B[1]; + T _B02 = B[2]; + T _B10 = B[3]; + T _B11 = B[4]; + T _B12 = B[5]; + T _B20 = B[6]; + T _B21 = B[7]; + T _B22 = B[8]; + + + C[0] = _A0011*_B22 - _A0012*_B21 - _A0021*_B12 + _A0022*_B11; + C[1] = _A0012*_B20 - _A0010*_B22 + _A0020*_B12 - _A0022*_B10; + C[2] = _A0010*_B21 - _A0011*_B20 - _A0020*_B11 + _A0021*_B10; + C[3] = _A0002*_B21 - _A0001*_B22 + _A0021*_B02 - _A0022*_B01; + C[4] = _A0000*_B22 - _A0002*_B20 - _A0020*_B02 + _A0022*_B00; + C[5] = _A0001*_B20 - _A0000*_B21 + _A0020*_B01 - _A0021*_B00; + C[6] = _A0001*_B12 - _A0002*_B11 - _A0011*_B02 + _A0012*_B01; + C[7] = _A0002*_B10 - _A0000*_B12 + _A0010*_B02 - _A0012*_B00; + C[8] = _A0000*_B11 - _A0001*_B10 - _A0010*_B01 + _A0011*_B00; + C[9] = _A0111*_B22 - _A0112*_B21 - _A0121*_B12 + _A0122*_B11; + C[10] = _A0112*_B20 - _A0110*_B22 + _A0120*_B12 - _A0122*_B10; + C[11] = _A0110*_B21 - _A0111*_B20 - _A0120*_B11 + _A0121*_B10; + C[12] = _A0102*_B21 - _A0101*_B22 + _A0121*_B02 - _A0122*_B01; + C[13] = _A0100*_B22 - _A0102*_B20 - _A0120*_B02 + _A0122*_B00; + C[14] = _A0101*_B20 - _A0100*_B21 + _A0120*_B01 - _A0121*_B00; + C[15] = _A0101*_B12 - _A0102*_B11 - _A0111*_B02 + _A0112*_B01; + C[16] = _A0102*_B10 - _A0100*_B12 + _A0110*_B02 - _A0112*_B00; + C[17] = _A0100*_B11 - _A0101*_B10 - _A0110*_B01 + _A0111*_B00; + C[18] = _A0211*_B22 - _A0212*_B21 - _A0221*_B12 + _A0222*_B11; + C[19] = _A0212*_B20 - _A0210*_B22 + _A0220*_B12 - _A0222*_B10; + C[20] = _A0210*_B21 - _A0211*_B20 - _A0220*_B11 + _A0221*_B10; + C[21] = _A0202*_B21 - _A0201*_B22 + _A0221*_B02 - _A0222*_B01; + C[22] = _A0200*_B22 - _A0202*_B20 - _A0220*_B02 + _A0222*_B00; + C[23] = _A0201*_B20 - _A0200*_B21 + _A0220*_B01 - _A0221*_B00; + C[24] = _A0201*_B12 - _A0202*_B11 - _A0211*_B02 + _A0212*_B01; + C[25] = _A0202*_B10 - _A0200*_B12 + _A0210*_B02 - _A0212*_B00; + C[26] = _A0200*_B11 - _A0201*_B10 - _A0210*_B01 + _A0211*_B00; + C[27] = _A1011*_B22 - _A1012*_B21 - _A1021*_B12 + _A1022*_B11; + C[28] = _A1012*_B20 - _A1010*_B22 + _A1020*_B12 - _A1022*_B10; + C[29] = _A1010*_B21 - _A1011*_B20 - _A1020*_B11 + _A1021*_B10; + C[30] = _A1002*_B21 - _A1001*_B22 + _A1021*_B02 - _A1022*_B01; + C[31] = _A1000*_B22 - _A1002*_B20 - _A1020*_B02 + _A1022*_B00; + C[32] = _A1001*_B20 - _A1000*_B21 + _A1020*_B01 - _A1021*_B00; + C[33] = _A1001*_B12 - _A1002*_B11 - _A1011*_B02 + _A1012*_B01; + C[34] = _A1002*_B10 - _A1000*_B12 + _A1010*_B02 - _A1012*_B00; + C[35] = _A1000*_B11 - _A1001*_B10 - _A1010*_B01 + _A1011*_B00; + C[36] = _A1111*_B22 - _A1112*_B21 - _A1121*_B12 + _A1122*_B11; + C[37] = _A1112*_B20 - _A1110*_B22 + _A1120*_B12 - _A1122*_B10; + C[38] = _A1110*_B21 - _A1111*_B20 - _A1120*_B11 + _A1121*_B10; + C[39] = _A1102*_B21 - _A1101*_B22 + _A1121*_B02 - _A1122*_B01; + C[40] = _A1100*_B22 - _A1102*_B20 - _A1120*_B02 + _A1122*_B00; + C[41] = _A1101*_B20 - _A1100*_B21 + _A1120*_B01 - _A1121*_B00; + C[42] = _A1101*_B12 - _A1102*_B11 - _A1111*_B02 + _A1112*_B01; + C[43] = _A1102*_B10 - _A1100*_B12 + _A1110*_B02 - _A1112*_B00; + C[44] = _A1100*_B11 - _A1101*_B10 - _A1110*_B01 + _A1111*_B00; + C[45] = _A1211*_B22 - _A1212*_B21 - _A1221*_B12 + _A1222*_B11; + C[46] = _A1212*_B20 - _A1210*_B22 + _A1220*_B12 - _A1222*_B10; + C[47] = _A1210*_B21 - _A1211*_B20 - _A1220*_B11 + _A1221*_B10; + C[48] = _A1202*_B21 - _A1201*_B22 + _A1221*_B02 - _A1222*_B01; + C[49] = _A1200*_B22 - _A1202*_B20 - _A1220*_B02 + _A1222*_B00; + C[50] = _A1201*_B20 - _A1200*_B21 + _A1220*_B01 - _A1221*_B00; + C[51] = _A1201*_B12 - _A1202*_B11 - _A1211*_B02 + _A1212*_B01; + C[52] = _A1202*_B10 - _A1200*_B12 + _A1210*_B02 - _A1212*_B00; + C[53] = _A1200*_B11 - _A1201*_B10 - _A1210*_B01 + _A1211*_B00; + C[54] = _A2011*_B22 - _A2012*_B21 - _A2021*_B12 + _A2022*_B11; + C[55] = _A2012*_B20 - _A2010*_B22 + _A2020*_B12 - _A2022*_B10; + C[56] = _A2010*_B21 - _A2011*_B20 - _A2020*_B11 + _A2021*_B10; + C[57] = _A2002*_B21 - _A2001*_B22 + _A2021*_B02 - _A2022*_B01; + C[58] = _A2000*_B22 - _A2002*_B20 - _A2020*_B02 + _A2022*_B00; + C[59] = _A2001*_B20 - _A2000*_B21 + _A2020*_B01 - _A2021*_B00; + C[60] = _A2001*_B12 - _A2002*_B11 - _A2011*_B02 + _A2012*_B01; + C[61] = _A2002*_B10 - _A2000*_B12 + _A2010*_B02 - _A2012*_B00; + C[62] = _A2000*_B11 - _A2001*_B10 - _A2010*_B01 + _A2011*_B00; + C[63] = _A2111*_B22 - _A2112*_B21 - _A2121*_B12 + _A2122*_B11; + C[64] = _A2112*_B20 - _A2110*_B22 + _A2120*_B12 - _A2122*_B10; + C[65] = _A2110*_B21 - _A2111*_B20 - _A2120*_B11 + _A2121*_B10; + C[66] = _A2102*_B21 - _A2101*_B22 + _A2121*_B02 - _A2122*_B01; + C[67] = _A2100*_B22 - _A2102*_B20 - _A2120*_B02 + _A2122*_B00; + C[68] = _A2101*_B20 - _A2100*_B21 + _A2120*_B01 - _A2121*_B00; + C[69] = _A2101*_B12 - _A2102*_B11 - _A2111*_B02 + _A2112*_B01; + C[70] = _A2102*_B10 - _A2100*_B12 + _A2110*_B02 - _A2112*_B00; + C[71] = _A2100*_B11 - _A2101*_B10 - _A2110*_B01 + _A2111*_B00; + C[72] = _A2211*_B22 - _A2212*_B21 - _A2221*_B12 + _A2222*_B11; + C[73] = _A2212*_B20 - _A2210*_B22 + _A2220*_B12 - _A2222*_B10; + C[74] = _A2210*_B21 - _A2211*_B20 - _A2220*_B11 + _A2221*_B10; + C[75] = _A2202*_B21 - _A2201*_B22 + _A2221*_B02 - _A2222*_B01; + C[76] = _A2200*_B22 - _A2202*_B20 - _A2220*_B02 + _A2222*_B00; + C[77] = _A2201*_B20 - _A2200*_B21 + _A2220*_B01 - _A2221*_B00; + C[78] = _A2201*_B12 - _A2202*_B11 - _A2211*_B02 + _A2212*_B01; + C[79] = _A2202*_B10 - _A2200*_B12 + _A2210*_B02 - _A2212*_B00; + C[80] = _A2200*_B11 - _A2201*_B10 - _A2210*_B01 + _A2211*_B00; +} +//--------------------------------------------------------------------------------------------------------// + + + +//--------------------------------------------------------------------------------------------------------// +//(AxB)_{iIjJ} = E_{ipq}E_{IPQ}A_{qQjJ}B_{pP} +// tensor cross product of 2nd and 4th order tensors +template +FASTOR_HINT_INLINE void _crossproduct24(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { + + T _A0000 = A[0]; + T _A0001 = A[1]; + T _A0002 = A[2]; + T _A0010 = A[3]; + T _A0011 = A[4]; + T _A0012 = A[5]; + T _A0020 = A[6]; + T _A0021 = A[7]; + T _A0022 = A[8]; + T _A0100 = A[9]; + T _A0101 = A[10]; + T _A0102 = A[11]; + T _A0110 = A[12]; + T _A0111 = A[13]; + T _A0112 = A[14]; + T _A0120 = A[15]; + T _A0121 = A[16]; + T _A0122 = A[17]; + T _A0200 = A[18]; + T _A0201 = A[19]; + T _A0202 = A[20]; + T _A0210 = A[21]; + T _A0211 = A[22]; + T _A0212 = A[23]; + T _A0220 = A[24]; + T _A0221 = A[25]; + T _A0222 = A[26]; + T _A1000 = A[27]; + T _A1001 = A[28]; + T _A1002 = A[29]; + T _A1010 = A[30]; + T _A1011 = A[31]; + T _A1012 = A[32]; + T _A1020 = A[33]; + T _A1021 = A[34]; + T _A1022 = A[35]; + T _A1100 = A[36]; + T _A1101 = A[37]; + T _A1102 = A[38]; + T _A1110 = A[39]; + T _A1111 = A[40]; + T _A1112 = A[41]; + T _A1120 = A[42]; + T _A1121 = A[43]; + T _A1122 = A[44]; + T _A1200 = A[45]; + T _A1201 = A[46]; + T _A1202 = A[47]; + T _A1210 = A[48]; + T _A1211 = A[49]; + T _A1212 = A[50]; + T _A1220 = A[51]; + T _A1221 = A[52]; + T _A1222 = A[53]; + T _A2000 = A[54]; + T _A2001 = A[55]; + T _A2002 = A[56]; + T _A2010 = A[57]; + T _A2011 = A[58]; + T _A2012 = A[59]; + T _A2020 = A[60]; + T _A2021 = A[61]; + T _A2022 = A[62]; + T _A2100 = A[63]; + T _A2101 = A[64]; + T _A2102 = A[65]; + T _A2110 = A[66]; + T _A2111 = A[67]; + T _A2112 = A[68]; + T _A2120 = A[69]; + T _A2121 = A[70]; + T _A2122 = A[71]; + T _A2200 = A[72]; + T _A2201 = A[73]; + T _A2202 = A[74]; + T _A2210 = A[75]; + T _A2211 = A[76]; + T _A2212 = A[77]; + T _A2220 = A[78]; + T _A2221 = A[79]; + T _A2222 = A[80]; + + + T _B00 = B[0]; + T _B01 = B[1]; + T _B02 = B[2]; + T _B10 = B[3]; + T _B11 = B[4]; + T _B12 = B[5]; + T _B20 = B[6]; + T _B21 = B[7]; + T _B22 = B[8]; + + + C[0] = _A1100*_B22 - _A1200*_B21 - _A2100*_B12 + _A2200*_B11; + C[1] = _A1101*_B22 - _A1201*_B21 - _A2101*_B12 + _A2201*_B11; + C[2] = _A1102*_B22 - _A1202*_B21 - _A2102*_B12 + _A2202*_B11; + C[3] = _A1110*_B22 - _A1210*_B21 - _A2110*_B12 + _A2210*_B11; + C[4] = _A1111*_B22 - _A1211*_B21 - _A2111*_B12 + _A2211*_B11; + C[5] = _A1112*_B22 - _A1212*_B21 - _A2112*_B12 + _A2212*_B11; + C[6] = _A1120*_B22 - _A1220*_B21 - _A2120*_B12 + _A2220*_B11; + C[7] = _A1121*_B22 - _A1221*_B21 - _A2121*_B12 + _A2221*_B11; + C[8] = _A1122*_B22 - _A1222*_B21 - _A2122*_B12 + _A2222*_B11; + C[9] = _A1200*_B20 - _A1000*_B22 + _A2000*_B12 - _A2200*_B10; + C[10] = _A1201*_B20 - _A1001*_B22 + _A2001*_B12 - _A2201*_B10; + C[11] = _A1202*_B20 - _A1002*_B22 + _A2002*_B12 - _A2202*_B10; + C[12] = _A1210*_B20 - _A1010*_B22 + _A2010*_B12 - _A2210*_B10; + C[13] = _A1211*_B20 - _A1011*_B22 + _A2011*_B12 - _A2211*_B10; + C[14] = _A1212*_B20 - _A1012*_B22 + _A2012*_B12 - _A2212*_B10; + C[15] = _A1220*_B20 - _A1020*_B22 + _A2020*_B12 - _A2220*_B10; + C[16] = _A1221*_B20 - _A1021*_B22 + _A2021*_B12 - _A2221*_B10; + C[17] = _A1222*_B20 - _A1022*_B22 + _A2022*_B12 - _A2222*_B10; + C[18] = _A1000*_B21 - _A1100*_B20 - _A2000*_B11 + _A2100*_B10; + C[19] = _A1001*_B21 - _A1101*_B20 - _A2001*_B11 + _A2101*_B10; + C[20] = _A1002*_B21 - _A1102*_B20 - _A2002*_B11 + _A2102*_B10; + C[21] = _A1010*_B21 - _A1110*_B20 - _A2010*_B11 + _A2110*_B10; + C[22] = _A1011*_B21 - _A1111*_B20 - _A2011*_B11 + _A2111*_B10; + C[23] = _A1012*_B21 - _A1112*_B20 - _A2012*_B11 + _A2112*_B10; + C[24] = _A1020*_B21 - _A1120*_B20 - _A2020*_B11 + _A2120*_B10; + C[25] = _A1021*_B21 - _A1121*_B20 - _A2021*_B11 + _A2121*_B10; + C[26] = _A1022*_B21 - _A1122*_B20 - _A2022*_B11 + _A2122*_B10; + C[27] = _A0200*_B21 - _A0100*_B22 + _A2100*_B02 - _A2200*_B01; + C[28] = _A0201*_B21 - _A0101*_B22 + _A2101*_B02 - _A2201*_B01; + C[29] = _A0202*_B21 - _A0102*_B22 + _A2102*_B02 - _A2202*_B01; + C[30] = _A0210*_B21 - _A0110*_B22 + _A2110*_B02 - _A2210*_B01; + C[31] = _A0211*_B21 - _A0111*_B22 + _A2111*_B02 - _A2211*_B01; + C[32] = _A0212*_B21 - _A0112*_B22 + _A2112*_B02 - _A2212*_B01; + C[33] = _A0220*_B21 - _A0120*_B22 + _A2120*_B02 - _A2220*_B01; + C[34] = _A0221*_B21 - _A0121*_B22 + _A2121*_B02 - _A2221*_B01; + C[35] = _A0222*_B21 - _A0122*_B22 + _A2122*_B02 - _A2222*_B01; + C[36] = _A0000*_B22 - _A0200*_B20 - _A2000*_B02 + _A2200*_B00; + C[37] = _A0001*_B22 - _A0201*_B20 - _A2001*_B02 + _A2201*_B00; + C[38] = _A0002*_B22 - _A0202*_B20 - _A2002*_B02 + _A2202*_B00; + C[39] = _A0010*_B22 - _A0210*_B20 - _A2010*_B02 + _A2210*_B00; + C[40] = _A0011*_B22 - _A0211*_B20 - _A2011*_B02 + _A2211*_B00; + C[41] = _A0012*_B22 - _A0212*_B20 - _A2012*_B02 + _A2212*_B00; + C[42] = _A0020*_B22 - _A0220*_B20 - _A2020*_B02 + _A2220*_B00; + C[43] = _A0021*_B22 - _A0221*_B20 - _A2021*_B02 + _A2221*_B00; + C[44] = _A0022*_B22 - _A0222*_B20 - _A2022*_B02 + _A2222*_B00; + C[45] = _A0100*_B20 - _A0000*_B21 + _A2000*_B01 - _A2100*_B00; + C[46] = _A0101*_B20 - _A0001*_B21 + _A2001*_B01 - _A2101*_B00; + C[47] = _A0102*_B20 - _A0002*_B21 + _A2002*_B01 - _A2102*_B00; + C[48] = _A0110*_B20 - _A0010*_B21 + _A2010*_B01 - _A2110*_B00; + C[49] = _A0111*_B20 - _A0011*_B21 + _A2011*_B01 - _A2111*_B00; + C[50] = _A0112*_B20 - _A0012*_B21 + _A2012*_B01 - _A2112*_B00; + C[51] = _A0120*_B20 - _A0020*_B21 + _A2020*_B01 - _A2120*_B00; + C[52] = _A0121*_B20 - _A0021*_B21 + _A2021*_B01 - _A2121*_B00; + C[53] = _A0122*_B20 - _A0022*_B21 + _A2022*_B01 - _A2122*_B00; + C[54] = _A0100*_B12 - _A0200*_B11 - _A1100*_B02 + _A1200*_B01; + C[55] = _A0101*_B12 - _A0201*_B11 - _A1101*_B02 + _A1201*_B01; + C[56] = _A0102*_B12 - _A0202*_B11 - _A1102*_B02 + _A1202*_B01; + C[57] = _A0110*_B12 - _A0210*_B11 - _A1110*_B02 + _A1210*_B01; + C[58] = _A0111*_B12 - _A0211*_B11 - _A1111*_B02 + _A1211*_B01; + C[59] = _A0112*_B12 - _A0212*_B11 - _A1112*_B02 + _A1212*_B01; + C[60] = _A0120*_B12 - _A0220*_B11 - _A1120*_B02 + _A1220*_B01; + C[61] = _A0121*_B12 - _A0221*_B11 - _A1121*_B02 + _A1221*_B01; + C[62] = _A0122*_B12 - _A0222*_B11 - _A1122*_B02 + _A1222*_B01; + C[63] = _A0200*_B10 - _A0000*_B12 + _A1000*_B02 - _A1200*_B00; + C[64] = _A0201*_B10 - _A0001*_B12 + _A1001*_B02 - _A1201*_B00; + C[65] = _A0202*_B10 - _A0002*_B12 + _A1002*_B02 - _A1202*_B00; + C[66] = _A0210*_B10 - _A0010*_B12 + _A1010*_B02 - _A1210*_B00; + C[67] = _A0211*_B10 - _A0011*_B12 + _A1011*_B02 - _A1211*_B00; + C[68] = _A0212*_B10 - _A0012*_B12 + _A1012*_B02 - _A1212*_B00; + C[69] = _A0220*_B10 - _A0020*_B12 + _A1020*_B02 - _A1220*_B00; + C[70] = _A0221*_B10 - _A0021*_B12 + _A1021*_B02 - _A1221*_B00; + C[71] = _A0222*_B10 - _A0022*_B12 + _A1022*_B02 - _A1222*_B00; + C[72] = _A0000*_B11 - _A0100*_B10 - _A1000*_B01 + _A1100*_B00; + C[73] = _A0001*_B11 - _A0101*_B10 - _A1001*_B01 + _A1101*_B00; + C[74] = _A0002*_B11 - _A0102*_B10 - _A1002*_B01 + _A1102*_B00; + C[75] = _A0010*_B11 - _A0110*_B10 - _A1010*_B01 + _A1110*_B00; + C[76] = _A0011*_B11 - _A0111*_B10 - _A1011*_B01 + _A1111*_B00; + C[77] = _A0012*_B11 - _A0112*_B10 - _A1012*_B01 + _A1112*_B00; + C[78] = _A0020*_B11 - _A0120*_B10 - _A1020*_B01 + _A1120*_B00; + C[79] = _A0021*_B11 - _A0121*_B10 - _A1021*_B01 + _A1121*_B00; + C[80] = _A0022*_B11 - _A0122*_B10 - _A1022*_B01 + _A1122*_B00; +} +//--------------------------------------------------------------------------------------------------------// + +} // end of namespace Fastor + + +#endif // TENSOR_CROSS_H + diff --git a/noarch/include/Fastor/backend/trace.h b/noarch/include/Fastor/backend/trace.h new file mode 100644 index 00000000..e7c4b46b --- /dev/null +++ b/noarch/include/Fastor/backend/trace.h @@ -0,0 +1,59 @@ +#ifndef TRACE_H +#define TRACE_H + +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" + +namespace Fastor { + + +template::type=0> +FASTOR_INLINE T _trace(const T * FASTOR_RESTRICT a) { + T sum = static_cast(0); + for (FASTOR_INDEX i=0; i +FASTOR_INLINE double _trace(const double * FASTOR_RESTRICT a) { + // AVX VERSION + // IVY 5 OPS / HW 7 OPS +// __m256d a_reg = _mm256_load_pd(a); +// __m128d a_high = _mm256_extractf128_pd(a_reg,0x1); +// return _mm_cvtsd_f64(_mm_add_sd(_mm256_castpd256_pd128(a_reg),_mm_shuffle_pd(a_high,a_high,0x1))); + + // SSE VERSION + // 3 OPS + __m128d a0 = _mm_load_sd(a); + __m128d a1 = _mm_load_sd(a+3); + return _mm_cvtsd_f64(_mm_add_pd(a0,a1)); +} + +template<> +FASTOR_INLINE double _trace(const double * FASTOR_RESTRICT a) { + // No benefit in AVX + return _mm_cvtsd_f64(_mm_add_sd(_mm_load_sd(a),_mm_add_sd(_mm_load_sd(a+4),_mm_load_sd(a+8)))); +} + +template<> +FASTOR_INLINE float _trace(const float * FASTOR_RESTRICT a) { + __m128 a_reg = _mm_load_ps(a); + return _mm_cvtss_f32(_mm_add_ss(a_reg,_mm_reverse_ps(a_reg))); +} +#endif + +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE float _trace(const float * FASTOR_RESTRICT a) { + __m256 a_reg = _mm256_load_ps(a); + __m128 sum_two = _mm_add_ps(_mm256_castps256_ps128(a_reg),_mm256_extractf128_ps(a_reg,0x1)); + return _mm_cvtss_f32(_mm_add_ss(sum_two,_mm_load_ss(a+8))); +} +#endif + +} + +#endif // TRACE_H + diff --git a/noarch/include/Fastor/backend/transpose/transpose.h b/noarch/include/Fastor/backend/transpose/transpose.h new file mode 100644 index 00000000..9e45cfae --- /dev/null +++ b/noarch/include/Fastor/backend/transpose/transpose.h @@ -0,0 +1,448 @@ +#ifndef TRANSPOSE_H +#define TRANSPOSE_H + + +#include "Fastor/config/config.h" +#include "Fastor/backend/transpose/transpose_kernels.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/SIMDVector.h" + +namespace Fastor { + +// Forward declare +namespace internal { +template +FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out); +} // internal + + +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_AVX_IMPL + +template +FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { + + using V = SIMDVector; + + // Block sizes of 8x8 i.e. numSIMDRows=1 + // numSIMDCols=1 and innerBlock=outerBlock=1 + // give a much greater speed up, but causes + // significant slow-down for issue #42 + +#ifndef FASTOR_TRANS_OUTER_BLOCK_SIZE + constexpr size_t numSIMDRows = 1UL; +#else + constexpr size_t numSIMDRows = FASTOR_TRANS_OUTER_BLOCK_SIZE; +#endif +#ifndef FASTOR_TRANS_INNER_BLOCK_SIZE + constexpr size_t numSIMDCols = 1UL; +#else + constexpr size_t numSIMDCols = FASTOR_TRANS_INNER_BLOCK_SIZE; +#endif + + constexpr size_t innerBlock = V::Size * numSIMDCols; + constexpr size_t outerBlock = V::Size * numSIMDRows; + + FASTOR_ARCH_ALIGN T pack_a[outerBlock*innerBlock]; + FASTOR_ARCH_ALIGN T pack_out[outerBlock*innerBlock]; + + constexpr size_t M0 = M / innerBlock * innerBlock; + constexpr size_t N0 = N / outerBlock * outerBlock; + V _vec; + + // For row-major matrices we go over N + // and then M to get contiguous writes + size_t j=0; + for (; j(pack_a,pack_out); + // Unpack pack_out to out + for (size_t jj=0; jj +FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { + for (size_t j=0; j +FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + __m128 a_reg = _mm_loadu_ps(a); + _mm_storeu_ps(out,_mm_shuffle_ps(a_reg,a_reg,_MM_SHUFFLE(3,1,2,0))); +} + +template<> +FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { +#ifndef FASTOR_AVX2_IMPL + // 5 OPS + __m128 row0 = _mm_loadu_ps(a); + __m128 row1 = _mm_loadu_ps(a+3); + __m128 row2 = _mm_loadu_ps(a+6); + + __m128 T0 = _mm_unpacklo_ps(row0,row1); + __m128 T1 = _mm_unpackhi_ps(row0,row1); + + row0 = _mm_movelh_ps ( T0,row2 ); + row1 = _mm_shuffle_ps( T0,row2, _MM_SHUFFLE(3,1,3,2) ); + row2 = _mm_shuffle_ps( T1,row2, _MM_SHUFFLE(3,2,1,0) ); + + _mm_storeu_ps(out,row0); + _mm_storeu_ps(out+3,row1); + _mm_storeu_ps(out+6,row2); // out of range for out[9] +#else + // 3 OPS + // gcc/clang emit vpermsps tht operate on (%rsp) + // less pressure on shuffle port perhaps + __m256 trans07 = _mm256_loadu_ps(a); + const __m256i trans_mask = _mm256_setr_epi32( + 0,3,6, + 1,4,7, + 2,5); + // does not shuffle across 256 lanes, only 128 lanes + // __m256 _res = _mm256_permutevar_ps(trans07, trans_mask); + // this one shuffles correctly + __m256 _res = _mm256_permutevar8x32_ps(trans07, trans_mask); + _mm256_storeu_ps(out,_res); + // out[8] = a[8]; + _mm_store_ss(out+8,_mm_load_ss(a+8)); +#endif +} + +template<> +FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { +#ifdef FASTOR_AVX512F_IMPL + __m512 amm = _mm512_loadu_ps(a); + __m512i idx = _mm512_setr_epi32( 0, 4, 8, 12, + 1, 5, 9, 13, + 2, 6, 10, 14, + 3, 7, 11, 15); + __m512 omm = _mm512_permutexvar_ps(idx, amm); + _mm512_storeu_ps(out, omm); +#else + __m128 row1 = _mm_loadu_ps(a); + __m128 row2 = _mm_loadu_ps(a+4); + __m128 row3 = _mm_loadu_ps(a+8); + __m128 row4 = _mm_loadu_ps(a+12); + _MM_TRANSPOSE4_PS(row1, row2, row3, row4); + _mm_storeu_ps(out , row1); + _mm_storeu_ps(out+4 , row2); + _mm_storeu_ps(out+8 , row3); + _mm_storeu_ps(out+12, row4); +#endif +} +#endif + +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + __m256 row1 = _mm256_loadu_ps(a); + __m256 row2 = _mm256_loadu_ps(a+8); + __m256 row3 = _mm256_loadu_ps(a+16); + __m256 row4 = _mm256_loadu_ps(a+24); + __m256 row5 = _mm256_loadu_ps(a+32); + __m256 row6 = _mm256_loadu_ps(a+40); + __m256 row7 = _mm256_loadu_ps(a+48); + __m256 row8 = _mm256_loadu_ps(a+56); + internal::_MM_TRANSPOSE8_PS(row1, row2, row3, row4, row5, row6, row7, row8); + _mm256_storeu_ps(out, row1); + _mm256_storeu_ps(out+8, row2); + _mm256_storeu_ps(out+16, row3); + _mm256_storeu_ps(out+24, row4); + _mm256_storeu_ps(out+32, row5); + _mm256_storeu_ps(out+40, row6); + _mm256_storeu_ps(out+48, row7); + _mm256_storeu_ps(out+56, row8); +} +#endif + +#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL) +template<> +FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + internal::_MM_TRANSPOSE16_PS(a,out); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + + +// Specialisations - double +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE void _transpose(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) { + /*-------------------------------------------------------*/ + // 2 OPS + __m128d row0 = _mm_loadu_pd(a); + __m128d row1 = _mm_loadu_pd(a+2); + __m128d tmp = row0; + row0 = _mm_shuffle_pd(row0,row1,0x0); + row1 = _mm_shuffle_pd(tmp ,row1,0x3); + _mm_storeu_pd(out ,row0); + _mm_storeu_pd(out+2,row1); + /*-------------------------------------------------------*/ + + /*-------------------------------------------------------*/ +// // AVX VERSION + // // IVY 4 OPS / HW 8 OPS + // __m256d a1 = _mm256_loadu_pd(a); + // __m128d a2 = _mm256_castpd256_pd128(a1); + // __m128d a3 = _mm256_extractf128_pd(a1,0x1); + // __m128d a4 = _mm_shuffle_pd(a2,a3,0x0); + // a3 = _mm_shuffle_pd(a2,a3,0x3); + // a1 = _mm256_castpd128_pd256(a4); + // a1 = _mm256_insertf128_pd(a1,a3,0x1); + // _mm256_storeu_pd(out,a1); + /*-------------------------------------------------------*/ +} +#endif + +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE void _transpose(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) { + // AVX512 is the fastest & AVX version despite more instructions is faster than SSE +#if defined(FASTOR_AVX512F_IMPL) + // AVX512 + /*-------------------------------------------------------*/ + __m512d a07 = _mm512_loadu_pd(a); + const __m512i trans_mask = _mm512_setr_epi64( + 0,3,6, + 1,4,7, + 2,5); + __m512d trans07 = _mm512_permutexvar_pd(trans_mask, a07); + _mm512_storeu_pd(out,trans07); + _mm_store_sd(out+8,_mm_load_sd(a+8)); + /*-------------------------------------------------------*/ +#elif defined(FASTOR_AVX_IMPL) + // AVX + /*-------------------------------------------------------*/ + __m256d row1 = _mm256_loadu_pd(a); + __m256d row2 = _mm256_loadu_pd(a+4); + + __m128d a11 = _mm256_castpd256_pd128(row1); + __m128d a12 = _mm256_extractf128_pd(row1,0x1); + __m128d a21 = _mm256_castpd256_pd128(row2); + __m128d a22 = _mm256_extractf128_pd(row2,0x1); + + row1 = _mm256_castpd128_pd256(_mm_shuffle_pd(a11,a12,0x2)); + row1 = _mm256_insertf128_pd(row1,_mm_shuffle_pd(a22,a11,0x2),0x1); + row2 = _mm256_castpd128_pd256(_mm_shuffle_pd(a21,a22,0x2)); + row2 = _mm256_insertf128_pd(row2,_mm_shuffle_pd(a12,a21,0x2),0x1); + + _mm256_storeu_pd(out,row1); + _mm256_storeu_pd(out+4,row2); + _mm_store_sd(out+8,_mm_load_sd(a+8)); + /*-------------------------------------------------------*/ +#else + // SSE + /*-------------------------------------------------------*/ + __m128d a11 = _mm_loadu_pd(a); + __m128d a12 = _mm_loadu_pd(a+2); + __m128d a21 = _mm_loadu_pd(a+4); + __m128d a22 = _mm_loadu_pd(a+6); + + _mm_storeu_pd(out ,_mm_shuffle_pd(a11,a12,0x2)); + _mm_storeu_pd(out+2,_mm_shuffle_pd(a22,a11,0x2)); + _mm_storeu_pd(out+4,_mm_shuffle_pd(a21,a22,0x2)); + _mm_storeu_pd(out+6,_mm_shuffle_pd(a12,a21,0x2)); + _mm_store_sd (out+8,_mm_load_sd(a+8)); + /*-------------------------------------------------------*/ +#endif +} +#endif + +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE void _transpose(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { +#ifdef FASTOR_AVX512F_IMPL + __m512d amm0 = _mm512_loadu_pd(a); + __m512d amm1 = _mm512_loadu_pd(a+8); + __m512i idx0 = _mm512_setr_epi64(0, 4, 8, 12, 1, 5, 9, 13); + __m512i idx1 = _mm512_setr_epi64(2, 6, 10, 14, 3, 7, 11, 15); + __m512d omm0 = _mm512_permutex2var_pd(amm0, idx0, amm1); + __m512d omm1 = _mm512_permutex2var_pd(amm0, idx1, amm1); + _mm512_storeu_pd(out , omm0); + _mm512_storeu_pd(out+8, omm1); +#else + __m256d row1 = _mm256_loadu_pd(a); + __m256d row2 = _mm256_loadu_pd(a+4); + __m256d row3 = _mm256_loadu_pd(a+8); + __m256d row4 = _mm256_loadu_pd(a+12); + internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); + _mm256_storeu_pd(out, row1); + _mm256_storeu_pd(out+4, row2); + _mm256_storeu_pd(out+8, row3); + _mm256_storeu_pd(out+12, row4); +#endif +} +#endif + +#if defined(FASTOR_AVX512F_IMPL) +template<> +FASTOR_INLINE void _transpose(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { + __m512d row0 = _mm512_loadu_pd(a); + __m512d row1 = _mm512_loadu_pd(a+8); + __m512d row2 = _mm512_loadu_pd(a+16); + __m512d row3 = _mm512_loadu_pd(a+24); + __m512d row4 = _mm512_loadu_pd(a+32); + __m512d row5 = _mm512_loadu_pd(a+40); + __m512d row6 = _mm512_loadu_pd(a+48); + __m512d row7 = _mm512_loadu_pd(a+56); + internal::_MM_TRANSPOSE8_PD(row0,row1,row2,row3,row4,row5,row6,row7); + _mm512_storeu_pd(out , row0); + _mm512_storeu_pd(out+8 , row1); + _mm512_storeu_pd(out+16, row2); + _mm512_storeu_pd(out+24, row3); + _mm512_storeu_pd(out+32, row4); + _mm512_storeu_pd(out+40, row5); + _mm512_storeu_pd(out+48, row6); + _mm512_storeu_pd(out+56, row7); +} +#elif defined(FASTOR_AVX_IMPL) +template<> +FASTOR_INLINE void _transpose(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { + + { + __m256d row1 = _mm256_loadu_pd(&a[0]); + __m256d row2 = _mm256_loadu_pd(&a[8]); + __m256d row3 = _mm256_loadu_pd(&a[16]); + __m256d row4 = _mm256_loadu_pd(&a[24]); + internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); + _mm256_storeu_pd(&out[0], row1); + _mm256_storeu_pd(&out[8], row2); + _mm256_storeu_pd(&out[16], row3); + _mm256_storeu_pd(&out[24], row4); + } + + { + __m256d row1 = _mm256_loadu_pd(&a[32]); + __m256d row2 = _mm256_loadu_pd(&a[40]); + __m256d row3 = _mm256_loadu_pd(&a[48]); + __m256d row4 = _mm256_loadu_pd(&a[56]); + internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); + _mm256_storeu_pd(&out[4], row1); + _mm256_storeu_pd(&out[12], row2); + _mm256_storeu_pd(&out[20], row3); + _mm256_storeu_pd(&out[28], row4); + } + + { + __m256d row1 = _mm256_loadu_pd(&a[4]); + __m256d row2 = _mm256_loadu_pd(&a[12]); + __m256d row3 = _mm256_loadu_pd(&a[20]); + __m256d row4 = _mm256_loadu_pd(&a[28]); + internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); + _mm256_storeu_pd(&out[32], row1); + _mm256_storeu_pd(&out[40], row2); + _mm256_storeu_pd(&out[48], row3); + _mm256_storeu_pd(&out[56], row4); + } + + { + __m256d row1 = _mm256_loadu_pd(&a[36]); + __m256d row2 = _mm256_loadu_pd(&a[44]); + __m256d row3 = _mm256_loadu_pd(&a[52]); + __m256d row4 = _mm256_loadu_pd(&a[60]); + internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); + _mm256_storeu_pd(&out[36], row1); + _mm256_storeu_pd(&out[44], row2); + _mm256_storeu_pd(&out[52], row3); + _mm256_storeu_pd(&out[60], row4); + } +} +#endif +//----------------------------------------------------------------------------------------------------------// + + + + +//----------------------------------------------------------------------------------------------------------// +namespace internal { +// To get around compilers recusive inlining depth issue +template +FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { + for (size_t j=0; j +FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { + _transpose(a,out); +} +template<> +FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { + _transpose(a,out); +} +} // internal +//----------------------------------------------------------------------------------------------------------// + +} + +#endif // TRANSPOSE_H + diff --git a/noarch/include/Fastor/backend/transpose/transpose_kernels.h b/noarch/include/Fastor/backend/transpose/transpose_kernels.h new file mode 100644 index 00000000..c0ecef99 --- /dev/null +++ b/noarch/include/Fastor/backend/transpose/transpose_kernels.h @@ -0,0 +1,233 @@ +#ifndef TRANSPOSE_KERNELS_H +#define TRANSPOSE_KERNELS_H + +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" + +namespace Fastor { + +namespace internal { + +#ifdef FASTOR_SSE_IMPL +// 4x4 PS - defined +// _MM_TRANSPOSE4_PS +#endif + +#ifdef FASTOR_AVX_IMPL + +// 4x4 PD +FASTOR_INLINE void _MM_TRANSPOSE4_PD(__m256d &row0, __m256d &row1, __m256d &row2, __m256d &row3) +{ + __m256d tmp3, tmp2, tmp1, tmp0; + tmp0 = _mm256_shuffle_pd((row0),(row1), 0x0); + tmp2 = _mm256_shuffle_pd((row0),(row1), 0xF); + tmp1 = _mm256_shuffle_pd((row2),(row3), 0x0); + tmp3 = _mm256_shuffle_pd((row2),(row3), 0xF); + row0 = _mm256_permute2f128_pd(tmp0, tmp1, 0x20); + row1 = _mm256_permute2f128_pd(tmp2, tmp3, 0x20); + row2 = _mm256_permute2f128_pd(tmp0, tmp1, 0x31); + row3 = _mm256_permute2f128_pd(tmp2, tmp3, 0x31); +} + +// 8x8 PS +FASTOR_INLINE void _MM_TRANSPOSE8_PS(__m256 &row0, __m256 &row1, __m256 &row2, __m256 &row3, + __m256 &row4, __m256 &row5, __m256 &row6, __m256 &row7) +{ + __m256 __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7; + __m256 __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7; + __t0 = _mm256_unpacklo_ps(row0, row1); + __t1 = _mm256_unpackhi_ps(row0, row1); + __t2 = _mm256_unpacklo_ps(row2, row3); + __t3 = _mm256_unpackhi_ps(row2, row3); + __t4 = _mm256_unpacklo_ps(row4, row5); + __t5 = _mm256_unpackhi_ps(row4, row5); + __t6 = _mm256_unpacklo_ps(row6, row7); + __t7 = _mm256_unpackhi_ps(row6, row7); + __tt0 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(1,0,1,0)); + __tt1 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(3,2,3,2)); + __tt2 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(1,0,1,0)); + __tt3 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(3,2,3,2)); + __tt4 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(1,0,1,0)); + __tt5 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(3,2,3,2)); + __tt6 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(1,0,1,0)); + __tt7 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(3,2,3,2)); + row0 = _mm256_permute2f128_ps(__tt0, __tt4, 0x20); + row1 = _mm256_permute2f128_ps(__tt1, __tt5, 0x20); + row2 = _mm256_permute2f128_ps(__tt2, __tt6, 0x20); + row3 = _mm256_permute2f128_ps(__tt3, __tt7, 0x20); + row4 = _mm256_permute2f128_ps(__tt0, __tt4, 0x31); + row5 = _mm256_permute2f128_ps(__tt1, __tt5, 0x31); + row6 = _mm256_permute2f128_ps(__tt2, __tt6, 0x31); + row7 = _mm256_permute2f128_ps(__tt3, __tt7, 0x31); +} +#endif + +#ifdef FASTOR_AVX512F_IMPL +// 8x8 PD +inline void _MM_TRANSPOSE8_PD(__m512d &row0, __m512d &row1, __m512d &row2, __m512d &row3, + __m512d &row4, __m512d &row5, __m512d &row6, __m512d &row7) +{ + __m512d __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7; + __m512d __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7; + + FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {0, 8 , 1 , 9 , 4 , 12, 5 , 13}; + FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {2, 10, 3 , 11, 6 , 14, 7 , 15}; + FASTOR_ARCH_ALIGN constexpr int64_t idx3[8] = {0, 1 , 8 , 9 , 4 , 5 , 12, 13}; + FASTOR_ARCH_ALIGN constexpr int64_t idx4[8] = {2, 3 , 10, 11, 6 , 7 , 14, 15}; + FASTOR_ARCH_ALIGN constexpr int64_t idx5[8] = {4, 5 , 6 , 7 , 12, 13, 14, 15}; + + __m512i vidx1 = _mm512_load_epi64(idx1); + __m512i vidx2 = _mm512_load_epi64(idx2); + __m512i vidx3 = _mm512_load_epi64(idx3); + __m512i vidx4 = _mm512_load_epi64(idx4); + __m512i vidx5 = _mm512_load_epi64(idx5); + + __t0 = _mm512_permutex2var_pd(row0, vidx1, row1); + __t1 = _mm512_permutex2var_pd(row0, vidx2, row1); + __t2 = _mm512_permutex2var_pd(row2, vidx1, row3); + __t3 = _mm512_permutex2var_pd(row2, vidx2, row3); + __t4 = _mm512_permutex2var_pd(row4, vidx1, row5); + __t5 = _mm512_permutex2var_pd(row4, vidx2, row5); + __t6 = _mm512_permutex2var_pd(row6, vidx1, row7); + __t7 = _mm512_permutex2var_pd(row6, vidx2, row7); + + __tt0 = _mm512_permutex2var_pd(__t0, vidx3, __t2); + __tt1 = _mm512_permutex2var_pd(__t0, vidx4, __t2); + __tt2 = _mm512_permutex2var_pd(__t1, vidx3, __t3); + __tt3 = _mm512_permutex2var_pd(__t1, vidx4, __t3); + __tt4 = _mm512_permutex2var_pd(__t4, vidx3, __t6); + __tt5 = _mm512_permutex2var_pd(__t4, vidx4, __t6); + __tt6 = _mm512_permutex2var_pd(__t5, vidx3, __t7); + __tt7 = _mm512_permutex2var_pd(__t5, vidx4, __t7); + + row0 = _mm512_insertf64x4(__tt0,_mm512_castpd512_pd256(__tt4),0x1); + row1 = _mm512_insertf64x4(__tt1,_mm512_castpd512_pd256(__tt5),0x1); + row2 = _mm512_insertf64x4(__tt2,_mm512_castpd512_pd256(__tt6),0x1); + row3 = _mm512_insertf64x4(__tt3,_mm512_castpd512_pd256(__tt7),0x1); + row4 = _mm512_permutex2var_pd(__tt0, vidx5, __tt4); + row5 = _mm512_permutex2var_pd(__tt1, vidx5, __tt5); + row6 = _mm512_permutex2var_pd(__tt2, vidx5, __tt6); + row7 = _mm512_permutex2var_pd(__tt3, vidx5, __tt7); +} +#endif + + +#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL) +// 16x16 +FASTOR_INLINE void _MM_TRANSPOSE16_PS(const float * FASTOR_RESTRICT mat, float * FASTOR_RESTRICT matT) +{ + __m512 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, ta, tb, tc, td, te, tf; + __m512 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, ra, rb, rc, rd, re, rf; + + int mask; + FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {2, 3, 0, 1, 6, 7, 4, 5}; + FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {1, 0, 3, 2, 5, 4, 7, 6}; + FASTOR_ARCH_ALIGN constexpr int32_t idx3[16] = {1, 0, 3, 2, 5 ,4 ,7 ,6 ,9 ,8 , 11, 10, 13, 12 ,15, 14}; + __m512i vidx1 = _mm512_load_epi64(idx1); + __m512i vidx2 = _mm512_load_epi64(idx2); + __m512i vidx3 = _mm512_load_epi32(idx3); + + t0 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+0])), _mm256_loadu_ps(&mat[ 8*16+0]), 1); + t1 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+0])), _mm256_loadu_ps(&mat[ 9*16+0]), 1); + t2 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+0])), _mm256_loadu_ps(&mat[10*16+0]), 1); + t3 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+0])), _mm256_loadu_ps(&mat[11*16+0]), 1); + t4 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+0])), _mm256_loadu_ps(&mat[12*16+0]), 1); + t5 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+0])), _mm256_loadu_ps(&mat[13*16+0]), 1); + t6 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+0])), _mm256_loadu_ps(&mat[14*16+0]), 1); + t7 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+0])), _mm256_loadu_ps(&mat[15*16+0]), 1); + + t8 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+8])), _mm256_loadu_ps(&mat[ 8*16+8]), 1); + t9 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+8])), _mm256_loadu_ps(&mat[ 9*16+8]), 1); + ta = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+8])), _mm256_loadu_ps(&mat[10*16+8]), 1); + tb = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+8])), _mm256_loadu_ps(&mat[11*16+8]), 1); + tc = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+8])), _mm256_loadu_ps(&mat[12*16+8]), 1); + td = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+8])), _mm256_loadu_ps(&mat[13*16+8]), 1); + te = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+8])), _mm256_loadu_ps(&mat[14*16+8]), 1); + tf = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+8])), _mm256_loadu_ps(&mat[15*16+8]), 1); + + mask= 0xcc; + r0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t0), (__mmask8)mask, vidx1, _mm512_castps_pd(t4))); + r1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t1), (__mmask8)mask, vidx1, _mm512_castps_pd(t5))); + r2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t2), (__mmask8)mask, vidx1, _mm512_castps_pd(t6))); + r3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t3), (__mmask8)mask, vidx1, _mm512_castps_pd(t7))); + r8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t8), (__mmask8)mask, vidx1, _mm512_castps_pd(tc))); + r9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t9), (__mmask8)mask, vidx1, _mm512_castps_pd(td))); + ra = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ta), (__mmask8)mask, vidx1, _mm512_castps_pd(te))); + rb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tb), (__mmask8)mask, vidx1, _mm512_castps_pd(tf))); + + mask= 0x33; + r4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t4), (__mmask8)mask, vidx1, _mm512_castps_pd(t0))); + r5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t5), (__mmask8)mask, vidx1, _mm512_castps_pd(t1))); + r6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t6), (__mmask8)mask, vidx1, _mm512_castps_pd(t2))); + r7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t7), (__mmask8)mask, vidx1, _mm512_castps_pd(t3))); + rc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tc), (__mmask8)mask, vidx1, _mm512_castps_pd(t8))); + rd = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(td), (__mmask8)mask, vidx1, _mm512_castps_pd(t9))); + re = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(te), (__mmask8)mask, vidx1, _mm512_castps_pd(ta))); + rf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tf), (__mmask8)mask, vidx1, _mm512_castps_pd(tb))); + + mask = 0xaa; + t0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r0), (__mmask8)mask, vidx2, _mm512_castps_pd(r2))); + t1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r1), (__mmask8)mask, vidx2, _mm512_castps_pd(r3))); + t4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r4), (__mmask8)mask, vidx2, _mm512_castps_pd(r6))); + t5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r5), (__mmask8)mask, vidx2, _mm512_castps_pd(r7))); + t8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r8), (__mmask8)mask, vidx2, _mm512_castps_pd(ra))); + t9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r9), (__mmask8)mask, vidx2, _mm512_castps_pd(rb))); + tc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rc), (__mmask8)mask, vidx2, _mm512_castps_pd(re))); + td = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rd), (__mmask8)mask, vidx2, _mm512_castps_pd(rf))); + + mask = 0x55; + t2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r2), (__mmask8)mask, vidx2, _mm512_castps_pd(r0))); + t3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r3), (__mmask8)mask, vidx2, _mm512_castps_pd(r1))); + t6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r6), (__mmask8)mask, vidx2, _mm512_castps_pd(r4))); + t7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r7), (__mmask8)mask, vidx2, _mm512_castps_pd(r5))); + ta = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ra), (__mmask8)mask, vidx2, _mm512_castps_pd(r8))); + tb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rb), (__mmask8)mask, vidx2, _mm512_castps_pd(r9))); + te = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(re), (__mmask8)mask, vidx2, _mm512_castps_pd(rc))); + tf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rf), (__mmask8)mask, vidx2, _mm512_castps_pd(rd))); + + mask = 0xaaaa; + r0 = _mm512_mask_permutexvar_ps(t0, (__mmask16)mask, vidx3, t1); + r2 = _mm512_mask_permutexvar_ps(t2, (__mmask16)mask, vidx3, t3); + r4 = _mm512_mask_permutexvar_ps(t4, (__mmask16)mask, vidx3, t5); + r6 = _mm512_mask_permutexvar_ps(t6, (__mmask16)mask, vidx3, t7); + r8 = _mm512_mask_permutexvar_ps(t8, (__mmask16)mask, vidx3, t9); + ra = _mm512_mask_permutexvar_ps(ta, (__mmask16)mask, vidx3, tb); + rc = _mm512_mask_permutexvar_ps(tc, (__mmask16)mask, vidx3, td); + re = _mm512_mask_permutexvar_ps(te, (__mmask16)mask, vidx3, tf); + + mask = 0x5555; + r1 = _mm512_mask_permutexvar_ps(t1, (__mmask16)mask, vidx3, t0); + r3 = _mm512_mask_permutexvar_ps(t3, (__mmask16)mask, vidx3, t2); + r5 = _mm512_mask_permutexvar_ps(t5, (__mmask16)mask, vidx3, t4); + r7 = _mm512_mask_permutexvar_ps(t7, (__mmask16)mask, vidx3, t6); + r9 = _mm512_mask_permutexvar_ps(t9, (__mmask16)mask, vidx3, t8); + rb = _mm512_mask_permutexvar_ps(tb, (__mmask16)mask, vidx3, ta); + rd = _mm512_mask_permutexvar_ps(td, (__mmask16)mask, vidx3, tc); + rf = _mm512_mask_permutexvar_ps(tf, (__mmask16)mask, vidx3, te); + + _mm512_storeu_ps(&matT[ 0*16], r0); + _mm512_storeu_ps(&matT[ 1*16], r1); + _mm512_storeu_ps(&matT[ 2*16], r2); + _mm512_storeu_ps(&matT[ 3*16], r3); + _mm512_storeu_ps(&matT[ 4*16], r4); + _mm512_storeu_ps(&matT[ 5*16], r5); + _mm512_storeu_ps(&matT[ 6*16], r6); + _mm512_storeu_ps(&matT[ 7*16], r7); + _mm512_storeu_ps(&matT[ 8*16], r8); + _mm512_storeu_ps(&matT[ 9*16], r9); + _mm512_storeu_ps(&matT[10*16], ra); + _mm512_storeu_ps(&matT[11*16], rb); + _mm512_storeu_ps(&matT[12*16], rc); + _mm512_storeu_ps(&matT[13*16], rd); + _mm512_storeu_ps(&matT[14*16], re); + _mm512_storeu_ps(&matT[15*16], rf); +} + +#endif + +} // end of namespace internal +} // end of namespace Fastor + + + +#endif // TRANSPOSE_KERNELS_H diff --git a/noarch/include/Fastor/backend/voigt.h b/noarch/include/Fastor/backend/voigt.h new file mode 100644 index 00000000..c4e436ee --- /dev/null +++ b/noarch/include/Fastor/backend/voigt.h @@ -0,0 +1,182 @@ +#ifndef VOIGT_H +#define VOIGT_H + +#include "Fastor/tensor/Tensor.h" + +// Conversion of tensors to symmetrised Voigt forms + +namespace Fastor { + +template +struct VoigtType; +template +struct VoigtType { + using return_type = Tensor; +}; +template +struct VoigtType { + using return_type = Tensor; +}; +template +struct VoigtType { + using return_type = Tensor; +}; +template +struct VoigtType { + using return_type = Tensor; +}; +template +struct VoigtType { + using return_type = Tensor; +}; +template +struct VoigtType { + using return_type = Tensor; +}; + + +template::value == 16 + ,bool>::type=0> +FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) { + VoigtA[0] = a_data[0]; + VoigtA[1] = a_data[3]; + VoigtA[2] = 0.5*(a_data[1]+a_data[2]); + VoigtA[3] = VoigtA[1]; + VoigtA[4] = a_data[15]; + VoigtA[5] = 0.5*(a_data[13]+a_data[14]); + VoigtA[6] = VoigtA[2]; + VoigtA[7] = VoigtA[5]; + VoigtA[8] = 0.5*(a_data[5]+a_data[6]); +} + + +template::value == 81 + ,bool>::type=0> +FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) { + + VoigtA[0] = a_data[0]; + VoigtA[1] = a_data[4]; + VoigtA[2] = a_data[8]; + VoigtA[3] = 0.5*(a_data[1]+a_data[3]); + VoigtA[4] = 0.5*(a_data[2]+a_data[6]); + VoigtA[5] = 0.5*(a_data[5]+a_data[7]); + VoigtA[6] = VoigtA[1]; + VoigtA[7] = a_data[40]; + VoigtA[8] = a_data[44]; + VoigtA[9] = 0.5*(a_data[37]+a_data[39]); + VoigtA[10] = 0.5*(a_data[38]+a_data[42]); + VoigtA[11] = 0.5*(a_data[41]+a_data[43]); + VoigtA[12] = VoigtA[2]; + VoigtA[13] = VoigtA[8]; + VoigtA[14] = a_data[80]; + VoigtA[15] = 0.5*(a_data[73]+a_data[75]); + VoigtA[16] = 0.5*(a_data[74]+a_data[78]); + VoigtA[17] = 0.5*(a_data[77]+a_data[79]); + VoigtA[18] = VoigtA[3]; + VoigtA[19] = VoigtA[9]; + VoigtA[20] = VoigtA[15]; + VoigtA[21] = 0.5*(a_data[10]+a_data[12]); + VoigtA[22] = 0.5*(a_data[11]+a_data[15]); + VoigtA[23] = 0.5*(a_data[14]+a_data[16]); + VoigtA[24] = VoigtA[4]; + VoigtA[25] = VoigtA[10]; + VoigtA[26] = VoigtA[16]; + VoigtA[27] = VoigtA[22]; + VoigtA[28] = 0.5*(a_data[20]+a_data[24]); + VoigtA[29] = 0.5*(a_data[23]+a_data[25]); + VoigtA[30] = VoigtA[5]; + VoigtA[31] = VoigtA[11]; + VoigtA[32] = VoigtA[17]; + VoigtA[33] = VoigtA[23]; + VoigtA[34] = VoigtA[29]; + VoigtA[35] = 0.5*(a_data[50]+a_data[52]); +} + + +template::value == 8 + ,bool>::type=0> +FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { + + // 3rd order tensor to 2nd order tensor + // 3rd order tensor should be symmetric in the first two indices + VoigtA[0] = e[0]; + VoigtA[1] = e[1]; + VoigtA[2] = e[6]; + VoigtA[3] = e[7]; + VoigtA[4] = 0.5*(e[2]+e[4]); + VoigtA[5] = 0.5*(e[3]+e[5]); +} + + +template::value == 27 + ,bool>::type=0> +FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { + + // 3rd order tensor to 2nd order tensor + // 3rd order tensor should be symmetric in the first two indices + VoigtA[0] = e[0]; + VoigtA[1] = e[1]; + VoigtA[2] = e[2]; + VoigtA[3] = e[12]; + VoigtA[4] = e[13]; + VoigtA[5] = e[14]; + VoigtA[6] = e[24]; + VoigtA[7] = e[25]; + VoigtA[8] = e[26]; + VoigtA[9] = 0.5*(e[3]+e[9]); + VoigtA[10] = 0.5*(e[4]+e[10]); + VoigtA[11] = 0.5*(e[5]+e[11]); + VoigtA[12] = 0.5*(e[6]+e[18]); + VoigtA[13] = 0.5*(e[7]+e[19]); + VoigtA[14] = 0.5*(e[8]+e[20]); + VoigtA[15] = 0.5*(e[15]+e[21]); + VoigtA[16] = 0.5*(e[16]+e[22]); + VoigtA[17] = 0.5*(e[17]+e[23]); +} + + +template::value == 4 + ,bool>::type=0> +FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { + + VoigtA[0] = e[0]; + VoigtA[1] = e[3]; + VoigtA[2] = 0.5*(e[1]+e[2]); +} + + +template::value == 9 + ,bool>::type=0> +FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { + + VoigtA[0] = e[0]; + VoigtA[1] = e[4]; + VoigtA[2] = e[8]; + VoigtA[3] = 0.5*(e[1]+e[3]); + VoigtA[4] = 0.5*(e[2]+e[6]); + VoigtA[5] = 0.5*(e[5]+e[7]); +} + + +template +FASTOR_INLINE auto voigt(const Tensor &a) +-> typename VoigtType::return_type { + T *a_data = a.data(); + using ret_type = typename VoigtType::return_type; + ret_type voigt_a; + T *VoigtA = voigt_a.data(); + _voigt(a_data,VoigtA); + + return voigt_a; +} + +} + +#endif // VOIGT_H + diff --git a/noarch/include/Fastor/config/config.h b/noarch/include/Fastor/config/config.h new file mode 100644 index 00000000..c10b3e67 --- /dev/null +++ b/noarch/include/Fastor/config/config.h @@ -0,0 +1,359 @@ +/* This file is part of the FASTOR library. {{{ +Copyright (c) 2020 Roman Poya + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +}}}*/ + +#ifndef CONFIG_H +#define CONFIG_H + +#include +#include +#include +#include +#include +#include + +//------------------------------------------------------------------------------------------------// +//------------------------------------------------------------------------------------------------// + +// Error out for pre-C++14 compiler versions +//------------------------------------------------------------------------------------------------// +#if defined(_MSC_VER) + #if _MSC_VER < 1920 + #error FASTOR REQUIRES AN ISO C++14 COMPLIANT COMPILER + #endif +#elif defined(__GNUC__) || defined(__GNUG__) + #if __cplusplus < 201402L + #error FASTOR REQUIRES AN ISO C++14 COMPLIANT COMPILER + #endif +#endif +//------------------------------------------------------------------------------------------------// + + +// Compiler define macros +//------------------------------------------------------------------------------------------------// +#ifdef __INTEL_COMPILER +#define FASTOR_INTEL __INTEL_COMPILER_BUILD_DATE +#elif defined(__clang__) && defined(__apple_build_version__) +#define FASTOR_APPLECLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) +#elif defined(__clang__) +#define FASTOR_CLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) +#elif defined(__GNUC__) +#define FASTOR_GCC (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#elif defined(_MSC_VER) +#define FASTOR_MSVC _MSC_FULL_VER +#else +#define FASTOR_UNSUPPORTED_COMPILER 1 +#endif +//------------------------------------------------------------------------------------------------// + + +// Operating system define macros +//------------------------------------------------------------------------------------------------// +#if defined(_WIN32) +#define FASTOR_WINDOWS32_OS 1 +#endif +#if defined(_WIN64) +#define FASTOR_WINDOWS64_OS 1 +#endif +#if defined(_WIN32) || defined(_WIN64) +#define FASTOR_WINDOWS_OS 1 +#endif +#if defined(__unix__) || defined(__unix) +#define FASTOR_UNIX_OS 1 +#endif +#if defined(__linux__) +#define FASTOR_LINUX_OS 1 +#endif +#if defined(__APPLE__) || defined(__MACH__) +#define FASTOR_APPLE_OS 1 +#endif +#if defined(__ANDROID__) || defined(ANDROID) +#define FASTOR_ANDROID_OS 1 +#endif +#if defined(__CYGWIN__) +#define FASTOR_CYGWIN_OS 1 +#endif +//------------------------------------------------------------------------------------------------// + + +// Determine CXX version +//------------------------------------------------------------------------------------------------// +#if defined(FASTOR_MSVC) +#if defined(_MSVC_LANG) + #if _MSVC_LANG == 199711L + #define FASTOR_CXX_VERSION 1998 + #elif _MSVC_LANG == 201103L + #define FASTOR_CXX_VERSION 2011 + #elif _MSVC_LANG == 201402L + #define FASTOR_CXX_VERSION 2014 + #elif _MSVC_LANG == 201703L + #define FASTOR_CXX_VERSION 2017 + #elif _MSVC_LANG > 201703L + #define FASTOR_CXX_VERSION 2020 + #endif +#endif +#else +#if defined(__cplusplus) + #if __cplusplus == 199711L + #define FASTOR_CXX_VERSION 1998 + #elif __cplusplus == 201103L + #define FASTOR_CXX_VERSION 2011 + #elif __cplusplus == 201402L + #define FASTOR_CXX_VERSION 2014 + #elif __cplusplus == 201703L + #define FASTOR_CXX_VERSION 2017 + #elif __cplusplus > 201703L + #define FASTOR_CXX_VERSION 2020 + #endif +#endif +#endif +//------------------------------------------------------------------------------------------------// + + +// Force inline and no-inline macros +//------------------------------------------------------------------------------------------------// +#if defined(__GNUC__) || defined(__GNUG__) + #define FASTOR_INLINE inline __attribute__((always_inline)) + #define FASTOR_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) + #define FASTOR_INLINE __forceinline + #define FASTOR_NOINLINE __declspec(noinline) +#endif + +#if defined(__GNUC__) || defined(__GNUG__) +#define FASTOR_RESTRICT __restrict__ +#elif defined(_MSC_VER) +#define FASTOR_RESTRICT __restrict +#endif + +// Traditional inline which works will and helps the compiler +// eliminate a lot of code +#define FASTOR_HINT_INLINE inline +//------------------------------------------------------------------------------------------------// + + +// C++17 [if constexpr] define +//------------------------------------------------------------------------------------------------// +#if FASTOR_CXX_VERSION >= 2017 + #define FASTOR_HAS_IF_CONSTEXPR 1 + #define FASTOR_IF_CONSTEXPR if constexpr +#else + #define FASTOR_HAS_IF_CONSTEXPR 0 + #define FASTOR_IF_CONSTEXPR if +#endif +//------------------------------------------------------------------------------------------------// + + + +// Intrinsics defines +//------------------------------------------------------------------------------------------------// +#ifdef FASTOR_MSVC + #ifdef _M_IX86_FP + #if _M_IX86_FP >= 1 + #ifndef __SSE__ + #define __SSE__ 1 + #endif + #endif + #if _M_IX86_FP >= 2 + #ifndef __SSE2__ + #define __SSE2__ 1 + #endif + #endif + #elif defined(_M_AMD64) + #ifndef __SSE__ + #define __SSE__ 1 + #endif + #ifndef __SSE2__ + #define __SSE2__ 1 + #endif + #endif +#endif + +#if defined(__MIC__) + #define FASTOR_MIC_IMPL 1 +#endif +#if defined(__AVX512F__) + #define FASTOR_AVX512F_IMPL 1 +#endif +#if defined(__AVX512CD__) + #define FASTOR_AVX512CD_IMPL 1 +#endif +#if defined(__AVX512BW__) + #define FASTOR_AVX512BW_IMPL 1 +#endif +#if defined(__AVX512DQ__) + #define FASTOR_AVX512DQ_IMPL 1 +#endif +#if defined(__AVX512VL__) + #define FASTOR_AVX512VL_IMPL 1 +#endif +#if defined(__AVX2__) + #define FASTOR_AVX2_IMPL 1 +#endif +#if defined(__AVX__) + #define FASTOR_AVX_IMPL 1 +#endif +#if defined(__SSE4_2__) + #define FASTOR_SSE4_2_IMPL 1 +#endif +#if defined(__SSE4_1__) + #define FASTOR_SSE4_1_IMPL 1 +#endif +#if defined(__SSE3__) + #define FASTOR_SSE3_IMPL 1 +#endif +#if defined(__SSSE3__) + #define FASTOR_SSSE3_IMPL 1 +#endif +#if defined(__SSE2__) + #define FASTOR_SSE2_IMPL 1 +#endif + +#if defined(__FMA4__) + #define FASTOR_FMA4_IMPL 1 +#endif +#if defined(__FMA__) + #define FASTOR_FMA_IMPL 1 +#endif +// #if !defined(__FMA__) && defined(__AVX2__) +// #define __FMA__ 1 +// #endif + + +// Get around MSVC issue +#if defined(FASTOR_WINDOWS_OS) || defined(FASTOR_MSVC) + #if defined(FASTOR_AVX512F_IMPL) + #if !defined(FASTOR_SSE2_IMPL) + #define FASTOR_SSE2_IMPL 1 + #endif + #if !defined(FASTOR_SSE3_IMPL) + #define FASTOR_SSE3_IMPL 1 + #endif + #if !defined(FASTOR_SSSE3_IMPL) + #define FASTOR_SSSE3_IMPL 1 + #endif + #if !defined(FASTOR_SSE4_1_IMPL) + #define FASTOR_SSE4_1_IMPL 1 + #endif + #if !defined(FASTOR_SSE4_2_IMPL) + #define FASTOR_SSE4_2_IMPL 1 + #endif + #if !defined(FASTOR_AVX_IMPL) + #define FASTOR_AVX_IMPL 1 + #endif + #if !defined(FASTOR_AVX2_IMPL) + #define FASTOR_AVX2_IMPL 1 + #endif + #elif defined(FASTOR_AVX_IMPL) || defined(FASTOR_AVX2_IMPL) + #if !defined(FASTOR_SSE2_IMPL) + #define FASTOR_SSE2_IMPL 1 + #endif + #if !defined(FASTOR_SSE3_IMPL) + #define FASTOR_SSE3_IMPL 1 + #endif + #if !defined(FASTOR_SSSE3_IMPL) + #define FASTOR_SSSE3_IMPL 1 + #endif + #if !defined(FASTOR_SSE4_1_IMPL) + #define FASTOR_SSE4_1_IMPL 1 + #endif + #if !defined(FASTOR_SSE4_2_IMPL) + #define FASTOR_SSE4_2_IMPL 1 + #endif + #endif +#endif + + +#if defined(FASTOR_AVX512F_IMPL) || defined(FASTOR_AVX512CD_IMPL) || defined(FASTOR_AVX512BW_IMPL) || defined(FASTOR_AVX512DQ_IMPL) || defined(FASTOR_AVX512VL_IMPL) + #define FASTOR_AVX512_IMPL 1 +#endif +#if defined(FASTOR_AVX2_IMPL) && !defined(FASTOR_AVX_IMPL) + #define FASTOR_AVX_IMPL 1 +#endif +#if defined(FASTOR_SSE2_IMPL) || defined(FASTOR_SSSE3_IMPL) || defined(FASTOR_SSE3_IMPL) || defined(FASTOR_SSE4_1_IMPL) || defined(FASTOR_SSE4_2_IMPL) + #define FASTOR_SSE_IMPL 1 +#endif + +#if !defined(FASTOR_MIC_IMPL) && !defined(FASTOR_AVX512_IMPL) && !defined(FASTOR_AVX_IMPL) && !defined(FASTOR_SSE_IMPL) +#define FASTOR_SCALAR_IMPL 1 +#endif + + +#ifdef FASTOR_SSE2_IMPL +#include +#endif +#ifdef FASTOR_SSE3_IMPL +#include +#endif +#ifdef FASTOR_SSSE3_IMPL +#include +#endif +#ifdef FASTOR_SSE4_1_IMPL +#include +#endif +#ifdef FASTOR_SSE4_2_IMPL +#include +#endif +#ifdef FASTOR_AVX_IMPL +#include +#endif + + +// Mask loading +//------------------------------------------------------------------------------------------------// +#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) + #define FASTOR_HAS_AVX512_MASKS 1 +#endif +//------------------------------------------------------------------------------------------------// + +// Horizontal add +//------------------------------------------------------------------------------------------------// +#if defined(FASTOR_AVX512F_IMPL) +#if defined(FASTOR_INTEL) + #define FASTOR_HAS_AVX512_REDUCE_ADD 1 +#elif defined (FASTOR_GCC) && __GNUC__ >= 7 + #define FASTOR_HAS_AVX512_REDUCE_ADD 1 +#elif defined (FASTOR_GCC) && __clang_major__ >= 4 + #define FASTOR_HAS_AVX512_REDUCE_ADD 1 +#endif +#endif +//------------------------------------------------------------------------------------------------// + +// AVX512 abs +//------------------------------------------------------------------------------------------------// +#if defined(FASTOR_AVX512F_IMPL) +#if defined(FASTOR_INTEL) + #define FASTOR_HAS_AVX512_ABS 1 +#elif defined (FASTOR_GCC) && __GNUC__ >= 7 && __GNUC_MINOR__ >= 4 + #define FASTOR_HAS_AVX512_ABS 1 +#elif defined (FASTOR_GCC) && __clang_major__ >= 4 + #define FASTOR_HAS_AVX512_ABS 1 +#endif +#endif +//------------------------------------------------------------------------------------------------// +//------------------------------------------------------------------------------------------------// + +#include "Fastor/config/macros.h" + +//------------------------------------------------------------------------------------------------// + +#endif // CONFIG_H diff --git a/noarch/include/Fastor/config/cpuid.h b/noarch/include/Fastor/config/cpuid.h new file mode 100644 index 00000000..498883fe --- /dev/null +++ b/noarch/include/Fastor/config/cpuid.h @@ -0,0 +1,71 @@ +/* This file is part of the FASTOR library. {{{ +Copyright (c) 2020 Roman Poya + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +}}}*/ + +#ifndef CPUID_H +#define CPUID_H + +#ifdef _WIN32 +#include +#include +typedef unsigned __int32 uint32_t; +#else +#include +#endif + + +namespace Fastor { + +class CPUID { + uint32_t regs[4]; + +public: + explicit CPUID(unsigned i) { +#ifdef _WIN32 + __cpuid((int *)regs, (int)i); + +#else + asm volatile + ("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3]) + : "a" (i), "c" (0)); + // ECX is set to zero for CPUID function 4 +#endif + } + + const uint32_t& EAX() const {return regs[0];} // cpu base frequency + const uint32_t& EBX() const {return regs[1];} // cpu max frequency + const uint32_t& ECX() const {return regs[2];} // bus (reference) frequency + const uint32_t& EDX() const {return regs[3];} +}; + +// Usage: +// CPUID cpuID(0); +// std::string vendor; +// vendor += std::string((const char *)&cpuID.EBX(), 4); +// vendor += std::string((const char *)&cpuID.EDX(), 4); +// vendor += std::string((const char *)&cpuID.ECX(), 4); +// std::cout << "CPU vendor = " << vendor << std::endl; + + +} // Fastor + +#endif // CPUID_H diff --git a/noarch/include/Fastor/config/macros.h b/noarch/include/Fastor/config/macros.h new file mode 100644 index 00000000..8e772443 --- /dev/null +++ b/noarch/include/Fastor/config/macros.h @@ -0,0 +1,292 @@ +/* This file is part of the FASTOR library. {{{ +Copyright (c) 2020 Roman Poya + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +}}}*/ + +#ifndef FASTOR_MACROS_H +#define FASTOR_MACROS_H + + +/* This file contains only the set of macros that Fastor defines and can be used or altered by the user +*/ +//------------------------------------------------------------------------------------------------// +// Compiler version +//------------------------------------------------------------------------------------------------// +#define FASTOR_MAJOR 0 +#define FASTOR_MINOR 6 +#define FASTOR_PATCHLEVEL 4 +#define FASTOR_VERSION (FASTOR_MAJOR * 10000 + FASTOR_MINOR * 100 + FASTOR_PATCHLEVEL) +//------------------------------------------------------------------------------------------------// + +//------------------------------------------------------------------------------------------------// +//------------------------------------------------------------------------------------------------// +//------------------------------------------------------------------------------------------------// +// ICC's default option is fast anyway (i.e. -fp-model fast=1) +// but it does not define the __FAST_MATH__ macro +#if defined(__FAST_MATH__) +#define FASTOR_FAST_MATH +// Use the following for unsafe math +// Only for GCC & Clang, activates fast div/rcp +//#define FASTOR_UNSAFE_MATH +#endif + +// Runtime checks - off by default under release +//------------------------------------------------------------------------------------------------// +#ifndef FASTOR_ENABLE_RUNTIME_CHECKS +#define FASTOR_ENABLE_RUNTIME_CHECKS 0 +#endif +//------------------------------------------------------------------------------------------------// + +// Bounds and shape checking - ON by default under debug +//------------------------------------------------------------------------------------------------// +#if !defined(NDEBUG) || FASTOR_ENABLE_RUNTIME_CHECKS +#ifndef FASTOR_BOUNDS_CHECK +#define FASTOR_BOUNDS_CHECK 1 +#endif +#ifndef FASTOR_SHAPE_CHECK +#define FASTOR_SHAPE_CHECK 1 +#endif +#else +#ifndef FASTOR_BOUNDS_CHECK +#define FASTOR_BOUNDS_CHECK 0 +#endif +#ifndef FASTOR_SHAPE_CHECK +#define FASTOR_SHAPE_CHECK 0 +#endif +#endif +//------------------------------------------------------------------------------------------------// + + +// Aliasing +//------------------------------------------------------------------------------------------------// +#ifndef FASTOR_NO_ALIAS +#define FASTOR_NO_ALIAS 0 +#endif +//------------------------------------------------------------------------------------------------// + + +// Macros used throughout Fastor +//------------------------------------------------------------------------------------------------// +//#define FASTOR_COPY_EXPR +//#define FASTOR_DONT_VECTORISE +//#define FASTOR_DONT_PERFORM_OP_MIN +//#define FASTOR_USE_OLD_OUTER +//#define FASTOR_USE_OLD_INTRINSICS +//#define FASTOR_USE_HADD +//#define FASTOR_USE_VECTORISED_EXPR_ASSIGN // to use vectorised expression assignment +//#define FASTOR_ZERO_INITIALISE +//#define FASTOR_USE_OLD_NDVIEWS +//#define FASTOR_DISPATCH_DIV_TO_MUL_EXPR // change BINARY_DIV_OP to BINARY_MUL_OP for Expression/Number +//#define FASTOR_DISABLE_SPECIALISED_CTR + +//FASTOR_MATMUL_OUTER_BLOCK_SIZE 2 +//FASTOR_MATMUL_INNER_BLOCK_SIZE 2 + +//FASTOR_TRANS_OUTER_BLOCK_SIZE 2 +//FASTOR_TRANS_INNER_BLOCK_SIZE 2 + +#ifndef FASTOR_BLAS_SWITCH_MATRIX_SIZE +#define FASTOR_BLAS_SWITCH_MATRIX_SIZE 16 +#endif + +// FASTOR_NIL +//------------------------------------------------------------------------------------------------// +#define FASTOR_NIL 0 +//------------------------------------------------------------------------------------------------// + + +// Bit sizes +//------------------------------------------------------------------------------------------------// +#define FASTOR_AVX512_BITSIZE 512 +#define FASTOR_AVX_BITSIZE 256 +#define FASTOR_SSE_BITSIZE 128 +#define FASTOR_DOUBLE_BITSIZE (sizeof(double)*8) +#define FASTOR_SINGLE_BITSIZE (sizeof(float)*8) +#ifndef FASTOR_SCALAR_BITSIZE +#define FASTOR_SCALAR_BITSIZE FASTOR_DOUBLE_BITSIZE +#endif +//------------------------------------------------------------------------------------------------// + + +// Alignment +//------------------------------------------------------------------------------------------------// +#ifndef FASTOR_MEMORY_ALIGNMENT_VALUE +#if defined(FASTOR_AVX512_IMPL) +#define FASTOR_MEMORY_ALIGNMENT_VALUE 64 +#elif defined(FASTOR_AVX_IMPL) +#define FASTOR_MEMORY_ALIGNMENT_VALUE 32 +#elif defined(FASTOR_SSE_IMPL) +#define FASTOR_MEMORY_ALIGNMENT_VALUE 16 +#else +#define FASTOR_MEMORY_ALIGNMENT_VALUE 8 +#endif +#endif + +/* User controllable alignment for Fastor containers */ +#define FASTOR_ALIGN alignas((FASTOR_MEMORY_ALIGNMENT_VALUE)) +/* Strict non-controllable alignment for Fastor's internal use */ +#define FASTOR_ARCH_ALIGN alignas((FASTOR_MEMORY_ALIGNMENT_VALUE)) + +// Conservative alignment for SIMD +#if defined(FASTOR_DONT_ALIGN) || defined(FASTOR_DONT_VECTORISE) + #define FASTOR_ALIGNED false +#else + #define FASTOR_ALIGNED true +#endif + +#define FASTOR_ISALIGNED(POINTER, BYTE_COUNT) \ + (((uintptr_t)(const void *)(POINTER)) % (BYTE_COUNT) == 0) +//------------------------------------------------------------------------------------------------// + + +// Assertions +//------------------------------------------------------------------------------------------------// +namespace Fastor { +// Strong unconditional assert +FASTOR_INLINE void FASTOR_EXIT_ASSERT(bool cond, const std::string &msg="") { + if (cond==false) { + throw std::runtime_error(msg); + } +} + +#if !defined(NDEBUG) || FASTOR_ENABLE_RUNTIME_CHECKS +#define FASTOR_ASSERT(COND, MESSAGE) FASTOR_EXIT_ASSERT(COND, MESSAGE) +#else +#define FASTOR_ASSERT(COND, MESSAGE) +#endif + +// Warn +FASTOR_INLINE void FASTOR_WARN(bool cond, const std::string &x) { + if (cond==false) { + std::cout << x << std::endl; + } +} +} // end of namespace Fastor +//------------------------------------------------------------------------------------------------// + + +// Warnings +//------------------------------------------------------------------------------------------------// +// Static warn +#ifndef FASTOR_NO_STATIC_WARNING +#if defined(__GNUC__) + #define FASTOR_DEPRECATE(foo, msg) foo __attribute__((deprecated(msg))) +#elif defined(_MSC_VER) + #define FASTOR_DEPRECATE(foo, msg) __declspec(deprecated(msg)) foo +#else + #error FASTOR STATIC WARNING DOES NOT SUPPORT THIS COMPILER +#endif + +#define FASTOR_CAT(x,y) FASTOR_CAT1_(x,y) +#define FASTOR_CAT1_(x,y) x##y + + +namespace Fastor { +namespace useless +{ + struct true_type {}; + struct false_type {}; + template struct converter : public true_type {}; + template <> struct converter<0> : public false_type {}; +} + +#define FASTOR_STATIC_WARN(cond, msg) \ +struct FASTOR_CAT(static_warning,__LINE__) { \ + FASTOR_DEPRECATE(void _(::useless::false_type const& ),msg) {}; \ + void _(::useless::true_type const& ) {}; \ + FASTOR_CAT(static_warning,__LINE__)() {_(::useless::converter<(cond)>());} \ +} + +} // end of namespace Fastor +#endif // FASTOR_NO_STATIC_WARNING +//------------------------------------------------------------------------------------------------// + + +// Token to string +//------------------------------------------------------------------------------------------------// +#define FASTOR_TOSTRING_(X) #X +#define FASTOR_TOSTRING(X) FASTOR_TOSTRING_(X) +//------------------------------------------------------------------------------------------------// + +// asm comment +//------------------------------------------------------------------------------------------------// +#define FASTOR_ASM(STR) asm(STR ::) +//------------------------------------------------------------------------------------------------// + +// unused +//------------------------------------------------------------------------------------------------// +namespace Fastor { +// clobber +template void unused(T &&x) { +#ifndef _WIN32 + asm("" ::"m"(x)); +#endif +} +template void unused(T&& x, U&& ...y) { unused(x); unused(y...); } +} // end of namespace Fastor +//------------------------------------------------------------------------------------------------// + + +// SIMD round-down +//------------------------------------------------------------------------------------------------// +#define ROUND_DOWN2(x, s) ((x) & ~((s)-1)) +#define ROUND_DOWN(x, s) ROUND_DOWN2(x,s) +//------------------------------------------------------------------------------------------------// + + +// FASTOR CONSTRUCTS +//------------------------------------------------------------------------------------------------// +#include + +namespace Fastor { + +using FASTOR_INDEX = size_t; +using Int64 = int64_t; +using DEFAULT_FLOAT_TYPE = double; +using DFT = DEFAULT_FLOAT_TYPE; +using FASTOR_VINDEX = volatile size_t; + +constexpr int RowMajor = 0; +constexpr int ColumnMajor = 1; + + +constexpr int FASTOR_Symmetric = -100; +constexpr int FASTOR_NonSymmetric = -101; +constexpr int FASTOR_AntiSymmetric = -102; +constexpr int FASTOR_Identity = -103; +constexpr int FASTOR_One = -104; +constexpr int FASTOR_Zero = -105; + +constexpr int FASTOR_ThreeD = -150; +constexpr int FASTOR_TwoD = -151; +constexpr int FASTOR_PlaneStrain = -152; +constexpr int FASTOR_PlaneStress = -153; +constexpr int FASTOR_Voigt = -106; + +constexpr int DepthFirst = -200; +constexpr int NoDepthFirst = -201; + +constexpr double PRECI_TOL = 1e-14; + +} +//------------------------------------------------------------------------------------------------// + +#endif // FASTOR_MACROS_H diff --git a/noarch/include/Fastor/experimental/SingleValueTensor.h b/noarch/include/Fastor/experimental/SingleValueTensor.h new file mode 100644 index 00000000..f59f5b9a --- /dev/null +++ b/noarch/include/Fastor/experimental/SingleValueTensor.h @@ -0,0 +1,274 @@ +#ifndef SINGLEVALUE_TENSOR_H +#define SINGLEVALUE_TENSOR_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/TensorIO.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/meta/tensor_meta.h" +#include + +namespace Fastor { + +template +class SingleValueTensor : public AbstractTensor,sizeof...(Rest)> { +public: + using scalar_type = T; + using simd_vector_type = choose_best_simd_vector_t; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = SingleValueTensor; + using dimension_t = std::integral_constant; + static constexpr FASTOR_INLINE FASTOR_INDEX rank() {return sizeof...(Rest);} + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return pack_prod::value;} + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX dim) const { +#if FASTOR_SHAPE_CHECK + FASTOR_ASSERT(dim>=0 && dim < sizeof...(Rest), "TENSOR SHAPE MISMATCH"); +#endif + const FASTOR_INDEX DimensionHolder[sizeof...(Rest)] = {Rest...}; + return DimensionHolder[dim]; + } + + template + SingleValueTensor(U num) : _data{(T)num} {} + SingleValueTensor(const SingleValueTensor &a) : _data{(T)a.data()[0]} {} + FASTOR_INLINE SingleValueTensor(const AbstractTensor,sizeof...(Rest)>& src_) : _data{T(0)} { + } + + constexpr FASTOR_INLINE T* data() const { return const_cast(this->_data.data());} + + // Index retriever + //----------------------------------------------------------------------------------------------------------// + template + FASTOR_INLINE int get_mem_index(U index) const { +#if FASTOR_BOUNDS_CHECK + FASTOR_ASSERT((index>=0 && index::value,bool>::type =0> + FASTOR_INLINE int get_flat_index(Args ... args) const { +#if FASTOR_BOUNDS_CHECK + int largs[sizeof...(Args)] = {args...}; + constexpr int DimensionHolder[dimension_t::value] = {Rest...}; + for (int i=0; i=0 && largs[i] &as) const { +#if FASTOR_BOUNDS_CHECK + constexpr std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; + int index = 0; + for (int i=0; i=0 && index +#define SCALAR_INDEXING_CONST_H + //----------------------------------------------------------------------------------------------------------// + + // Expression templates evaluators + //----------------------------------------------------------------------------------------------------------// +#undef TENSOR_EVALUATOR_H + #include "Fastor/tensor/TensorEvaluator.h" +#define TENSOR_EVALUATOR_H + //----------------------------------------------------------------------------------------------------------// + + // Tensor methods + //----------------------------------------------------------------------------------------------------------// +#undef TENSOR_METHODS_CONST_H + #include "Fastor/tensor/TensorMethods.h" +#define TENSOR_METHODS_CONST_H + //----------------------------------------------------------------------------------------------------------// + + // Converters + //----------------------------------------------------------------------------------------------------------// +#undef PODCONVERTERS_H + #include "Fastor/tensor/PODConverters.h" +#define PODCONVERTERS_H + //----------------------------------------------------------------------------------------------------------// + + // Cast method + //----------------------------------------------------------------------------------------------------------// + template + FASTOR_INLINE SingleValueTensor cast() const { + SingleValueTensor out(static_cast(_data[0])); + return out; + } + //----------------------------------------------------------------------------------------------------------// + + //----------------------------------------------------------------------------------------------------------// +private: + const FASTOR_ALIGN std::array _data; + //----------------------------------------------------------------------------------------------------------// +}; + +// template +// constexpr const T SingleValueTensor::_data[pack_prod::value]; + +template +struct tensor_type_finder> { + using type = SingleValueTensor; +}; + +template +struct scalar_type_finder> { + using type = T; +}; + + +FASTOR_MAKE_OS_STREAM_TENSOR0(SingleValueTensor) +FASTOR_MAKE_OS_STREAM_TENSOR1(SingleValueTensor) +FASTOR_MAKE_OS_STREAM_TENSOR2(SingleValueTensor) +FASTOR_MAKE_OS_STREAM_TENSORn(SingleValueTensor) + + +template +FASTOR_INLINE SingleValueTensor transpose(const SingleValueTensor &a) { + return SingleValueTensor(a(0,0)); +} + +template +T trace(const SingleValueTensor &a) { + return M*a(0,0); +} + +template +FASTOR_INLINE T determinant(const SingleValueTensor &a) { + // determinant of a single value tensor is 0 + return 0.; +} + +template +FASTOR_INLINE double norm(const SingleValueTensor &a) { + return a(0,0)*std::sqrt(double(M*N)); +} + +template +FASTOR_INLINE Tensor inverse(const SingleValueTensor &a) { + // A single value tensor is not invertible + Tensor out(std::numeric_limits::quiet_NaN()); + return out; +} + +template +FASTOR_INLINE Tensor matmul(const Tensor &a, const SingleValueTensor &b) { + + using V = SIMDVector; + + Tensor out; + T *out_data = out.data(); + const T *a_data = a.data(); + const T b_value = b(0,0); + + for (size_t i=0; i +FASTOR_INLINE Tensor matmul(const SingleValueTensor &a, const Tensor &b) { + return transpose(matmul(transpose(b),transpose(a))); +} + +template +FASTOR_INLINE SingleValueTensor matmul(const SingleValueTensor &a, const SingleValueTensor &b) { + + const T a_value = a(0,0); + const T b_value = b(0,0); + // matmul is just this + SingleValueTensor out(a_value*b_value*K); + + // Not necessary + // using V = SIMDVector; + // V vec_out; + // size_t j=0; + // for (; j out(out_value); + + return out; +} + + + + +// This one is almost like a compile time einsum +template +FASTOR_INLINE +typename contraction_impl::type,SingleValueTensor, + typename std_ext::make_index_sequence::type>::type +einsum(const SingleValueTensor &a, const SingleValueTensor &b) { + + static_assert(einsum_index_checker::type>::value, + "INDICES FOR EINSUM FUNCTION CANNOT APPEAR MORE THAN TWICE. USE CONTRACTION INSTEAD"); + + std::array idx0; std::copy_n(Index_I::_IndexHolder,Index_I::NoIndices,idx0.begin()); + std::array idx1; std::copy_n(Index_J::_IndexHolder,Index_J::NoIndices,idx1.begin()); + std::array dims0 = {Rest0...}; + + // n^2 but it is okay as this is a small loop with compile time spans + size_t total = 1; + for (int i=0; i::type,SingleValueTensor, + typename std_ext::make_index_sequence::type>::type; + OutTensor out(out_value); + return out; +} + + +} // end of namespace Fastor + + +#endif // SINGLEVALUE_TENSOR_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_add_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_add_op.h new file mode 100644 index 00000000..4f9d6cd8 --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_add_op.h @@ -0,0 +1,247 @@ +#ifndef BINARY_ADD_OP_H +#define BINARY_ADD_OP_H + +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/expressions/expression_traits.h" + + + +namespace Fastor { + +template +struct BinaryAddOp: public AbstractTensor,DIM0> { +private: + expression_t _lhs; + expression_t _rhs; +public: + + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = binary_op_simd_vector_t >; + using simd_abi_type = typename simd_vector_type::abi_type; + + FASTOR_INLINE BinaryAddOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} + + FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} + template::value && + !std::is_arithmetic::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); +#endif + return _rhs.size(); + } + + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} + template::value && + !std::is_arithmetic::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); +#endif + return _rhs.dimension(i); + } + constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} + constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} + + // Generic version of eval + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + // Delay evaluation using a helper function to fully inform BinaryOp about _lhs and _rhs + return helper(i); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return _lhs.template eval(i) + _rhs.template eval(i); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return (U)_lhs + _rhs.template eval(i); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return _lhs.template eval(i) + (U)_rhs; + } + + + // scalar based + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return helper_s(i); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) + _rhs.template eval_s(i); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return (U)_lhs + _rhs.template eval_s(i); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) + (U)_rhs; + } + + + // for 2D tensors + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper(i,j); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval(i,j) + _rhs.template eval(i,j); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return (U)_lhs + _rhs.template eval(i,j); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval(i,j) + (U)_rhs; + } + + // scalar based (for 2D tensors) + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper_s(i,j); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) + _rhs.template eval_s(i,j); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return (U)_lhs + _rhs.template eval_s(i,j); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) + (U)_rhs; + } + + // for nD tensors + template + FASTOR_INLINE SIMDVector teval(const std::array &as) const { + return thelper(as); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) + _rhs.template teval(as); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return (U)_lhs + _rhs.template teval(as); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) + (U)_rhs; + } + + // scalar based (for nD tensors) + template + FASTOR_INLINE U teval_s(const std::array &as) const { + return thelper_s(as); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) + _rhs.template teval_s(as); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return (U)_lhs + _rhs.template teval_s(as); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) + (U)_rhs; + } +}; + +template::value && + !std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinaryAddOp operator+(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinaryAddOp(_lhs.self(), _rhs.self()); +} +template::value && + std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinaryAddOp operator+(const AbstractTensor &_lhs, TRhs bb) { + return BinaryAddOp(_lhs.self(), bb); +} +template::value && + !std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinaryAddOp operator+(TLhs bb, const AbstractTensor &_rhs) { + return BinaryAddOp(bb,_rhs.self()); +} + +template::value && + !std::is_arithmetic::value && + DIM0!=DIM1,bool>::type = 0 > +FASTOR_INLINE BinaryAddOp::value> +operator+(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinaryAddOp::value>(_lhs.self(), _rhs.self()); +} + +} // end of namespace Fastor + + +#endif // BINARY_ADD_OP_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_assignment.h b/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_assignment.h new file mode 100644 index 00000000..cdc73fec --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_assignment.h @@ -0,0 +1,156 @@ +#ifndef BINARY_ARITHMETIC_ASSIGNMENT_H +#define BINARY_ARITHMETIC_ASSIGNMENT_H + + +#include "Fastor/expressions/binary_ops/binary_arithmetic_ops.h" +#include "Fastor/tensor/Aliasing.h" + +namespace Fastor { + +// Create assign for all binrary arithmetic ops +#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(NAME, ASSIGN_TYPE, OP_ASSIGN_TYPE)\ +template && !is_primitive_v_ && !(requires_evaluation_v || requires_evaluation_v), bool> = false >\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template && !is_primitive_v_ && (requires_evaluation_v || requires_evaluation_v), bool> = false >\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ + assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ +}\ +template && !is_primitive_v_ && !requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template && !is_primitive_v_ && requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + assign ##ASSIGN_TYPE (dst.self(), src.lhs());\ + assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ +}\ +template && is_primitive_v_ && !requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template && is_primitive_v_ && requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ + assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs());\ +}\ + +// Create assign_add, assign_sub for BinaryAddOp and BinarySubOp +#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(NAME, ASSIGN_TYPE, OP_ASSIGN_TYPE)\ +template && !is_primitive_v_ && !(requires_evaluation_v || requires_evaluation_v), bool> = false >\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template && !is_primitive_v_ && (requires_evaluation_v || requires_evaluation_v), bool> = false >\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + if (!does_alias(dst.self(),src.rhs().self())) {\ + assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ + assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ + }\ + else{\ + const Derived tmp(dst.self());\ + assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ + assign ##OP_ASSIGN_TYPE (dst.self(), tmp);\ + }\ +}\ +template && !is_primitive_v_ && !requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template && !is_primitive_v_ && requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + assign ##ASSIGN_TYPE (dst.self(), src.lhs());\ + assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ +}\ +template && is_primitive_v_ && !requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template && is_primitive_v_ && requires_evaluation_v, bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ + assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs());\ +}\ + + +// Create assign_add, assign_sub for BinaryMulOp and BinaryDivOp +// Create assign_mul, assign_div for all binary ops +#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(NAME, ASSIGN_TYPE)\ +template || requires_evaluation_v),bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template || requires_evaluation_v),bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + using result_type = typename Binary ##NAME ## Op::result_type;\ + const result_type a(src.self());\ + trivial_assign ##ASSIGN_TYPE (dst.self(), a);\ +}\ + +// assign +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Add, , _add) +// assign_add +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Add, _add, _add) +// assign_sub +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Add, _sub, _sub) +// assign_mul +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Add, _mul) +// assign_div +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Add, _div) + +// assign +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Sub, , _sub) +// assign_add +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Sub, _add, _sub) +// assign_sub +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Sub, _sub, _add) +// assign_mul +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Sub, _mul) +// assign_div +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Sub, _div) + + +// assign +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Mul, , _mul) +// assign_add +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _add) +// assign_sub +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _sub) +// assign_mul +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _mul) +// assign_div +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _div) + + +// assign +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Div, , _div) +// assign_add +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _add) +// assign_sub +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _sub) +// assign_mul +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _mul) +// assign_div +FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _div) + + +} // end of namespace Fastor + + +#endif // BINARY_ARITHMETIC_ASSIGNMENT_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_ops.h b/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_ops.h new file mode 100644 index 00000000..664692cc --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_ops.h @@ -0,0 +1,229 @@ +#ifndef BINARY_ARITHMETIC_OP_H +#define BINARY_ARITHMETIC_OP_H + +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + + +#define FASTOR_MAKE_BINARY_ARITHMETIC_OPS(OP, NAME, EVAL_TYPE) \ +template\ +struct Binary ##NAME ## Op: public AbstractTensor,DIM0> {\ + expression_t _lhs;\ + expression_t _rhs;\ +public:\ + static constexpr FASTOR_INDEX Dimension = DIM0;\ + static constexpr FASTOR_INDEX rank() {return DIM0;}\ + using scalar_type = typename scalar_type_finder>::type;\ + using simd_vector_type = binary_op_simd_vector_t< Binary ##NAME ## Op >;\ + using simd_abi_type = typename simd_vector_type::abi_type;\ + using result_type = binary_arithmetic_result_t< Binary ##NAME ## Op >;\ + FASTOR_INLINE Binary ##NAME ## Op(expression_t inlhs, expression_t inrhs) : _lhs(inlhs), _rhs(inrhs) {}\ + FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\ + template &&\ + !is_primitive_v_,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {\ + FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");\ + return _rhs.size();\ + }\ + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\ + template &&\ + !is_primitive_v_,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\ + FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");\ + return _rhs.dimension(i);\ + }\ + FASTOR_INLINE expression_t lhs() const {return _lhs;}\ + FASTOR_INLINE expression_t rhs() const {return _rhs;}\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ + return helper(i);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return _lhs.template eval(i) OP _rhs.template eval(i);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval(i);\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return _lhs.template eval(i) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\ + return helper_s(i);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ + return _lhs.template eval_s(i) OP _rhs.template eval_s(i);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i);\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ + return _lhs.template eval_s(i) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return helper(i,j);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval(i,j) OP _rhs.template eval(i,j);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval(i,j);\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval(i,j) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return helper_s(i,j);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval_s(i,j) OP _rhs.template eval_s(i,j);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i,j);\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval_s(i,j) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ + return thelper(as);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return _lhs.template teval(as) OP _rhs.template teval(as);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template teval(as);\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return _lhs.template teval(as) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE teval_s(const std::array &as) const {\ + return thelper_s(as);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ + return _lhs.template teval_s(as) OP _rhs.template teval_s(as);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template teval_s(as);\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ + return _lhs.template teval_s(as) OP (EVAL_TYPE)_rhs;\ + }\ +};\ +template &&\ + !is_primitive_v_,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ + return Binary ##NAME ## Op(_lhs.self(), _rhs.self());\ +}\ +template &&\ + is_primitive_v_,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op operator OP(const AbstractTensor &_lhs, TRhs bb) {\ + return Binary ##NAME ## Op(_lhs.self(), bb);\ +}\ +template &&\ + !is_primitive_v_,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op operator OP(TLhs bb, const AbstractTensor &_rhs) {\ + return Binary ##NAME ## Op(bb,_rhs.self());\ +}\ +template &&\ + !is_primitive_v_ &&\ + DIM0!=DIM1,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op::value>\ +operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ + return Binary ##NAME ## Op::value>(_lhs.self(), _rhs.self());\ +}\ + +// Dispatch based on type of expressions not the tensor +FASTOR_MAKE_BINARY_ARITHMETIC_OPS(+, Add, scalar_type) +FASTOR_MAKE_BINARY_ARITHMETIC_OPS(-, Sub, scalar_type) +FASTOR_MAKE_BINARY_ARITHMETIC_OPS(*, Mul, scalar_type) +// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(/, Div, scalar_type) // Dont create div as it is a special case + +// Dispatch based on the type of tensor and not the expression +// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(+, Add, U) +// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(-, Sub, U) +// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(*, Mul, U) +// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(/, Div, U) + +} + + +#endif // BINARY_ARITHMETIC_OP_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_cmp_ops.h b/noarch/include/Fastor/expressions/binary_ops/binary_cmp_ops.h new file mode 100644 index 00000000..da277aa2 --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_cmp_ops.h @@ -0,0 +1,262 @@ +#ifndef BINARY_CMP_OPS +#define BINARY_CMP_OPS + +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" + +namespace Fastor { + +#define FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(OP, NAME, EVAL_TYPE) \ +template\ +struct BinaryCmpOp##NAME: public AbstractTensor,DIM0> {\ + expression_t _lhs;\ + expression_t _rhs;\ + static constexpr FASTOR_INDEX Dimension = DIM0;\ + static constexpr FASTOR_INDEX rank() {return DIM0;}\ + using scalar_type = typename scalar_type_finder>::type;\ + using result_type = to_bool_tensor_t>>;\ + using simd_vector_type = binary_op_simd_vector_t >;\ + using simd_abi_type = typename simd_vector_type::abi_type;\ + using ABI = simd_abi::fixed_size::Size>;\ + using UU = bool /*this needs to change to U once masks are implemented*/;\ + FASTOR_INLINE BinaryCmpOp##NAME(expression_t inlhs, expression_t inrhs) : _lhs(inlhs), _rhs(inrhs) {}\ + FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();}\ + template::value,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\ + template::value,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\ + template::value &&\ + !std::is_arithmetic::value,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {\ + return _rhs.size();\ + }\ + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);}\ + template::value,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\ + template::value,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\ + template::value &&\ + !std::is_arithmetic::value,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\ + return _rhs.dimension(i);\ + }\ + FASTOR_INLINE expression_t lhs() const {return _lhs;}\ + FASTOR_INLINE expression_t rhs() const {return _rhs;}\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ + return helper(i);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return _lhs.template eval(i) OP _rhs.template eval(i);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval(i);\ + }\ + template::value &&\ + std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return _lhs.template eval(i) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE UU eval_s(FASTOR_INDEX i) const {\ + return helper_s(i);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\ + return _lhs.template eval_s(i) OP _rhs.template eval_s(i);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i);\ + }\ + template::value &&\ + std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\ + return _lhs.template eval_s(i) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return helper(i,j);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval(i,j) OP _rhs.template eval(i,j);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval(i,j);\ + }\ + template::value &&\ + std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval(i,j) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE UU eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return helper_s(i,j);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval_s(i,j) OP _rhs.template eval_s(i,j);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i,j);\ + }\ + template::value &&\ + std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return _lhs.template eval_s(i,j) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ + return thelper(as);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return _lhs.template teval(as) OP _rhs.template teval(as);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template teval(as);\ + }\ + template::value &&\ + std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return _lhs.template teval(as) OP (EVAL_TYPE)_rhs;\ + }\ + template\ + FASTOR_INLINE UU teval_s(const std::array &as) const {\ + return thelper_s(as);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU thelper_s(const std::array &as) const {\ + return _lhs.template teval_s(as) OP _rhs.template teval_s(as);\ + }\ + template::value &&\ + !std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU thelper_s(const std::array &as) const {\ + return (EVAL_TYPE)_lhs OP _rhs.template teval_s(as);\ + }\ + template::value &&\ + std::is_arithmetic::value,bool>::type = 0>\ + FASTOR_INLINE UU thelper_s(const std::array &as) const {\ + return _lhs.template teval_s(as) OP (EVAL_TYPE)_rhs;\ + }\ +};\ +template::value &&\ + !std::is_arithmetic::value,bool>::type = 0 >\ +FASTOR_INLINE BinaryCmpOp##NAME operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ + return BinaryCmpOp##NAME(_lhs.self(), _rhs.self());\ +}\ +template::value &&\ + std::is_arithmetic::value,bool>::type = 0 >\ +FASTOR_INLINE BinaryCmpOp##NAME operator OP(const AbstractTensor &_lhs, TRhs bb) {\ + return BinaryCmpOp##NAME(_lhs.self(), bb);\ +}\ +template::value &&\ + !std::is_arithmetic::value,bool>::type = 0 >\ +FASTOR_INLINE BinaryCmpOp##NAME operator OP(TLhs bb, const AbstractTensor &_rhs) {\ + return BinaryCmpOp##NAME(bb,_rhs.self());\ +}\ +template::value &&\ + !std::is_arithmetic::value &&\ + DIM0!=DIM1,bool>::type = 0 >\ +FASTOR_INLINE BinaryCmpOp##NAME::value>\ +operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ + return BinaryCmpOp##NAME::value>(_lhs.self(), _rhs.self());\ +}\ + + +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(== ,EQ, scalar_type) +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(!= ,NEQ,scalar_type) +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(< ,LT, scalar_type) +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(> ,GT, scalar_type) +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(<= ,LE, scalar_type) +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(>= ,GE, scalar_type) +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(&& ,AND,scalar_type) +FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(|| ,OR, scalar_type) + + + + + +#define FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(NAME, ASSIGN_TYPE)\ +template || requires_evaluation_v),bool> = false >\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const BinaryCmpOp ##NAME &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template || requires_evaluation_v),bool> = false >\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const BinaryCmpOp ##NAME &src) {\ + using lhs_type = typename get_binary_arithmetic_result_type::type;\ + using rhs_type = typename get_binary_arithmetic_result_type::type;\ + const lhs_type a(src.lhs());\ + const rhs_type b(src.rhs());\ + trivial_assign ##ASSIGN_TYPE (dst.self(), BinaryCmpOp ##NAME (a,b));\ +}\ + +#define FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS(ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(EQ, ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(NEQ, ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(LT, ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(GT, ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(LE, ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(GE, ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(AND, ASSIGN_TYPE)\ +FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(OR, ASSIGN_TYPE)\ + +FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( ) +FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _add) +FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _sub) +FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _mul) +FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _div) + + +} // end of namespace Fastor + + +#endif // BINARY_CMP_OPS diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_div_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_div_op.h new file mode 100644 index 00000000..f9ecd195 --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_div_op.h @@ -0,0 +1,306 @@ +#ifndef BINARY_DIV_OP_H +#define BINARY_DIV_OP_H + +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/expressions/expression_traits.h" + +namespace Fastor { + +// Dispatch based on type of expressions not the tensor +#define FASTOR_BD_OP_EVAL_TYPE scalar_type + +// Dispatch based on the type of tensor and not the expression +// #define FASTOR_BD_OP_EVAL_TYPE U + + +template +struct BinaryDivOp: public AbstractTensor,DIM0> { +private: + expression_t _lhs; + expression_t _rhs; +public: + + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = binary_op_simd_vector_t >; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = binary_arithmetic_result_t< BinaryDivOp >; + + FASTOR_INLINE BinaryDivOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} + + FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} + template,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} + template,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} + template && + !is_primitive_v_,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); +#endif + return _rhs.size(); + } + + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} + template,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} + template,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} + template && + !is_primitive_v_,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); +#endif + return _rhs.dimension(i); + } + constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} + constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + return helper(i); + } + + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { +#ifndef FASTOR_UNSAFE_MATH + return _lhs.template eval(i) / _rhs.template eval(i); +#else + return _lhs.template eval(i) * rcp(_rhs.template eval(i)); +#endif + } + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { +#ifndef FASTOR_UNSAFE_MATH + return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval(i); +#else + return (FASTOR_BD_OP_EVAL_TYPE)_lhs * rcp(_rhs.template eval(i)); +#endif + } + template && + is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { +#ifndef FASTOR_UNSAFE_MATH + return _lhs.template eval(i) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; +#else + return _lhs.template eval(i) * rcp(SIMDVector(_rhs)); +#endif + } + + // scalar based + template + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE eval_s(FASTOR_INDEX i) const { + return helper_s(i); + } + + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) / _rhs.template eval_s(i); + } + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const { + return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval_s(i); + } + template && + is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; + } + + // for 2D tensors + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper(i,j); + } + + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { +#ifndef FASTOR_UNSAFE_MATH + return _lhs.template eval(i,j) / _rhs.template eval(i,j); +#else + return _lhs.template eval(i,j) * rcp(_rhs.template eval(i,j)); +#endif + } + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { +#ifndef FASTOR_UNSAFE_MATH + return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval(i,j); +#else + return (FASTOR_BD_OP_EVAL_TYPE)_lhs * rcp(_rhs.template eval(i,j)); +#endif + } + template && + is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { +#ifndef FASTOR_UNSAFE_MATH + return _lhs.template eval(i,j) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; +#else + return _lhs.template eval(i,j) * rcp(SIMDVector(_rhs)); +#endif + } + + // scalar based (for 2D tensors) + template + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper_s(i,j); + } + + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) / _rhs.template eval_s(i,j); + } + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval_s(i,j); + } + template && + is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; + } + + // for nD tensors + template + FASTOR_INLINE SIMDVector teval(const std::array &as) const { + return thelper(as); + } + + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) / _rhs.template teval(as); + } + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template teval(as); + } + template && + is_primitive_v_,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; + } + + // scalar based (for nD tensors) + template + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE teval_s(const std::array &as) const { + return thelper_s(as); + } + + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) / _rhs.template teval_s(as); + } + template && + !is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array &as) const { + return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template teval_s(as); + } + template && + is_primitive_v_,bool>::type = 0> + FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; + } +}; + +template && + !is_primitive_v_,bool>::type = 0 > +FASTOR_INLINE BinaryDivOp operator/(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinaryDivOp(_lhs.self(), _rhs.self()); +} + +#ifndef FASTOR_DISPATCH_DIV_TO_MUL_EXPR +template && + is_primitive_v_,bool>::type = 0 > +FASTOR_INLINE BinaryDivOp operator/(const AbstractTensor &_lhs, TRhs bb) { + return BinaryDivOp(_lhs.self(), bb); +} +template && + !is_primitive_v_,bool>::type = 0 > +FASTOR_INLINE BinaryDivOp operator/(TLhs bb, const AbstractTensor &_rhs) { + return BinaryDivOp(bb,_rhs.self()); +} +#else +template && + is_primitive_v_ && !std::is_integral::value,bool>::type = 0 > +FASTOR_INLINE BinaryMulOp operator/(const AbstractTensor &_lhs, TRhs bb) { + return BinaryMulOp( + _lhs.self(), + static_cast::type>(1)/static_cast::type>(bb)); +} +template && !std::is_integral::value && + !is_primitive_v_,bool>::type = 0 > +FASTOR_INLINE BinaryMulOp operator/(TLhs bb, const AbstractTensor &_rhs) { + return BinaryMulOp( + static_cast::type>(1)/static_cast::type>(bb), + _rhs.self()); +} +// Special case for integral types +template && + is_primitive_v_ && std::is_integral::value,bool>::type = 0 > +FASTOR_INLINE BinaryDivOp operator/(const AbstractTensor &_lhs, TRhs bb) { + return BinaryDivOp(_lhs.self(), bb); +} +template && std::is_integral::value && + !is_primitive_v_,bool>::type = 0 > +FASTOR_INLINE BinaryDivOp operator/(TLhs bb, const AbstractTensor &_rhs) { + return BinaryDivOp(bb,_rhs.self()); +} +#endif + +template && + !is_arithmetic_v_ && + DIM0!=DIM1,bool> = false > +FASTOR_INLINE BinaryDivOp::value> +operator/(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinaryDivOp::value>(_lhs.self(), _rhs.self()); +} + +} + +#endif // BINARY_DIV_OP_H + diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_math_ops.h b/noarch/include/Fastor/expressions/binary_ops/binary_math_ops.h new file mode 100644 index 00000000..c37e14d9 --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_math_ops.h @@ -0,0 +1,261 @@ +#ifndef BINARY_MATH_OP_H +#define BINARY_MATH_OP_H + +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + + +#define FASTOR_MAKE_BINARY_MATH_OPS(OP_NAME, SIMD_OP, OP, NAME, EVAL_TYPE) \ +template\ +struct Binary ##NAME ## Op: public AbstractTensor,DIM0> {\ + expression_t _lhs;\ + expression_t _rhs;\ +public:\ + static constexpr FASTOR_INDEX Dimension = DIM0;\ + static constexpr FASTOR_INDEX rank() {return DIM0;}\ + using scalar_type = typename scalar_type_finder>::type;\ + using simd_vector_type = binary_op_simd_vector_t< Binary ##NAME ## Op >;\ + using simd_abi_type = typename simd_vector_type::abi_type;\ + using result_type = binary_arithmetic_result_t< Binary ##NAME ## Op >;\ + FASTOR_INLINE Binary ##NAME ## Op(expression_t inlhs, expression_t inrhs) : _lhs(inlhs), _rhs(inrhs) {}\ + FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\ + template &&\ + !is_primitive_v_,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_size() const {\ + FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");\ + return _rhs.size();\ + }\ + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\ + template,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\ + template &&\ + !is_primitive_v_,bool>::type =0 >\ + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\ + FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");\ + return _rhs.dimension(i);\ + }\ + FASTOR_INLINE expression_t lhs() const {return _lhs;}\ + FASTOR_INLINE expression_t rhs() const {return _rhs;}\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ + return helper(i);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return SIMD_OP(_lhs.template eval(i), _rhs.template eval(i));\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template eval(i));\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ + return SIMD_OP(_lhs.template eval(i), (EVAL_TYPE)_rhs);\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\ + return helper_s(i);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ + return OP(_lhs.template eval_s(i), _rhs.template eval_s(i));\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ + return OP((EVAL_TYPE)_lhs, _rhs.template eval_s(i));\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ + return OP(_lhs.template eval_s(i), (EVAL_TYPE)_rhs);\ + }\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return helper(i,j);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return SIMD_OP(_lhs.template eval(i,j), _rhs.template eval(i,j));\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template eval(i,j));\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return SIMD_OP(_lhs.template eval(i,j), (EVAL_TYPE)_rhs);\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return helper_s(i,j);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return OP(_lhs.template eval_s(i,j), _rhs.template eval_s(i,j));\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return OP((EVAL_TYPE)_lhs, _rhs.template eval_s(i,j));\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return OP(_lhs.template eval_s(i,j), (EVAL_TYPE)_rhs);\ + }\ + template\ + FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ + return thelper(as);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return SIMD_OP(_lhs.template teval(as), _rhs.template teval(as));\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template teval(as));\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ + return SIMD_OP(_lhs.template teval(as), (EVAL_TYPE)_rhs);\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE teval_s(const std::array &as) const {\ + return thelper_s(as);\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ + return OP(_lhs.template teval_s(as), _rhs.template teval_s(as));\ + }\ + template &&\ + !is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ + return OP((EVAL_TYPE)_lhs, _rhs.template teval_s(as));\ + }\ + template &&\ + is_primitive_v_,bool>::type = 0>\ + FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ + return OP(_lhs.template teval_s(as), (EVAL_TYPE)_rhs);\ + }\ +};\ +template &&\ + !is_primitive_v_,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op OP_NAME(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ + return Binary ##NAME ## Op(_lhs.self(), _rhs.self());\ +}\ +template &&\ + is_primitive_v_,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op OP_NAME(const AbstractTensor &_lhs, TRhs bb) {\ + return Binary ##NAME ## Op(_lhs.self(), bb);\ +}\ +template &&\ + !is_primitive_v_,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op OP_NAME(TLhs bb, const AbstractTensor &_rhs) {\ + return Binary ##NAME ## Op(bb,_rhs.self());\ +}\ +template &&\ + !is_primitive_v_ &&\ + DIM0!=DIM1,bool>::type = 0 >\ +FASTOR_INLINE Binary ##NAME ## Op::value>\ +OP_NAME(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ + return Binary ##NAME ## Op::value>(_lhs.self(), _rhs.self());\ +}\ + +// Dispatch based on type of expressions not the tensor +FASTOR_MAKE_BINARY_MATH_OPS(min, min, std::min, Min, scalar_type) +FASTOR_MAKE_BINARY_MATH_OPS(max, max, std::max, Max, scalar_type) +FASTOR_MAKE_BINARY_MATH_OPS(pow, pow, std::pow, Pow, scalar_type) +FASTOR_MAKE_BINARY_MATH_OPS(atan2, atan2, std::atan2, Atan2, scalar_type) +FASTOR_MAKE_BINARY_MATH_OPS(hypot, hypot, std::hypot, Hypot, scalar_type) + +// Dispatch based on the type of tensor and not the expression +// FASTOR_MAKE_BINARY_MATH_OPS(min, min, std::min, Min, U) +// FASTOR_MAKE_BINARY_MATH_OPS(max, max, std::max, Max, U) +// FASTOR_MAKE_BINARY_MATH_OPS(pow, pow, std::pow, Pow, U) +// FASTOR_MAKE_BINARY_MATH_OPS(atan2, atan2, std::atan2, Atan2, U) +// FASTOR_MAKE_BINARY_MATH_OPS(hypot, hypot, std::hypot, Hypot, U) + + +// Create assignment for all binary math_ops +#define FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, ASSIGN_TYPE)\ +template || requires_evaluation_v),bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template || requires_evaluation_v),bool> = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + using result_type = typename Binary ##NAME ## Op::result_type;\ + const result_type a(src.self());\ + trivial_assign ##ASSIGN_TYPE (dst.self(), a);\ +}\ + + +#define FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(NAME)\ +FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, )\ +FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _add)\ +FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _sub)\ +FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _mul)\ +FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _div)\ + +FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Min) +FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Max) +FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Pow) +FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Atan2) +FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Hypot) + +} + + +#endif // BINARY_MATH_OP_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_mul_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_mul_op.h new file mode 100644 index 00000000..ab3c90f6 --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_mul_op.h @@ -0,0 +1,245 @@ +#ifndef BINARY_MUL_OP_H +#define BINARY_MUL_OP_H + +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + +template +struct BinaryMulOp: public AbstractTensor,DIM0> { +private: + expression_t _lhs; + expression_t _rhs; +public: + + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = binary_op_simd_vector_t >; + using simd_abi_type = typename simd_vector_type::abi_type; + + FASTOR_INLINE BinaryMulOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} + + + FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} + template::value && + !std::is_arithmetic::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); +#endif + return _rhs.size(); + } + + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} + template::value && + !std::is_arithmetic::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); +#endif + return _rhs.dimension(i); + } + constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} + constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + return helper(i); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return _lhs.template eval(i) * _rhs.template eval(i); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return (U)_lhs * _rhs.template eval(i); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return _lhs.template eval(i) * (U)_rhs; + } + + + // scalar based + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return helper_s(i); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) * _rhs.template eval_s(i); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return (U)_lhs * _rhs.template eval_s(i); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) * (U)_rhs; + } + + // for 2D tensors + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper(i,j); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval(i,j) * _rhs.template eval(i,j); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return (U)_lhs * _rhs.template eval(i,j); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval(i,j) * (U)_rhs; + } + + // scalar based (for 2D tensors) + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper_s(i,j); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) * _rhs.template eval_s(i,j); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return (U)_lhs * _rhs.template eval_s(i,j); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) * (U)_rhs; + } + + // for nD tensors + template + FASTOR_INLINE SIMDVector teval(const std::array &as) const { + return thelper(as); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) * _rhs.template teval(as); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return (U)_lhs * _rhs.template teval(as); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) * (U)_rhs; + } + + // scalar based (for nD tensors) + template + FASTOR_INLINE U teval_s(const std::array &as) const { + return thelper_s(as); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) * _rhs.template teval_s(as); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return (U)_lhs * _rhs.template teval_s(as); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) * (U)_rhs; + } +}; + +template::value && + !std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinaryMulOp operator*(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinaryMulOp(_lhs.self(), _rhs.self()); +} +template::value && + std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinaryMulOp operator*(const AbstractTensor &_lhs, TRhs bb) { + return BinaryMulOp(_lhs.self(), bb); +} +template::value && + !std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinaryMulOp operator*(TLhs bb, const AbstractTensor &_rhs) { + return BinaryMulOp(bb,_rhs.self()); +} + + +template::value && + !std::is_arithmetic::value && + DIM0!=DIM1,bool>::type = 0 > +FASTOR_INLINE BinaryMulOp::value> +operator*(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinaryMulOp::value>(_lhs.self(), _rhs.self()); +} + +} + +#endif // BINARY_MUL_OP_H + diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_sub_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_sub_op.h new file mode 100644 index 00000000..624004bc --- /dev/null +++ b/noarch/include/Fastor/expressions/binary_ops/binary_sub_op.h @@ -0,0 +1,246 @@ +#ifndef BINARY_SUB_OP_H +#define BINARY_SUB_OP_H + +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/expressions/expression_traits.h" + + + +namespace Fastor { + +template +struct BinarySubOp: public AbstractTensor,DIM0> { +private: + expression_t _lhs; + expression_t _rhs; +public: + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = binary_op_simd_vector_t >; + using simd_abi_type = typename simd_vector_type::abi_type; + + FASTOR_INLINE BinarySubOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} + + FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} + template::value && + !std::is_arithmetic::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_size() const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); +#endif + return _rhs.size(); + } + + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} + template::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} + template::value && + !std::is_arithmetic::value,bool>::type =0 > + FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { +#ifndef NDEBUG + FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); +#endif + return _rhs.dimension(i); + } + constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} + constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + return helper(i); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return _lhs.template eval(i) - _rhs.template eval(i); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return (U)_lhs - _rhs.template eval(i); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { + return _lhs.template eval(i) - (U)_rhs; + } + + + // scalar based + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return helper_s(i); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) - _rhs.template eval_s(i); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return (U)_lhs - _rhs.template eval_s(i); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { + return _lhs.template eval_s(i) - (U)_rhs; + } + + + // for 2D tensors + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper(i,j); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval(i,j) - _rhs.template eval(i,j); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return (U)_lhs - _rhs.template eval(i,j); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval(i,j) - (U)_rhs; + } + + // scalar based (for 2D tensors) + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return helper_s(i,j); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) - _rhs.template eval_s(i,j); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return (U)_lhs - _rhs.template eval_s(i,j); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _lhs.template eval_s(i,j) - (U)_rhs; + } + + // for nD tensors + template + FASTOR_INLINE SIMDVector teval(const std::array &as) const { + return thelper(as); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) - _rhs.template teval(as); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return (U)_lhs - _rhs.template teval(as); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE SIMDVector thelper(const std::array &as) const { + return _lhs.template teval(as) - (U)_rhs; + } + + // scalar based (for nD tensors) + template + FASTOR_INLINE U teval_s(const std::array &as) const { + return thelper_s(as); + } + + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) - _rhs.template teval_s(as); + } + template::value && + !std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return (U)_lhs - _rhs.template teval_s(as); + } + template::value && + std::is_arithmetic::value,bool>::type = 0> + FASTOR_INLINE U thelper_s(const std::array &as) const { + return _lhs.template teval_s(as) - (U)_rhs; + } +}; + +template::value && + !std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinarySubOp operator-(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinarySubOp(_lhs.self(), _rhs.self()); +} +template::value && + std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinarySubOp operator-(const AbstractTensor &_lhs, TRhs bb) { + return BinarySubOp(_lhs.self(), bb); +} +template::value && + !std::is_arithmetic::value,bool>::type = 0 > +FASTOR_INLINE BinarySubOp operator-(TLhs bb, const AbstractTensor &_rhs) { + return BinarySubOp(bb,_rhs.self()); +} + +template::value && + !std::is_arithmetic::value && + DIM0!=DIM1,bool>::type = 0 > +FASTOR_INLINE BinarySubOp::value> +operator-(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { + return BinarySubOp::value>(_lhs.self(), _rhs.self()); +} + +} + + + +#endif // BINARY_SUB_OP_H + diff --git a/noarch/include/Fastor/expressions/expression_traits.h b/noarch/include/Fastor/expressions/expression_traits.h new file mode 100644 index 00000000..0c7ae3c2 --- /dev/null +++ b/noarch/include/Fastor/expressions/expression_traits.h @@ -0,0 +1,327 @@ +#ifndef EXPRESSION_TRAITS_H +#define EXPRESSION_TRAITS_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include + +namespace Fastor { + +//------------------------------------------------------------------------------------------------// +template +struct is_expression { + static constexpr bool value = false; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct is_expression> { + static constexpr bool value = true; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct is_expression> { + static constexpr bool value = true; +}; + +template +static constexpr bool is_expression_v = is_expression::value; +//------------------------------------------------------------------------------------------------// + + +// Expression binder - by value or ref +//------------------------------------------------------------------------------------------------// +template +struct expression_binder_type { +#ifndef FASTOR_COPY_EXPR + using type = conditional_t_ || is_arithmetic_v_, const T, const T&>; +#else + using type = const T; +#endif +}; + +template +using expression_t = typename expression_binder_type::type; +//------------------------------------------------------------------------------------------------// + + + +template +class Tensor; + +template +class TensorMap; + +template +class SingleValueTensor; + +// traits +//----------------------------------------------------------------------------------------------------------// +template +struct is_unary_bool_op { + static constexpr bool value = false; +}; + +template +struct is_unary_bool_op> { + static constexpr bool value = true; +}; +template +struct is_unary_bool_op> { + static constexpr bool value = true; +}; +template +struct is_unary_bool_op> { + static constexpr bool value = true; +}; +template +struct is_unary_bool_op> { + static constexpr bool value = true; +}; + +template +struct is_unary_bool_op> { + static constexpr bool value = true; +}; +template +struct is_unary_bool_op> { + static constexpr bool value = true; +}; +template +struct is_unary_bool_op> { + static constexpr bool value = true; +}; + +template +static constexpr bool is_unary_bool_op_v = is_unary_bool_op::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template +struct is_binary_cmp_op { + static constexpr bool value = false; +}; +#define FASTOR_MAKE_IS_BINARY_CMP_OP(NAME) \ +template\ +struct is_binary_cmp_op> {\ + static constexpr bool value = true;\ +};\ + +FASTOR_MAKE_IS_BINARY_CMP_OP(EQ) +FASTOR_MAKE_IS_BINARY_CMP_OP(NEQ) +FASTOR_MAKE_IS_BINARY_CMP_OP(LT) +FASTOR_MAKE_IS_BINARY_CMP_OP(GT) +FASTOR_MAKE_IS_BINARY_CMP_OP(LE) +FASTOR_MAKE_IS_BINARY_CMP_OP(GE) +FASTOR_MAKE_IS_BINARY_CMP_OP(AND) +FASTOR_MAKE_IS_BINARY_CMP_OP(OR) + +template +struct is_binary_cmp_op> { + static constexpr bool value = true; +}; +template +struct is_binary_cmp_op> { + static constexpr bool value = true; +}; +template +struct is_binary_cmp_op> { + static constexpr bool value = true; +}; + +template +static constexpr bool is_binary_cmp_op_v = is_binary_cmp_op::value; +//----------------------------------------------------------------------------------------------------------// + + +/* Is boolean expresssion */ +//----------------------------------------------------------------------------------------------------------// +template +struct is_boolean_expression { + static constexpr bool value = is_unary_bool_op::value || is_binary_cmp_op::value; +}; + +template +static constexpr bool is_boolean_expression_v = is_boolean_expression::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template +struct is_tensor_view { + static constexpr bool value = false; +}; +template +struct is_tensor_view,DIMS>> { + static constexpr bool value = true; +}; +template +struct is_tensor_view,DIMS>> { + static constexpr bool value = true; +}; + +template +struct has_tensor_view { + static constexpr bool value = is_tensor_view::value ? true : false; +}; +template +struct has_tensor_view,DIMS>> { + static constexpr bool value = true; +}; +template +struct has_tensor_view,DIMS>> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_tensor_view> { + static constexpr bool value = has_tensor_view::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_tensor_view> { + static constexpr bool value = has_tensor_view::value || has_tensor_view::value; +}; + +template +static constexpr bool has_tensor_view_v = has_tensor_view::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template +struct is_tensor_fixed_view_2d { + static constexpr bool value = false; +}; +template +struct is_tensor_fixed_view_2d,Seq0,Seq1,2>> { + static constexpr bool value = true; +}; +template +struct is_tensor_fixed_view_2d,Seq0,Seq1,2>> { + static constexpr bool value = true; +}; + +template +struct has_tensor_fixed_view_2d { + static constexpr bool value = is_tensor_fixed_view_2d::value ? true : false; +}; +template +struct has_tensor_fixed_view_2d,Seq0,Seq1,2>> { + static constexpr bool value = true; +}; +template +struct has_tensor_fixed_view_2d,Seq0,Seq1,2>> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_tensor_fixed_view_2d> { + static constexpr bool value = has_tensor_fixed_view_2d::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_tensor_fixed_view_2d> { + static constexpr bool value = has_tensor_fixed_view_2d::value || has_tensor_fixed_view_2d::value; +}; + +template +static constexpr bool has_tensor_fixed_view_2d_v = has_tensor_fixed_view_2d::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template +struct is_tensor_fixed_view_nd { + static constexpr bool value = false; +}; +template +struct is_tensor_fixed_view_nd,Fseq...>> { + static constexpr bool value = true; +}; +template +struct is_tensor_fixed_view_nd,Fseq...>> { + static constexpr bool value = true; +}; + +template +struct has_tensor_fixed_view_nd { + static constexpr bool value = is_tensor_fixed_view_nd::value ? true : false; +}; +template +struct has_tensor_fixed_view_nd,Fseq...>> { + static constexpr bool value = true; +}; +template +struct has_tensor_fixed_view_nd,Fseq...>> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_tensor_fixed_view_nd> { + static constexpr bool value = has_tensor_fixed_view_nd::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_tensor_fixed_view_nd> { + static constexpr bool value = has_tensor_fixed_view_nd::value || has_tensor_fixed_view_nd::value; +}; + +template +static constexpr bool has_tensor_fixed_view_nd_v = has_tensor_fixed_view_nd::value; +//----------------------------------------------------------------------------------------------------------// + + + +//------------------------------------------------------------------------------------------------// +template +struct get_binary_arithmetic_result_type; +template +struct get_binary_arithmetic_result_type > > { + using type = T; +}; +template +struct get_binary_arithmetic_result_type > > { + using type = typename T::result_type; +}; + +template +struct binary_arithmetic_result_type; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIM0> +struct binary_arithmetic_result_type< BinaryExpr > { + using type = conditional_t_, + typename get_binary_arithmetic_result_type::type, + typename get_binary_arithmetic_result_type::type >; +}; +template +using binary_arithmetic_result_t = typename binary_arithmetic_result_type::type; +//------------------------------------------------------------------------------------------------// + + + +//------------------------------------------------------------------------------------------------// +template +struct get_binary_op_simd_vector_type; +template +struct get_binary_op_simd_vector_type > > { + using type = T; +}; +template +struct get_binary_op_simd_vector_type > > { + using type = typename T::simd_vector_type; +}; + +template +struct binary_op_simd_vector_type; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIM0> +struct binary_op_simd_vector_type< BinaryExpr > { + using type = conditional_t_, + typename get_binary_op_simd_vector_type::type, + typename get_binary_op_simd_vector_type::type >; +}; +template +using binary_op_simd_vector_t = typename binary_op_simd_vector_type::type; +//------------------------------------------------------------------------------------------------// + + + + +template +struct scalar_type_finder; + +} // end of namespace Fastor + + +#endif // EXPRESSION_TRAITS_H diff --git a/noarch/include/Fastor/expressions/expressions.h b/noarch/include/Fastor/expressions/expressions.h new file mode 100644 index 00000000..7623c738 --- /dev/null +++ b/noarch/include/Fastor/expressions/expressions.h @@ -0,0 +1,27 @@ +#ifndef EXPRESSIONS_H +#define EXPRESSIONS_H + +#include "Fastor/expressions/binary_ops/binary_arithmetic_ops.h" +// #include "Fastor/expressions/binary_ops/binary_add_op.h" +// #include "Fastor/expressions/binary_ops/binary_sub_op.h" +// #include "Fastor/expressions/binary_ops/binary_mul_op.h" +#include "Fastor/expressions/binary_ops/binary_div_op.h" +#include "Fastor/expressions/binary_ops/binary_arithmetic_assignment.h" +#include "Fastor/expressions/binary_ops/binary_math_ops.h" +#include "Fastor/expressions/binary_ops/binary_cmp_ops.h" +#include "Fastor/expressions/unary_ops/unary_math_ops.h" +#include "Fastor/expressions/unary_ops/unary_bool_ops.h" +#include "Fastor/expressions/linalg_ops/linalg_ops.h" + +#include "Fastor/expressions/views/tensor_fixed_views_1d.h" +#include "Fastor/expressions/views/tensor_fixed_views_2d.h" +#include "Fastor/expressions/views/tensor_fixed_views_nd.h" +#include "Fastor/expressions/views/tensor_random_views.h" +#include "Fastor/expressions/views/tensor_filter_views.h" +#include "Fastor/expressions/views/tensor_diag_views.h" +#include "Fastor/expressions/views/tensor_views.h" + +#include "Fastor/expressions/views/tensor_views_assignment.h" + +#endif // EXPRESSIONS_H + diff --git a/noarch/include/Fastor/expressions/linalg_ops/binary_cross_op.h b/noarch/include/Fastor/expressions/linalg_ops/binary_cross_op.h new file mode 100644 index 00000000..00838ea1 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/binary_cross_op.h @@ -0,0 +1,191 @@ +#ifndef BINARY_CROSS_OP_H +#define BINARY_CROSS_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/backend/tensor_cross.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/expressions/expression_traits.h" + +namespace Fastor { + +// classic vector cross product +template = false> +FASTOR_INLINE Tensor cross(const Tensor &a, const Tensor &b) { + Tensor out; + _vector_crossproduct(a.data(),b.data(),out.data()); + return out; +} +template = false> +FASTOR_INLINE Tensor cross(const Tensor &a, const Tensor &b) { + Tensor out; + _vector_crossproduct(a.data(),b.data(),out.data()); + return out; +} + +// Tensor cross product of two 2nd order tensors +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { + Tensor out; + _crossproduct(a.data(),b.data(),out.data()); + return out; +} + +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { + Tensor out; + _crossproduct(a.data(),b.data(),out.data()); + return out; +} + +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { + // Plane strain case + Tensor out; + Tensor a3, b3; + a3(0,0) = a(0,0); + a3(0,1) = a(0,1); + a3(1,0) = a(1,0); + a3(1,1) = a(1,1); + a3(2,2) = 1; + b3(0,0) = b(0,0); + b3(0,1) = b(0,1); + b3(1,0) = b(1,0); + b3(1,1) = b(1,1); + b3(2,2) = 1; + _crossproduct(a.data(),b.data(),out.data()); + return out; +} + +// Tensor cross product of a vector with 2nd order tensor +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { + Tensor out; + _crossproduct(a.data(),b.data(),out.data()); + return out; +} + +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { + Tensor out; + _crossproduct(a.data(),b.data(),out.data()); + return out; +} + +// Tensor cross product of a 2nd order tensor with a vector +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { + Tensor out; + _crossproduct(a.data(),b.data(),out.data()); + return out; +} + +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { + Tensor out; + _crossproduct(a.data(),b.data(),out.data()); + return out; +} + + +// Tensor cross product of a 3rd order tensors +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { + Tensor C; + _crossproduct(A.data(),B.data(),C.data()); + return C; +} + + +// Tensor cross product of a 4th order tensors +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { + Tensor C; + _crossproduct(A.data(),B.data(),C.data()); + return C; +} + + +// Tensor cross product of a 4th order tensor with 2nd order tensor +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { + Tensor C; + _crossproduct42(A.data(),B.data(),C.data()); + return C; +} + + +// Tensor cross product of a 2nd order tensor with 4th order tensor +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { + Tensor C; + _crossproduct24(B.data(),A.data(),C.data()); + return C; +} + + +// Tensor cross product of a 3rd order tensor with 2nd order tensor +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { + Tensor C; + _crossproduct32(A.data(),B.data(),C.data()); + return C; +} + + +// Tensor cross product of a 2nd order tensor with 3rd order tensor +template::type=0> +FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { + Tensor C; + _crossproduct23(B.data(),A.data(),C.data()); + return C; +} + + +#if FASTOR_CXX_VERSION >= 2014 +// Tensor cross product for generic expressions +template && !is_tensor_v,bool> = 0 > +FASTOR_INLINE +auto +cross(const AbstractTensor &a, const AbstractTensor &b) { + using rhs_type = typename Derived1::result_type; + const rhs_type tmp_b(b); + return cross(a.self(),tmp_b); +} + +template && is_tensor_v,bool> = 0 > +FASTOR_INLINE +auto +cross(const AbstractTensor &a, const AbstractTensor &b) { + using lhs_type = typename Derived0::result_type; + const lhs_type tmp_a(a); + return cross(tmp_a,b.self()); +} + +template && !is_tensor_v,bool> = 0 > +FASTOR_INLINE +auto +cross(const AbstractTensor &a, const AbstractTensor &b) { + using lhs_type = typename Derived0::result_type; + using rhs_type = typename Derived1::result_type; + const lhs_type tmp_a(a); + const rhs_type tmp_b(b); + return cross(tmp_a,tmp_b); +} +#endif + +} // end of namespace Fastor + + +#endif // BINARY_CROSS_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/binary_matmul_op.h b/noarch/include/Fastor/expressions/linalg_ops/binary_matmul_op.h new file mode 100644 index 00000000..ca0dbfa4 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/binary_matmul_op.h @@ -0,0 +1,627 @@ +#ifndef BINARY_MATMUL_OP_H +#define BINARY_MATMUL_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/backend/inner.h" +#include "Fastor/backend/matmul/matmul.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + +template +struct BinaryMatMulOp: public AbstractTensor,DIM0> { + using lhs_expr_type = expression_t; + using rhs_expr_type = expression_t; + using lhs_type = typename TLhs::result_type; + using rhs_type = typename TRhs::result_type; + static constexpr FASTOR_INDEX lhs_rank = lhs_type::dimension_t::value; + static constexpr FASTOR_INDEX rhs_rank = rhs_type::dimension_t::value; + static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; + static constexpr FASTOR_INDEX K = get_tensor_dimension_v<1,lhs_type>; + static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,rhs_type>; + static constexpr FASTOR_INDEX K_other = get_tensor_dimension_v<0,rhs_type>; + static constexpr FASTOR_INDEX flop_count = M*N*K; + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + // does not matter which one as matmul bypasses this + using simd_vector_type = typename TLhs::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = conditional_t_ , + conditional_t_, + Tensor // matrix-matrix + > + >; + + FASTOR_INLINE BinaryMatMulOp(lhs_expr_type inlhs, rhs_expr_type inrhs) : _lhs(inlhs), _rhs(inrhs) { + static_assert(lhs_rank <=2 && rhs_rank <=2, "EXPRESSIONS FOR MATRIX MULTIPLICATION HAVE TO BE 2-DIMENSIONAL"); + static_assert(K == K_other, "INVALID MATMUL OPERANDS. COLUMNS(A)!=ROWS(B)"); + } + + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i==0 ? M : N;} + + constexpr FASTOR_INLINE lhs_expr_type lhs() const {return _lhs;} + constexpr FASTOR_INLINE rhs_expr_type rhs() const {return _rhs;} + +private: + lhs_expr_type _lhs; + rhs_expr_type _rhs; +}; + +template &&!is_arithmetic_v_,bool> = 0 > +constexpr FASTOR_INLINE BinaryMatMulOp::value> +operator %(const AbstractTensor &lhs, const AbstractTensor &rhs) { + return BinaryMatMulOp::value>(lhs.self(), rhs.self()); +} + + + + +// helper dispatcher functions +namespace internal { +// till streaming gemm is implemented +template +FASTOR_INLINE +void _gemm(const T alpha, const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, const T beta, T * FASTOR_RESTRICT c) { + + FASTOR_ARCH_ALIGN T tmp[M*N]; + if (beta == 0) { + // non-streaming + _matmul(a,b,tmp); + for (size_t i = 0; i(a,b,tmp); + for (size_t i = 0; i +FASTOR_INLINE +void _gemm_mul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + FASTOR_ARCH_ALIGN T tmp[M*N]; + _matmul(a,b,tmp); + for (size_t i = 0; i +FASTOR_INLINE +void _gemm_div(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { + FASTOR_ARCH_ALIGN T tmp[M*N]; + _matmul(a,b,tmp); + for (size_t i = 0; i +FASTOR_INLINE void matmul_dispatcher(const Tensor &a, const Tensor &b, Tensor &out) { + FASTOR_IF_CONSTEXPR(J==1) { + _dyadic(a.data(),b.data(),out.data()); + } + else FASTOR_IF_CONSTEXPR(I==1 && J!=1 && K==1) { + out.data()[0] = _inner(a.data(),b.data()); + } + else { + _matmul(a.data(),b.data(),out.data()); + } +} +template +FASTOR_INLINE void matmul_dispatcher(const Tensor &a, const Tensor &b, Tensor &out) { + _matmul(a.data(),b.data(),out.data()); +} +template +FASTOR_INLINE void matmul_dispatcher(const Tensor &a, const Tensor &b, Tensor &out) { + _matmul(a.data(),b.data(),out.data()); +} + +template +FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor &a, const Tensor &b, const T beta, Tensor &out) { + _gemm(alpha,a.data(),b.data(),beta,out.data()); +} +template +FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor &a, const Tensor &b, const T beta, Tensor &out) { + _gemm(alpha,a.data(),b.data(),beta,out.data()); +} +template +FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor &a, const Tensor &b, const T beta, Tensor &out) { + _gemm(alpha,a.data(),b.data(),beta,out.data()); +} + +template +FASTOR_INLINE void matmul_dispatcher_mul(const Tensor &a, const Tensor &b, Tensor &out) { + _gemm_mul(a.data(),b.data(),out.data()); +} +template +FASTOR_INLINE void matmul_dispatcher_mul(const Tensor &a, const Tensor &b, Tensor &out) { + _gemm_mul(a.data(),b.data(),out.data()); +} +template +FASTOR_INLINE void matmul_dispatcher_mul(const Tensor &a, const Tensor &b, Tensor &out) { + _gemm_mul(a.data(),b.data(),out.data()); +} + +template +FASTOR_INLINE void matmul_dispatcher_div(const Tensor &a, const Tensor &b, Tensor &out) { + _gemm_div(a.data(),b.data(),out.data()); +} +template +FASTOR_INLINE void matmul_dispatcher_div(const Tensor &a, const Tensor &b, Tensor &out) { + _gemm_div(a.data(),b.data(),out.data()); +} +template +FASTOR_INLINE void matmul_dispatcher_div(const Tensor &a, const Tensor &b, Tensor &out) { + _gemm_div(a.data(),b.data(),out.data()); +} + +} // internal + + +// For tensors +template +FASTOR_INLINE Tensor matmul(const Tensor &a, const Tensor &b) { + Tensor out; + FASTOR_IF_CONSTEXPR(J==1) { + _dyadic(a.data(),b.data(),out.data()); + } + else FASTOR_IF_CONSTEXPR(I==1 && J!=1 && K==1) { + out.data()[0] = _inner(a.data(),b.data()); + } + else { + _matmul(a.data(),b.data(),out.data()); + } + return out; +} +template +FASTOR_INLINE Tensor matmul(const Tensor &a, const Tensor &b) { + Tensor out; + _matmul(a.data(),b.data(),out.data()); + return out; +} +template +FASTOR_INLINE Tensor matmul(const Tensor &a, const Tensor &b) { + Tensor out; + _matmul(a.data(),b.data(),out.data()); + return out; +} + + +// Generic matmul function for AbstractTensor types are provided here +template && is_less_equal_v_ + && !is_tensor_v && !is_tensor_v,bool> = 0 > +FASTOR_INLINE +conditional_t_ >, + conditional_t_ >, + Tensor , + get_tensor_dimension_v<1,typename Derived1::result_type> > + > +> +matmul(const AbstractTensor &a, const AbstractTensor &b) { + using lhs_type = typename Derived0::result_type; + using rhs_type = typename Derived1::result_type; + const lhs_type tmp_a(a); + const rhs_type tmp_b(b); + return matmul(tmp_a,tmp_b); +} + +template && is_less_equal_v_ + && !is_tensor_v && is_tensor_v,bool> = 0 > +FASTOR_INLINE +conditional_t_ >, + conditional_t_ >, + Tensor , + get_tensor_dimension_v<1,typename Derived1::result_type> > + > +> +matmul(const AbstractTensor &a, const AbstractTensor &b) { + using lhs_type = typename Derived0::result_type; + const lhs_type tmp_a(a); + return matmul(tmp_a,b.self()); +} + +template && is_less_equal_v_ + && is_tensor_v && !is_tensor_v,bool> = 0 > +FASTOR_INLINE +conditional_t_ >, + conditional_t_ >, + Tensor , + get_tensor_dimension_v<1,typename Derived1::result_type> > + > +> +matmul(const AbstractTensor &a, const AbstractTensor &b) { + using rhs_type = typename Derived1::result_type; + const rhs_type tmp_b(b); + return matmul(a.self(),tmp_b); +} + + + +// triangular matmul functions +template +Tensor tmatmul(const Tensor &a, const Tensor &b) { + Tensor out; + _tmatmul(a.data(),b.data(),out.data()); + return out; +} +template +FASTOR_INLINE Tensor tmatmul(const Tensor &a, const Tensor &b) { + Tensor out; + _tmatmul(a.data(),b.data(),out.data()); + return out; +} +template +FASTOR_INLINE Tensor tmatmul(const Tensor &a, const Tensor &b) { + Tensor out; + _tmatmul(a.data(),b.data(),out.data()); + return out; +} + +#if FASTOR_CXX_VERSION >= 2014 +// tmatmul for generic expressions +template && is_less_equal_v_ + && !is_tensor_v && !is_tensor_v,bool> = 0 > +FASTOR_INLINE +decltype(auto) +tmatmul(const AbstractTensor &a, const AbstractTensor &b) { + using lhs_type = typename Derived0::result_type; + using rhs_type = typename Derived1::result_type; + const lhs_type tmp_a(a); + const rhs_type tmp_b(b); + return tmatmul(tmp_a,tmp_b); +} + +template && is_less_equal_v_ + && !is_tensor_v && is_tensor_v,bool> = 0 > +FASTOR_INLINE +decltype(auto) +tmatmul(const AbstractTensor &a, const AbstractTensor &b) { + using lhs_type = typename Derived0::result_type; + const lhs_type tmp_a(a); + return tmatmul(tmp_a,b.self()); +} + +template && is_less_equal_v_ + && is_tensor_v && !is_tensor_v,bool> = 0 > +FASTOR_INLINE +decltype(auto) +tmatmul(const AbstractTensor &a, const AbstractTensor &b) { + using rhs_type = typename Derived1::result_type; + const rhs_type tmp_b(b); + return tmatmul(a.self(),tmp_b); +} +#endif + + + + + + + + + +// assignments +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { + // dst = matmul(src.lhs().self(),src.rhs().self()); // this makes a copy for dst, compiler emits a memcpy + internal::matmul_dispatcher(src.lhs().self(),src.rhs().self(),dst.self()); +} +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { + using lhs_t = typename BinaryMatMulOp::lhs_type; + lhs_t a(src.lhs().self()); + // dst = matmul(a,src.rhs().self()); // this makes a copy for dst, compiler emits a memcpy + internal::matmul_dispatcher(a,src.rhs().self(),dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { + using rhs_t = typename BinaryMatMulOp::rhs_type; + rhs_t b(src.rhs().self()); + // dst = matmul(src.lhs().self(),b); // this makes a copy for dst, compiler emits a memcpy + internal::matmul_dispatcher(src.lhs().self(),b,dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { + using lhs_t = typename BinaryMatMulOp::lhs_type; + using rhs_t = typename BinaryMatMulOp::rhs_type; + lhs_t a(src.lhs().self()); + rhs_t b(src.rhs().self()); + // dst = matmul(a,b); // this makes a copy for dst, compiler emits a memcpy + internal::matmul_dispatcher(a,b,dst.self()); +} + + + +// assignments add +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + internal::matmul_dispatcher((T)1,src.lhs().self(),src.rhs().self(),(T)1,dst.self()); +} +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + using lhs_t = typename BinaryMatMulOp::lhs_type; + lhs_t a(src.lhs().self()); + internal::matmul_dispatcher((T)1,a,src.rhs().self(),(T)1,dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + using rhs_t = typename BinaryMatMulOp::rhs_type; + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher((T)1,src.lhs().self(),b,(T)1,dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + using lhs_t = typename BinaryMatMulOp::lhs_type; + using rhs_t = typename BinaryMatMulOp::rhs_type; + lhs_t a(src.lhs().self()); + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher((T)1,a,b,(T)1,dst.self()); +} + + +// assignments sub +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + internal::matmul_dispatcher((T)-1,src.lhs().self(),src.rhs().self(),(T)1,dst.self()); +} +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + using lhs_t = typename BinaryMatMulOp::lhs_type; + lhs_t a(src.lhs().self()); + internal::matmul_dispatcher((T)-1,a,src.rhs().self(),(T)1,dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + using rhs_t = typename BinaryMatMulOp::rhs_type; + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher((T)-1,src.lhs().self(),b,(T)1,dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { + using T = typename BinaryMatMulOp::scalar_type; + using lhs_t = typename BinaryMatMulOp::lhs_type; + using rhs_t = typename BinaryMatMulOp::rhs_type; + lhs_t a(src.lhs().self()); + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher((T)-1,a,b,(T)1,dst.self()); +} + + +// assignments mul +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { + internal::matmul_dispatcher_mul(src.lhs().self(),src.rhs().self(),dst.self()); +} +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { + using lhs_t = typename BinaryMatMulOp::lhs_type; + lhs_t a(src.lhs().self()); + internal::matmul_dispatcher_mul(a,src.rhs().self(),dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { + using rhs_t = typename BinaryMatMulOp::rhs_type; + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher_mul(src.lhs().self(),b,dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { + using lhs_t = typename BinaryMatMulOp::lhs_type; + using rhs_t = typename BinaryMatMulOp::rhs_type; + lhs_t a(src.lhs().self()); + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher_mul(a,b,dst.self()); +} + + +// assignments div +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { + internal::matmul_dispatcher_div(src.lhs().self(),src.rhs().self(),dst.self()); +} +template::lhs_expr_type>> && + is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { + using lhs_t = typename BinaryMatMulOp::lhs_type; + lhs_t a(src.lhs().self()); + internal::matmul_dispatcher_div(a,src.rhs().self(),dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { + using rhs_t = typename BinaryMatMulOp::rhs_type; + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher_div(src.lhs().self(),b,dst.self()); +} +template::lhs_expr_type>> && + !is_tensor_v::rhs_expr_type>>, bool >::type = false> +FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { + using lhs_t = typename BinaryMatMulOp::lhs_type; + using rhs_t = typename BinaryMatMulOp::rhs_type; + lhs_t a(src.lhs().self()); + rhs_t b(src.rhs().self()); + internal::matmul_dispatcher_div(a,b,dst.self()); +} + + + + + + + +// recursive greedy-like +template +FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { + FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); + assign(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); + } + else + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); + assign(dst, BinaryMatMulOp(tmp,src.rhs().self())); + } +} + +template +FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { + FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); + assign_add(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); + } + else + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); + assign_add(dst, BinaryMatMulOp(tmp,src.rhs().self())); + } +} + +template +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { + FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); + assign_sub(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); + } + else + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); + assign_sub(dst, BinaryMatMulOp(tmp,src.rhs().self())); + } +} + +template +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { + FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); + assign_mul(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); + } + else + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); + assign_mul(dst, BinaryMatMulOp(tmp,src.rhs().self())); + } +} + +template +FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { + FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); + assign_div(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); + } + else + { + using result_t = typename BinaryMatMulOp::result_type; + result_t tmp; + assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); + assign_div(dst, BinaryMatMulOp(tmp,src.rhs().self())); + } +} + + +} // end of namespace Fastor + + +#endif // BINARY_MATMUL_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/binary_solve_op.h b/noarch/include/Fastor/expressions/linalg_ops/binary_solve_op.h new file mode 100644 index 00000000..3a0fc5bc --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/binary_solve_op.h @@ -0,0 +1,197 @@ +#ifndef BINARY_SOLVE_OP_H +#define BINARY_SOLVE_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" +#include "Fastor/expressions/linalg_ops/binary_matmul_op.h" +#include "Fastor/expressions/linalg_ops/unary_inv_op.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" +#include "Fastor/expressions/linalg_ops/unary_piv_op.h" + + +namespace Fastor { + +// Solving using LU decomposition is in the LU module [unary_lu_op] +// For tensors +// Solve - no pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { + return matmul(inverse(A),b); +} +// Solve - pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { + // We need to post multiply - swap columns using row permutation vector + Tensor p; + pivot_inplace(A,p); + auto tmp(apply_pivot(A,p)); + Tensor invA = inverse(tmp); + return matmul(reconstruct_colwise(invA,p),b); + + // // matrix version + // Tensor p; + // pivot_inplace(A,p); + // auto tmp(apply_pivot(A,p)); + // Tensor invA = inverse(tmp); + // return matmul(matmul(invA,p),b); +} + +// Multiple right hand sides +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { + return matmul(inverse(A),B); +} + +// Solve - pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { + Tensor p; + pivot_inplace(A,p); + auto tmp(apply_pivot(A,p)); + Tensor invA = inverse(tmp); + return matmul(reconstruct(invA,p),B); +} + + + +// For expressions +template && is_tensor_v && DIM0==2,bool> = false> +FASTOR_INLINE +conditional_t_ + >, + Tensor< + typename TLhs::scalar_type, + get_tensor_dimension_v<0,typename TLhs::result_type>, + get_tensor_dimension_v<1,typename TRhs::result_type> + > +> +solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { + + using lhs_type = typename TLhs::result_type; + using rhs_type = typename TRhs::result_type; + constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; + constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; + constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; + + static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); + static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); + + return solve(lhs.self(),rhs.self()); +} + + +template && is_tensor_v && DIM0==2,bool> = false> +FASTOR_INLINE +conditional_t_ + >, + Tensor< + typename TLhs::scalar_type, + get_tensor_dimension_v<0,typename TLhs::result_type>, + get_tensor_dimension_v<1,typename TRhs::result_type> + > +> +solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { + + using lhs_type = typename TLhs::result_type; + using rhs_type = typename TRhs::result_type; + constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; + constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; + constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; + + static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); + static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); + + const lhs_type A(lhs.self()); + + return solve(A,rhs.self()); +} + + +template && !is_tensor_v && DIM0==2,bool> = false> +FASTOR_INLINE +conditional_t_ + >, + Tensor< + typename TLhs::scalar_type, + get_tensor_dimension_v<0,typename TLhs::result_type>, + get_tensor_dimension_v<1,typename TRhs::result_type> + > +> +solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { + + using lhs_type = typename TLhs::result_type; + using rhs_type = typename TRhs::result_type; + constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; + constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; + constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; + + static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); + static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); + + const rhs_type b(rhs.self()); + + return solve(lhs.self(),b); +} + + +template && !is_tensor_v && DIM0==2,bool> = false> +FASTOR_INLINE +conditional_t_ + >, + Tensor< + typename TLhs::scalar_type, + get_tensor_dimension_v<0,typename TLhs::result_type>, + get_tensor_dimension_v<1,typename TRhs::result_type> + > +> +solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { + + using lhs_type = typename TLhs::result_type; + using rhs_type = typename TRhs::result_type; + constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; + constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; + constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; + + static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); + static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); + + const lhs_type A(lhs.self()); + const rhs_type b(rhs.self()); + + return solve(A,b); +} + + +} // end of namespace Fastor + + +#endif // BINARY_SOLVE_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/linalg_computation_types.h b/noarch/include/Fastor/expressions/linalg_ops/linalg_computation_types.h new file mode 100644 index 00000000..8af6685f --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/linalg_computation_types.h @@ -0,0 +1,66 @@ +#ifndef LINALG_COMPUTATION_TYPES_H +#define LINALG_COMPUTATION_TYPES_H + +namespace Fastor { + +// Pivot types +enum class PivType : int +{ + V = 0, /* The permutation vector */ + M, /* The permutatoin matrix */ +}; + + +// QR factorisation computation types +enum class QRCompType : int +{ + MGSR = 0, /* Modified Gram-Schmidt Row-wise */ + MGSRPiv, /* Modified Gram-Schmidt Row-wise with pivot */ + HHR /* House Holder Reflections */ +}; + +// LU factorisation computation types +enum class LUCompType : int +{ + BlockLU = 0, /* Block LU decomposition wiht no pivot */ + BlockLUPiv, /* Block LU decomposition wiht pivot */ + SimpleLU, /* Simple LU decomposition wiht no pivot */ + SimpleLUPiv, /* Simple LU decomposition wiht pivot */ +}; + +// Determinant computation type +enum class DetCompType : int +{ + Simple = 0, /* Using simple hand-optimised calculations */ + LU, /* Using LU factorisation */ + QR, /* Using QR factorisation */ +}; + +// Inverse computation type +enum class InvCompType : int +{ + SimpleInv = 0,/* Using simple hand-optimised calculations */ + SimpleInvPiv, /* Simple with pivot */ + BlockLU, /* Using block LU factorisation */ + BlockLUPiv, /* Using block LU factorisation with pivot */ + SimpleLU, /* Using simple LU factorisation */ + SimpleLUPiv, /* Using simple LU factorisation with pivot */ +}; + +// Solve computation type +enum class SolveCompType : int +{ + SimpleInv = 0, /* Using optimised inversion */ + SimpleInvPiv, /* Using optimised inversion with pivot */ + BlockLU, /* Using block LU factorisation */ + BlockLUPiv, /* Using block LU factorisation with pivot */ + SimpleLU, /* Using simple LU factorisation */ + SimpleLUPiv, /* Using simple LU factorisation with pivot */ + QR, /* Using QR factorisation */ + Chol, /* Using Cholesky factorisation */ +}; + + +} // end of namespace Fastor + +#endif // LINALG_COMPUTATION_TYPES_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/linalg_ops.h b/noarch/include/Fastor/expressions/linalg_ops/linalg_ops.h new file mode 100644 index 00000000..e59a6835 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/linalg_ops.h @@ -0,0 +1,24 @@ +#ifndef LINALG_OPS_H +#define LINALG_OPS_H + +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" + +#include "Fastor/expressions/linalg_ops/binary_matmul_op.h" +#include "Fastor/expressions/linalg_ops/unary_piv_op.h" +#include "Fastor/expressions/linalg_ops/unary_lu_op.h" +#include "Fastor/expressions/linalg_ops/binary_solve_op.h" +#include "Fastor/expressions/linalg_ops/unary_trans_op.h" +#include "Fastor/expressions/linalg_ops/unary_ctrans_op.h" +#include "Fastor/expressions/linalg_ops/unary_adj_op.h" +#include "Fastor/expressions/linalg_ops/unary_cof_op.h" +#include "Fastor/expressions/linalg_ops/unary_inv_op.h" +#include "Fastor/expressions/linalg_ops/unary_trace_op.h" +#include "Fastor/expressions/linalg_ops/unary_norm_op.h" +#include "Fastor/expressions/linalg_ops/unary_qr_op.h" +#include "Fastor/expressions/linalg_ops/unary_det_op.h" +#include "Fastor/expressions/linalg_ops/unary_svd_op.h" +#include "Fastor/expressions/linalg_ops/binary_cross_op.h" + +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +#endif // LINALG_OPS_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/linalg_traits.h b/noarch/include/Fastor/expressions/linalg_ops/linalg_traits.h new file mode 100644 index 00000000..3774029f --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/linalg_traits.h @@ -0,0 +1,273 @@ +#ifndef LINALG_TRAITS_H +#define LINALG_TRAITS_H + + +#include "Fastor/expressions/linalg_ops/binary_matmul_op.h" +#include "Fastor/expressions/linalg_ops/unary_trans_op.h" +#include "Fastor/expressions/linalg_ops/unary_ctrans_op.h" +#include "Fastor/expressions/linalg_ops/unary_adj_op.h" +#include "Fastor/expressions/linalg_ops/unary_cof_op.h" +#include "Fastor/expressions/linalg_ops/unary_inv_op.h" +#include "Fastor/expressions/linalg_ops/binary_solve_op.h" +#include + + +namespace Fastor { + + +// Is a binary matmul expression +//----------------------------------------------------------------------------------------------------------// +template +struct is_binary_matmul_op { + static constexpr bool value = false; +}; +template +struct is_binary_matmul_op,Tensor,DIM>> { + static constexpr bool value = true; +}; +template +struct is_binary_matmul_op> { + static constexpr bool value = true; +}; + +template +struct has_binary_matmul_op { + static constexpr bool value = is_binary_matmul_op::value ? true : false; +}; +template +struct has_binary_matmul_op,Tensor,DIM>> { + static constexpr bool value = true; +}; +template +struct has_binary_matmul_op> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_binary_matmul_op> { + static constexpr bool value = has_binary_matmul_op::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_binary_matmul_op> { + static constexpr bool value = has_binary_matmul_op::value || has_binary_matmul_op::value; +}; + +// helper +template +static constexpr bool is_binary_matmul_op_v = is_binary_matmul_op::value; +template +static constexpr bool has_binary_matmul_op_v = has_binary_matmul_op::value; +//----------------------------------------------------------------------------------------------------------// + + +// Is unary trans expression +//----------------------------------------------------------------------------------------------------------// +template +struct is_unary_trans_op { + static constexpr bool value = false; +}; +template +struct is_unary_trans_op> { + static constexpr bool value = true; +}; + +template +struct has_unary_trans_op { + static constexpr bool value = is_unary_trans_op::value ? true : false; +}; +template +struct has_unary_trans_op> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_unary_trans_op> { + static constexpr bool value = has_unary_trans_op::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_unary_trans_op> { + static constexpr bool value = has_unary_trans_op::value || has_unary_trans_op::value; +}; + +// helper +template +static constexpr bool is_unary_trans_op_v = is_unary_trans_op::value; +template +static constexpr bool has_unary_trans_op_v = has_unary_trans_op::value; +//----------------------------------------------------------------------------------------------------------// + + +// Is unary ctrans expression +//----------------------------------------------------------------------------------------------------------// +template +struct is_unary_ctrans_op { + static constexpr bool value = false; +}; +template +struct is_unary_ctrans_op> { + static constexpr bool value = true; +}; + +template +struct has_unary_ctrans_op { + static constexpr bool value = is_unary_ctrans_op::value ? true : false; +}; +template +struct has_unary_ctrans_op> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_unary_ctrans_op> { + static constexpr bool value = has_unary_ctrans_op::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_unary_ctrans_op> { + static constexpr bool value = has_unary_ctrans_op::value || has_unary_ctrans_op::value; +}; + +// helper +template +static constexpr bool is_unary_ctrans_op_v = is_unary_ctrans_op::value; +template +static constexpr bool has_unary_ctrans_op_v = has_unary_ctrans_op::value; +//----------------------------------------------------------------------------------------------------------// + + +// Is unary adj expression +//----------------------------------------------------------------------------------------------------------// +template +struct is_unary_adj_op { + static constexpr bool value = false; +}; +template +struct is_unary_adj_op> { + static constexpr bool value = true; +}; + +template +struct has_unary_adj_op { + static constexpr bool value = is_unary_adj_op::value ? true : false; +}; +template +struct has_unary_adj_op> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_unary_adj_op> { + static constexpr bool value = has_unary_adj_op::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_unary_adj_op> { + static constexpr bool value = has_unary_adj_op::value || has_unary_adj_op::value; +}; + +// helper +template +static constexpr bool is_unary_adj_op_v = is_unary_adj_op::value; +template +static constexpr bool has_unary_adj_op_v = has_unary_adj_op::value; +//----------------------------------------------------------------------------------------------------------// + +// Is unary adj expression +//----------------------------------------------------------------------------------------------------------// +template +struct is_unary_cof_op { + static constexpr bool value = false; +}; +template +struct is_unary_cof_op> { + static constexpr bool value = true; +}; + +template +struct has_unary_cof_op { + static constexpr bool value = is_unary_cof_op::value ? true : false; +}; +template +struct has_unary_cof_op> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_unary_cof_op> { + static constexpr bool value = has_unary_cof_op::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_unary_cof_op> { + static constexpr bool value = has_unary_cof_op::value || has_unary_cof_op::value; +}; + +// helper +template +static constexpr bool is_unary_cof_op_v = is_unary_cof_op::value; +template +static constexpr bool has_unary_cof_op_v = has_unary_cof_op::value; +//----------------------------------------------------------------------------------------------------------// + + +// Is unary inv expression +//----------------------------------------------------------------------------------------------------------// +template +struct is_unary_inv_op { + static constexpr bool value = false; +}; +template +struct is_unary_inv_op> { + static constexpr bool value = true; +}; + +template +struct has_unary_inv_op { + static constexpr bool value = is_unary_inv_op::value ? true : false; +}; +template +struct has_unary_inv_op> { + static constexpr bool value = true; +}; +template class UnaryExpr, typename Expr, size_t DIM> +struct has_unary_inv_op> { + static constexpr bool value = has_unary_inv_op::value; +}; +template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> +struct has_unary_inv_op> { + static constexpr bool value = has_unary_inv_op::value || has_unary_inv_op::value; +}; + +// helper +template +static constexpr bool is_unary_inv_op_v = is_unary_inv_op::value; +template +static constexpr bool has_unary_inv_op_v = has_unary_inv_op::value; +//----------------------------------------------------------------------------------------------------------// + + +// Is a linear algebra expression +//----------------------------------------------------------------------------------------------------------// +template +struct has_linalg_op { + static constexpr bool value = has_binary_matmul_op::value || has_unary_trans_op::value || + has_unary_ctrans_op::value || has_unary_adj_op::value || + has_unary_cof_op::value || has_unary_inv_op::value; +}; + +// helper +template +static constexpr bool has_linalg_op_v = has_linalg_op::value; +//----------------------------------------------------------------------------------------------------------// + + +// Requires immediate evaluation +//----------------------------------------------------------------------------------------------------------// +template +struct requires_evaluation { + static constexpr bool value = has_linalg_op::value; +}; + +// helper +template +static constexpr bool requires_evaluation_v = requires_evaluation::value; +//----------------------------------------------------------------------------------------------------------// + + + +} // end of namespace Fastor + + +#endif // LINALG_TRAITS_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_adj_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_adj_op.h new file mode 100644 index 00000000..fbe19793 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_adj_op.h @@ -0,0 +1,160 @@ +#ifndef UNARY_ADJ_OP_H +#define UNARY_ADJ_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/adjoint.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + +template +struct UnaryAdjOp: public AbstractTensor,DIM0> { + using expr_type = expression_t; + using result_type = typename Expr::result_type; + static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; + static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = typename Expr::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + + FASTOR_INLINE UnaryAdjOp(expr_type inexpr) : _expr(inexpr) { + static_assert(M==N, "MATRIX MUST BE SQUARE"); + } + + FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} + + constexpr FASTOR_INLINE expr_type expr() const {return _expr;} + +private: + expr_type _expr; +}; + +template +FASTOR_INLINE UnaryAdjOp +adj(const AbstractTensor &src) { + return UnaryAdjOp(src.self()); +} + + +namespace internal { +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void adjoint_dispatcher(const Tensor &in, Tensor& out) { + _adjoint(in.data(),out.data()); +} +} // internal + + +// For tensors +template +FASTOR_INLINE Tensor adjoint(const Tensor &a) { + Tensor out; + _adjoint(a.data(),out.data()); + return out; +} + +// For high order tensors +template=3,bool>::type=0> +FASTOR_INLINE Tensor +adjoint(const Tensor &a) { + + constexpr size_t remaining_product = last_matrix_extracter, + typename std_ext::make_index_sequence::type>::remaining_product; + + constexpr size_t I = get_value::value; + constexpr size_t J = get_value::value; + static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); + + Tensor out; + T *a_data = a.data(); + T *out_data = out.data(); + + for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); + } + + return out; +} + +// Adjoint for generic expressions is provided here +template +FASTOR_INLINE +typename Derived::result_type +adjoint(const AbstractTensor &src) { + // If we are here Derived is already an expression + using result_type = typename Derived::result_type; + const result_type tmp(src.self()); + return adjoint(tmp); +} + + + + + + +// assignments +template +FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryAdjOp &src) { + using result_type = typename Expr::result_type; + const result_type& tmp = evaluate(src.expr().self()); + internal::adjoint_dispatcher(tmp,dst.self()); +} + +template +FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryAdjOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::adjoint_dispatcher(tmp,tmp_inv); + trivial_assign_add(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryAdjOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::adjoint_dispatcher(tmp,tmp_inv); + trivial_assign_sub(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryAdjOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::adjoint_dispatcher(tmp,tmp_inv); + trivial_assign_mul(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryAdjOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::adjoint_dispatcher(tmp,tmp_inv); + trivial_assign_div(dst.self(),tmp_inv); +} + + +} // end of namespace Fastor + + +#endif // UNARY_ADJ_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_cof_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_cof_op.h new file mode 100644 index 00000000..9bbfac20 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_cof_op.h @@ -0,0 +1,160 @@ +#ifndef UNARY_COF_OP_H +#define UNARY_COF_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/cofactor.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + +template +struct UnaryCofOp: public AbstractTensor,DIM0> { + using expr_type = expression_t; + using result_type = typename Expr::result_type; + static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; + static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = typename Expr::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + + FASTOR_INLINE UnaryCofOp(expr_type inexpr) : _expr(inexpr) { + static_assert(M==N, "MATRIX MUST BE SQUARE"); + } + + FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} + + constexpr FASTOR_INLINE expr_type expr() const {return _expr;} + +private: + expr_type _expr; +}; + +template +FASTOR_INLINE UnaryCofOp +cof(const AbstractTensor &src) { + return UnaryCofOp(src.self()); +} + + +namespace internal { +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void cofactor_dispatcher(const Tensor &in, Tensor& out) { + _cofactor(in.data(),out.data()); +} +} // internal + + +// For tensors +template +FASTOR_INLINE Tensor cofactor(const Tensor &a) { + Tensor out; + _cofactor(a.data(),out.data()); + return out; +} + +// For high order tensors +template=3,bool>::type=0> +FASTOR_INLINE Tensor +cofactor(const Tensor &a) { + + constexpr size_t remaining_product = last_matrix_extracter, + typename std_ext::make_index_sequence::type>::remaining_product; + + constexpr size_t I = get_value::value; + constexpr size_t J = get_value::value; + static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); + + Tensor out; + T *a_data = a.data(); + T *out_data = out.data(); + + for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); + } + + return out; +} + +// Cofactor for generic expressions is provided here +template +FASTOR_INLINE +typename Derived::result_type +cofactor(const AbstractTensor &src) { + // If we are here Derived is already an expression + using result_type = typename Derived::result_type; + const result_type tmp(src.self()); + return cofactor(tmp); +} + + + + + + +// assignments +template +FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryCofOp &src) { + using result_type = typename Expr::result_type; + const result_type& tmp = evaluate(src.expr().self()); + internal::cofactor_dispatcher(tmp,dst.self()); +} + +template +FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryCofOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::cofactor_dispatcher(tmp,tmp_inv); + trivial_assign_add(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryCofOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::cofactor_dispatcher(tmp,tmp_inv); + trivial_assign_sub(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryCofOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::cofactor_dispatcher(tmp,tmp_inv); + trivial_assign_mul(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryCofOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::cofactor_dispatcher(tmp,tmp_inv); + trivial_assign_div(dst.self(),tmp_inv); +} + + +} // end of namespace Fastor + + +#endif // UNARY_COF_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_ctrans_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_ctrans_op.h new file mode 100644 index 00000000..18d226c3 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_ctrans_op.h @@ -0,0 +1,178 @@ +#ifndef UNARY_CTRANS_OP_H +#define UNARY_CTRANS_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + +template +struct UnaryCTransOp: public AbstractTensor,DIM0> { + using expr_type = expression_t; + static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,typename Expr::result_type>; + static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,typename Expr::result_type>; + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = typename Expr::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + + FASTOR_INLINE UnaryCTransOp(expr_type inexpr) : _expr(inexpr) { + } + + FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i == 0 ? N : M;} + + constexpr FASTOR_INLINE expr_type expr() const {return _expr;} + +private: + expr_type _expr; +}; + +/* Tensor conjugate transpose returning a tensor expression */ +template +FASTOR_INLINE UnaryCTransOp +ctrans(const AbstractTensor &src) { + return UnaryCTransOp(src.self()); +} + +/* Backend implementation */ +template +FASTOR_INLINE void _ctranspose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { + for (size_t j=0; j +FASTOR_INLINE Tensor ctranspose(const Tensor &a) { + Tensor out; + _ctranspose(a.data(),out.data()); + return out; +} + +/* Tensor conjugate transpose for higher order tensors immediately returning a tensor */ +template=3,bool>::type=0> +FASTOR_INLINE Tensor +ctranspose(const Tensor &a) { + + constexpr size_t remaining_product = last_matrix_extracter, + typename std_ext::make_index_sequence::type>::remaining_product; + + constexpr size_t I = get_value::value; + constexpr size_t J = get_value::value; + static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); + + Tensor out; + T *a_data = a.data(); + T *out_data = out.data(); + + for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); + } + + return out; +} + +// Conjagate transpose for generic expressions +template +FASTOR_INLINE +Tensor< + typename scalar_type_finder::type, + get_tensor_dimension_v<1,typename Expr::result_type>, + get_tensor_dimension_v<0,typename Expr::result_type>> +ctranspose(const AbstractTensor &src) { + // If we are here Expr is already an expression + using result_type = typename Expr::result_type; + const result_type tmp(src.self()); + return ctranspose(tmp); +} + + + + +// assignments +template +FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryCTransOp &src) { + using result_type = typename Expr::result_type; + const result_type& tmp = evaluate(src.expr().self()); + using T = typename UnaryCTransOp::scalar_type; + static constexpr size_t M = UnaryCTransOp::M; + static constexpr size_t N = UnaryCTransOp::N; + _ctranspose(tmp.data(),dst.self().data()); +} + +template +FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryCTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryCTransOp::scalar_type; + using result_t = typename UnaryCTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryCTransOp::M; + static constexpr size_t N = UnaryCTransOp::N; + _ctranspose(tmp.data(),tmp_trans.data()); + trivial_assign_add(dst.self(),tmp_trans); +} + +template +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryCTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryCTransOp::scalar_type; + using result_t = typename UnaryCTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryCTransOp::M; + static constexpr size_t N = UnaryCTransOp::N; + _ctranspose(tmp.data(),tmp_trans.data()); + trivial_assign_sub(dst.self(),tmp_trans); +} + +template +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryCTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryCTransOp::scalar_type; + using result_t = typename UnaryCTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryCTransOp::M; + static constexpr size_t N = UnaryCTransOp::N; + _ctranspose(tmp.data(),tmp_trans.data()); + trivial_assign_mul(dst.self(),tmp_trans); +} + +template +FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryCTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryCTransOp::scalar_type; + using result_t = typename UnaryCTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryCTransOp::M; + static constexpr size_t N = UnaryCTransOp::N; + _ctranspose(tmp.data(),tmp_trans.data()); + trivial_assign_div(dst.self(),tmp_trans); +} + + +} // end of namespace Fastor + + +#endif // UNARY_CTRANS_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_det_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_det_op.h new file mode 100644 index 00000000..2b53d147 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_det_op.h @@ -0,0 +1,86 @@ +#ifndef UNARY_DET_OP_H +#define UNARY_DET_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/backend/determinant.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +#include + +namespace Fastor { + +// Computing determinant using QR/LU decompositions are in those respective modules +// For tensors +template && DetType==DetCompType::Simple,bool> = false> +FASTOR_INLINE T determinant(const Tensor &a) { + return _det(static_cast(a.data())); +} +template && DetType == DetCompType::Simple,bool> = false> +FASTOR_INLINE T determinant(const Tensor &a) { + // Dispatch to LU + return determinant(a); +} + +// For high order tensors +template=3 && DetType == DetCompType::Simple,bool> = false> +FASTOR_INLINE +typename last_matrix_extracter, typename std_ext::make_index_sequence::type>::type +determinant(const Tensor &a) { + + using OutTensor = typename last_matrix_extracter, + typename std_ext::make_index_sequence::type>::type; + constexpr size_t remaining_product = last_matrix_extracter, + typename std_ext::make_index_sequence::type>::remaining_product; + + constexpr size_t I = get_value::value; + constexpr size_t J = get_value::value; + static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); + + OutTensor out; + T *a_data = a.data(); + T *out_data = out.data(); + + for (size_t i=0; i(static_cast(a_data+i*J*J)); + } + + return out; +} + +// Find determinant of a tensor expression - dispatches to determinant for tensors +template +FASTOR_INLINE typename Derived::scalar_type determinant(const AbstractTensor &_src) { + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out = evaluate(src); + return determinant(out); +} + + +template +FASTOR_INLINE typename Derived::scalar_type det(const AbstractTensor &_src) { + return determinant(_src.self()); +} + +template +FASTOR_INLINE typename Derived::scalar_type absdet(const AbstractTensor &_src) { + return std::abs(determinant(_src.self())); +} + +template +FASTOR_INLINE typename Derived::scalar_type logdet(const AbstractTensor &_src) { + return std::log(std::abs(determinant(_src.self()))); +} + + +} // end of namespace Fastor + +#endif // UNARY_DET_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_inv_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_inv_op.h new file mode 100644 index 00000000..a4d32768 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_inv_op.h @@ -0,0 +1,731 @@ +#ifndef UNARY_INV_OP_H +#define UNARY_INV_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/inverse.h" +#include "Fastor/backend/lut_inverse.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" +#include "Fastor/expressions/linalg_ops/unary_piv_op.h" + + +namespace Fastor { + +template +struct UnaryInvOp: public AbstractTensor,DIM0> { + using expr_type = expression_t; + using result_type = typename Expr::result_type; + static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; + static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = typename Expr::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + + FASTOR_INLINE UnaryInvOp(expr_type inexpr) : _expr(inexpr) { + static_assert(M==N, "MATRIX MUST BE SQUARE"); + } + + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} + + constexpr FASTOR_INLINE expr_type expr() const {return _expr;} + +private: + expr_type _expr; +}; + +template +FASTOR_INLINE UnaryInvOp +inv(const AbstractTensor &src) { + return UnaryInvOp(src.self()); +} + + +// Upper triangular block matrix inversion +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +namespace internal { + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { + _inverse(in.data(),out.data()); +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + constexpr size_t N = 4UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + // Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + ut_inverse_dispatcher(a, inv_a); + + Tensor inv_d; + ut_inverse_dispatcher(d, inv_d); + + Tensor b_invd = matmul(b, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = -matmul(inv_a, b_invd); + out(fseq(),fseq<0,N>()) = 0; + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + constexpr size_t N = 8UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + // Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + ut_inverse_dispatcher(a, inv_a); + + Tensor inv_d; + ut_inverse_dispatcher(d, inv_d); + + Tensor b_invd = matmul(b, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = -matmul(inv_a, b_invd); + out(fseq(),fseq<0,N>()) = 0; + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 16UL; // start size + constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + // Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + ut_inverse_dispatcher(a, inv_a); + + Tensor inv_d; + ut_inverse_dispatcher(d, inv_d); + + Tensor b_invd = matmul(b, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = -matmul(inv_a, b_invd); + out(fseq(),fseq<0,N>()) = 0; + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 32UL; // start size + constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + // Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + ut_inverse_dispatcher(a, inv_a); + + Tensor inv_d; + ut_inverse_dispatcher(d, inv_d); + + Tensor b_invd = tmatmul(b, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = -tmatmul(inv_a, b_invd); + out(fseq(),fseq<0,N>()) = 0; + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 64UL; // start size + constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + // Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + ut_inverse_dispatcher(a, inv_a); + + Tensor inv_d; + ut_inverse_dispatcher(d, inv_d); + + Tensor b_invd = tmatmul(b, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = -tmatmul(inv_a, b_invd); + out(fseq(),fseq<0,N>()) = 0; + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 128UL; // start size + constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + // Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + ut_inverse_dispatcher(a, inv_a); + + Tensor inv_d; + ut_inverse_dispatcher(d, inv_d); + + Tensor b_invd = tmatmul(b, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = -tmatmul(inv_a, b_invd); + out(fseq(),fseq<0,N>()) = 0; + out(fseq(),fseq()) = inv_d; +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +} // internal + + + + +// Lower uni-triangular block matrix inversion +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +namespace internal { + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { + // This restricts the recursive block to lower uni-triangualar matrices + _lowunitri_inverse(in.data(),out.data()); +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + constexpr size_t N = 4UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + // Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + lut_inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor inv_d; + lut_inverse_dispatcher(d, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = 0; + out(fseq(),fseq<0,N>()) = -matmul(inv_d, c_inva); + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + constexpr size_t N = 8UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + // Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + lut_inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor inv_d; + lut_inverse_dispatcher(d, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = 0; + out(fseq(),fseq<0,N>()) = -matmul(inv_d, c_inva); + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 16UL; // start size + constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + // Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + lut_inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor inv_d; + lut_inverse_dispatcher(d, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = 0; + out(fseq(),fseq<0,N>()) = -matmul(inv_d, c_inva); + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 32UL; // start size + constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + // Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + lut_inverse_dispatcher(a, inv_a); + + Tensor c_inva = tmatmul(c, inv_a); + + Tensor inv_d; + lut_inverse_dispatcher(d, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = 0; + out(fseq(),fseq<0,N>()) = -tmatmul(inv_d, c_inva); + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 64UL; // start size + constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + // Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + lut_inverse_dispatcher(a, inv_a); + + Tensor c_inva = tmatmul(c, inv_a); + + Tensor inv_d; + lut_inverse_dispatcher(d, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = 0; + out(fseq(),fseq<0,N>()) = -tmatmul(inv_d, c_inva); + out(fseq(),fseq()) = inv_d; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 128UL; // start size + constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + // Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + lut_inverse_dispatcher(a, inv_a); + + Tensor c_inva = tmatmul(c, inv_a); + + Tensor inv_d; + lut_inverse_dispatcher(d, inv_d); + + out(fseq<0,N>(),fseq<0,N>()) = inv_a; + out(fseq<0,N>(),fseq()) = 0; + out(fseq(),fseq<0,N>()) = -tmatmul(inv_d, c_inva); + out(fseq(),fseq()) = inv_d; +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +} // internal + + + + +// General recursive block matrix inversion +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +namespace internal { + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { + _inverse(in.data(),out.data()); +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { + + constexpr size_t N = 4UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a = inverse(a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor block_bb = inverse(static_cast>(d - matmul(c_inva, b))); + + Tensor inva_b = matmul(inv_a, b); + Tensor bb_c_inva = matmul(block_bb, c_inva); + + Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); + Tensor block_ab = -matmul(inva_b, block_bb); + Tensor block_ba = -bb_c_inva; + + out(fseq<0,N>(),fseq<0,N>()) = block_aa; + out(fseq<0,N>(),fseq()) = block_ab; + out(fseq(),fseq<0,N>()) = block_ba; + out(fseq(),fseq()) = block_bb; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { + + constexpr size_t N = 8UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor block_bb; + inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); + + Tensor inva_b = matmul(inv_a, b); + Tensor bb_c_inva = matmul(block_bb, c_inva); + + Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); + Tensor block_ab = -matmul(inva_b, block_bb); + Tensor block_ba = -bb_c_inva; + + out(fseq<0,N>(),fseq<0,N>()) = block_aa; + out(fseq<0,N>(),fseq()) = block_ab; + out(fseq(),fseq<0,N>()) = block_ba; + out(fseq(),fseq()) = block_bb; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 16UL; // start size + constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor block_bb; + inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); + + Tensor inva_b = matmul(inv_a, b); + Tensor bb_c_inva = matmul(block_bb, c_inva); + + Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); + Tensor block_ab = -matmul(inva_b, block_bb); + Tensor block_ba = -bb_c_inva; + + out(fseq<0,N>(),fseq<0,N>()) = block_aa; + out(fseq<0,N>(),fseq()) = block_ab; + out(fseq(),fseq<0,N>()) = block_ba; + out(fseq(),fseq()) = block_bb; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 32UL; // start size + constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor block_bb; + inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); + + Tensor inva_b = matmul(inv_a, b); + Tensor bb_c_inva = matmul(block_bb, c_inva); + + Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); + Tensor block_ab = -matmul(inva_b, block_bb); + Tensor block_ba = -bb_c_inva; + + out(fseq<0,N>(),fseq<0,N>()) = block_aa; + out(fseq<0,N>(),fseq()) = block_ab; + out(fseq(),fseq<0,N>()) = block_ba; + out(fseq(),fseq()) = block_bb; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 64UL; // start size + constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor block_bb; + inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); + + Tensor inva_b = matmul(inv_a, b); + Tensor bb_c_inva = matmul(block_bb, c_inva); + + Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); + Tensor block_ab = -matmul(inva_b, block_bb); + Tensor block_ba = -bb_c_inva; + + out(fseq<0,N>(),fseq<0,N>()) = block_aa; + out(fseq<0,N>(),fseq()) = block_ab; + out(fseq(),fseq<0,N>()) = block_ba; + out(fseq(),fseq()) = block_bb; +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { + + // constexpr size_t N = 128UL; // start size + constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size + Tensor a = in(fseq<0,N>(),fseq<0,N>()); + Tensor b = in(fseq<0,N>(),fseq()); + Tensor c = in(fseq(),fseq<0,N>()); + Tensor d = in(fseq(),fseq()); + + Tensor inv_a; + inverse_dispatcher(a, inv_a); + + Tensor c_inva = matmul(c, inv_a); + + Tensor block_bb; + inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); + + Tensor inva_b = matmul(inv_a, b); + Tensor bb_c_inva = matmul(block_bb, c_inva); + + Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); + Tensor block_ab = -matmul(inva_b, block_bb); + Tensor block_ba = -bb_c_inva; + + out(fseq<0,N>(),fseq<0,N>()) = block_aa; + out(fseq<0,N>(),fseq()) = block_ab; + out(fseq(),fseq<0,N>()) = block_ba; + out(fseq(),fseq()) = block_bb; +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +} // internal + + + +// Inversion using LU decomposition is in the LU module [unary_lu_op] +// For tensors +// SimpleInv - no pivot +template && InvType == InvCompType::SimpleInv,bool> = false> +FASTOR_INLINE Tensor inverse(const Tensor &a) { + Tensor out; + _inverse(a.data(),out.data()); + return out; +} +template && InvType == InvCompType::SimpleInv,bool> = false> +FASTOR_INLINE Tensor inverse(const Tensor &in) { + Tensor out; + internal::inverse_dispatcher(in,out); + return out; +} +// SimpleInv pivot +template = false> +FASTOR_INLINE Tensor inverse(const Tensor &in) { + Tensor out; + // We need to post multiply - swap columns using row permutation vector + Tensor P; + pivot_inplace(in,P); + auto A(apply_pivot(in,P)); + internal::inverse_dispatcher(A,out); + return reconstruct_colwise(out,P); + + // // matrix version + // Tensor P; + // pivot_inplace(in,P); + // auto A(apply_pivot(in,P)); + // internal::inverse_dispatcher(A,out); + // return matmul(out,P); +} + +// For high order tensors +template=3 && InvType == InvCompType::SimpleInv,bool> = false > +FASTOR_INLINE Tensor +inverse(const Tensor &a) { + + constexpr size_t remaining_product = last_matrix_extracter, + typename std_ext::make_index_sequence::type>::remaining_product; + + constexpr size_t I = get_value::value; + constexpr size_t J = get_value::value; + static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); + + Tensor out; + T *a_data = a.data(); + T *out_data = out.data(); + + for (size_t i=0; i(static_cast(a_data+i*J*J)); + _inverse(a_data+i*J*J,out_data+i*J*J); + } + + return out; +} + +// Inverse for generic expressions is provided here +template +FASTOR_INLINE +typename Derived::result_type +inverse(const AbstractTensor &src) { + // If we are here Derived is already an expression + using result_type = typename Derived::result_type; + const result_type tmp(src.self()); + return inverse(tmp); +} + + +// Inverse of lower uni-triangular matrices +template,bool> = false> +FASTOR_INLINE Tensor tinverse(const Tensor &in) { + Tensor out; + internal::lut_inverse_dispatcher(in,out); + return out; +} + +// Inverse of lower uni-triangular matrices +template,bool> = false> +FASTOR_INLINE Tensor tinverse(const Tensor &in) { + Tensor out; + internal::ut_inverse_dispatcher(in,out); + return out; +} +// Inverse for generic expressions of lower uni-triangular matrices is provided here +template +FASTOR_INLINE +typename Derived::result_type +tinverse(const AbstractTensor &src) { + // If we are here Derived is already an expression + using result_type = typename Derived::result_type; + const result_type tmp(src.self()); + return tinverse(tmp); +} + + + + + + +// assignments +template +FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryInvOp &src) { + using result_type = typename Expr::result_type; + const result_type& tmp = evaluate(src.expr().self()); + internal::inverse_dispatcher(tmp,dst.self()); +} + +template +FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryInvOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::inverse_dispatcher(tmp,tmp_inv); + trivial_assign_add(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryInvOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::inverse_dispatcher(tmp,tmp_inv); + trivial_assign_sub(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryInvOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::inverse_dispatcher(tmp,tmp_inv); + trivial_assign_mul(dst.self(),tmp_inv); +} + +template +FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryInvOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + result_type tmp_inv; + internal::inverse_dispatcher(tmp,tmp_inv); + trivial_assign_div(dst.self(),tmp_inv); +} + + +} // end of namespace Fastor + + +#endif // UNARY_INV_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_lu_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_lu_op.h new file mode 100644 index 00000000..d34dc502 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_lu_op.h @@ -0,0 +1,970 @@ +#ifndef UNARY_LU_OP_H +#define UNARY_LU_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/inner.h" +#include "Fastor/backend/lufact.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" +#include "Fastor/expressions/linalg_ops/unary_piv_op.h" + + +namespace Fastor { + +namespace internal { + +/* Compile time recursive loop with inner for forward substitution of b/B given the lower unitriangular matrix L. + The following meta functions implements L * y = b for single or multiple right sides +*/ +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +template +struct forward_subs_impl { + + template + static FASTOR_INLINE void do_single_rhs(const Tensor &L, const Tensor &b, Tensor &y) { + y(from) = b(from) - _inner(&L.data()[from*M],y.data()); + forward_subs_impl::do_single_rhs(L, b, y); + } + template + static FASTOR_INLINE void do_single_rhs_pivot(const Tensor &L, const Tensor &b, const Tensor &p, Tensor &y) { + y(from) = b(p(from)) - _inner(&L.data()[from*M],y.data()); + forward_subs_impl::do_single_rhs_pivot(L, b, p, y); + } + + template + static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &L, const Tensor &B, Tensor &y, Tensor &X) { + y(from) = B(from,j) - _inner(&L.data()[from*M],y.data()); + X(from,j) = y(from); + forward_subs_impl::do_multi_rhs(j, L, B, y, X); + } + + template + static FASTOR_INLINE void do_multi_rhs_pivot(const size_t j, const Tensor &L, const Tensor &B, + const Tensor &p, Tensor &y, Tensor &X) { + y(from) = B(p(from),j) - _inner(&L.data()[from*M],y.data()); + X(from,j) = y(from); + forward_subs_impl::do_multi_rhs_pivot(j, L, B, p, y, X); + } +}; +template +struct forward_subs_impl { + + template + static FASTOR_INLINE void do_single_rhs(const Tensor &L, const Tensor &b, Tensor &y) { + y(from) = b(from) - _inner(&L.data()[from*M],y.data()); + } + template + static FASTOR_INLINE void do_single_rhs_pivot(const Tensor &L, const Tensor &b, const Tensor &p, Tensor &y) { + y(from) = b(p(from)) - _inner(&L.data()[from*M],y.data()); + } + + template + static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &L, const Tensor &B, Tensor &y, Tensor &X) { + y(from) = B(from,j) - _inner(&L.data()[from*M],y.data()); + X(from,j) = y(from); + } + template + static FASTOR_INLINE void do_multi_rhs_pivot(const size_t j, const Tensor &L, const Tensor &B, + const Tensor &p, Tensor &y, Tensor &X) { + y(from) = B(p(from),j) - _inner(&L.data()[from*M],y.data()); + X(from,j) = y(from); + } +}; + +template +FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &b) { + + Tensor y(0); + + forward_subs_impl<0,M-1>::do_single_rhs(L, b, y); +#if 0 + // The run-time loop version + // Solve for L * y = b + for (size_t i=0; i< M; ++i) { + T value = 0; + for (size_t k=0; k +FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &B) { + + // We keep a separate output tensor X from y [y is columns of X] + // to avoid strided access in X for the inner product + Tensor X; + + for (size_t j=0; j < N; ++j) { + Tensor y(0); + forward_subs_impl<0,M-1>::do_multi_rhs(j, L, B, y, X); + } +#if 0 + // The run-time loop version - X needs to be zeroed out for this version + for (size_t j=0; j < N; ++j) { + Tensor y(0); + // Solve for L * y = b + for (size_t i=0; i< M; ++i) { + T value = 0; + for (size_t k=0; k +FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &p, const Tensor &b) { + + Tensor y(0); + + forward_subs_impl<0,M-1>::do_single_rhs_pivot(L, b, p, y); +#if 0 + // The run-time loop version + // Solve for L * y = b + for (size_t i=0; i< M; ++i) { + T value = 0; + for (size_t k=0; k +FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &p, const Tensor &B) { + + // We keep a separate output tensor X from y [y is columns of X] + // to avoid strided access in X for the inner product + Tensor X; + + for (size_t j=0; j < N; ++j) { + Tensor y(0); + forward_subs_impl<0,M-1>::do_multi_rhs_pivot(j, L, B, p, y, X); + } +#if 0 + // The run-time loop version - X needs to be zeroed out for this version + for (size_t j=0; j < N; ++j) { + Tensor y(0); + // Solve for L * y = b + for (size_t i=0; i< M; ++i) { + T value = 0; + for (size_t k=0; k +struct backward_subs_impl { + + template + static FASTOR_INLINE void do_single_rhs(const Tensor &U, const Tensor &y, Tensor &x) { + constexpr int idx = (int)M - from; + const T value = _inner(&U.data()[from*M+from],&x.data()[from]); + x(from) = ( y(from) - value) / U(from, from); + backward_subs_impl::do_single_rhs(U, y, x); + } + + template + static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &U, const Tensor &Y, Tensor &x, Tensor &X) { + constexpr int idx = (int)M - from; + const T value = _inner(&U.data()[from*M+from],&x.data()[from]); + x(from) = ( Y(from,j) - value ) / U(from, from); + X(from,j) = x(from); + backward_subs_impl::do_multi_rhs(j, U, Y, x, X); + } +}; + +template <> +struct backward_subs_impl<0,0> { + + template + static FASTOR_INLINE void do_single_rhs(const Tensor &U, const Tensor &y, Tensor &x) { + constexpr int idx = (int)M; + const T value = _inner(&U.data()[0*M+0],&x.data()[0]); + x(0) = ( y(0) - value) / U(0, 0); + } + + template + static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &U, const Tensor &Y, Tensor &x, Tensor &X) { + constexpr int idx = (int)M; + const T value = _inner(&U.data()[0*M+0],&x.data()[0]); + x(0) = ( Y(0,j) - value ) / U(0, 0); + X(0,j) = x(0); + } +}; + +template +FASTOR_INLINE Tensor backward_subs(const Tensor &U, const Tensor &y) { + + // We keep a separate output tensor X from x [x is columns of X] + // to avoid strided access in X for the inner product + Tensor x(0); + backward_subs_impl::do_single_rhs(U, y, x); +#if 0 + // The run-time loop version + // Solve for of U * x = y + for (int i= int(M) - 1; i>=0; --i) { + T value = 0; + for (int k=i; k +FASTOR_INLINE Tensor backward_subs(const Tensor &U, const Tensor &Y) { + + // We keep a separate output tensor X from x [x is columns of X] + // to avoid strided access in X for the inner product + Tensor X; + + for (size_t j=0; j < N; ++j) { + Tensor x(0); + backward_subs_impl::do_multi_rhs(j, U, Y, x, X); + } +#if 0 + // The run-time loop version - X needs to be zeroed out for this version + Tensor X(0); + for (size_t j=0; j < 1; ++j) { + // Solve for of U * x = y + for (int i= int(M) - 1; i>=0; --i) { + T value = 0; + for (int k=i; k && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lu_simple_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + _lufact(A.data(),L.data(),U.data()); +} + +template ,bool> = false> +FASTOR_INLINE void lu_simple_dispatcher(const Tensor& A1, Tensor& L, Tensor& U) { + L.fill(0); + U.fill(0); + for (size_t j = 0; j < M; ++j) { + L(j, j) = 1; + for (size_t i = 0; i <= j; ++i) { + T value = A1(i, j); + for (size_t k = 0; k < i; ++k) { + value -= L(i, k) * U(k, j); + } + U(i, j) = value; + } + for (size_t i = j; i < M; ++i) { + T value = A1(i, j); + for (size_t k = 0; k < j; ++k) { + value -= L(i, k) * U(k, j); + } + value /= U(j, j); + L(i, j) = value; + } + } +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + +// Recursive non-modifying LU using matmul/outer product +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +/* This in essence implements the following loop but with static views + as dynamic views cannot be dispatched to matmul or a tensor cannot be + constructed to be sent to matmul + + for(size_t j=0; j +struct recursive_lu_impl { + template + static FASTOR_INLINE void Do(Tensor &L, Tensor &U) { + L(fseq(),fix) = U(fseq(),fix) / U.data()[from*N+from]; + U(fseq(),fix) = 0; + U(fseq(),fseq()) -= matmul(L(fseq(),fix), U(fix,fseq())); + recursive_lu_impl::Do(L, U); + } +}; +template +struct recursive_lu_impl { + template + static FASTOR_INLINE void Do(Tensor &L, Tensor &U) { + L(fseq(),fix) = U(fseq(),fix) / U.data()[from*N+from]; + U(fseq(),fix) = 0; + U(fseq(),fseq()) -= matmul(L(fseq(),fix), U(fix,fseq())); + } +}; + +template ,bool> = false> +FASTOR_INLINE void recursive_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + L(0,0) = 1; U(0,0) = A(0,0); +} +template ,bool> = false> +FASTOR_INLINE void recursive_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + // Requires M >=2 + U = A; + L.eye2(); + recursive_lu_impl<0,M-2>::Do(L, U); +} + + +#if 0 +// Recursive in-place LU using matmul/outer product +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +/* This in essence implements the following loop but with static views + as dynamic views cannot be dispatched to matmul or a tensor cannot be + constructed to be sent to matmul + + for(size_t j=0; j +struct recursive_lu_impl_inplace { + template + static FASTOR_INLINE void Do(Tensor &LU) { + LU(fseq(),fix) /= LU.data()[from*N+from]; + LU(fseq(),fseq()) -= matmul(LU(fseq(),fix), LU(fix,fseq())); + recursive_lu_impl_inplace::Do(LU); + } +}; +template +struct recursive_lu_impl_inplace { + template + static FASTOR_INLINE void Do(Tensor &LU) { + LU(fseq(),fix) /= LU.data()[from*N+from]; + LU(fseq(),fseq()) -= matmul(LU(fseq(),fix), LU(fix,fseq())); + } +}; + + +template ,bool> = false> +FASTOR_INLINE void recursive_inplace_lu_dispatcher(Tensor& A) { + return; +} +template ,bool> = false> +FASTOR_INLINE void recursive_inplace_lu_dispatcher(Tensor& A) { + // Requires M >=2 + recursive_lu_impl_inplace<0,M-2>::Do(A); +} + +template ,bool> = false> +FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& LU) { + LU(0,0) = A(0,0); +} +template ,bool> = false> +FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& LU) { + // Requires M >=2 + LU =A; + recursive_lu_impl_inplace<0,M-2>::Do(LU); +} + +template ,bool> = false> +FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + L(0,0) = 1; U(0,0) = A(0,0); +} +template ,bool> = false> +FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + // Requires M >=2 + Tensor LU(A); + recursive_lu_impl_inplace<0,M-2>::Do(LU); + + // Extracting L and U after this recursive decomposition is done + // almost beats the purpose performance wise + for (size_t i=0; i && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + _lufact(A.data(),L.data(),U.data()); +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + recursive_lu_dispatcher(A, L, U); +} + +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + + // We will compute LU decomposition block-wise assuming that A11 and A22 are invertible + // + // [A11 A12] [L11 0] [U11 U12] + // [A21 A22] [L21 L22] [0 U22] + // + // This results in + // + // A11 = L11 * U11 + // A12 = L11 * U12 + // A21 = L21 * U11 + // A22 = L21 * U12 - L22 * U22 + // + // Hence we need to do LU factorisation once for A11 and once for A22 + + // This is to avoid odd sizes for instance for size 35 we would + // want to do 35 = 16 + 19 rather than 35 = 32 + 3 if the start size was 32 + constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size + Tensor A11 = A(fseq<0,N>(),fseq<0,N>()); + Tensor A12 = A(fseq<0,N>(),fseq()); + Tensor A21 = A(fseq(),fseq<0,N>()); + Tensor A22 = A(fseq(),fseq()); + + Tensor L11(0), U11(0); + lu_block_dispatcher(A11, L11, U11); + + // Solve for U12 = {L11}^(-1)*A12 + Tensor U12 = tmatmul(tinverse(L11),A12); + // Solve for L21 = A21*{U11}^(-1) + Tensor L21 = tmatmul(A21,tinverse(U11)); + + Tensor S = A22 - matmul(L21,U12); + + Tensor L22(0), U22(0); + lu_block_dispatcher(S, L22, U22); + + L(fseq<0,N>(),fseq<0,N>()) = L11; + // L(fseq<0,N>(),fseq()) = 0; + L(fseq(),fseq<0,N>()) = L21; + L(fseq(),fseq()) = L22; + + U(fseq<0,N>(),fseq<0,N>()) = U11; + U(fseq<0,N>(),fseq()) = U12; + // U(fseq(),fseq<0,N>()) = 0; + U(fseq(),fseq()) = U22; +} + +// Conditional dispatch +namespace useless { +template && is_less_equal_v_,bool> = false> +FASTOR_INLINE void lu_block_simple_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + lu_block_dispatcher(A, L, U); + return; +} +template ,bool> = false> +FASTOR_INLINE void lu_block_simple_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + // lu_simple_dispatcher(A, L, U); + recursive_lu_dispatcher(A, L, U); + return; +} +} // useless + +/* For sizes greater than 64 we tile differently to avoid too many recursions +*/ +template ,bool> = false> +FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { + + // We will compute LU decomposition block-wise assuming that A11 and A22 are invertible + // + // [A11 A12] [L11 0] [U11 U12] + // [A21 A22] [L21 L22] [0 U22] + // + // This results in + // + // A11 = L11 * U11 + // A12 = L11 * U12 + // A21 = L21 * U11 + // A22 = L21 * U12 - L22 * U22 + // + // Hence we need to do LU factorisation once for A11 and once for A22 + + // This is to avoid odd sizes for instance for size 65 we would + // want to do 65 = 32 + 33 rather than 65 = 64 + 1 if the start size was 64 + constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size + Tensor A11 = A(fseq<0,N>(),fseq<0,N>()); + Tensor A12 = A(fseq<0,N>(),fseq()); + Tensor A21 = A(fseq(),fseq<0,N>()); + Tensor A22 = A(fseq(),fseq()); + + Tensor L11(0), U11(0); + useless::lu_block_simple_dispatcher(A11, L11, U11); + + // Solve for U12 = {L11}^(-1)*A12 + Tensor U12 = tmatmul(tinverse(L11),A12); + // Ideally use forward_subs but its iterative nature makes it less efficient than tmatmul + // Tensor U12 = forward_subs(L11,A12); + // Solve for L21 = A21*{U11}^(-1) + Tensor L21 = tmatmul(A21,tinverse(U11)); + // Not quite performant as we can't avoid the matmul here + // Tensor I; I.eye2(); + // Tensor L21 = matmul(A21, backward_subs(U11, I)); + + Tensor S = A22 - matmul(L21,U12); + + Tensor L22(0), U22(0); + useless::lu_block_simple_dispatcher(S, L22, U22); + + L(fseq<0,N>(),fseq<0,N>()) = L11; + // L(fseq<0,N>(),fseq()) = 0; + L(fseq(),fseq<0,N>()) = L21; + L(fseq(),fseq()) = L22; + + U(fseq<0,N>(),fseq<0,N>()) = U11; + U(fseq<0,N>(),fseq()) = U12; + // U(fseq(),fseq<0,N>()) = 0; + U(fseq(),fseq()) = U22; +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +} // internal + + +/* Block LU factorisation overloads */ +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +// BlockLU - no pivot +template && LUType == LUCompType::BlockLU,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U) { + L.fill(0); + U.fill(0); + internal::lu_block_dispatcher(src.self(),L,U); +} +template && LUType == LUCompType::BlockLU,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U) { + L.fill(0); + U.fill(0); + typename Expr::result_type tmp(src.self()); + internal::lu_block_dispatcher(tmp,L,U); +} + +// BlockLU - vector pivot +template && LUType == LUCompType::BlockLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + L.fill(0); + U.fill(0); + pivot_inplace(src.self(),P); + auto A(apply_pivot(src.self(),P)); + internal::lu_block_dispatcher(A,L,U); +} +template && LUType == LUCompType::BlockLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + L.fill(0); + U.fill(0); + typename Expr::result_type A(src.self()); + pivot_inplace(A,P); + // Modify A as A is a temporary anyway + apply_pivot_inplace(A,P); + internal::lu_block_dispatcher(A,L,U); +} + +// BlockLU - matrix pivot +template && LUType == LUCompType::BlockLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + L.fill(0); + U.fill(0); + pivot_inplace(src.self(),P); + auto A(apply_pivot(src.self(),P)); + internal::lu_block_dispatcher(A,L,U); +} +template && LUType == LUCompType::BlockLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + L.fill(0); + U.fill(0); + typename Expr::result_type A(src.self()); + pivot_inplace(A,P); + // Modify A as A is a temporary anyway + apply_pivot_inplace(A,P); + internal::lu_block_dispatcher(A,L,U); +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + + +/* Simple LU factorisation overloads */ +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +// SimpleLU - no pivot +template && LUType == LUCompType::SimpleLU,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U) { + internal::lu_simple_dispatcher(src.self(),L,U); +} +template && LUType == LUCompType::SimpleLU,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U) { + typename Expr::result_type tmp(src.self()); + internal::lu_simple_dispatcher(tmp,L,U); +} + +// SimpleLU - vector pivot +template && LUType == LUCompType::SimpleLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + pivot_inplace(src.self(),P); + auto A(apply_pivot(src.self(),P)); + internal::lu_simple_dispatcher(A,L,U); +} +template && LUType == LUCompType::SimpleLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + typename Expr::result_type A(src.self()); + pivot_inplace(A,P); + // Modify A as A is a temporary anyway + apply_pivot_inplace(A,P); + internal::lu_simple_dispatcher(A,L,U); +} + +// SimpleLU - matrix pivot +template && LUType == LUCompType::SimpleLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + pivot_inplace(src.self(),P); + auto A(apply_pivot(src.self(),P)); + internal::lu_simple_dispatcher(A,L,U); +} +template && LUType == LUCompType::SimpleLUPiv,bool> = false> +FASTOR_INLINE +void +lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { + typename Expr::result_type A(src.self()); + pivot_inplace(A,P); + // Modify A as A is a temporary anyway + apply_pivot_inplace(A,P); + internal::lu_simple_dispatcher(A,L,U); +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + + + + + + + +// Inversion using LU +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +namespace internal { + +template +FASTOR_INLINE Tensor get_lu_inverse(const Tensor &L, const Tensor &U) { + // We will solve for multiple RHS [B = I] + // Loop over columns of B = I + Tensor I; I.eye2(); + Tensor Y = forward_subs(L, I); + Tensor X = backward_subs(U, Y); + return X; +} + +template +FASTOR_INLINE Tensor get_lu_inverse(Tensor &L, const Tensor &U, const Tensor &p) { + // We will solve for multiple RHS [B = I] + // Loop over columns of B = I + Tensor I; I.eye2(); + Tensor Y = forward_subs(L, p, I); + Tensor X = backward_subs(U, Y); + return X; +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +} // internal + + +// SimpleLU +template = false> +FASTOR_INLINE Tensor inverse(const Tensor &A) { + Tensor L, U; + lu(A, L, U); + return internal::get_lu_inverse(L, U); +} + +// BlockLU +template = false> +FASTOR_INLINE Tensor inverse(const Tensor &A) { + Tensor L, U; + lu(A, L, U); + return internal::get_lu_inverse(L, U); +} + +// SimpleLUPiv +template = false> +FASTOR_INLINE Tensor inverse(const Tensor &A) { + Tensor L, U; + Tensor p; + lu(A, L, U, p); + return internal::get_lu_inverse(L, U, p); +} + +// BlockLUPiv +template = false> +FASTOR_INLINE Tensor inverse(const Tensor &A) { + Tensor L, U; + Tensor p; + lu(A, L, U, p); + return internal::get_lu_inverse(L, U, p); +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + + + + + + + + +// Solving linear system of equations using LU +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +namespace internal { + +template +FASTOR_INLINE Tensor get_lu_solve(const Tensor &L, const Tensor &U, const Tensor &b) { + Tensor y = forward_subs(L, b); + Tensor x = backward_subs(U, y); + return x; +} + +template +FASTOR_INLINE Tensor get_lu_solve(Tensor &L, const Tensor &U, const Tensor &p, const Tensor &b) { + Tensor y = forward_subs(L, p, b); + Tensor x = backward_subs(U, y); + return x; +} + +// Multiple RHS +template +FASTOR_INLINE Tensor get_lu_solve(const Tensor &L, const Tensor &U, const Tensor &B) { + Tensor Y = forward_subs(L, B); + Tensor X = backward_subs(U, Y); + return X; +} + +template +FASTOR_INLINE Tensor get_lu_solve(const Tensor &L, const Tensor &U, const Tensor &p, const Tensor &B) { + Tensor Y = forward_subs(L, p, B); + Tensor X = backward_subs(U, Y); + return X; +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +} // internal + + +// Single RHS +// SimpleLU - no pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { + Tensor L, U; + lu(A, L, U); + return internal::get_lu_solve(L, U, b); +} + +// SimpleLU - pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { + Tensor L, U; + Tensor p; + lu(A, L, U, p); + return internal::get_lu_solve(L, U, p, b); +} + +// BlockLU - no pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { + Tensor L, U; + lu(A, L, U); + return internal::get_lu_solve(L, U, b); +} + +// BlockLU - pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { + Tensor L, U; + Tensor p; + lu(A, L, U, p); + return internal::get_lu_solve(L, U, p, b); +} + +// Multiple RHS +// SimpleLU - no pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { + Tensor L, U; + lu(A, L, U); + return internal::get_lu_solve(L, U, B); +} + +// SimpleLU - pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { + Tensor L, U; + Tensor p; + lu(A, L, U, p); + return internal::get_lu_solve(L, U, p, B); +} + +// SimpleLU - no pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { + Tensor L, U; + lu(A, L, U); + return internal::get_lu_solve(L, U, B); +} + +// SimpleLU - pivot +template = false> +FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { + Tensor L, U; + Tensor p; + lu(A, L, U, p); + return internal::get_lu_solve(L, U, p, B); +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + + + + + + + + + +// Computing determinant using LU +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +template = false> +FASTOR_INLINE T determinant(const Tensor &A) { + int nswaps = internal::count_swaps(A) % 2UL == 0 ? 1 : -1; + Tensor L, U; + Tensor p; + lu(A, L, U, p); + return product(diag(U)) * nswaps; +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + +} // end of namespace Fastor + + +#endif // UNARY_LU_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_norm_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_norm_op.h new file mode 100644 index 00000000..bcd83f71 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_norm_op.h @@ -0,0 +1,106 @@ +#ifndef UNARY_NORM_OP_H +#define UNARY_NORM_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/backend/norm.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +namespace Fastor { + +// For tensors +template,bool> = false> +FASTOR_INLINE T norm(const T &a) { + return std::abs(a); +} +template +FASTOR_INLINE T norm(const Tensor &a) { + if (sizeof...(Rest) == 0) + return *a.data(); + return _norm::value>(a.data()); +} + +// For generic expressions +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type norm(const AbstractTensor &_src) { + + const Derived &src = _src.self(); + using T = typename Derived::scalar_type; + using V = choose_best_simd_vector_t; + T _scal=0; +#ifdef FASTOR_AVX512_IMPL + V omm0, omm1, omm2, omm3, omm4, omm5, omm6, omm7; +#else + V omm0, omm1, omm2, omm3; +#endif + FASTOR_INDEX i = 0; + + // With AVX utilises all the 16 registers but hurts the performance + // due to spill if eval has created temporary registers so only + // activated for AVX512 +#ifdef FASTOR_AVX512_IMPL + for (; i < ROUND_DOWN(src.size(),8*V::Size); i+=8*V::Size) { + const auto smm0 = src.template eval(i); + const auto smm1 = src.template eval(i+V::Size); + const auto smm2 = src.template eval(i+2*V::Size); + const auto smm3 = src.template eval(i+3*V::Size); + const auto smm4 = src.template eval(i+4*V::Size); + const auto smm5 = src.template eval(i+5*V::Size); + const auto smm6 = src.template eval(i+6*V::Size); + const auto smm7 = src.template eval(i+7*V::Size); + omm0 = fmadd(smm0,smm0,omm0); + omm1 = fmadd(smm1,smm1,omm1); + omm2 = fmadd(smm2,smm2,omm2); + omm3 = fmadd(smm3,smm3,omm3); + omm4 = fmadd(smm4,smm4,omm4); + omm5 = fmadd(smm5,smm5,omm5); + omm6 = fmadd(smm6,smm6,omm6); + omm7 = fmadd(smm7,smm7,omm7); + } +#endif + for (; i < ROUND_DOWN(src.size(),4*V::Size); i+=4*V::Size) { + const auto smm0 = src.template eval(i); + const auto smm1 = src.template eval(i+V::Size); + const auto smm2 = src.template eval(i+2*V::Size); + const auto smm3 = src.template eval(i+3*V::Size); + omm0 = fmadd(smm0,smm0,omm0); + omm1 = fmadd(smm1,smm1,omm1); + omm2 = fmadd(smm2,smm2,omm2); + omm3 = fmadd(smm3,smm3,omm3); + } + for (; i < ROUND_DOWN(src.size(),2*V::Size); i+=2*V::Size) { + const auto smm0 = src.template eval(i); + const auto smm1 = src.template eval(i+V::Size); + omm0 = fmadd(smm0,smm0,omm0); + omm1 = fmadd(smm1,smm1,omm1); + } + for (; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { + const auto smm0 = src.template eval(i); + omm0 = fmadd(smm0,smm0,omm0); + } + for (; i < src.size(); ++i) { + const auto smm0 = src.template eval_s(i); + _scal += smm0*smm0; + } +#ifdef FASTOR_AVX512_IMPL + return sqrts( (omm0 + omm1 + omm2 + omm3 + omm4 + omm5 + omm6 + omm7).sum() + _scal); +#else + return sqrts( (omm0 + omm1 + omm2 + omm3).sum() + _scal); +#endif +} + + +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type norm(const AbstractTensor &_src) { + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out(src); + return norm(out); +} + +} // end of namespace Fastor + +#endif // UNARY_NORM_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_piv_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_piv_op.h new file mode 100644 index 00000000..42c5b12b --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_piv_op.h @@ -0,0 +1,409 @@ +#ifndef UNARY_PIV_OP_H +#define UNARY_PIV_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" + +#include + + +namespace Fastor { + +namespace internal { + +template +FASTOR_INLINE size_t count_swaps(const Tensor& A) { + size_t count = 0; + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A(i, j)) > std::abs(A(max_index, j))) + max_index = i; + } + if (j != max_index) + count++; + } + return count; +} + +} + + +template = false> +FASTOR_INLINE Tensor pivot(const Tensor& A) { + Tensor perm; + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A(i, j)) > std::abs(A(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } + Tensor P(0); + for (size_t i = 0; i < M; ++i) + P(i, perm(i)) = 1; + return P; +} + +template = false> +FASTOR_INLINE Tensor pivot(const Tensor& A) { + Tensor perm; + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A(i, j)) > std::abs(A(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } + return perm; +} + + +// For generic expressions +template, bool> = false> +FASTOR_INLINE +Tensor, + get_tensor_dimension_v<1,typename Derived::result_type> +> +pivot(const AbstractTensor& src) { + typename Derived::result_type tmp(src.self()); + return pivot(tmp); +} +template, bool> = false> +FASTOR_INLINE +Tensor, + get_tensor_dimension_v<1,typename Derived::result_type> +> +pivot(const AbstractTensor& src) { + + static_assert(DIM==2, "TENSOR MUST BE SQUARE FOR PIVOT COMPUTATION"); + + using T = typename Derived::scalar_type; + using result_type = typename Derived::result_type; + constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; + constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; + + const Derived &A = src.self(); + + Tensor perm; + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } + Tensor P(0); + for (size_t i = 0; i < M; ++i) + P(i, perm(i)) = 1; + return P; +} + +template, bool> = false> +FASTOR_INLINE +Tensor +> +pivot(const AbstractTensor& src) { + typename Derived::result_type tmp(src.self()); + return pivot(tmp); +} +template, bool> = false> +FASTOR_INLINE +Tensor +> +pivot(const AbstractTensor& src) { + + static_assert(DIM==2, "TENSOR MUST BE SQUARE FOR PIVOT COMPUTATION"); + + using T = typename Derived::scalar_type; + using result_type = typename Derived::result_type; + constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; + constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; + + const Derived &A = src.self(); + + Tensor perm; + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } + return perm; +} + +/* In place versions - Given an evaluated pivot tensor populates it +*/ +template +FASTOR_INLINE void pivot_inplace(const Tensor& A, Tensor& perm) { + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A(i, j)) > std::abs(A(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } +} + +template +FASTOR_INLINE void pivot_inplace(const Tensor& A, Tensor &P) { + Tensor perm; + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A(i, j)) > std::abs(A(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } + P.fill(0); + for (size_t i = 0; i < M; ++i) + P(i, perm(i)) = 1; +} + +template, bool> = false> +FASTOR_INLINE +void +pivot_inplace(const AbstractTensor& src, Tensor &perm) { + typename Derived::result_type tmp(src.self()); + pivot_inplace(tmp,perm); +} +template, bool> = false> +FASTOR_INLINE +void +pivot_inplace(const AbstractTensor& src, Tensor &perm) { + + static_assert(DIM==2, "TENSOR MUST BE 2D FOR PIVOT COMPUTATION"); + + using T = typename Derived::scalar_type; + + const Derived &A = src.self(); + + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } +} + +template, bool> = false> +FASTOR_INLINE +void +pivot_inplace(const AbstractTensor& src, Tensor &P) { + typename Derived::result_type tmp(src.self()); + pivot_inplace(tmp,P); +} +template, bool> = false> +FASTOR_INLINE +void +pivot_inplace(const AbstractTensor& src, Tensor &P) { + + static_assert(DIM==2, "TENSOR MUST BE 2D FOR PIVOT COMPUTATION"); + + const Derived &A = src.self(); + + Tensor perm; + perm.iota(); + for (size_t j = 0; j < M; ++j) { + size_t max_index = j; + for (size_t i = j; i < M; ++i) { + // std::abs is necessary only for complex valued numbers + if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) + max_index = i; + } + if (j != max_index) + std::swap(perm(j), perm(max_index)); + } + for (size_t i = 0; i < M; ++i) + P(i, perm(i)) = 1; +} + + + + +/* Apply a pivot on a tensor/matrix + Applying pivot can only work on evaluated tensors and not expression + as non-evaluated expression do not have storage/data +*/ +template +FASTOR_INLINE Tensor apply_pivot(const Tensor& A, const Tensor& P) { + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + if (P(i) != i) { + std::copy_n(&A.data()[P(i)*N],N,©A.data()[i*N]); + } + } + return copyA; +} +template +FASTOR_INLINE Tensor apply_pivot(const Tensor& A, const Tensor& P) { + // The output tensor is just matmul(P,A), but we are going to avoid + // the call to matmul + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1)); + size_t p = std::distance(&P.data()[i*N],it); + // Swap row p and i + if (p != i) { + std::copy_n(&A.data()[p*N],N,©A.data()[i*N]); + } + } + return copyA; +} + +template +FASTOR_INLINE void apply_pivot_inplace(Tensor& A, const Tensor& P) { + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + if (P(i) != i) { + std::copy_n(©A.data()[P(i)*N],N,&A.data()[i*N]); + } + } +} +template +FASTOR_INLINE void apply_pivot_inplace(Tensor& A, const Tensor& P) { + // The output tensor is just matmul(P,A), but we are going to avoid + // the call to matmul + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1)); + size_t p = std::distance(&P.data()[i*N],it); + // Swap row p and i + if (p != i) { + std::copy_n(©A.data()[p*N],N,&A.data()[i*N]); + } + } +} + + + + +/* Reconstructing the matrix back from a pivoted factorisation + Reconstructs from LU/QR/QL etc +*/ +template +FASTOR_INLINE Tensor reconstruct(const Tensor& L, Tensor& U) { + return matmul(L,U); +} + +/* Reconstructing the matrix back from a pivoted factorisation + Reconstructs from an PLU decomposition given P, L, U where P is + integral permutation vector + A = {P}^(-1)*L*U +*/ +template +FASTOR_INLINE Tensor reconstruct(const Tensor& L, Tensor& U, const Tensor& P) { + Tensor A = matmul(L,U); + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + if (P(i) != i) { + std::copy_n(©A.data()[i*N],N,&A.data()[P(i)*N]); + } + } + return A; +} + +/* Reconstructing the matrix back from a pivoted factorisation + Reconstructs from an PLU decomposition given P, L, U + A = {P}^(-1)*L*U +*/ +template +FASTOR_INLINE Tensor reconstruct(const Tensor& L, Tensor& U, const Tensor& P) { + // To avoid computing the inverse of P in {P}^(-1)*L*U + Tensor A = matmul(L,U); + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1)); + size_t p = std::distance(&P.data()[i*N],it); + if (p != i) { + std::copy_n(©A.data()[i*N],N,&A.data()[p*N]); + } + } + return A; +} + + +/* Reconstructing the matrix back from a pivot only +*/ +template +FASTOR_INLINE Tensor reconstruct(const Tensor& A, const Tensor& P) { + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + if (P(i) != i) { + std::copy_n(&A.data()[i*N],N,©A.data()[P(i)*N]); + } + } + return copyA; +} + +/* Reconstructing the matrix back from a pivot only - column-wise or rather post multiplication + used for computing inverse +*/ +template +FASTOR_INLINE Tensor reconstruct_colwise(const Tensor& A, const Tensor& P) { + Tensor copyA(A); + for (size_t i=0; i< M; ++i) { + if (P(i) != i) { + copyA(all,P(i)) = A(all,i); + } + } + return copyA; +} + + + +} // end of namespace Fastor + + +#endif // UNARY_PIV_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_qr_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_qr_op.h new file mode 100644 index 00000000..f83a90fd --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_qr_op.h @@ -0,0 +1,198 @@ +#ifndef UNARY_QR_OP_H +#define UNARY_QR_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" +#include "Fastor/expressions/linalg_ops/unary_piv_op.h" + + +namespace Fastor { + +namespace internal { + +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +/* Modified Gram-Schmidt Row-wise [MGSR] QR factorisation + that provides numerical stabilitiy. + This simple implementation provided for now is just for + convenience and not tuned for performance. Given that Tensor + types have fixed dimensions the compiler typically does a good + job + + See: + Thomas Jakobs et. al. "Performance and energy consumption of the SIMD + Gram–Schmidt process for vector orthogonalization" + for implementation details including AVX/AVX512 implementation. A straight- + forward explicit vectorisation of this does not yield good performance for + row-major tensors, as step 1-2 need gather instructions and step 3-4 require + dynamic masking to gain performance +*/ +template +FASTOR_INLINE void qr_mgsr_dispatcher(const Tensor &A0, Tensor& Q, Tensor& R) { + + // Copy incoming tensor as we will modify A + Tensor A(A0); + // Zero out + R.fill(0); + + for (size_t i=0; i< N; ++i) { + // step 1 + T R_ii = 0; + for (size_t k=0; k< M; ++k) { + R_ii += A(k,i)*A(k,i); + } + R_ii = sqrts(R_ii); + R(i,i) = R_ii; + + // step 2 + for (size_t k=0; k< M; ++k) { + Q(k,i) = A(k,i) / R_ii; + } + + // step 3 + for (size_t k=0; k< M; ++k) { + for (size_t j=i+1; j && QRType == QRCompType::MGSR,bool> = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R) { + internal::qr_mgsr_dispatcher(src.self(),Q,R); +} + +template && QRType == QRCompType::MGSR,bool> = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R) { + Tensor A(src.self()); + internal::qr_mgsr_dispatcher(A,Q,R); +} + +// QR MGSR - vector pivot +template && QRType == QRCompType::MGSRPiv,bool> = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { + pivot_inplace(src.self(),P); + auto A(apply_pivot(src.self(),P)); + internal::qr_mgsr_dispatcher(A,Q,R); +} + +template && QRType == QRCompType::MGSRPiv,bool> = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { + typename Expr::result_type A(src.self()); + pivot_inplace(A,P); + // Modify A as A is a temporary anyway + apply_pivot_inplace(A,P); + internal::qr_mgsr_dispatcher(A,Q,R); +} + +// QR MGSR - matrix pivot +template && QRType == QRCompType::MGSRPiv,bool> = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { + pivot_inplace(src.self(),P); + auto A(apply_pivot(src.self(),P)); + internal::qr_mgsr_dispatcher(A,Q,R); +} + +template && QRType == QRCompType::MGSRPiv,bool> = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { + typename Expr::result_type A(src.self()); + pivot_inplace(A,P); + // Modify A as A is a temporary anyway + apply_pivot_inplace(A,P); + internal::qr_mgsr_dispatcher(A,Q,R); +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +// QR householder reflections etc +template = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R) { + static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET"); +} +template = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { + static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET"); +} +template = false> +FASTOR_INLINE +void +qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { + static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET"); +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + + + + +// Computing determinant using QR +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// +template = false> +FASTOR_INLINE T determinant(const Tensor &a) { + Tensor Q, R; + qr(a, Q, R); + return product(diag(R)); +} +//-----------------------------------------------------------------------------------------------------------// +//-----------------------------------------------------------------------------------------------------------// + + + +} // end of namespace Fastor + + +#endif // UNARY_QR_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_svd_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_svd_op.h new file mode 100644 index 00000000..ce1ba59d --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_svd_op.h @@ -0,0 +1,163 @@ +#ifndef UNARY_SVD_OP_H +#define UNARY_SVD_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/inner.h" +#include "Fastor/backend/lufact.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" +#include "Fastor/expressions/linalg_ops/unary_det_op.h" +#include "Fastor/expressions/linalg_ops/unary_trans_op.h" + + +namespace Fastor { + +// SVD +template = false > +FASTOR_INLINE void svd(const Tensor &A, Tensor &U, Tensor &S, Tensor &V) { + + constexpr T Epsilon_v = std::numeric_limits::epsilon(); + + const T f00 = A(0, 0); + const T f01 = A(0, 1); + const T f10 = A(1, 0); + const T f11 = A(1, 1); + + // If matrix is diagonal, SVD is trivial + if (std::abs(f01 - f10) < Epsilon_v && std::abs(f01) < Epsilon_v) + { + // Compute U + U(0,0) = f00 < 0 ? -1. : 1.; + U(0,1) = 0.; + U(1,0) = 0.; + U(1,1) = f11 < 0. ? -1. : 1.; + + // Compute S + S(0,0) = std::abs(f00); + S(0,1) = 0; + S(1,0) = 0; + S(1,1) = std::abs(f11); + + // Compute V + V.eye2(); + } + // Otherwise, we need to compute A^T*A + else + { + T j = f00 * f00 + f01 * f01; + T k = f10 * f10 + f11 * f11; + T v_c = f00 * f10 + f01 * f11; + // Check to see if A^T*A is diagonal + if (std::abs(v_c) < Epsilon_v) + { + // Compute S + T s1 = std::sqrt(j); + T s2 = std::abs(j - k) < Epsilon_v ? s1 : std::sqrt(k); + S(0,0) = s1; + S(0,1) = 0; + S(1,0) = 0; + S(1,1) = s2; + + // Compute U + U.eye2(); + + // Compute V + V(0,0) = f00 / s1; + V(0,1) = f10 / s2; + V(1,0) = f01 / s1; + V(1,1) = f11 / s2; + } + // Otherwise, solve quadratic equation for eigenvalues + else + { + T jmk = j - k; + T jpk = j + k; + T root = std::sqrt(jmk * jmk + 4. * v_c * v_c); + T eig1 = (jpk + root) * 0.5; + T eig2 = (jpk - root) * 0.5; + + // Compute S + T s1 = std::sqrt(eig1); + T s2 = std::abs(root) < Epsilon_v ? s1 : ( eig2 > 0 ? std::sqrt(eig2) : Epsilon_v); + S(0,0) = s1; + S(0,1) = 0; + S(1,0) = 0; + S(1,1) = s2; + + // Compute U - use eigenvectors of A^T*A as U + T v_s = eig1 - j; + T len = std::max(std::sqrt(v_s * v_s + v_c * v_c), Epsilon_v); + v_c /= len; + v_s /= len; + + U(0,0) = v_c; + U(0,1) = -v_s; + U(1,0) = v_s; + U(1,1) = v_c; + + // Compute V - as A * U / s + const T cc = (f00 * v_c + f10 * v_s) / s1; + const T cs = (f01 * v_c + f11 * v_s) / s1; + if (std::abs(s2) > Epsilon_v) + { + V(0,0) = cc; + V(0,1) = (f10* v_c - f00 * v_s) / s2; + V(1,0) = cs; + V(1,1) = (f11 * v_c - f01 * v_s) / s2; + } + else + { + V(0,0) = cc; + V(0,1) = cs; + V(1,0) = cs; + V(1,1) = -cc; + } + } + } +} + + + +// Signed SVD +template +FASTOR_INLINE void ssvd(const Tensor &A, Tensor &U, Tensor &S, Tensor &V) { + + // Same as above but avoiding the L matrix + svd(A, U, S, V); + + // See where to pull the reflection out of + const T detU = determinant(U); + const T detV = determinant(V); + + if (detU >= 0 && detV >= 0) + { + // No reflection svd == svd_rv, return + return; + } + + Tensor L = matmul(U, transpose(V)); + const T lastColumn = determinant(L); + + if (detU < 0 && detV > 0) + { + U(all, M - 1) *= lastColumn; + } + else if (detU > 0 && detV < 0) + { + V(all, M - 1) *= lastColumn; + } + + // Push the reflection to the diagonal + S(M - 1, M - 1) *= lastColumn; +} +//-----------------------------------------------------------------------------------------------------------// + +} // end of namespace Fastor + + +#endif // UNARY_SVD_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_trace_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_trace_op.h new file mode 100644 index 00000000..95a62781 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_trace_op.h @@ -0,0 +1,75 @@ +#ifndef UNARY_TRACE_OP_H +#define UNARY_TRACE_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +namespace Fastor { + +// For tensors +template +FASTOR_INLINE T trace(const Tensor &a) { + return _trace(static_cast(a.data())); +} + +// For high order tensors +template=3,bool>::type=0> +FASTOR_INLINE +typename last_matrix_extracter, typename std_ext::make_index_sequence::type>::type +trace(const Tensor &a) { + + using OutTensor = typename last_matrix_extracter, + typename std_ext::make_index_sequence::type>::type; + constexpr size_t remaining_product = last_matrix_extracter, + typename std_ext::make_index_sequence::type>::remaining_product; + + constexpr size_t I = get_value::value; + constexpr size_t J = get_value::value; + static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); + + OutTensor out; + T *a_data = a.data(); + T *out_data = out.data(); + + for (size_t i=0; i(static_cast(a_data+i*J*J)); + } + + return out; +} + +// Trace for generic expressions +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type trace(const AbstractTensor &_src) { + + const Derived& src = _src.self(); + using T = typename Derived::scalar_type; + using tensor_type = typename Derived::result_type; + constexpr size_t M = get_tensor_dimensions::dims[0]; + constexpr size_t N = get_tensor_dimensions::dims[1]; + static_assert(DIMS==2,"TENSOR EXPRESSION SHOULD BE UNIFORM (SQUARE)"); + static_assert(M==N,"TENSOR EXPRESSION SHOULD BE TWO DIMENSIONAL"); + + FASTOR_INDEX i; + T _scal=0; + for (i = 0; i < M; ++i) { + _scal += src.template eval_s(i*(N+1)); + } + return _scal; +} + +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type trace(const AbstractTensor &_src) { + + using result_type = typename Derived::result_type; + const result_type out(_src.self()); + return trace(out); +} + +} // end of namespace Fastor + +#endif // UNARY_TRACE_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_trans_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_trans_op.h new file mode 100644 index 00000000..20858785 --- /dev/null +++ b/noarch/include/Fastor/expressions/linalg_ops/unary_trans_op.h @@ -0,0 +1,173 @@ +#ifndef UNARY_TRANS_OP_H +#define UNARY_TRANS_OP_H + +#include "Fastor/meta/meta.h" +#include "Fastor/backend/transpose/transpose.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Aliasing.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/TensorTraits.h" +#include "Fastor/expressions/expression_traits.h" + + +namespace Fastor { + +template +struct UnaryTransOp: public AbstractTensor,DIM0> { + using expr_type = expression_t; + static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,typename Expr::result_type>; + static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,typename Expr::result_type>; + static constexpr FASTOR_INDEX Dimension = DIM0; + static constexpr FASTOR_INDEX rank() {return DIM0;} + using scalar_type = typename scalar_type_finder>::type; + using simd_vector_type = typename Expr::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + + FASTOR_INLINE UnaryTransOp(expr_type inexpr) : _expr(inexpr) { + } + + FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i == 0 ? N : M;} + + constexpr FASTOR_INLINE expr_type expr() const {return _expr;} + +private: + expr_type _expr; +}; + + +/* Tensor transpose returning a tensor expression */ +template +FASTOR_INLINE UnaryTransOp +trans(const AbstractTensor &src) { + return UnaryTransOp(src.self()); +} + + +/* Tensor transpose immediately returning a tensor */ +template +FASTOR_INLINE Tensor transpose(const Tensor &a) { + Tensor out; + _transpose(a.data(),out.data()); + return out; +} + +/* Tensor transpose for higher order tensors immediately returning a tensor */ +template=3,bool>::type=0> +FASTOR_INLINE Tensor +transpose(const Tensor &a) { + + constexpr size_t remaining_product = last_matrix_extracter, + typename std_ext::make_index_sequence::type>::remaining_product; + + constexpr size_t I = get_value::value; + constexpr size_t J = get_value::value; + static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); + + Tensor out; + T *a_data = a.data(); + T *out_data = out.data(); + + for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); + } + + return out; +} + +// Transpose for generic expressions +template +FASTOR_INLINE +Tensor< + typename scalar_type_finder::type, + get_tensor_dimension_v<1,typename Expr::result_type>, + get_tensor_dimension_v<0,typename Expr::result_type>> +transpose(const AbstractTensor &src) { + // If we are here Expr is already an expression + using result_type = typename Expr::result_type; + const result_type tmp(src.self()); + return transpose(tmp); +} + + + + +// assignments +template +FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryTransOp &src) { + using result_type = typename Expr::result_type; + const result_type& tmp = evaluate(src.expr().self()); + using T = typename UnaryTransOp::scalar_type; + static constexpr size_t M = UnaryTransOp::M; + static constexpr size_t N = UnaryTransOp::N; + _transpose(tmp.data(),dst.self().data()); +} + +template +FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryTransOp::scalar_type; + using result_t = typename UnaryTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryTransOp::M; + static constexpr size_t N = UnaryTransOp::N; + _transpose(tmp.data(),tmp_trans.data()); + trivial_assign_add(dst.self(),tmp_trans); +} + +template +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryTransOp::scalar_type; + using result_t = typename UnaryTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryTransOp::M; + static constexpr size_t N = UnaryTransOp::N; + _transpose(tmp.data(),tmp_trans.data()); + trivial_assign_sub(dst.self(),tmp_trans); +} + +template +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryTransOp::scalar_type; + using result_t = typename UnaryTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryTransOp::M; + static constexpr size_t N = UnaryTransOp::N; + _transpose(tmp.data(),tmp_trans.data()); + trivial_assign_mul(dst.self(),tmp_trans); +} + +template +FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryTransOp &src) { + using result_type = typename Expr::result_type; + // no copies if expr is a tensor + const result_type& tmp = evaluate(src.expr().self()); + // one copy for the inverse + using T = typename UnaryTransOp::scalar_type; + using result_t = typename UnaryTransOp::result_type; + result_t tmp_trans; + static constexpr size_t M = UnaryTransOp::M; + static constexpr size_t N = UnaryTransOp::N; + _transpose(tmp.data(),tmp_trans.data()); + trivial_assign_div(dst.self(),tmp_trans); +} + + +} // end of namespace Fastor + + +#endif // UNARY_TRANS_OP_H diff --git a/noarch/include/Fastor/expressions/unary_ops/unary_bool_ops.h b/noarch/include/Fastor/expressions/unary_ops/unary_bool_ops.h new file mode 100644 index 00000000..0ce51021 --- /dev/null +++ b/noarch/include/Fastor/expressions/unary_ops/unary_bool_ops.h @@ -0,0 +1,101 @@ +#ifndef UNARY_BOOL_OP_H +#define UNARY_BOOL_OP_H + + +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/simd_math/simd_math.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" +#include "Fastor/expressions/expression_traits.h" + +namespace Fastor { + +// All unary bool ops +#define FASTOR_MAKE_UNARY_BOOL_OPS(OP_NAME, SIMD_OP, SCALAR_OP, STRUCT_NAME, EVAL_TYPE)\ +template\ +struct Unary ##STRUCT_NAME ## Op: public AbstractTensor,DIM0> {\ +private:\ + expression_t _expr;\ +public:\ + using scalar_type = typename Expr::scalar_type;\ + using simd_vector_type = typename Expr::simd_vector_type;\ + using simd_abi_type = typename simd_vector_type::abi_type;\ + using result_type = to_bool_tensor_t;\ + using result_scalar_type = typename result_type::scalar_type;\ + using result_simd_abi_type = typename result_type::simd_abi_type;\ + using result_simd_vector_type = typename result_type::simd_vector_type;\ + using UU = bool /*this needs to change to U once masks are implemented*/;\ + using ABI = simd_abi::fixed_size::Size>;\ + static constexpr FASTOR_INDEX Dimension = DIM0;\ + static constexpr FASTOR_INDEX rank() {return DIM0;}\ + FASTOR_INLINE FASTOR_INDEX size() const {return _expr.size();}\ + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _expr.dimension(i);}\ + Unary ##STRUCT_NAME ## Op(expression_t inexpr) : _expr(inexpr) {}\ + FASTOR_INLINE expression_t expr() const {return _expr;}\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ + return SIMD_OP(_expr.template eval(i));\ + }\ + template\ + FASTOR_INLINE UU eval_s(FASTOR_INDEX i) const {\ + return SCALAR_OP(_expr.template eval_s(i));\ + }\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return SIMD_OP(_expr.template eval(i,j));\ + }\ + template\ + FASTOR_INLINE UU eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return SCALAR_OP(_expr.template eval_s(i,j));\ + }\ + template\ + FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ + return SIMD_OP(_expr.template teval(as));\ + }\ + template\ + FASTOR_INLINE UU teval_s(const std::array &as) const {\ + return SCALAR_OP(_expr.template teval_s(as));\ + }\ +};\ +template::value,bool>::type = 0 >\ +FASTOR_INLINE Unary ##STRUCT_NAME ## Op OP_NAME(const AbstractTensor &_expr) {\ + return Unary ##STRUCT_NAME ## Op(_expr.self());\ +}\ + +FASTOR_MAKE_UNARY_BOOL_OPS(operator!, ! , ! , Not , scalar_type) +FASTOR_MAKE_UNARY_BOOL_OPS(isinf , isinf , std::isinf , Isinf , scalar_type) +FASTOR_MAKE_UNARY_BOOL_OPS(isnan , isnan , std::isnan , Isnan , scalar_type) +FASTOR_MAKE_UNARY_BOOL_OPS(isfinite, isfinite, std::isfinite, Isfinite, scalar_type) + + + + +#define FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\ +template,bool>::type = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ + using result_type = typename OtherDerived::result_type;\ + const result_type tmp(src.expr().self());\ + trivial_assign ##ASSIGN_TYPE (dst.self(), OP(tmp));\ +}\ +template,bool>::type = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ + +// only assignment +FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(! , Not , ) +FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isinf , Isinf , ) +FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isnan , Isnan , ) +FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isfinite, Isfinite, ) + + + + +} // end of namespace Fastor + + +#endif // UNARY_BOOL_OP_H + diff --git a/noarch/include/Fastor/expressions/unary_ops/unary_math_ops.h b/noarch/include/Fastor/expressions/unary_ops/unary_math_ops.h new file mode 100644 index 00000000..98af613c --- /dev/null +++ b/noarch/include/Fastor/expressions/unary_ops/unary_math_ops.h @@ -0,0 +1,209 @@ +#ifndef UNARY_MATH_OP_H +#define UNARY_MATH_OP_H + + +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/simd_math/simd_math.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" +#include "Fastor/expressions/expression_traits.h" + +namespace Fastor { + +// All unary math ops +#define FASTOR_MAKE_UNARY_MATH_OPS(OP_NAME, SIMD_OP, SCALAR_OP, STRUCT_NAME, EVAL_TYPE)\ +template\ +struct Unary ##STRUCT_NAME ## Op: public AbstractTensor,DIM0> {\ +private:\ + expression_t _expr;\ +public:\ + using scalar_type = typename Expr::scalar_type;\ + using simd_vector_type = typename Expr::simd_vector_type;\ + using simd_abi_type = typename simd_vector_type::abi_type;\ + using result_type = typename Expr::result_type;\ + static constexpr FASTOR_INDEX Dimension = DIM0;\ + static constexpr FASTOR_INDEX rank() {return DIM0;}\ + FASTOR_INLINE FASTOR_INDEX size() const {return _expr.size();}\ + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _expr.dimension(i);}\ + Unary ##STRUCT_NAME ## Op(expression_t inexpr) : _expr(inexpr) {}\ + FASTOR_INLINE expression_t expr() const {return _expr;}\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ + return SIMD_OP(_expr.template eval(i));\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\ + return SCALAR_OP(_expr.template eval_s(i));\ + }\ + template\ + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return SIMD_OP(_expr.template eval(i,j));\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ + return SCALAR_OP(_expr.template eval_s(i,j));\ + }\ + template\ + FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ + return SIMD_OP(_expr.template teval(as));\ + }\ + template\ + FASTOR_INLINE EVAL_TYPE teval_s(const std::array &as) const {\ + return SCALAR_OP(_expr.template teval_s(as));\ + }\ +};\ +template::value,bool>::type = 0 >\ +FASTOR_INLINE Unary ##STRUCT_NAME ## Op OP_NAME(const AbstractTensor &_expr) {\ + return Unary ##STRUCT_NAME ## Op(_expr.self());\ +}\ + + +FASTOR_MAKE_UNARY_MATH_OPS(operator+, , , Add, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(operator-, -, -, Sub, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(abs, abs, std::abs, Abs, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(sqrt, sqrt, sqrts, Sqrt, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(cbrt, cbrt, std::cbrt, Cbrt, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(exp, exp, std::exp, Exp, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(exp2, exp2, std::exp2, Exp2, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(expm1, expm1, std::expm1, Expm1, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(log, log, std::log, Log, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(log10, log10, std::log10, Log10, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(log2, log2, std::log2, Log2, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(log1p, log1p, std::log1p, Log1p, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(sin, sin, std::sin, Sin, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(cos, cos, std::cos, Cos, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(tan, tan, std::tan, Tan, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(asin, asin, std::asin, Asin, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(acos, acos, std::acos, Acos, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(atan, atan, std::atan, Atan, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(sinh, sinh, std::sinh, Sinh, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(cosh, cosh, std::cosh, Cosh, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(tanh, tanh, std::tanh, Tanh, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(asinh, asinh, std::asinh, Asinh, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(acosh, acosh, std::acosh, Acosh, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(atanh, atanh, std::atanh, Atanh, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(erf, erf, std::erf, Erf, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(tgamma, tgamma, std::tgamma, Tgamma, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(lgamma, lgamma, std::lgamma, Lgamma, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(ceil, ceil, std::ceil, Ceil, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(round, round, std::round, Round, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(floor, floor, std::floor, Floor, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(trunc, trunc, std::trunc, Trunc, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(conj, conj, std::conj, Conj, scalar_type) +FASTOR_MAKE_UNARY_MATH_OPS(arg , arg , std::arg , Arg , scalar_type) + + + + +#define FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\ +template,bool>::type = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ + assign ##ASSIGN_TYPE (dst.self(), src.expr().self());\ + trivial_assign(dst.self(), OP(dst.self()));\ +}\ +template,bool>::type = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ + +// only assignment +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT( , Add, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(-, Sub, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(abs, Abs, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sqrt, Sqrt, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cbrt, Cbrt, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(exp, Exp, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(exp2, Exp2, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(expm1, Expm1, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log, Log, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log10, Log10, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log2, Log2, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log1p, Log1p, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sin, Sin, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cos, Cos, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tan, Tan, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(asin, Asin, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(acos, Acos, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(atan, Atan, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sinh, Sinh, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cosh, Cosh, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tanh, Tanh, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(asinh, Asinh, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(acosh, Acosh, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(atanh, Atanh, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(erf, Erf, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tgamma, Tgamma, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(lgamma, Lgamma, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(ceil, Ceil, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(round, Round, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(floor, Floor, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(trunc, Trunc, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(conj, Conj, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(arg , Arg , ) + + + +// arithmetic assignments +#define FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\ +template,bool>::type = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ + using result_type = typename Unary ##NAME ## Op::result_type;\ + const result_type tmp(src.expr().self());\ + trivial_assign ##ASSIGN_TYPE (dst.self(), OP(tmp));\ +}\ +template,bool>::type = false>\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ + +#define FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT( , Add, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(-, Sub, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(abs, Abs, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sqrt, Sqrt, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cbrt, Cbrt, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(exp, Exp, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(exp2, Exp2, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(expm1, Expm1, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log, Log, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log10, Log10, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log2, Log2, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log1p, Log1p, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sin, Sin, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cos, Cos, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tan, Tan, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(asin, Asin, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(acos, Acos, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(atan, Atan, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sinh, Sinh, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cosh, Cosh, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tanh, Tanh, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(asinh, Asinh, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(acosh, Acosh, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(atanh, Atanh, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(erf, Erf, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tgamma, Tgamma, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(lgamma, Lgamma, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(ceil, Ceil, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(round, Round, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(floor, Floor, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(trunc, Trunc, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(conj, Conj, ASSIGN_TYPE)\ +FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(arg , Arg , ASSIGN_TYPE)\ + +// FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, ) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _add) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _sub) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _mul) +FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _div) + +} // end of namespace Fastor + + +#endif // UNARY_MATH_OP_H + diff --git a/noarch/include/Fastor/expressions/views/tensor_diag_views.h b/noarch/include/Fastor/expressions/views/tensor_diag_views.h new file mode 100644 index 00000000..8b9d8237 --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_diag_views.h @@ -0,0 +1,317 @@ +#ifndef TENSOR_DIAG_VIEWS_H +#define TENSOR_DIAG_VIEWS_H + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/TensorMap.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +namespace Fastor { + +template class TensorType, typename T, size_t M, size_t N, size_t DIM> +struct TensorDiagViewExpr,DIM> : public AbstractTensor,DIM>,DIM> { +private: + TensorType& _expr; + bool _does_alias = false; + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + using V = SIMDVector; + static constexpr FASTOR_INDEX Dimension = 1; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return 1;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} + constexpr const TensorType& expr() const {return _expr;} + + FASTOR_INLINE TensorViewExpr,2>& noalias() { + _does_alias = true; + return *this; + } + + FASTOR_INLINE TensorDiagViewExpr(Tensor &_ex) : _expr(_ex) { + static_assert(M==N, "MATRIX MUST BE SQUARE FOR DIAGONAL VIEW"); + } + + // Copy assignment + //---------------------------------------------------------------------------------// + FASTOR_INLINE void operator=(const TensorDiagViewExpr,DIM>& other_src) { + // Diagonal views don't alias +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + // Get around recursive inlining issue + const result_type tmp(other_src); + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i,i); + } + } + + // AbstractTensor binders + //---------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) = other_src.template eval_s(i); + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) += other_src.template eval_s(i); + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) -= other_src.template eval_s(i); + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) *= other_src.template eval_s(i); + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) /= other_src.template eval_s(i); + } + } + + // Scalar binders + //---------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) = num; + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) += num; + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) -= num; + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) *= num; + } + } + + template && !is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + T inum = T(1) / T(num); + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) *= inum; + } + } + template && is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + for (FASTOR_INDEX i = 0; i < M; ++i) { + _expr(i,i) /= num; + } + } + + // Evaluators + //---------------------------------------------------------------------------------// + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + V _vec; + std::array inds; + for (auto j=0; j + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + return _expr(idx,idx); + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + V _vec; + vector_setter(_vec,_expr.data(),i*N+j); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr(i,j); + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + V _vec; + vector_setter(_vec,_expr.data(),as[0]*N+as[1]); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.eval_s(as); + } +}; + + + +// Actual definition of diag function +// Note that like other views the input can only be an evaluated tensor +template +TensorDiagViewExpr,2> diag(Tensor &a) { + return TensorDiagViewExpr,2>(a); +} + +template +TensorDiagViewExpr,2> diag(TensorMap &a) { + return TensorDiagViewExpr,2>(a); +} + + +} // end of namespace Fastor + + +#endif diff --git a/noarch/include/Fastor/expressions/views/tensor_filter_views.h b/noarch/include/Fastor/expressions/views/tensor_filter_views.h new file mode 100644 index 00000000..900f5661 --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_filter_views.h @@ -0,0 +1,281 @@ +#ifndef TENSOR_FILTER_VIEWS_H +#define TENSOR_FILTER_VIEWS_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + + +namespace Fastor { + +template +struct TensorFilterViewExpr,Tensor,DIMS>: + public AbstractTensor,Tensor,DIMS>,DIMS> { +private: + Tensor &_expr; + const Tensor &fl_expr; + bool _does_alias = false; + + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = DIMS; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return DIMS;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return pack_prod::value;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return fl_expr.dimension(i);} + constexpr const Tensor& expr() const {return _expr;} + + FASTOR_INLINE TensorFilterViewExpr,Tensor,DIMS>& noalias() { + _does_alias = true; + return *this; + } + + constexpr FASTOR_INLINE TensorFilterViewExpr(Tensor &_ex, const Tensor &_it) : _expr(_ex), fl_expr(_it) { + static_assert(sizeof...(Rest)==DIMS, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + } + + // In place operators + //------------------------------------------------------------------------------------// + void operator=(const TensorFilterViewExpr,Tensor,DIMS> &src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + } + } + } + + + template,bool> = false> + void operator=(const AbstractTensor &src) { + const typename Derived::result_type& tmp = evaluate(src.self()); + this->operator=(tmp); + } + template,bool> = false> + void operator=(const AbstractTensor &src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + } + } + } + + template,bool> = false> + void operator+=(const AbstractTensor &src) { + const typename Derived::result_type& tmp = evaluate(src.self()); + this->operator+=(tmp); + } + template,bool> = false> + void operator+=(const AbstractTensor &src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + } + } + } + + template,bool> = false> + void operator-=(const AbstractTensor &src) { + const typename Derived::result_type& tmp = evaluate(src.self()); + this->operator-=(tmp); + } + template,bool> = false> + void operator-=(const AbstractTensor &src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + } + } + } + + template,bool> = false> + void operator*=(const AbstractTensor &src) { + const typename Derived::result_type& tmp = evaluate(src.self()); + this->operator*=(tmp); + } + template,bool> = false> + void operator*=(const AbstractTensor &src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + } + } + } + + template,bool> = false> + void operator/=(const AbstractTensor &src) { + const typename Derived::result_type& tmp = evaluate(src.self()); + this->operator/=(tmp); + } + template,bool> = false> + void operator/=(const AbstractTensor &src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + } + } + } + + //------------------------------------------------------------------------------------// + template,bool> = false> + void operator=(U num) { + T tnum = (T)num; + for (FASTOR_INDEX i = 0; i ,bool> = false> + void operator+=(U num) { + T tnum = (T)num; + for (FASTOR_INDEX i = 0; i ,bool> = false> + void operator-=(U num) { + T tnum = (T)num; + for (FASTOR_INDEX i = 0; i ,bool> = false> + void operator*=(U num) { + T tnum = (T)num; + for (FASTOR_INDEX i = 0; i && !is_integral_v_,bool> = false> + void operator/=(U num) { + T tnum = T(1)/(T)num; + for (FASTOR_INDEX i = 0; i && is_integral_v_,bool> = false> + void operator/=(U num) { + T tnum = (T)num; + for (FASTOR_INDEX i = 0; i + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + constexpr size_t _Stride = SIMDVector::Size; + SIMDVector _vec; + U inds[_Stride]; + for (FASTOR_INDEX j=0; j<_Stride; ++j) + inds[j] = fl_expr.eval_s(i+j) ? _expr.eval_s(i+j) : 0; + _vec.load(inds,false); + return _vec; + } + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return fl_expr.eval_s(i) ? _expr.eval_s(i) : 0; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { + constexpr size_t _Stride = SIMDVector::Size; + SIMDVector _vec; + U inds[_Stride]; + for (FASTOR_INDEX j=0; j<_Stride; ++j) + inds[j] = fl_expr.eval_s(i,k+j) ? _expr.eval_s(i,k+j) : 0; + _vec.load(inds,false); + return _vec; + } + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX k) const { + return fl_expr.eval_s(i,k) ? _expr.eval_s(i,k) : 0; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + constexpr size_t _Stride = SIMDVector::Size; + SIMDVector _vec; + U inds[_Stride]; + for (FASTOR_INDEX j=0; j<_Stride; ++j) + inds[j] = fl_expr.teval_s(as) ? _expr.teval_s(as) : 0; + _vec.load(inds,false); + return _vec; + } + + template + FASTOR_INLINE U teval_s(const std::array& as) const { + return fl_expr.teval_s(as) ? _expr.teval_s(as) : 0; + } + //------------------------------------------------------------------------------------// + +}; + +} // end of namespace Fastor + + +#endif // TENSOR_FILTER_VIEWS_H diff --git a/noarch/include/Fastor/expressions/views/tensor_fixed_views_1d.h b/noarch/include/Fastor/expressions/views/tensor_fixed_views_1d.h new file mode 100644 index 00000000..f606772d --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_fixed_views_1d.h @@ -0,0 +1,681 @@ +#ifndef TENSOR_FIXED_VIEWS_1D_H +#define TENSOR_FIXED_VIEWS_1D_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + + +namespace Fastor { + + +// 1D const fixed views +//----------------------------------------------------------------------------------------------// +template +struct TensorConstFixedViewExpr1D,fseq,1> : + public AbstractTensor,fseq,1>,1> { +private: + const Tensor &_expr; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor::value>; + static constexpr FASTOR_INDEX Dimension = 1; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INLINE FASTOR_INDEX size() { return range_detector::value;} + static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) {return range_detector::value;} + constexpr const Tensor& expr() const {return _expr;} + + constexpr FASTOR_INLINE TensorConstFixedViewExpr1D(const Tensor &_ex) : _expr(_ex) {} + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),S0*i+F0,S0); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr.data()[S0*i+F0]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),S0*(i+j)+F0,S0); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[S0*(i+j)+F0]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),S0*as[0]+F0,S0); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[S0*as[0]+F0]; + } +}; + + + + + +// 1D non-const fixed views +//----------------------------------------------------------------------------------------------// +template +struct TensorFixedViewExpr1D,fseq,1> : + public AbstractTensor,fseq,1>,1> { +private: + Tensor &_expr; + bool _does_alias = false; + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor::value>; + static constexpr FASTOR_INDEX Dimension = 1; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return range_detector::value;} + static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) {return range_detector::value;} + constexpr const Tensor& expr() const {return _expr;} + + FASTOR_INLINE TensorFixedViewExpr1D,fseq,1>& noalias() { + _does_alias = true; + return *this; + } + + constexpr FASTOR_INLINE TensorFixedViewExpr1D(Tensor &_ex) : _expr(_ex) {} + + + //----------------------------------------------------------------------------------// + FASTOR_HINT_INLINE void operator=(const TensorFixedViewExpr1D,fseq,1> &other_src) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); + // Assign other to temporary + tmp = other_src; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + _vec_other.store(&_data[i+F0],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[S0*(i+j) + F0] = _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + //----------------------------------------------------------------------------------// + + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i (i); + } + return; + } + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i (i); + _vec_other.store(&_data[i+F0],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[S0*(i+j) + F0] = _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) + other_src.template eval(i); + _vec.store(&_data[i+F0],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[S0*(i+j) + F0] += _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) - other_src.template eval(i); + _vec.store(&_data[i+F0],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[S0*(i+j) + F0] -= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) * other_src.template eval(i); + _vec.store(&_data[i+F0],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[S0*(i+j) + F0] *= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) / other_src.template eval(i); + _vec.store(&_data[i+F0],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[S0*(i+j) + F0] /= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + //----------------------------------------------------------------------------------// + + + // Scalar binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _data[S0*(i+j) + F0] = _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) + _vec_other; + _vec.store(&_data[i+F0],false); + } + for (; i ::Size; ++j) { + _data[S0*(i+j) + F0] += _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) - _vec_other; + _vec.store(&_data[i+F0],false); + } + for (; i ::Size; ++j) { + _data[S0*(i+j) + F0] -= _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) * _vec_other; + _vec.store(&_data[i+F0],false); + } + for (; i ::Size; ++j) { + _data[S0*(i+j) + F0] *= _vec_other[j]; + } + } + for (; i && !is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + T inum = T(1) / static_cast(num); + simd_vector_type _vec_other(inum); + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) * _vec_other; + _vec.store(&_data[i+F0],false); + } + for (; i ::Size; ++j) { + _data[S0*(i+j) + F0] *= _vec_other[j]; + } + } + for (; i && is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR (S0 == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) / _vec_other; + _vec.store(&_data[i+F0],false); + } + for (; i ::Size; ++j) { + _data[S0*(i+j) + F0] /= _vec_other[j]; + } + } + for (; i + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),S0*i+F0,S0); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr.data()[S0*i+F0]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),S0*(i+j)+F0,S0); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[S0*(i+j)+F0]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),S0*as[0]+F0,S0); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[S0*as[0]+F0]; + } +}; + + +} + + +#endif // TENSOR_FIXED_VIEWS_1D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_fixed_views_2d.h b/noarch/include/Fastor/expressions/views/tensor_fixed_views_2d.h new file mode 100644 index 00000000..d56a6adc --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_fixed_views_2d.h @@ -0,0 +1,1121 @@ +#ifndef TENSOR_FIXED_VIEWS_2D_H +#define TENSOR_FIXED_VIEWS_2D_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +namespace Fastor { + + +// 2D const fixed views +//----------------------------------------------------------------------------------------------// +template +struct TensorConstFixedViewExpr2D,fseq,fseq,2> : + public AbstractTensor,fseq,fseq,2>,2> { +private: + const Tensor &_expr; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor::value, range_detector::value>; + static constexpr FASTOR_INDEX Dimension = 2; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INLINE FASTOR_INDEX size() { + return range_detector::value*range_detector::value; + } + static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) { + return i==0 ? range_detector::value : range_detector::value; + } + constexpr const Tensor& expr() const {return _expr;} + static constexpr FASTOR_INDEX Padding = F0*N+F1; + + constexpr FASTOR_INLINE TensorConstFixedViewExpr2D(const Tensor &_ex) : _expr(_ex) {} + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) { + auto it = (idx+j) / range_detector::value, jt = (idx+j) % range_detector::value; + inds[j] = S0*it*N+S1*jt + Padding; + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + auto it = idx / range_detector::value, jt = idx % range_detector::value; + auto ind = S0*it*N+S1*jt + Padding; + return _expr.data()[ind]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*i*N+j + Padding, false); + else vector_setter(_vec,_expr.data(),S0*i*N+S1*j + Padding,S1); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[S0*i*N+S1*j + Padding]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*as[0]*N+as[1] + Padding, false); + else vector_setter(_vec,_expr.data(),S0*as[0]*N+S1*as[1] + Padding,S1); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[S0*as[0]*N+S1*as[1] + Padding]; + } +}; + + + + + +// 2D non-const fixed views +//----------------------------------------------------------------------------------------------// +template +struct TensorFixedViewExpr2D,fseq,fseq,2> : + public AbstractTensor,fseq,fseq,2>,2> { + +private: + Tensor &_expr; + bool _does_alias = false; + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor::value, range_detector::value>; + static constexpr FASTOR_INDEX Dimension = 2; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INLINE FASTOR_INDEX size() { + return range_detector::value*range_detector::value; + } + static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) { + return i==0 ? range_detector::value : range_detector::value; + } + constexpr const Tensor& expr() const {return _expr;} + static constexpr FASTOR_INDEX Padding = F0*N+F1; + + FASTOR_INLINE TensorFixedViewExpr2D,fseq,fseq,2>& noalias() { + _does_alias = true; + return *this; + } + + constexpr FASTOR_INLINE TensorFixedViewExpr2D(Tensor &_ex) : _expr(_ex) {} + + //----------------------------------------------------------------------------------// + FASTOR_HINT_INLINE void operator=(const TensorFixedViewExpr2D,fseq,fseq,2> &other_src) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other_src; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i,j); + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j (i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i (i,j); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j (i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(i,j); + } + } +#endif + } + } + + // AbstractTensor binders - [equal order] + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i) { + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(i,j); + } + } + return; + } + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i (i,j); + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j (i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i (i,j); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j (i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) + other_src.template eval(i,j); + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j (i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) + other_src.template eval(i,j); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j (i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) += other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) - other_src.template eval(i,j); + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j (i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) - other_src.template eval(i,j); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j (i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) -= other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) * other_src.template eval(i,j); + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j (i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) * other_src.template eval(i,j); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j (i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) *= other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) / other_src.template eval(i,j); + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j (i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) / other_src.template eval(i,j); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j (i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) /= other_src.template eval_s(i,j); + } + } +#endif + } + } + //----------------------------------------------------------------------------------// + + + // AbstractTensor binders for other nth rank tensors + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(counter); + counter++; + } + } + return; + } + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i (counter); + _vec.store(&_data[S0*i*N+j+Padding],false); + counter += Stride; + } + for (; j (counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i (counter); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + counter += Stride; + } + for (; j (counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) + other_src.template eval(counter); + _vec.store(&_data[S0*i*N+j+Padding],false); + counter += Stride; + } + for (; j (counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) + other_src.template eval(counter); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + counter += Stride; + } + for (; j (counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) += other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) - other_src.template eval(counter); + _vec.store(&_data[S0*i*N+j+Padding],false); + counter += Stride; + } + for (; j (counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) - other_src.template eval(counter); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + counter += Stride; + } + for (; j (counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) -= other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) * other_src.template eval(counter); + _vec.store(&_data[S0*i*N+j+Padding],false); + counter += Stride; + } + for (; j (counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) * other_src.template eval(counter); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + counter += Stride; + } + for (; j (counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) *= other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#ifndef FASTOR_NO_ALIAS + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) / other_src.template eval(counter); + _vec.store(&_data[S0*i*N+j+Padding],false); + counter += Stride; + } + for (; j (counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i template eval(i,j) / other_src.template eval(counter); + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + counter += Stride; + } + for (; j (counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) /= other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + //----------------------------------------------------------------------------------// + + + // Scalar binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { + T *FASTOR_RESTRICT _data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) = num; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { + T *FASTOR_RESTRICT _data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) + _vec_other; + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j template eval(i,j) + _vec_other; + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) += num; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { + T *FASTOR_RESTRICT _data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) - _vec_other; + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j template eval(i,j) - _vec_other; + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) -= num; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { + T *FASTOR_RESTRICT _data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) * _vec_other; + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j template eval(i,j) * _vec_other; + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) *= num; + } + } +#endif + } + } + + template && !is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + T *FASTOR_RESTRICT _data = _expr.data(); + T inum = T(1)/num; + SIMDVector _vec_other(static_cast(inum)); + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) * _vec_other; + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j template eval(i,j) * _vec_other; + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) *= inum; + } + } +#endif + } + } + template && is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + T *FASTOR_RESTRICT _data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + FASTOR_IF_CONSTEXPR (S1==1) { + for (FASTOR_INDEX i = 0; i template eval(i,j) / _vec_other; + _vec.store(&_data[S0*i*N+j+Padding],false); + } + for (; j template eval(i,j) / _vec_other; + data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); + } + for (; j ::value; i++) { + for (FASTOR_INDEX j = 0; j ::value; j++) { + _expr(S0*i+F0,S1*j+F1) /= num; + } + } +#endif + } + } + //----------------------------------------------------------------------------------// + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) { + auto it = (idx+j) / range_detector::value, jt = (idx+j) % range_detector::value; + inds[j] = S0*it*N+S1*jt + Padding; + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + auto it = idx / range_detector::value, jt = idx % range_detector::value; + auto ind = S0*it*N+S1*jt + Padding; + return _expr.data()[ind]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*i*N+j + Padding, false); + else vector_setter(_vec,_expr.data(),S0*i*N+S1*j + Padding,S1); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[S0*i*N+S1*j + Padding]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*as[0]*N+as[1] + Padding, false); + else vector_setter(_vec,_expr.data(),S0*as[0]*N+S1*as[1] + Padding,S1); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[S0*as[0]*N+S1*as[1] + Padding]; + } +}; + + +} + + +#endif // TENSOR_FIXED_VIEWS_2D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_fixed_views_nd.h b/noarch/include/Fastor/expressions/views/tensor_fixed_views_nd.h new file mode 100644 index 00000000..e40cbe27 --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_fixed_views_nd.h @@ -0,0 +1,1645 @@ +#ifndef TENSOR_FIXED_VIEWS_ND_H +#define TENSOR_FIXED_VIEWS_ND_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + + +namespace Fastor { + + +// Generic const fixed tensor views based on sequences/slices +//----------------------------------------------------------------------------------------------// +template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> +struct TensorConstFixedViewExprnD,Fseqs...> : + public AbstractTensor,Fseqs...>,sizeof...(Fseqs)> { + +private: + const TensorType &_expr; + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; + static constexpr std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; +public: + using scalar_type = T; + using simd_vector_type = typename TensorType::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using dimension_helper = get_fixed_sequence_pack_dimensions< + TensorType, + typename std_ext::make_index_sequence::type, + Fseqs...>; + using result_type = typename dimension_helper::type; + using V = simd_vector_type; + // Get nth to_positive type + template using get_nth_pt = get_nth_type...>; + static constexpr FASTOR_INDEX Dimension = sizeof...(Fseqs); + static constexpr FASTOR_INDEX DIMS = sizeof...(Fseqs); + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + constexpr FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} + constexpr FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return dimension_helper::Size;} + static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) {return dimension_helper::dims[i];} + constexpr const TensorType& expr() const {return _expr;} +private: + static constexpr std::array _dims = dimension_helper::dims; + static constexpr bool _is_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step==1) ? true : false; + static constexpr bool _is_strided_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step!=1) ? true : false; +public: + + constexpr FASTOR_INLINE TensorConstFixedViewExprnD(const TensorType &_ex) : _expr(_ex) {} + //----------------------------------------------------------------------------------------------// + + // Evals + //----------------------------------------------------------------------------------------------// + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + + SIMDVector _vec; + std::array as = {}; + std::array inds; + for (auto j=0; j::Do(j,inds, as); + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + + std::array as = {}; + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + return _expr.eval_s(ind); + } + + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { + + idx += j; + V _vec; + std::array as = {}; + + std::array inds; + for (auto j=0; j::Do(j, inds, as); + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { + idx += j; + std::array as = {}; + + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + + return _expr.eval_s(ind); + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + FASTOR_IF_CONSTEXPR (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); + else FASTOR_IF_CONSTEXPR (_is_strided_vectorisable) { + V _vec; + vector_setter(_vec,_expr.data(),ind,get_nth_pt::_step); + return _vec; + } + else { + + V _vec; + std::array inds = {}; + std::array as_ = as; + for (auto j=0; j::Do(j, inds, as); + + for(int jt = (int)DIMS-1; jt>=0; jt--) + { + as_[jt] +=1; + if(as_[jt]<_dims[jt]) + break; + else + as_[jt]=0; + } + } + + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + } + + template + FASTOR_INLINE U teval_s(const std::array& as) const { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + return _expr.eval_s(ind); + } + //----------------------------------------------------------------------------------------------// + + + + //----------------------------------------------------------------------------------------------// +private: + template + struct get_index_v { + template + static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { + inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + get_index_v::Do(j, inds, as); + } + }; + template + struct get_index_v { + template + static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { + inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + } + }; + + template + struct get_index_s { + template + static FASTOR_INLINE void Do(int &ind, const std::array& as) { + ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + get_index_s::Do(ind, as); + } + }; + template + struct get_index_s { + template + static FASTOR_INLINE void Do(int &ind, const std::array& as) { + ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + } + }; + //----------------------------------------------------------------------------------------------// +}; + + +template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> +constexpr std::array TensorConstFixedViewExprnD,Fseqs...>::products_; + +template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> +constexpr std::array TensorConstFixedViewExprnD,Fseqs...>::_dims; + + + + + + + +//----------------------------------------------------------------------------------------------// +// Generic non-const fixed tensor views based on sequences/slices +//----------------------------------------------------------------------------------------------// +template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> +struct TensorFixedViewExprnD,Fseqs...> : + public AbstractTensor,Fseqs...>,sizeof...(Fseqs)> { + +private: + TensorType &_expr; + bool _does_alias = false; + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; + static constexpr std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; +public: + using scalar_type = T; + using simd_vector_type = typename TensorType::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using dimension_helper = get_fixed_sequence_pack_dimensions< + TensorType, + typename std_ext::make_index_sequence::type, + Fseqs...>; + using result_type = typename dimension_helper::type; + using V = simd_vector_type; + // Get nth to_positive type + template using get_nth_pt = get_nth_type...>; + static constexpr FASTOR_INDEX Dimension = sizeof...(Fseqs); + static constexpr FASTOR_INDEX DIMS = sizeof...(Fseqs); + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + constexpr FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} + constexpr FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return dimension_helper::Size;} + static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) {return dimension_helper::dims[i];} + constexpr const TensorType& expr() const {return _expr;} +private: + static constexpr std::array _dims = dimension_helper::dims; + static constexpr bool _is_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step==1) ? true : false; + static constexpr bool _is_strided_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step!=1) ? true : false; +public: + + FASTOR_INLINE TensorFixedViewExprnD,Fseqs...>& noalias() { + _does_alias = true; + return *this; + } + + constexpr FASTOR_INLINE TensorFixedViewExprnD(TensorType &_ex) : _expr(_ex) {} + //----------------------------------------------------------------------------------------------// + + // View evalution operators + // Copy assignment operators [Needed in addition to generic AbstractTensor overload] + //----------------------------------------------------------------------------------// + FASTOR_HINT_INLINE void operator=(const TensorFixedViewExprnD,Fseqs...> &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + // constexpr FASTOR_INDEX stride = V::Size; + V _vec; + while(counter < total) + { + int ind = 0; + // for(int it = 0; it< DIMS; it++) { + // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + // } + get_index_s<0,DIMS-1>::Do(ind, as); + _vec = other.template teval(as); + _vec.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + // for(int it = 0; it< DIMS; it++) { + // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + // } + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] = other.template teval_s(as); + // print(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] += 1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + //----------------------------------------------------------------------------------------------// + + // AbstractTensor binders [equal order] + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + while(counter < total) + { + int ind = 0; + // for(int it = 0; it< DIMS; it++) { + // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + // } + get_index_s<0,DIMS-1>::Do(ind, as); + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + // for(int it = 0; it< DIMS; it++) { + // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + // } + get_index_s<0,DIMS-1>::Do(ind, as); + // _data[ind] = other_src.template eval_s(counter); + _data[ind] = other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + + // // Generic vectorised version that takes care of the remainder scalar ops + // using V=SIMDVector; + // while(counter < total) + // { + // int ind = 0; + // for(int it = 0; it< DIMS; it++) { + // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + // } + // get_index_s<0,DIMS-1>::Do(ind, as); + // if (_dims[DIMS-1] - as[DIMS-1] % V::Size == 0) { + // // V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + // _vec.store(&_data[ind],false); + // counter+=V::Size; + // } + // else { + // // _data[ind] = other_src.template eval_s(counter); + // _data[ind] = other_src.template teval_s(as); + // counter++; + // } + + // for(jt = DIMS-1; jt>=0; jt--) + // { + // if (jt == _dims.size()-1) as[jt]+=V::Size; + // else as[jt] +=1; + // if(as[jt]<_dims[jt]) + // break; + // else + // as[jt]=0; + // } + // if(jt<0) + // break; + // } + } + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out += _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // _data[ind] += other_src.template eval_s(counter); + _data[ind] += other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out -= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // _data[ind] -= other_src.template eval_s(counter); + _data[ind] -= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out *= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // _data[ind] *= other_src.template eval_s(counter); + _data[ind] *= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out /= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + // _data[ind] /= other_src.template eval_s(counter); + _data[ind] /= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + //----------------------------------------------------------------------------------------------// + + + // AbstractTensor binders [non-equal orders] + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] = other_src.template eval_s(counter); + // _data[ind] = other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out += _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] += other_src.template eval_s(counter); + // _data[ind] += other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out -= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] -= other_src.template eval_s(counter); + // _data[ind] -= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out *= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] *= other_src.template eval_s(counter); + // _data[ind] *= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out /= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] /= other_src.template eval_s(counter); + // _data[ind] /= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + // scalar binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _vec.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] = (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _vec_out.load(&_data[ind],false); + _vec_out += _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] += (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _vec_out.load(&_data[ind],false); + _vec_out -= _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] -= (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _vec_out.load(&_data[ind],false); + _vec_out *= _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] *= (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _vec_out.load(&_data[ind],false); + _vec_out /= _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + _data[ind] /= (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + //----------------------------------------------------------------------------------// + + // Evals + //----------------------------------------------------------------------------------------------// + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + + SIMDVector _vec; + std::array as = {}; + std::array inds; + for (auto j=0; j::Do(j,inds, as); + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + + std::array as = {}; + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + return _expr.eval_s(ind); + } + + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { + + idx += j; + V _vec; + std::array as = {}; + + std::array inds; + for (auto j=0; j::Do(j, inds, as); + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { + idx += j; + std::array as = {}; + + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + + return _expr.eval_s(ind); + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + FASTOR_IF_CONSTEXPR (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); + else FASTOR_IF_CONSTEXPR (_is_strided_vectorisable) { + V _vec; + vector_setter(_vec,_expr.data(),ind,get_nth_pt::_step); + return _vec; + } + else { + + V _vec; + std::array inds = {}; + std::array as_ = as; + for (auto j=0; j::Do(j, inds, as); + + for(int jt = (int)DIMS-1; jt>=0; jt--) + { + as_[jt] +=1; + if(as_[jt]<_dims[jt]) + break; + else + as_[jt]=0; + } + } + + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + } + + template + FASTOR_INLINE U teval_s(const std::array& as) const { + int ind = 0; + get_index_s<0,DIMS-1>::Do(ind, as); + return _expr.eval_s(ind); + } + //----------------------------------------------------------------------------------------------// + + + + //----------------------------------------------------------------------------------------------// +private: + template + struct get_index_v { + template + static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { + inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + get_index_v::Do(j, inds, as); + } + }; + template + struct get_index_v { + template + static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { + inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + } + }; + + template + struct get_index_s { + template + static FASTOR_INLINE void Do(int &ind, const std::array& as) { + ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + get_index_s::Do(ind, as); + } + }; + template + struct get_index_s { + template + static FASTOR_INLINE void Do(int &ind, const std::array& as) { + ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; + } + }; + //----------------------------------------------------------------------------------------------// +}; + + +template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> +constexpr std::array TensorFixedViewExprnD,Fseqs...>::products_; + +template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> +constexpr std::array TensorFixedViewExprnD,Fseqs...>::_dims; + + + + + + + +} // end of namespace Fastor + + +#endif // TENSOR_FIXED_VIEWS_ND_H diff --git a/noarch/include/Fastor/expressions/views/tensor_random_views.h b/noarch/include/Fastor/expressions/views/tensor_random_views.h new file mode 100644 index 00000000..1aa795a0 --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_random_views.h @@ -0,0 +1,1152 @@ +#ifndef TENSOR_RANDOM_VIEWS_H +#define TENSOR_RANDOM_VIEWS_H + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + + +namespace Fastor { + + +// Const versions +//----------------------------------------------------------------------------------// +template +struct TensorConstRandomViewExpr,Tensor,1>: + public AbstractTensor,Tensor,1>,1> { +private: + const Tensor &_expr; + const Tensor &it_expr; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = 1; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return 1;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return IterSize;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return IterSize;} + constexpr const Tensor& expr() const {return _expr;} + + constexpr FASTOR_INLINE TensorConstRandomViewExpr(const Tensor &_ex, const Tensor &_it) : _expr(_ex), it_expr(_it) {} + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr.data()[it_expr.data()[i]]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { + i += k; + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr(it_expr(i+j)); + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[as[0]+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[it_expr.data()[as[0]]]; + } +}; + + +//----------------------------------------------------------------------------------// +template +struct TensorConstRandomViewExpr,Tensor,DIMS>: + public AbstractTensor,Tensor,DIMS>,DIMS> { +private: + const Tensor &_expr; + // The index tensor needs to be copied for the purpose of indexing a + // multidimensional tensor with multiple 1D or any other lower other + // tensors. This issue could be solved by providing another type parameter/flag + // such as CopyIndex to the class which would use std::conditional to choose + // between "const Tensor &" and "Tensor" + + // const Tensor &it_expr; + Tensor it_expr; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = DIMS; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return DIMS;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return pack_prod::value;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return it_expr.dimension(i);} + constexpr const Tensor& expr() const {return _expr;} + + constexpr FASTOR_INLINE TensorConstRandomViewExpr(const Tensor &_ex, + const Tensor &_it) : _expr(_ex), it_expr(_it) { + static_assert(sizeof...(Rest)==DIMS && sizeof...(IterSizes)==DIMS, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr.data()[it_expr.data()[i]]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { + i += k; + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[it_expr.data()[i+j]]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + int i = std::accumulate(as.begin(), as.end(), 0); + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + FASTOR_INLINE U teval_s(const std::array& as) const { + int i = std::accumulate(as.begin(), as.end(), 0); + return _expr.data()[it_expr.data()[i]]; + } + +}; + +//----------------------------------------------------------------------------------// + + + + + +template +struct TensorRandomViewExpr,Tensor,1>: + public AbstractTensor,Tensor,1>,1> { +private: + Tensor &_expr; + const Tensor &it_expr; + bool _does_alias = false; + + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = 1; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return 1;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return IterSize;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return IterSize;} + constexpr const Tensor& expr() const {return _expr;} + + FASTOR_INLINE TensorRandomViewExpr,Tensor,1>& noalias() { + _does_alias = true; + return *this; + } + + constexpr FASTOR_INLINE TensorRandomViewExpr(Tensor &_ex, const Tensor &_it) : _expr(_ex), it_expr(_it) {} + + + // View evalution operators + // Copy assignment operators + //----------------------------------------------------------------------------------// + FASTOR_HINT_INLINE void operator=(const TensorRandomViewExpr,Tensor,1> &other_src) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other_src; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + for (auto j=0; j::Size; ++j) { + _expr(it_expr(i+j)) = _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + + // AbstractTensor binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i (i); + } + return; + } + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _expr(it_expr(i+j)) = _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _expr(it_expr(i+j)) += _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _expr(it_expr(i+j)) -= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _expr(it_expr(i+j)) *= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _expr(it_expr(i+j)) /= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + //----------------------------------------------------------------------------------// + + + // scalar binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(static_cast(num)); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _expr(it_expr(i+j)) = _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(static_cast(num)); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _expr(it_expr(i+j)) += _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(static_cast(num)); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _expr(it_expr(i+j)) -= _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(static_cast(num)); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _expr(it_expr(i+j)) *= _vec_other[j]; + } + } + for (; i && !is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + T inum = T(1)/(T)num; + SIMDVector _vec_other(inum); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _expr(it_expr(i+j)) *= _vec_other[j]; + } + } + for (; i && is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(static_cast(num)); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _expr(it_expr(i+j)) /= _vec_other[j]; + } + } + for (; i + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr(i+j); + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr(it_expr(i)); + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { + i += k; + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr(i+j); + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr(it_expr(i+j)); + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[as[0]+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[it_expr.data()[as[0]]]; + } + +}; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +template +struct TensorRandomViewExpr,Tensor,DIMS>: + public AbstractTensor,Tensor,DIMS>,DIMS> { +private: + Tensor &_expr; + // The index tensor needs to be copied for the purpose of indexing a + // multidimensional tensor with multiple 1D or any other lower other + // tensors. + + // const Tensor &it_expr; + Tensor it_expr; + bool _does_alias = false; + + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = DIMS; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return DIMS;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return pack_prod::value;} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return it_expr.dimension(i);} + constexpr const Tensor& expr() const {return _expr;}; + + FASTOR_INLINE TensorRandomViewExpr,Tensor,DIMS>& noalias() { + _does_alias = true; + return *this; + } + + constexpr FASTOR_INLINE TensorRandomViewExpr(Tensor &_ex, const Tensor &_it) : _expr(_ex), it_expr(_it) { + static_assert(sizeof...(Rest)==DIMS && sizeof...(IterSizes)==DIMS, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + } + + // Copy assignment + //------------------------------------------------------------------------------------// + FASTOR_HINT_INLINE void operator=(const TensorRandomViewExpr,Tensor,DIMS> &other_src) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other_src; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i); + for (auto j=0; j::Size; ++j) { + _data[it_expr.data()[i+j]] = _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + // AbstractTensor overloads + //------------------------------------------------------------------------------------// + template + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i (i); + } + return; + } + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[it_expr.data()[i+j]] = _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[it_expr.data()[i+j]] += _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[it_expr.data()[i+j]] -= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[it_expr.data()[i+j]] *= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + template + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j::Size; ++j) { + _data[it_expr.data()[i+j]] /= _vec_other[j]; + } + } + for (; i (i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + + // Scalar binders + //------------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(num); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _data[it_expr.data()[i+j]] = _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(num); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _data[it_expr.data()[i+j]] += _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(num); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _data[it_expr.data()[i+j]] -= _vec_other[j]; + } + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other(num); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _data[it_expr.data()[i+j]] *= _vec_other[j]; + } + } + for (; i && !is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + T inum = T(1.)/(T)num; + SIMDVector _vec_other(inum); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _data[it_expr.data()[i+j]] *= _vec_other[j]; + } + } + for (; i && is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + T *_data = _expr.data(); +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + SIMDVector _vec_other((T)num); + FASTOR_INDEX i; + for (i = 0; i ::Size; ++j) { + _data[it_expr.data()[i+j]] /= _vec_other[j]; + } + } + for (; i + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr.data()[it_expr.data()[i]]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { + i += k; + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[it_expr.data()[i+j]]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + int i = std::accumulate(as.begin(), as.end(), 0); + SIMDVector _vec; + std::array::Size> inds; + for (FASTOR_INDEX j=0; j::Size; ++j) + inds[j] = it_expr.data()[i+j]; + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + FASTOR_INLINE U teval_s(const std::array& as) const { + int i = std::accumulate(as.begin(), as.end(), 0); + return _expr.data()[it_expr.data()[i]]; + } +}; + + + + +} + + + + +#endif // TENSOR_RANDOM_VIEWS_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views.h b/noarch/include/Fastor/expressions/views/tensor_views.h new file mode 100644 index 00000000..08688a9e --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_views.h @@ -0,0 +1,10 @@ +#ifndef TENSOR_VIEWS_H +#define TENSOR_VIEWS_H + +#include "Fastor/expressions/views/tensor_views_1d.h" +#include "Fastor/expressions/views/tensor_views_2d.h" +#include "Fastor/expressions/views/tensor_views_nd.h" + +#include "Fastor/expressions/views/tensor_views_assignment.h" + +#endif // TENSOR_VIEWS_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_1d.h b/noarch/include/Fastor/expressions/views/tensor_views_1d.h new file mode 100644 index 00000000..6289517f --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_views_1d.h @@ -0,0 +1,739 @@ +#ifndef TENSOR_VIEWS_1D_H +#define TENSOR_VIEWS_1D_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +namespace Fastor { + + + +// 1D const views +//-------------------------------------------------------------------------------------// +template +struct TensorConstViewExpr,1>: public AbstractTensor,1>,1> { +private: + const Tensor &_expr; + seq _seq; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using V = simd_vector_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = 1; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return 1;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq.size();} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return _seq.size();} + constexpr const Tensor& expr() const {return _expr;} + + FASTOR_INLINE TensorConstViewExpr(const Tensor &_ex, const seq &_s) : _expr(_ex), _seq(_s) { + if (_seq._last < 0) _seq._last += N + /*including the end point*/ 1; + if (_seq._first < 0) _seq._first += N + /*including the end point*/ 1; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),i*_seq._step+_seq._first,_seq._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr.data()[i*_seq._step+_seq._first]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),(i+j)*_seq._step+_seq._first,_seq._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[(i+j)*_seq._step+_seq._first]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),as[0]*_seq._step+_seq._first,_seq._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[as[0]*_seq._step+_seq._first]; + } +}; + + + + + + + + + + + + + + + +// 1D non-const views +//-------------------------------------------------------------------------------------// +template +struct TensorViewExpr,1>: public AbstractTensor,1>,1> { +private: + Tensor &_expr; + seq _seq; + bool _does_alias = false; + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using V = simd_vector_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = 1; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return 1;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq.size();} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return _seq.size();} + constexpr const Tensor& expr() const {return _expr;} + + FASTOR_INLINE TensorViewExpr,1>& noalias() { + _does_alias = true; + return *this; + } + + FASTOR_INLINE TensorViewExpr(Tensor &_ex, const seq &_s) : _expr(_ex), _seq(_s) { + if (_seq._last < 0) _seq._last += N + /*including the end point*/ 1; + if (_seq._first < 0) _seq._first += N + /*including the end point*/ 1; + } + + // View evalution operators + // Copy assignment operators [Needed in addition to generic AbstractTensor overload] + //----------------------------------------------------------------------------------// + FASTOR_HINT_INLINE void operator=(const TensorViewExpr,1> &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match - for this 1D case for loop unnecessary + for (FASTOR_INDEX i=0; i(i); + _vec.store(&_data[i+_seq._first],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j(i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + //----------------------------------------------------------------------------------// + + // AbstractTensor binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i (i); + } + return; + } + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i (i); + _vec.store(&_data[i+_seq._first],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j(i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) + other_src.template eval(i); + _vec.store(&_data[i+_seq._first],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j(i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) - other_src.template eval(i); + _vec.store(&_data[i+_seq._first],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j(i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) * other_src.template eval(i); + _vec.store(&_data[i+_seq._first],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j(i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) / other_src.template eval(i); + _vec.store(&_data[i+_seq._first],false); + } + for (; i (i); + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + FASTOR_INDEX i; + for (i = 0; i (i); + for (auto j=0; j(i); + } +#else + for (FASTOR_INDEX i = 0; i (i); + } +#endif + } + } + //----------------------------------------------------------------------------------// + + // scalar binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i ,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) + _vec_other; + _vec.store(&_data[i+_seq._first],false); + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) - _vec_other; + _vec.store(&_data[i+_seq._first],false); + } + for (; i ,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) * _vec_other; + _vec.store(&_data[i+_seq._first],false); + } + for (; i && !is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + T inum = T(1) / static_cast(num); + simd_vector_type _vec_other(inum); + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) * _vec_other; + _vec.store(&_data[i+_seq._first],false); + } + for (; i && is_integral_v_,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + simd_vector_type _vec_other(static_cast(num)); + T *FASTOR_RESTRICT _data = _expr.data(); + if (_seq._step == 1) { + FASTOR_INDEX i; + for (i = 0; i template eval(i) / _vec_other; + _vec.store(&_data[i+_seq._first],false); + } + for (; i + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),i*_seq._step+_seq._first,_seq._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { + return _expr.data()[i*_seq._step+_seq._first]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),(i+j)*_seq._step+_seq._first,_seq._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr.data()[(i+j)*_seq._step+_seq._first]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),as[0]*_seq._step+_seq._first,_seq._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr.data()[as[0]*_seq._step+_seq._first]; + } +}; + + +} // end of namespace Fastor + + +#endif // TENSOR_VIEWS_1D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_2d.h b/noarch/include/Fastor/expressions/views/tensor_views_2d.h new file mode 100644 index 00000000..70e01f00 --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_views_2d.h @@ -0,0 +1,1163 @@ +#ifndef TENSOR_VIEWS_2D_H +#define TENSOR_VIEWS_2D_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +namespace Fastor { + + +// 2D non-const views +//-------------------------------------------------------------------------------------// +template +struct TensorConstViewExpr,2>: public AbstractTensor,2>,2> { +private: + const Tensor& _expr; + seq _seq0; + seq _seq1; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = 2; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return 2;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq0.size()*_seq1.size();} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i==0 ? _seq0.size() : _seq1.size();} + constexpr const Tensor& expr() const {return _expr;} + + FASTOR_INLINE TensorConstViewExpr(const Tensor &_ex, seq _s0, seq _s1) : _expr(_ex), _seq0(std::move(_s0)), _seq1(std::move(_s1)) { + if (_seq0._last < 0 && _seq0._first >= 0) {_seq0._last += M + 1;} + else if (_seq0._last==0 && _seq0._first==-1) {_seq0._first=M-1; _seq0._last=M;} + else if (_seq0._last < 0 && _seq0._first < 0) {_seq0._first += M +1; _seq0._last += M+1;} + if (_seq1._last < 0 && _seq1._first >= 0) {_seq1._last += N + 1;} + else if (_seq1._last==0 && _seq1._first==-1) {_seq1._first=N-1; _seq1._last=N;} + else if (_seq1._last < 0 && _seq1._first < 0) {_seq1._first += N +1; _seq1._last += N+1;} +#ifndef NDEBUG + FASTOR_ASSERT(_seq0._last <= M && _seq0._first + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + SIMDVector _vec; + std::array::Size> inds; + for (auto j=0; j::Size; ++j) { + auto it = (idx+j) / _seq1.size(), jt = (idx+j) % _seq1.size(); + inds[j] = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + auto it = idx / _seq1.size(), jt = idx % _seq1.size(); + auto ind = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; + return _expr.data()[ind]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*i*N+j + _seq0._first*N + _seq1._first,false); + else vector_setter(_vec,_expr.data(),_seq0._step*i*N+_seq1._step*j + _seq0._first*N + _seq1._first,_seq1._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first); + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*as[0]*N+as[1] + _seq0._first*N + _seq1._first,false); + else vector_setter(_vec,_expr.data(),_seq0._step*as[0]*N+_seq1._step*as[1] + _seq0._first*N + _seq1._first,_seq1._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr(_seq0._step*as[0]+_seq0._first,_seq1._step*as[1]+_seq1._first); + } +}; + + + + + + + + + + + + + +// 2D non-const views +//-------------------------------------------------------------------------------------// +template +struct TensorViewExpr,2>: public AbstractTensor,2>,2> { +private: + Tensor& _expr; + seq _seq0; + seq _seq1; + bool _does_alias = false; + // std::array _dims; + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; +public: + using scalar_type = T; + using simd_vector_type = typename Tensor::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + static constexpr FASTOR_INDEX Dimension = 2; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr FASTOR_INDEX rank() {return 2;} + constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq0.size()*_seq1.size();} + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i==0 ? _seq0.size() : _seq1.size();} + constexpr const Tensor& expr() const {return _expr;} + + FASTOR_INLINE TensorViewExpr,2>& noalias() { + _does_alias = true; + return *this; + } + + FASTOR_INLINE TensorViewExpr(Tensor &_ex, seq _s0, seq _s1) : + _expr(_ex), _seq0(std::move(_s0)), _seq1(std::move(_s1)) { + + if (_seq0._last < 0 && _seq0._first >= 0) {_seq0._last += M + 1;} + else if (_seq0._last==0 && _seq0._first==-1) {_seq0._first=M-1; _seq0._last=M;} + else if (_seq0._last < 0 && _seq0._first < 0) {_seq0._first += M +1; _seq0._last += M+1;} + if (_seq1._last < 0 && _seq1._first >= 0) {_seq1._last += N + 1;} + else if (_seq1._last==0 && _seq1._first==-1) {_seq1._first=N-1; _seq1._last=N;} + else if (_seq1._last < 0 && _seq1._first < 0) {_seq1._first += N +1; _seq1._last += N+1;} +#ifndef NDEBUG + FASTOR_ASSERT(_seq0._last <= M && _seq0._first,2> &other_src) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other_src; + // assign temporary to this + this->operator=(tmp); + return; + // Alternatively one could do, but slower + // _does_alias = false; + // // Evaluate this into a temporary + // auto tmp_this_tensor = get_tensor(); + // tmp_this_tensor(_seq0,_seq1) = other_src; + // // assign temporary to this + // this->operator=(tmp_this_tensor(_seq0,_seq1)); + // return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i(i,j); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) = other_src.template eval_s(i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j (i,j); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); + } + } +#endif + } + } + //----------------------------------------------------------------------------------// + + + + // AbstractTensor binders - equal order + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); + } + } + return; + } + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j (i,j); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) = other_src.template eval_s(i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j (i,j); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) + other_src.template eval(i,j); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) += other_src.template eval_s(i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) + other_src.template eval(i,j); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) - other_src.template eval(i,j); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) -= other_src.template eval_s(i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) - other_src.template eval(i,j); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) * other_src.template eval(i,j); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= other_src.template eval_s(i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) * other_src.template eval(i,j); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(i,j); + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; itemplate eval(i,j) / other_src.template eval(i,j); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) /= other_src.template eval_s(i,j); + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) / other_src.template eval(i,j); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(i,j); + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(i,j); + } + } +#endif + } + } + //----------------------------------------------------------------------------------// + + + // AbstractTensor binders [non-equal order] + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(counter); + counter++; + } + } + return; + } + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j (counter); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) = other_src.template eval_s(counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j (counter); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) + other_src.template eval(counter); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) += other_src.template eval_s(counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) + other_src.template eval(counter); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) - other_src.template eval(counter); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) -= other_src.template eval_s(counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) - other_src.template eval(counter); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) * other_src.template eval(counter); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= other_src.template eval_s(counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) * other_src.template eval(counter); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *FASTOR_RESTRICT _data = _expr.data(); + FASTOR_INDEX counter = 0; + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) / other_src.template eval(counter); + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) /= other_src.template eval_s(counter); + counter++; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) / other_src.template eval(counter); + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + counter+=Stride; + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(counter); + counter++; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(counter); + counter++; + } + } +#endif + } + } + //----------------------------------------------------------------------------------// + + + // Scalar binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_INLINE void operator=(U num) { + T *_data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j ,bool> = false> + FASTOR_INLINE void operator+=(U num) { + T *_data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) + _vec_other; + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) += num; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) + _vec_other; + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += num; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += num; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + T *_data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) - _vec_other; + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) -= num; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) - _vec_other; + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= num; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= num; + } + } +#endif + } + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + T *_data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) * _vec_other; + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= num; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) * _vec_other; + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= num; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= num; + } + } +#endif + } + } + + template && !is_integral_v_,bool> = false> + FASTOR_INLINE void operator/=(U num) { + T *_data = _expr.data(); + T inum = T(1.0)/T(num); + SIMDVector _vec_other((inum)); + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) * _vec_other; + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= inum; + } + } + } +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + else { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) * _vec_other; + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= inum; + } + } + } +#else + else { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= inum; + } + } + } +#endif + } + template && is_integral_v_,bool> = false> + FASTOR_INLINE void operator/=(U num) { + T *_data = _expr.data(); + SIMDVector _vec_other(static_cast(num)); + if (_seq1._step == 1) { + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) / _vec_other; + _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,j+_seq1._first) /= num; + } + } + } + else { +#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + FASTOR_INDEX j; + for (j = 0; j template eval(i,j) / _vec_other; + data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); + } + for (; j <_seq1.size(); ++j) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= num; + } + } +#else + for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { + for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { + _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= num; + } + } +#endif + } + } + //----------------------------------------------------------------------------------// + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + SIMDVector _vec; + std::array::Size> inds; + for (auto j=0; j::Size; ++j) { + auto it = (idx+j) / _seq1.size(), jt = (idx+j) % _seq1.size(); + inds[j] = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + auto it = idx / _seq1.size(), jt = idx % _seq1.size(); + auto ind = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; + return _expr.data()[ind]; + } + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*i*N+j + _seq0._first*N + _seq1._first,false); + else vector_setter(_vec,_expr.data(),_seq0._step*i*N+_seq1._step*j + _seq0._first*N + _seq1._first,_seq1._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first); + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + SIMDVector _vec; + if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*as[0]*N+as[1] + _seq0._first*N + _seq1._first,false); + else vector_setter(_vec,_expr.data(),_seq0._step*as[0]*N+_seq1._step*as[1] + _seq0._first*N + _seq1._first,_seq1._step); + return _vec; + } + + template + constexpr FASTOR_INLINE U teval_s(const std::array& as) const { + return _expr(_seq0._step*as[0]+_seq0._first,_seq1._step*as[1]+_seq1._first); + } +}; + + +} // end of namespace Fastor + + +#endif // TENSOR_VIEWS_2D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_assignment.h b/noarch/include/Fastor/expressions/views/tensor_views_assignment.h new file mode 100644 index 00000000..ebefc27c --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_views_assignment.h @@ -0,0 +1,72 @@ +#ifndef TENSOR_VIEWS_ASSIGNMENT_H +#define TENSOR_VIEWS_ASSIGNMENT_H + + +#include "Fastor/expressions/views/tensor_views.h" +#include "Fastor/expressions/views/tensor_fixed_views_1d.h" +#include "Fastor/expressions/views/tensor_fixed_views_2d.h" +#include "Fastor/expressions/views/tensor_random_views.h" + + +namespace Fastor { + +#define FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(ASSIGN_TYPE)\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstFixedViewExpr1D& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFixedViewExpr1D& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstFixedViewExpr2D& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFixedViewExpr2D& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstFixedViewExprnD& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFixedViewExprnD& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstViewExpr& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorViewExpr& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstRandomViewExpr& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorRandomViewExpr& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFilterViewExpr& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ +template\ +FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorDiagViewExpr& src) {\ + trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ +}\ + +FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT( ) +FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_add) +FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_sub) +FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_mul) +FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_div) + +} // end of namespace Fastor + + +#endif // TENSOR_VIEWS_ASSIGNMENT_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_nd.h b/noarch/include/Fastor/expressions/views/tensor_views_nd.h new file mode 100644 index 00000000..05818d27 --- /dev/null +++ b/noarch/include/Fastor/expressions/views/tensor_views_nd.h @@ -0,0 +1,1680 @@ +#ifndef TENSOR_VIEWS_ND_H +#define TENSOR_VIEWS_ND_H + + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/expressions/linalg_ops/linalg_traits.h" + +namespace Fastor { + + + +// Generic const tensor views based on sequences/slices +//----------------------------------------------------------------------------------------------// +template class TensorType, typename T, size_t DIMS, size_t ... Rest> +struct TensorConstViewExpr,DIMS>: public AbstractTensor,DIMS>,DIMS> { +private: + const TensorType &_expr; + std::array _seqs; + std::array _dims; + bool _is_vectorisable; + bool _is_strided_vectorisable; +public: + using scalar_type = T; + using simd_vector_type = typename TensorType::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = TensorType; + static constexpr FASTOR_INDEX Dimension = DIMS; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; + + FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} + FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} + FASTOR_INLINE FASTOR_INDEX size() const { + int sizer = 1; + for (auto &_seq: _seqs) sizer *= _seq.size(); + return sizer; + } + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _seqs[i].size();} + constexpr const TensorType& expr() const {return _expr;} + + FASTOR_INLINE TensorConstViewExpr(const TensorType &_ex, std::array _s) : _expr(_ex), _seqs(std::move(_s)) { + static_assert(DIMS==sizeof...(Rest),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + auto counter = 0; + for (auto &_seq: _seqs) { + if (_seq._last < 0 && _seq._first>=0) { + _seq._last += _expr.dimension(counter) + 1; + } + // take care of scalar indexing with -1 + else if (_seq._last == 0 && _seq._first==-1) { + auto dim = _expr.dimension(counter); + _seq._first = dim-1; + _seq._last = dim; + } + else if (_seq._last < 0 && _seq._first < 0) { + auto dim = _expr.dimension(counter); + _seq._first += dim + 1; + _seq._last += dim + 1; + } +#ifndef NDEBUG + FASTOR_ASSERT(_seq._last <= _expr.dimension(counter) && _seq._first<_expr.dimension(counter),"INDEX OUT OF BOUNDS"); +#endif + counter++; + } + + for (FASTOR_INDEX i=0; i && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step==1) ? true : false; + _is_strided_vectorisable = !is_same_v_ && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step!=1) ? true : false; + } + + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + + SIMDVector _vec; + std::array as = {}; + std::array::Size> inds; + for (auto j=0; j::Size; ++j) { + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( int(idx+j) / remaining ) % _dims[n]; + } + inds[j] = 0; + for(int it = 0; it< DIMS; it++) { + inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + + std::array as = {}; + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + + return _expr.data()[ind]; + } + + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { + + idx += j; + SIMDVector _vec; + std::array as = {}; + + std::array::Size> inds; + for (auto j=0; j::Size; ++j) { + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( int(idx+j) / remaining ) % _dims[n]; + } + inds[j] = 0; + for(int it = 0; it< DIMS; it++) { + inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { + idx += j; + std::array as = {}; + + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + + return _expr.data()[ind]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + if (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); + else if (_is_strided_vectorisable) { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),ind,_seqs[DIMS-1]._step); + return _vec; + } + else { + // return eval(ind); + + SIMDVector _vec; + std::array::Size> inds; + std::array as_ = as; + for (auto j=0; j::Size; ++j) { + int _sum = 0; + for(int it = 0; it< DIMS; it++) { + _sum += products_[it]*as_[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + inds[j] = _sum; + + for(int jt = (int)DIMS-1; jt>=0; jt--) + { + as_[jt] +=1; + if(as_[jt]<_dims[jt]) + break; + else + as_[jt]=0; + } + } + + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + } + + template + FASTOR_INLINE U teval_s(const std::array& as) const { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + return _expr.data()[ind]; + } +}; +//----------------------------------------------------------------------------------------------// + + + + + + +// Generic non-const tensor views based on sequences/slices +//----------------------------------------------------------------------------------------------// +template class TensorType, typename T, size_t DIMS, size_t ... Rest> +struct TensorViewExpr,DIMS>: public AbstractTensor,DIMS>,DIMS> { +private: + TensorType &_expr; + std::array _seqs; + bool _does_alias = false; + std::array _dims; + bool _is_vectorisable; + bool _is_strided_vectorisable; + + constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; + constexpr FASTOR_INLINE std::array get_sequences() const {return _seqs;} + +public: + using scalar_type = T; + using simd_vector_type = typename TensorType::simd_vector_type; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = TensorType; + static constexpr FASTOR_INDEX Dimension = DIMS; + static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; + static constexpr std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; + + FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} + FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} + FASTOR_INLINE FASTOR_INDEX size() const { + int sizer = 1; + for (auto &_seq: _seqs) sizer *= _seq.size(); + return sizer; + } + constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _seqs[i].size();} + constexpr const TensorType& expr() const {return _expr;} + + FASTOR_INLINE TensorViewExpr,DIMS>& noalias() { + _does_alias = true; + return *this; + } + + TensorViewExpr(TensorType &_ex, std::array _s) : _expr(_ex), _seqs(std::move(_s)) { + static_assert(DIMS==sizeof...(Rest),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + auto counter = 0; + for (auto &_seq: _seqs) { + if (_seq._last < 0 && _seq._first>=0) { + _seq._last += _expr.dimension(counter) + 1; + } + // take care of scalar indexing with -1 + else if (_seq._last == 0 && _seq._first==-1) { + auto dim = _expr.dimension(counter); + _seq._first = dim-1; + _seq._last = dim; + } + else if (_seq._last < 0 && _seq._first < 0) { + auto dim = _expr.dimension(counter); + _seq._first += dim + 1; + _seq._last += dim + 1; + } +#ifndef NDEBUG + FASTOR_ASSERT(_seq._last <= _expr.dimension(counter) && _seq._first<_expr.dimension(counter),"INDEX OUT OF BOUNDS"); +#endif + counter++; + } + + for (FASTOR_INDEX i=0; i && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step==1) ? true : false; + _is_strided_vectorisable = !is_same_v_ && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step!=1) ? true : false; + } + + // View evalution operators + // Copy assignment operators [Needed in addition to generic AbstractTensor overload] + //----------------------------------------------------------------------------------// + FASTOR_HINT_INLINE void operator=(const TensorViewExpr,DIMS> &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif +#ifndef NDEBUG + FASTOR_ASSERT(other.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + // constexpr FASTOR_INDEX stride = V::Size; + V _vec; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _vec = other.template teval(as); + _vec.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] = other.template teval_s(as); + // print(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] += 1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + //----------------------------------------------------------------------------------// + + // AbstractTensor binders [equal order] + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // _data[ind] = other_src.template eval_s(counter); + _data[ind] = other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + + // // Generic vectorised version that takes care of the remainder scalar ops + // using V=SIMDVector; + // while(counter < total) + // { + // int ind = 0; + // for(int it = 0; it< DIMS; it++) { + // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + // } + // if (_dims[DIMS-1] - as[DIMS-1] % V::Size == 0) { + // // V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + // _vec.store(&_data[ind],false); + // counter+=V::Size; + // } + // else { + // // _data[ind] = other_src.template eval_s(counter); + // _data[ind] = other_src.template teval_s(as); + // counter++; + // } + + // for(jt = DIMS-1; jt>=0; jt--) + // { + // if (jt == _dims.size()-1) as[jt]+=V::Size; + // else as[jt] +=1; + // if(as[jt]<_dims[jt]) + // break; + // else + // as[jt]=0; + // } + // if(jt<0) + // break; + // } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out += _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // _data[ind] += other_src.template eval_s(counter); + _data[ind] += other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out -= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // _data[ind] -= other_src.template eval_s(counter); + _data[ind] -= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out *= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // _data[ind] *= other_src.template eval_s(counter); + _data[ind] *= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // V _vec = other_src.template eval(counter); + V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out /= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + // _data[ind] /= other_src.template eval_s(counter); + _data[ind] /= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + //----------------------------------------------------------------------------------// + + // AbstractTensor binders [non-equal orders] + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else + { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] = other_src.template eval_s(counter); + // _data[ind] = other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator+=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator+=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out += _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] += other_src.template eval_s(counter); + // _data[ind] += other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator-=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator-=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out -= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] -= other_src.template eval_s(counter); + // _data[ind] -= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator*=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator*=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out *= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] *= other_src.template eval_s(counter); + // _data[ind] *= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { + const typename Derived::result_type& tmp = evaluate(other.self()); + this->operator/=(tmp); + } + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { +#if !(FASTOR_NO_ALIAS) + if (_does_alias) { + _does_alias = false; + // Evaluate this into a temporary + auto tmp_this_tensor = get_tensor(); + auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); + // Assign other to temporary + tmp = other; + // assign temporary to this + this->operator/=(tmp); + return; + } +#endif + const Derived& other_src = other.self(); +#ifndef NDEBUG + FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + V _vec = other_src.template eval(counter); + // V _vec = other_src.template teval(as); + _vec_out.load(&_data[ind],false); + _vec_out /= _vec; + _vec_out.store(&_data[ind],false); + + counter+=V::Size; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] /= other_src.template eval_s(counter); + // _data[ind] /= other_src.template teval_s(as); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + // scalar binders + //----------------------------------------------------------------------------------// + template,bool> = false> + FASTOR_HINT_INLINE void operator=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _vec.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] = (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator+=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _vec_out.load(&_data[ind],false); + _vec_out += _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] += (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator-=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _vec_out.load(&_data[ind],false); + _vec_out -= _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] -= (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator*=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _vec_out.load(&_data[ind],false); + _vec_out *= _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] *= (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + + template,bool> = false> + FASTOR_HINT_INLINE void operator/=(U num) { + + T *_data = _expr.data(); + std::array as = {}; + int total = size(); + int jt, counter = 0; + + if (_is_vectorisable) { + using V=SIMDVector; + constexpr FASTOR_INDEX stride = V::Size; + V _vec = (T)num; + V _vec_out; + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _vec_out.load(&_data[ind],false); + _vec_out /= _vec; + _vec_out.store(&_data[ind],false); + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + if (jt == _dims.size()-1) as[jt]+=stride; + else as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + else { + while(counter < total) + { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); + } + _data[ind] /= (T)num; + + counter++; + for(jt = DIMS-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]<_dims[jt]) + break; + else + as[jt]=0; + } + if(jt<0) + break; + } + } + } + //----------------------------------------------------------------------------------// + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { + + SIMDVector _vec; + std::array as = {}; + std::array::Size> inds; + for (auto j=0; j::Size; ++j) { + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( int(idx+j) / remaining ) % _dims[n]; + } + inds[j] = 0; + for(int it = 0; it< DIMS; it++) { + inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { + + std::array as = {}; + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + + return _expr.data()[ind]; + } + + + template + FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { + + idx += j; + SIMDVector _vec; + std::array as = {}; + + std::array::Size> inds; + for (auto j=0; j::Size; ++j) { + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( int(idx+j) / remaining ) % _dims[n]; + } + inds[j] = 0; + for(int it = 0; it< DIMS; it++) { + inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + } + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + + + template + FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { + + idx += j; + std::array as = {}; + int remaining = size(); + for (int n = 0; n < DIMS; ++n) { + remaining /= _dims[n]; + as[n] = ( (int)idx / remaining ) % _dims[n]; + } + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + + return _expr.data()[ind]; + } + + template + FASTOR_INLINE SIMDVector teval(const std::array& as) const { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + if (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); + else if (_is_strided_vectorisable) { + SIMDVector _vec; + vector_setter(_vec,_expr.data(),ind,_seqs[DIMS-1]._step); + return _vec; + } + else { + // return eval(ind); + + SIMDVector _vec; + std::array::Size> inds; + std::array as_ = as; + for (auto j=0; j::Size; ++j) { + int _sum = 0; + for(int it = 0; it< DIMS; it++) { + _sum += products_[it]*as_[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + inds[j] = _sum; + + for(int jt = (int)DIMS-1; jt>=0; jt--) + { + as_[jt] +=1; + if(as_[jt]<_dims[jt]) + break; + else + as_[jt]=0; + } + } + + vector_setter(_vec,_expr.data(),inds); + return _vec; + } + } + + template + FASTOR_INLINE U teval_s(const std::array& as) const { + int ind = 0; + for(int it = 0; it< DIMS; it++) { + ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; + } + return _expr.data()[ind]; + } +}; + + +template class TensorType, typename T, size_t DIMS, size_t ... Rest> +constexpr std::array TensorViewExpr,DIMS>::products_; + + +} // end of namespace Fastor + + +#endif // TENSOR_VIEWS_ND_H diff --git a/noarch/include/Fastor/meta/einsum_meta.h b/noarch/include/Fastor/meta/einsum_meta.h new file mode 100644 index 00000000..ebe348cd --- /dev/null +++ b/noarch/include/Fastor/meta/einsum_meta.h @@ -0,0 +1,1133 @@ +#ifndef EINSUM_META_H +#define EINSUM_META_H + +#include "Fastor/meta/meta.h" +#include "Fastor/config/config.h" +#include "Fastor/tensor/Tensor.h" +#include + +namespace Fastor { + +//namespace detail { + + +template +struct Index; + + +// Find return type of tensor contraction +//------------------------------------------------------------------------------------------------------------// +// is ind[i] unique in ind? +template +constexpr bool is_uniq(const int (&ind)[N], size_t i, size_t cur = 0){ + return cur == N ? true : + (cur == i || ind[cur] != ind[i]) ? is_uniq(ind, i, cur + 1) : false; +} + +// For every i where ind[i] == index, is dim[i] == dimension? +template +constexpr bool check_all_eq(int index, int dimension, + const int (&ind)[N], const int (&dim)[N], size_t cur = 0) { + return cur == N ? true : + (ind[cur] != index || dim[cur] == dimension) ? + check_all_eq(index, dimension, ind, dim, cur + 1) : false; +} + +// if position i should be contracted away, return 1001001, otherwise return dim[i]. +// triggers a compile-time error on mismatch. +template +constexpr int calc(size_t i, const int (&ind)[N], const int (&dim)[N]){ + return is_uniq(ind, i) ? dim[i] : + check_all_eq(ind[i], dim[i], ind, dim) ? 1001001 : throw "dimension mismatch"; +} + +// if position i should be contracted away, return 1001001, otherwise return ind[i]. +// triggers a compile-time error on mismatch. +template +constexpr int calc_idx(size_t i, const int (&ind)[N], const int (&dim)[N]){ + return is_uniq(ind, i) ? ind[i] : + check_all_eq(ind[i], dim[i], ind, dim) ? 1001001 : throw "dimension mismatch"; +} + +//Now we need a way to get rid of the 1001001s: +template +struct concat_ { using type = Ind; }; +template +struct concat_, Index, Inds...> + : concat_, Inds...> {}; + +template +struct filter_ + : concat_, Index>::type...> {}; + +template +using filter_t = typename filter_::type; + + +//Use them: +template +struct contraction_impl; + +template class Derived, typename T, size_t... Ind, size_t... Dim, size_t... Seq> +struct contraction_impl, Derived, std_ext::index_sequence>{ + static constexpr int ind[sizeof...(Ind)] = { Ind... }; + static constexpr int dim[sizeof...(Dim)] = { Dim... }; + static constexpr int result[sizeof...(Seq)] = {calc(Seq, ind, dim)...}; + + template + static auto unpack_helper(Index) -> Derived; + + using type = decltype(unpack_helper(typename filter_<1001001, result[Seq]...>::type{})); + + // Get the indices instead of values + static constexpr int result2[sizeof...(Seq)] = {calc_idx(Seq, ind, dim)...}; + template + static auto unpack_helper2(Index) -> Index; + using indices = decltype(unpack_helper2(typename filter_<1001001, result2[Seq]...>::type{})); +}; +//------------------------------------------------------------------------------------------------------------// + + + + +// Generate product of dimensions e.g. products_a, products_b and products_out +//------------------------------------------------------------------------------------------------------------// +// products generator +template +constexpr int products(const size_t (&seq)[N], int i = N-1) { + return i == (N-1) ? seq[N-1] : products(seq, i+1)*seq[i]; +} + +template +constexpr int shifter(const size_t (&seq)[N], int i) { + return i < N-1 ? seq[i+1] : shifter(seq, i-1); +} + +template +constexpr int zeroer(const size_t (&seq)[N], int i) { + return i == N-1 ? 0 : seq[i]; +} + + +template +struct nprods; + +template +struct nprods,std_ext::index_sequence> { + constexpr static size_t vals[sizeof...(Rest)] = {Rest...}; + static constexpr size_t pvals[sizeof...(Rest)] = {products(vals,ss)...}; + static constexpr size_t svals[sizeof...(Rest)] = {shifter(pvals,ss)...}; + static constexpr std::array values = {zeroer(svals,ss)...}; +}; + +template +constexpr std::array nprods,std_ext::index_sequence>::values; +//------------------------------------------------------------------------------------------------------------// + + +// For views only +//------------------------------------------------------------------------------------------------------------// +template +constexpr int _oner(const size_t (&seq)[N], int i) { + return i == N-1 ? 1 : seq[i]; +} + +template +struct nprods_views; + +template +struct nprods_views,std_ext::index_sequence> { + constexpr static size_t vals[sizeof...(Rest)] = {Rest...}; + static constexpr size_t pvals[sizeof...(Rest)] = {products(vals,ss)...}; + static constexpr size_t svals[sizeof...(Rest)] = {shifter(pvals,ss)...}; + static constexpr std::array values = {_oner(svals,ss)...}; +}; + +template +constexpr std::array nprods_views,std_ext::index_sequence>::values; +//------------------------------------------------------------------------------------------------------------// + + + + + + +// Auxilary meta functions for tensor contraction +//------------------------------------------------------------------------------------------------------------// +// this is a meta-function equivalent to numpy's "where" +template +constexpr int find_index(const size_t (&ind)[N], int num, size_t i=0){ + return (i==N) ? N : (static_cast(ind[i])==num ? i : find_index(ind,num,i+1)); +} +template +constexpr int find_index(const std::array &ind, int num, size_t i=0){ + return (i==N) ? N : (static_cast(ind[i])==num ? i : find_index(ind,num,i+1)); +} + +// check if a given value is ind1 and not ind0 +template +constexpr bool check_diff(const size_t (&ind0)[M], const size_t (&ind1)[N], int num){ + return (find_index(ind0,num) == static_cast(M)) & (find_index(ind1,num) < static_cast(N)); +} + +// this is a meta-function somewhat equivalent to numpy's "setdiff1d" +// if a given value is in ind1 and not in ind0, then it returns the index in to the array ind1 such that +// ind1[index] = value (num) +template +constexpr int find_index_diff(const size_t (&ind0)[M], const size_t (&ind1)[N], int num){ + return check_diff(ind0,ind1,num) ? find_index(ind1,num) : N; +} + +// based on index from find_index_diff retrieve the actual value +template +constexpr int retrieve_value(const size_t (&ind0)[M], const size_t (&ind1)[N], const int (&nums1)[N], size_t i=0){ + return find_index_diff(ind0,ind1,ind1[i]) == static_cast(N) + ? 1 : nums1[find_index(ind1,ind1[i])]; +} + + +// does an array ind contain a number num (bool equivalent of find_index) +template +constexpr bool contains(const size_t (&ind)[N], int num){ + return find_index(ind,num)!=N; +} + + +template +struct put_dims_in_Index; + +template class Derived, size_t ... Rest, typename T> +struct put_dims_in_Index> { + using type = Index; +}; +//------------------------------------------------------------------------------------------------------------// + + +//! Checks vectorisability and returns a stride and a type +//------------------------------------------------------------------------------------------------------------// +template +struct is_vectorisable; + +template +struct is_vectorisable,Index,Tensor> { + template using _vec_size = internal::get_simd_vector_size>; + static constexpr size_t fastest_changing_index = get_value::value; + static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; + static constexpr bool does_2nd_tensor_disappear = ((int)no_of_unique::value == (int)sizeof...(Idx0) - (int)sizeof...(Idx1)); + static constexpr bool last_index_contracted = contains(idx,get_value::value); + static constexpr bool is_reducible = does_2nd_tensor_disappear && last_index_contracted; + static constexpr bool value = (!last_index_contracted) && (fastest_changing_index % _vec_size::value==0); + static constexpr bool sse_vectorisability = (!last_index_contracted) && + (fastest_changing_index % _vec_size::value==0 && fastest_changing_index % _vec_size::value!=0); + static constexpr bool avx_vectorisability = (!last_index_contracted) && + (fastest_changing_index % _vec_size::value==0 && fastest_changing_index % _vec_size::value==0); + static constexpr int stride = (avx_vectorisability ? _vec_size::value : (sse_vectorisability ? _vec_size::value : 1)); + + using type = typename std::conditional, + typename std::conditional,SIMDVector>::type>::type; +}; + + +template +struct is_vectorisable,Index,Tensor> { + static constexpr size_t fastest_changing_index = get_value::value; + static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; + static constexpr bool does_2nd_tensor_disappear = ((int)no_of_unique::value == (int)sizeof...(Idx0) - (int)sizeof...(Idx1)); + static constexpr bool last_index_contracted = contains(idx,get_value::value); + static constexpr bool is_reducible = does_2nd_tensor_disappear && last_index_contracted; + static constexpr bool value = (!last_index_contracted) && (fastest_changing_index % 4==0); + static constexpr bool sse_vectorisability = (!last_index_contracted) && (fastest_changing_index % 4==0 && fastest_changing_index % 8!=0); + static constexpr bool avx_vectorisability = (!last_index_contracted) && (fastest_changing_index % 4==0 && fastest_changing_index % 8==0); + static constexpr int stride = (avx_vectorisability ? 8 : (sse_vectorisability ? 4 : 1)); + + using type = typename std::conditional, + typename std::conditional,SIMDVector>::type>::type; +}; + +template +struct is_vectorisable,Index,Tensor> { + static constexpr size_t fastest_changing_index = get_value::value; + static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; + static constexpr bool does_2nd_tensor_disappear = ((int)no_of_unique::value == (int)sizeof...(Idx0) - (int)sizeof...(Idx1)); + static constexpr bool last_index_contracted = contains(idx,get_value::value); + static constexpr bool is_reducible = does_2nd_tensor_disappear && last_index_contracted; + static constexpr bool value = (!last_index_contracted) && (fastest_changing_index % 2==0); + static constexpr bool sse_vectorisability = (!last_index_contracted) && (fastest_changing_index % 2==0 && fastest_changing_index % 4!=0); + static constexpr bool avx_vectorisability = (!last_index_contracted) && (fastest_changing_index % 2==0 && fastest_changing_index % 4==0); + static constexpr int stride = (avx_vectorisability ? 4 : (sse_vectorisability ? 2 : 1)); + + using type = typename std::conditional, + typename std::conditional,SIMDVector>::type>::type; +}; + +//------------------------------------------------------------------------------------------------------------// + + +//! Checks reducible vectorisability and returns a stride and a type (use this for working on general strides) +//------------------------------------------------------------------------------------------------------------// +template +struct is_reducibly_vectorisable; + +template +struct is_reducibly_vectorisable,Tensor> { + template using _vec_size = internal::get_simd_vector_size>; + static constexpr size_t fastest_changing_index = get_value::value; + static constexpr bool value = (fastest_changing_index % _vec_size::value==0); + static constexpr bool sse_vectorisability = (fastest_changing_index % _vec_size::value==0 && + fastest_changing_index % _vec_size::value!=0); + static constexpr bool avx_vectorisability = (fastest_changing_index % _vec_size::value==0 && + fastest_changing_index % _vec_size::value==0); + static constexpr int stride = (avx_vectorisability ? _vec_size::value : + (sse_vectorisability ? _vec_size::value : 1)); + + using type = typename std::conditional, + typename std::conditional,SIMDVector>::type>::type; +}; +//------------------------------------------------------------------------------------------------------------// + + + + +//! Check if indices in Einstein summation appear more than twice +//------------------------------------------------------------------------------------------------------------// +template +struct einsum_index_checker; + +template +struct einsum_index_checker> { + static constexpr bool value = no_more_than_two::value; +}; + +//template +//struct einsum_index_checker,Index> { +// static constexpr bool value = no_more_than_two::value; +//}; +//------------------------------------------------------------------------------------------------------------// + + + + + + + + + +//------------------------------------------------------------------------------------------------------------// +template +struct get_resuling_tensor; + +template +struct get_resuling_tensor,Index,Tensor,Tensor> { + using type = typename contraction_impl, Tensor, + typename std_ext::make_index_sequence::type>::type; +}; + + +template +struct get_resuling_index; + +template +struct get_resuling_index,Index,Tensor,Tensor> { + using type = typename contraction_impl, Tensor, + typename std_ext::make_index_sequence::type>::indices; +}; +//------------------------------------------------------------------------------------------------------------// + + + +// How many nested loops should be set up +//------------------------------------------------------------------------------------------------------------// +template +struct no_of_loops_to_set; + +template +struct no_of_loops_to_set,Index,Tensor,Tensor,std_ext::index_sequence> { + + using index_temp = apply_typelist_t, + uniq_t>>; + + static constexpr size_t concat_idx[sizeof...(Idx0)+sizeof...(Idx1)] = {Idx0...,Idx1...}; + static constexpr size_t concat_nums[sizeof...(Idx0)+sizeof...(Idx1)] = {Rest0...,Rest1...}; + static constexpr std::array idx_in_concat = {find_index(concat_idx,index_temp::values[ss])...}; + static constexpr std::array dims = {concat_nums[idx_in_concat[ss]]...}; + static constexpr int value = pack_prod::value; + + using type = Tensor; + using indices = Index; + + using dims_type = Index; +}; + +template +constexpr std::array +no_of_loops_to_set,Index,Tensor,Tensor,std_ext::index_sequence>::dims; +//------------------------------------------------------------------------------------------------------------// + + + + + + +// Find indices from the number of loops to set up e.g. idx_a, idx_b and idx_out +//------------------------------------------------------------------------------------------------------------// +template +struct IndexFirstTensor; + +template +struct IndexFirstTensor,Index,Tensor,Tensor,std_ext::index_sequence> { + + using index_temp = typename no_of_loops_to_set,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::value>::type>::indices; + + static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; + static constexpr std::array + indices = {find_index(index_temp::values, idx[ss])...}; + + using type = Tensor; + +}; + +template +constexpr std::array +IndexFirstTensor,Index,Tensor,Tensor,std_ext::index_sequence>::indices; + + +template +struct IndexSecondTensor; + +template +struct IndexSecondTensor,Index,Tensor,Tensor,std_ext::index_sequence> { + + using index_temp = typename no_of_loops_to_set,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::value>::type>::indices; + + static constexpr size_t idx[sizeof...(Idx1)] = {Idx1...}; + static constexpr std::array + indices = {find_index(index_temp::values, idx[ss])...}; + + using type = Tensor; + +}; + +template +constexpr std::array +IndexSecondTensor,Index,Tensor,Tensor,std_ext::index_sequence>::indices; + + + +template +struct IndexResultingTensor; + +template +struct IndexResultingTensor,Index,Tensor,Tensor,std_ext::index_sequence> { + + using index_temp = typename no_of_loops_to_set,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::value>::type>::indices; + + using resulting_index_0 = typename get_resuling_index,Index, + Tensor,Tensor>::type; + + static constexpr std::array + indices = {find_index(index_temp::values, resulting_index_0::values[ss])...}; + + using type = Tensor; +}; + +template +constexpr std::array +IndexResultingTensor,Index,Tensor,Tensor,std_ext::index_sequence>::indices; +//------------------------------------------------------------------------------------------------------------// + + + +//------------------------------------------------------------------------------------------------------------// +// This is for pure inner reduction and not permuted reduction +template +struct is_pair_reduction; + +template +struct is_pair_reduction,Index> { + static constexpr bool value = is_same_v_,Index>; +}; + +// helper +template +constexpr bool is_pair_reduction_v = is_pair_reduction::value; + + +// Reduction for a single tensor +template +struct is_single_reduction; + +template +struct is_single_reduction, Tensor> { + using resulting_tensor = typename contraction_impl, Tensor, + typename std_ext::make_index_sequence::type>::type; + static constexpr bool value = resulting_tensor::dimension_t::value == 0; +}; + +// helper +template +constexpr bool is_single_reduction_v = is_single_reduction::value; +//------------------------------------------------------------------------------------------------------------// + + + +//--------------------------------------------------------------------------------------------------------------// +template +constexpr int contain_prod(const size_t (&ind)[N], const size_t (&sseq)[N], size_t num){ + return (ind[num]==0) ? static_cast(sseq[num]) : -1; +} + +template +constexpr int last_indices_prod(const int (&sseq)[N], int num){ + return num > 0 ? ( (sseq[num-1]!=-1) ? sseq[num-1]*last_indices_prod(sseq,num-1) : 1) : 1; +} + + +template +struct general_stride_finder; + +template +struct general_stride_finder,Index,Tensor,Tensor, std_ext::index_sequence> { + + using OutIndices = typename contraction_impl, Tensor, + typename std_ext::make_index_sequence::type>::indices; + + static constexpr size_t b_idx[sizeof...(Idx1)] = {Idx1...}; + static constexpr size_t b_dim[sizeof...(Rest1)] = {Rest1...}; +// static constexpr std::array container_idx = {contains(OutIndices::values,b_idx[ss])...}; + static constexpr size_t container_idxx[sizeof...(Rest1)] = {contains(OutIndices::values,b_idx[ss])...}; +// static constexpr std::array container_dim = {contain_prod(container_idxx,b_dim,ss)...}; + static constexpr int container_dim[sizeof...(Rest1)] = {contain_prod(container_idxx,b_dim,ss)...}; + static constexpr int value = last_indices_prod(container_dim,sizeof...(Rest1)); +}; + +//template +//constexpr std::array +//general_stride_finder,Index,Tensor,Tensor,std_ext::index_sequence>::container_idx; +//template +//constexpr std::array +//general_stride_finder,Index,Tensor,Tensor,std_ext::index_sequence>::container_dim; +//--------------------------------------------------------------------------------------------------------------------// + + + + + + +namespace internal { +//--------------------------------------------------------------------------------------------------------------------// +//! Given the einsum indices of two tensors, matches their indices to see +//! if it is a genearalised matrix-vector multiplication. If either the indices +//! of the first tensor or the second tensor disappear while matching indices +//! from the end, then it is a genearalised matrix-vector multiplication. +//! Note that this function matches indices from the end in that it can detect +//! generalised matrix-vector product of the forms or (it can detect swapping) +//! but it cannot detect or which is vector-matrix product +template +constexpr +inline bool match_indices_from_end(const T (&ind0)[N0], + const T (&ind1)[N1], T num0=N0-1, T num1=N1-1) { + return ind1[num1] == ind0[num0] ? ( num1 == 0 ? ind1[num1] == ind0[num0] : + (num0 == 0 ? ind1[num1] == ind0[num0] : match_indices_from_end(ind0, ind1, num0 - 1, num1 - 1))) : + false; +} + +//! Same as above but gives the index up to which the higher order tensor (generalised matrix) +//! matches the lower order tensor (generalised vector) from the end. The index is for higher +//! higher order tensor counting from the start including the index itself for instance +//! will return 0 (i.e. index 0 does not match) and will return 1 (i.e. indices 0 and 1 do no match) +//! The function in essence gives the remainder indices that don't match the vector +//! +//! This function only works (gives the correct index) if the accompanying boolean +//! function (match_indices_from_end) is true i.e. it only works if we have a true +//! generalised matrix-vector product otherwise gives an incorrect index +template +constexpr +inline T match_indices_from_end_index(const T (&ind0)[N0], + const T (&ind1)[N1], T num0=N0-1, T num1=N1-1) { + return ind1[num1] == ind0[num0] ? ( num1 == 0 ? (ind1[num1] == ind0[num0] ? num0-1 : num0) : + num0 == 0 ? (ind1[num1] == ind0[num0] ? num1-1 : num1) : + match_indices_from_end_index(ind0, ind1, num0 - 1, num1 - 1) ) : num0-1; +} + + +//! Given the einsum indices of two tensors, matches their indices to see +//! if it is a genearalised vector-matrix multiplication. If either the indices +//! of the first tensor or the second tensor disappear while matching indices +//! from the beggining, then it is a genearalised vector-matrix multiplication. +//! Note that this function matches indices from the beggingin in that it can detect +//! generalised vector-matrix product of the forms or (it can detect swapping) +//! but it cannot detect or which is matrix-vector product +template +constexpr +inline bool match_indices_from_start(const T (&ind0)[N0], + const T (&ind1)[N1], T num0=0, T num1=0) { + return ind1[num1] == ind0[num0] ? ( num1 == N1-1 ? ind1[num1] == ind0[num0] : + (num0 == N0-1 ? ind1[num1] == ind0[num0] : match_indices_from_start(ind0, ind1, num0 + 1, num1 + 1))) : + false; +} + +//! Same as above but gives the index up to which the higher order tensor (generalised matrix) +//! matches the lower order tensor (generalised vector) from the start. The index is for higher +//! higher order tensor counting from the end excluding the index itself for instance +//! will return 1 (only 0 matches) and will return 2 (i.e. indices 0 and 1 match) +//! The function in essence gives the remainder indices that don't match the vector +//! +//! function (match_indices_from_start) is true i.e. it only works if we have a true +//! generalised vector-matrix product otherwise gives an incorrect index +template +constexpr +inline T match_indices_from_start_index(const T (&ind0)[N0], + const T (&ind1)[N1], T num0=0, T num1=0) { + return ind1[num1] == ind0[num0] ? ( num1 == N1-1 ? (ind1[num1] == ind0[num0] ? num0+1 : num0) : + num0 == N0-1 ? (ind1[num1] == ind0[num0] ? num1+1 : num1) : + match_indices_from_start_index(ind0, ind1, num0 + 1, num1 + 1) ) : num0; +} + + +//! Given the einsum indices of two tensors, matches their indices to see +//! if it is a standard genearalised matrix-matrix multiplication. +//! By standard we mean . This function cannot detect transposed cases of +//! gemm such as and so on. +//! Note that this function matches indices from the two ends but it also detects +//! as gemm. So it should be used in conjunction with is_vector_matrix/is_matrix_vec +//! and is_inner. Look at the corresponding struct that uses it +template +constexpr +inline bool match_indices_from_two_ends(const T (&ind0)[N0], + const T (&ind1)[N1], int ncontracted, T num0=N0-1, T num1=0) { + return ncontracted != 0 ? + (ncontracted == 1 ? (ind1[num1] == ind0[num0 - ncontracted + 1] ? true : false) + : (ind1[num1] == ind0[num0 - ncontracted + 1] ? + match_indices_from_two_ends(ind0, ind1, ncontracted-1, num0, num1 + 1) : + false)) : + false; +} + + + + +template +struct is_generalised_matrix_vector; + +template +struct is_generalised_matrix_vector,Index > { + static constexpr size_t which_one_is_vector = sizeof...(Idx0) > sizeof...(Idx1) ? 1 : 0; + static constexpr size_t idx0[sizeof...(Idx0)] = {Idx0...}; + static constexpr size_t idx1[sizeof...(Idx1)] = {Idx1...}; + static constexpr bool value = match_indices_from_end(idx0, idx1) && sizeof...(Idx0) != sizeof...(Idx1); + static constexpr size_t matches_up_to = match_indices_from_end_index(idx0, idx1); +}; + + +template +struct is_generalised_vector_matrix; + +template +struct is_generalised_vector_matrix,Index > { + static constexpr size_t which_one_is_vector = sizeof...(Idx0) > sizeof...(Idx1) ? 1 : 0; + static constexpr size_t idx0[sizeof...(Idx0)] = {Idx0...}; + static constexpr size_t idx1[sizeof...(Idx1)] = {Idx1...}; + static constexpr bool value = match_indices_from_start(idx0, idx1) && sizeof...(Idx0) != sizeof...(Idx1); + static constexpr size_t matches_up_to = match_indices_from_start_index(idx0, idx1); +}; + + +template +struct is_generalised_matrix_matrix; + +template +struct is_generalised_matrix_matrix,Index > { + static constexpr bool is_mat_vec = is_generalised_matrix_vector,Index>::value; + static constexpr bool is_vec_mat = is_generalised_vector_matrix,Index>::value; + static constexpr int ncontracted = sizeof...(Idx0) + sizeof...(Idx1) - no_of_unique::value; + static constexpr bool is_inner = sizeof...(Idx0) == sizeof...(Idx1) && no_of_unique::value == sizeof...(Idx1); + static constexpr size_t idx0[sizeof...(Idx0)] = {Idx0...}; + static constexpr size_t idx1[sizeof...(Idx1)] = {Idx1...}; + static constexpr bool value = !is_mat_vec && !is_vec_mat && !is_inner && match_indices_from_two_ends(idx0, idx1, ncontracted); +}; +//--------------------------------------------------------------------------------------------------------------------// +} // namespace internal + + + + + + + +// A complete tensor contraction meta-engine +//--------------------------------------------------------------------------------------------------------------// +//--------------------------------------------------------------------------------------------------------------// +#if FASTOR_CXX_VERSION >= 2017 +template +inline constexpr std::array find_remaining(const std::array &maxes_out, int total) { + std::array remainings = {}; + remainings[0] = total / maxes_out[0]; + for (int i=1; i +inline constexpr std::array,(size_t)total> cartesian_product_2(const std::array &maxes_out) { + std::array,(size_t)total> as_all = {}; + for (int i=0; i +constexpr int find_remaining(const int (&maxes_out)[N], int remaining, int i) { + return i==0 ? remaining/maxes_out[0] : find_remaining(maxes_out,remaining,i-1) / maxes_out[i]; +} + +template +constexpr int cartesian_product_single(const int (&maxes_out)[N], int remaining, int i, int n=0) { + return (i/(find_remaining(maxes_out,remaining,n))) % maxes_out[n]; +} + +template +struct gen_single_cartesian_product; + +template +struct gen_single_cartesian_product,std_ext::index_sequence> { + static constexpr int vals[sizeof...(Rest)] = {Rest...}; + static constexpr std::array values = {cartesian_product_single(vals,pack_prod::value,I,ss)...}; +}; + +template +constexpr std::array gen_single_cartesian_product,std_ext::index_sequence>::values; + + +template +constexpr std::array all_cartesian_product() { + return gen_single_cartesian_product,typename std_ext::make_index_sequence::type>::values; +} + + +template +struct cartesian_product; + +template +struct cartesian_product,std_ext::index_sequence> { + static constexpr std::array,sizeof...(ss)> values = {all_cartesian_product()...}; + +}; + +template +constexpr std::array,sizeof...(ss)> cartesian_product,std_ext::index_sequence>::values; + + +template +constexpr int get_indices(const std::array &products, + const std::array& idx, + const std::array,All> &as_all, + int i, + int it) { + return it==0 ? as_all[i][idx[static_cast(N)-1]] + products[0]*as_all[i][idx[0]] : + products[it]*as_all[i][idx[it]]+get_indices(products,idx,as_all,i,it-1); +} + +// Blowing compilation time and memory usage 101 +//--------------------------------------------------------------------------------------------------------------// +template +struct contract_meta_engine; + +template +struct contract_meta_engine,Index,Tensor,Tensor,std_ext::index_sequence> { + + using OutTensor = typename contraction_impl, Tensor, + typename std_ext::make_index_sequence::type>::type; + using OutIndices = typename contraction_impl, Tensor, + typename std_ext::make_index_sequence::type>::indices; + + static constexpr int a_dim = sizeof...(Rest0); + static constexpr int b_dim = sizeof...(Rest1); + static constexpr int out_dim = OutTensor::Dimension; + static constexpr int total = sizeof...(ss); + + static constexpr auto& idx_a = IndexFirstTensor,Index, Tensor,Tensor, + typename std_ext::make_index_sequence::type>::indices; + static constexpr auto& idx_b = IndexSecondTensor,Index, Tensor,Tensor, + typename std_ext::make_index_sequence::type>::indices; + static constexpr auto& idx_out = IndexResultingTensor,Index, Tensor,Tensor, + typename std_ext::make_index_sequence::type>::indices; + + static constexpr int uniques = no_of_unique::value; + using uniques_type = typename std_ext::make_index_sequence::type; + static constexpr auto& maxes_out = no_of_loops_to_set,Index,Tensor,Tensor, + uniques_type>::dims; + + using maxes_out_type = typename no_of_loops_to_set,Index,Tensor,Tensor, + uniques_type>::type; + + static constexpr std::array products_a = nprods,typename std_ext::make_index_sequence::type>::values; + static constexpr std::array products_b = nprods,typename std_ext::make_index_sequence::type>::values; + using Index_with_dims = typename put_dims_in_Index::type; + static constexpr std::array products_out = nprods::type>::values; + + // Generate the cartesian product + static constexpr auto& as_all = cartesian_product::type>::values; + // Alternatively you can pass the ss... directly into cartesian_product but that does not change anything in terms of + // memory usage or compilation time + //using maxes_out_indices = typename no_of_loops_to_set,Index,Tensor,Tensor, + // uniques_type>::indices; + //static constexpr std::array,total> as_all = {all_cartesian_product()...}; + + static constexpr std::array index_a = {get_indices(products_a,idx_a,as_all,ss,a_dim-1)...}; + static constexpr std::array index_b = {get_indices(products_b,idx_b,as_all,ss,b_dim-1)...}; + static constexpr std::array index_out = {get_indices(products_out,idx_out,as_all,ss,out_dim-1)...}; +}; + +template +constexpr std::array +contract_meta_engine,Index, +Tensor,Tensor, +std_ext::index_sequence>::index_a; + +template +constexpr std::array +contract_meta_engine,Index, +Tensor,Tensor, +std_ext::index_sequence>::index_b; + +template +constexpr std::array +contract_meta_engine,Index, +Tensor,Tensor, +std_ext::index_sequence>::index_out; + +//--------------------------------------------------------------------------------------------------------------// +//--------------------------------------------------------------------------------------------------------------// + + + + + + + + +// The followings are generic implemenations that work for any type of complex tensor network +//------------------------------------------------------------------------------------------------------------// +//------------------------------------------------------------------------------------------------------------// +//------------------------------------------------------------------------------------------------------------// + + +// Find how many loops needs to be set. Works for any complex tensor network +//------------------------------------------------------------------------------------------------------------// +// This is equivalent to no_of_loops_to_set_up but more generic (works for arbitrary tensor networks) +template +struct loop_setter; + +template +struct loop_setter,Tensor,std_ext::index_sequence> { + + using index_temp = apply_typelist_t, + uniq_t>>; + + static constexpr size_t concat_idx[sizeof...(Idx)] = {Idx...}; + static constexpr size_t concat_nums[sizeof...(Rest)] = {Rest...}; + static constexpr std::array idx_in_concat = {find_index(concat_idx,index_temp::values[ss])...}; + static constexpr std::array dims = {concat_nums[idx_in_concat[ss]]...}; + static constexpr int value = pack_prod::value; + + using type = Tensor; + using indices = Index; + + using dims_type = Index; +}; + +template +constexpr std::array +loop_setter,Tensor,std_ext::index_sequence>::dims; +//------------------------------------------------------------------------------------------------------------// + + +// Get indices of every individual tensor in the network. Works for any complex tensor network +//------------------------------------------------------------------------------------------------------------// +template +struct IndexTensors; + +template class Derived0, + template class Derived1, + typename T, size_t... Idx, size_t... Idx_t, size_t ...Rest, size_t ...Rest_t, size_t ... ss> +struct IndexTensors,Derived0,Index,Derived1,std_ext::index_sequence> { + + using index_temp = typename loop_setter,Tensor, + typename std_ext::make_index_sequence::value>::type>::indices; + + static constexpr size_t idx[sizeof...(Idx_t)] = {Idx_t...}; + static constexpr std::array + indices = {find_index(index_temp::values, idx[ss])...}; + + using type = Tensor; + +}; + +template class Derived0, + template class Derived1, + typename T, size_t... Idx, size_t... Idx_t, size_t ...Rest, size_t ...Rest_t, size_t ... ss> +constexpr std::array +IndexTensors,Derived0,Index,Derived1,std_ext::index_sequence>::indices; +//------------------------------------------------------------------------------------------------------------// + + + +namespace internal { +//------------------------------------------------------------------------------------------------------------// +template +struct meta_argmin_wrapper; +template +struct meta_argmin_wrapper> { + static constexpr int value = meta_argmin::value; +}; +template +struct meta_argmin_wrapper> { + static constexpr int value = idx; +}; + + +// Complete compile-time sorting algorithm: Does not work if there a duplicate entries in a pack +template +struct meta_sort; +template +struct meta_sort> { + static constexpr int least_value_idx = meta_argmin_wrapper>::value; + static constexpr int least_value = get_value::value; + using reduced_seq = typename filter_::type; + using new_seq = typename concat_,typename meta_sort::new_seq>::type; +}; +template +struct meta_sort> { + using reduced_seq = Index; + using new_seq = Index; +}; + + +// Complete compile-time arg-sorting algorithm: Does not work if there a duplicate entries in a pack +template +struct meta_argsort; +template +struct meta_argsort,Index> { + static constexpr int least_value_idx = meta_argmin_wrapper>::value; + static constexpr int least_value = get_value::value; + using reduced_seq = typename filter_::type; + using new_seq = typename concat_,typename meta_sort::new_seq>::type; + + static constexpr int least_index = get_value::value; + + using reduced_argseq = typename filter_::type; + using new_argseq = typename concat_,typename meta_argsort::new_argseq>::type; +}; +template +struct meta_argsort,Index> { + using reduced_seq = Index; + using new_seq = Index; + + using reduced_argseq = Index; + using new_argseq = Index; +}; +//------------------------------------------------------------------------------------------------------------// + + +// Permutation functions +//------------------------------------------------------------------------------------------------------------// +template +constexpr size_t count_less(const size_t (&seq)[N], size_t i, size_t cur = 0) { + return cur == N ? 0 : (count_less(seq, i, cur + 1) + (seq[cur] < i ? 1 : 0)); +} + +/* Check if a compile time array is sequential */ +template +constexpr bool is_sequential(const size_t (&seq)[N], size_t i=0) { + return i+1 == N ? true : ( seq[i] + 1 == seq[i+1] ? is_sequential(seq, i+1) : false ) ; +} +template +constexpr bool is_sequential(const std::array &seq, size_t i=0) { + return i+1 == N ? true : ( seq[i] + 1 == seq[i+1] ? is_sequential(seq, i+1) : false ) ; +} + +// permutation helper class +template +struct permute_impl; + +template +struct permute_impl, Tensor, std_ext::index_sequence> { + constexpr static size_t lst[sizeof...(ls)] = { ls... }; + constexpr static size_t fvals[sizeof...(ls)] = {fs...}; + using resulting_tensor = Tensor; + using resulting_index = typename meta_argsort,Index>::new_argseq; + using maxes_out_type = Index,Index>::new_argseq::values[ss]]...>; + static constexpr bool requires_permutation = !(is_same_v_> && + is_sequential(resulting_index::values)); +}; + +// permute helper class +template +struct new_permute_impl; + +template +struct new_permute_impl, Tensor, std_ext::index_sequence> { + constexpr static size_t lst[sizeof...(ls)] = { ls... }; + constexpr static size_t fvals[sizeof...(ls)] = {fs...}; + using resulting_tensor = Tensor; + constexpr static size_t aranger[sizeof...(ss)] = { ss... }; + using resulting_index = Index; + static constexpr bool requires_permutation = !(is_same_v_> && + is_sequential(resulting_index::values)); +}; +//------------------------------------------------------------------------------------------------------------// + + +//------------------------------------------------------------------------------------------------------------// +#if FASTOR_CXX_VERSION >= 2017 +template +constexpr std::array get_floor_map(const std::array &idx) { + std::array out = {}; + for (size_t i=0; i +constexpr std::array find_permuation(const std::array &idx0, const std::array &idx1) { + std::array out = {}; + for (size_t i=0; i,OIndex>(a)]. + In the above example the resulting index from einsum is already Index but the user decides to force this + nevertheless using OIndex. Now if permuation is simply performed using OIndex the final output would + be incorrect since permuation only works with what indices it is given and does not have the knowledge of the context. + This meta function is responsible for creating a mapping between einsum and permutation. It takes the resulting index + of einsum and the permutation index [the output index OIndex] and figures out how the tensor should be permuted. +*/ +template +struct permute_mapped_index_impl; + +// // specialisation for when the input and output indices are the same - the no permutation case +// template +// struct permute_mapped_index_impl,Index,std_ext::index_sequence> { +// using resulting_index = Index; +// }; + +template +struct permute_mapped_index_impl,Index,std_ext::index_sequence> { + constexpr static size_t einsum_idx[sizeof...(Idx0)] = { Idx0... }; + constexpr static size_t to_be_permuted_idx[sizeof...(Idx1)] = { Idx1... }; + /* Given that the resulting index from einsum can be discontinuous like Index<5,8,2> we need to first create a continuous + Index starting from zero. This is done through [get_floor_map] function. We floor both resulting einsum and output + indices and we then call the [find_permuation] function to know how many swaps/permutations are necessary going from + the resulting einsum index to the output index + */ + static constexpr std::array argsort_idx0 = meta_argsort,Index>::new_argseq::values; + static constexpr std::array argsort_idx1 = meta_argsort,Index>::new_argseq::values; + static constexpr std::array mapped_idx0 = get_floor_map(argsort_idx0); + static constexpr std::array mapped_idx1 = get_floor_map(argsort_idx1); + static constexpr std::array mapped_idx = find_permuation(mapped_idx0,mapped_idx1); + using resulting_index = Index; +}; + +/* Provided for debugging */ +template +constexpr std::array +permute_mapped_index_impl,Index,std_ext::index_sequence>::argsort_idx0; +template +constexpr std::array +permute_mapped_index_impl,Index,std_ext::index_sequence>::argsort_idx1; +template +constexpr std::array +permute_mapped_index_impl,Index,std_ext::index_sequence>::mapped_idx0; +template +constexpr std::array +permute_mapped_index_impl,Index,std_ext::index_sequence>::mapped_idx1; +template +constexpr std::array +permute_mapped_index_impl,Index,std_ext::index_sequence>::mapped_idx; + + +template +struct permute_mapped_index { + using resulting_index = typename permute_mapped_index_impl::type>::resulting_index; +}; + +template +using permute_mapped_index_t = typename permute_mapped_index::resulting_index; + +#endif // CXX 2017 +//------------------------------------------------------------------------------------------------------------// + +} // internal + + +template +struct requires_permutation; +template +struct requires_permutation, Tensor> { + using _permute_impl = internal::permute_impl, Tensor, + typename std_ext::make_index_sequence::type>; + static constexpr bool value = _permute_impl::requires_permutation; +}; + +// helper +template +constexpr bool requires_permutation_v = requires_permutation::value; + + +template +struct requires_permute; +template +struct requires_permute, Tensor> { + using _permute_impl = internal::new_permute_impl, Tensor, + typename std_ext::make_index_sequence::type>; + static constexpr bool value = _permute_impl::requires_permutation; +}; + +// helper +template +constexpr bool requires_permute_v = requires_permute::value; +//------------------------------------------------------------------------------------------------------------// + + +// einsum helper to extract the resulting index and the resulting tensor +//------------------------------------------------------------------------------------------------------------// +template +struct permute_helper; + +template +struct permute_helper> { + using resulting_index = typename internal::new_permute_impl, + typename std_ext::make_index_sequence::type>::resulting_index; + using resulting_tensor = typename internal::new_permute_impl, + typename std_ext::make_index_sequence::type>::resulting_tensor; +}; +//------------------------------------------------------------------------------------------------------------// + + + +} + +#endif // EINSUM_META_H + diff --git a/noarch/include/Fastor/meta/meta.h b/noarch/include/Fastor/meta/meta.h new file mode 100644 index 00000000..f8fcf7da --- /dev/null +++ b/noarch/include/Fastor/meta/meta.h @@ -0,0 +1,344 @@ +#ifndef META_H_ +#define META_H_ + + +#include +#include + + +namespace Fastor { + +//----------------------------------------------------------------------------------------------------------// +template< bool B, class T = void > +using enable_if_t_ = typename std::enable_if::type; + +template< bool B, class T = void > +using disable_if_t_ = typename std::enable_if::type; + +template< bool B, class T, class F > +using conditional_t_ = typename std::conditional::type; + +template< class T, class U > +constexpr bool is_same_v_ = std::is_same::value; + +template< class T > +constexpr bool is_fundamental_v_ = std::is_fundamental::value; + +template< class T > +constexpr bool is_arithmetic_v_ = std::is_arithmetic::value; + +template< class T > +constexpr bool is_integral_v_ = std::is_integral::value; + +template< class T > +constexpr bool is_floating_v_ = std::is_floating_point::value; + +template< class T > +constexpr bool is_array_v_ = std::is_array::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +// If BLAS compatible type +template< class T > struct is_numeric { static constexpr bool value = false; }; +template<> struct is_numeric { static constexpr bool value = true; }; +template<> struct is_numeric { static constexpr bool value = true; }; +template<> struct is_numeric> { static constexpr bool value = true; }; +template<> struct is_numeric> { static constexpr bool value = true; }; + +template< class T > +constexpr bool is_numeric_v_ = is_numeric::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +// If complex type +template< class T > struct is_complex { static constexpr bool value = false; }; +template< class T > struct is_complex> { static constexpr bool value = true; }; + +template< class T > +constexpr bool is_complex_v_ = is_complex::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +// If a type is std::is_fundamental + std::complex + any type that specialises this trait class +// This class is provided because the behaviour of any code that specialises std::is_fundamental/arithmetic/ +// integral/floating_point is undefined +template< class T > struct is_primitive { + static constexpr bool value = std::is_fundamental::value || is_numeric::value; +}; + +template< class T > +constexpr bool is_primitive_v_ = is_primitive::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +// Checks if all parameters in a variadic parameter pack are arithmetic +template +struct is_arithmetic_pack; +template +struct is_arithmetic_pack { + static constexpr bool value = std::is_arithmetic::value && is_arithmetic_pack::value; +}; +template +struct is_arithmetic_pack { static constexpr bool value = std::is_arithmetic::value; }; + +template +static constexpr bool is_arithmetic_pack_v = is_arithmetic_pack::value; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +// Remove cv-qualified and their refs +template struct remove_cvref_ { using type = T; }; +template struct remove_cvref_ { using type = typename remove_cvref_::type; }; +template struct remove_cvref_ { using type = typename remove_cvref_::type; }; +template struct remove_cvref_ { using type = typename remove_cvref_::type; }; +template struct remove_cvref_ { using type = typename remove_cvref_::type; }; +template struct remove_cvref_ { using type = typename remove_cvref_::type; }; + +template +using remove_cv_ref_t = typename remove_cvref_::type; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template struct remove_all { using type = T; }; +template struct remove_all { using type = typename remove_all::type; }; +template struct remove_all { using type = typename remove_all::type; }; +template struct remove_all { using type = typename remove_all::type; }; +template struct remove_all { using type = typename remove_all::type; }; +template struct remove_all { using type = typename remove_all::type; }; +template struct remove_all { using type = typename remove_all::type; }; +template struct remove_all { using type = typename remove_all::type; }; +template struct remove_all { using type = typename remove_all::type; }; + +template +using remove_all_t = typename remove_all::type; +//----------------------------------------------------------------------------------------------------------// + + +// Sum/multiply reduce all elements of a variadic pack +//----------------------------------------------------------------------------------------------------------// +// Sum +template struct pack_add; +template +struct pack_add { static const size_t value = Head+pack_add::value;}; +template<> struct pack_add<> { static const size_t value = 0;}; + +// Multiply +template struct pack_prod; +template +struct pack_prod { static const size_t value = Head*pack_prod::value;}; +template<> struct pack_prod<> { static const size_t value = 1;}; + +// Multiply first n elements +template struct prod_nel; +template +struct prod_nel { static const size_t value = prod_nel::value;}; +template +struct prod_nel<1, First, Rest...> { static const size_t value = pack_prod::value;}; + +//! Partial product of a sequence up to an index up_to +template +constexpr +inline T partial_prod(const T (&ind)[N], T up_to, T num=0) { + return num == up_to ? ind[num] : ind[num]*partial_prod(ind, up_to, num+1); +} + +//! Partial product of a sequence up to an index num from the end +template +constexpr +inline T partial_prod_reverse(const T (&ind)[N], T up_to, T num=N-1) { + return num == up_to ? ind[num] : ind[num]*partial_prod_reverse(ind, up_to, num-1); +} + +template +constexpr T size_proder_(T one){return one.size();} +template +constexpr T size_proder_(T one, T two){return one.size()*two.size();} +template +constexpr T size_proder_(T one, T two, Ts ... ts) {return _proder_(_proder_(one,two),ts...);} +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template +struct get_value; +template +struct get_value { + static const size_t value = get_value::value; +}; +template +struct get_value<1,First,Rest...> { + static const size_t value = First; +}; +template +// Work around to avoid compiler errors +struct get_value { + static const size_t value = 0; +}; + +template +constexpr inline auto get_index(Args&&... as) +-> decltype(std::get(std::forward_as_tuple(std::forward(as)...))) { + return std::get(std::forward_as_tuple(std::forward(as)...)); +} + +template +using get_nth_type = typename std::tuple_element>::type; + +template +struct get_all { + static const size_t indices[sizeof...(Rest)]; +}; +//----------------------------------------------------------------------------------------------------------// + + +// comparitors +//----------------------------------------------------------------------------------------------------------// +template +struct is_equal { + static constexpr bool value = I == J; +}; +template +static constexpr bool is_equal_v_ = is_equal::value; + +template +struct is_less { + static constexpr bool value = (I < J); +}; +template +static constexpr bool is_less_v_ = is_less::value; + +template +struct is_less_equal { + static constexpr bool value = I <= J; +}; +template +static constexpr bool is_less_equal_v_ = is_less_equal::value; + +template +struct is_greater { + static constexpr bool value = I > J; +}; +template +static constexpr bool is_greater_v_ = is_greater::value; + +template +struct is_greater_equal { + static constexpr bool value = I >= J; +}; +template +static constexpr bool is_greater_equal_v_ = is_greater_equal::value; +//----------------------------------------------------------------------------------------------------------// + + +// min-max +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// +template +struct meta_min; +template +struct meta_min { + static constexpr size_t pval = meta_min::value; + static const size_t value = (pval <= meta_min::value) ? + pval : meta_min::value; +}; +template +struct meta_min { + static const size_t value = (m<=n) ? m : n; +}; + +template +struct meta_max; +template +struct meta_max { + static constexpr size_t pval = meta_max::value; + static const size_t value = (pval >= meta_max::value) ? + pval : meta_max::value; +}; +template +struct meta_max { + static const size_t value = (m>=n) ? m : n; +}; + +namespace internal { +// namespace to avoid clash with MSVC macros +template constexpr FASTOR_INLINE T min_(const T a, const T b) {return a < b ? a : b;} +template constexpr FASTOR_INLINE T max_(const T a, const T b) {return a > b ? a : b;} +} +//----------------------------------------------------------------------------------------------------------// + +//----------------------------------------------------------------------------------------------------------// +template +struct meta_argmin; +template +struct meta_argmin { + static constexpr size_t pval = meta_min::value; + static const size_t value = (pval <= meta_min::value) ? + meta_argmin::value : meta_argmin::value+1; +}; +template +struct meta_argmin { + static const size_t value = (m +struct meta_argmax; +template +struct meta_argmax { + static constexpr size_t pval = meta_max::value; + static const size_t value = (pval >= meta_max::value) ? + meta_argmax::value : meta_argmax::value+1; +}; +template +struct meta_argmax { + static const size_t value = (m>n) ? 0 : 1; +}; +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// + + +// square/cube +//----------------------------------------------------------------------------------------------------------// +namespace internal { +template struct meta_square { static constexpr size_t value = Val*Val;}; +template struct meta_cube { static constexpr size_t value = Val*Val*Val;}; +} +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +//square root of integers at compile time, use like meta_sqrt<36>::ret +template Y))) > + // use ?: instead of || just to shut up a gcc 4.3 warning +class meta_sqrt +{ + enum { + MidX = (InfX+SupX)/2, + TakeInf = MidX*MidX > Y ? 1 : 0, + NewInf = int(TakeInf) ? InfX : int(MidX), + NewSup = int(TakeInf) ? int(MidX) : SupX +}; + public: + enum { ret = meta_sqrt::ret }; +}; + +template +class meta_sqrt +{ + public: enum { ret = (SupX*SupX <= Y) ? SupX : InfX }; +}; +//----------------------------------------------------------------------------------------------------------// + + +} // end of namespace Fastor + +#endif // META_H_ diff --git a/noarch/include/Fastor/meta/opmin_meta.h b/noarch/include/Fastor/meta/opmin_meta.h new file mode 100644 index 00000000..24738d4a --- /dev/null +++ b/noarch/include/Fastor/meta/opmin_meta.h @@ -0,0 +1,620 @@ +#ifndef OPMIN_META_H +#define OPMIN_META_H + +#ifndef FASTOR_DONT_PERFORM_OP_MIN + +#include "tensor_meta.h" +#include "einsum_meta.h" + +namespace Fastor { + + +// Cost model for by-pair tensor contraction +//------------------------------------------------------------------------------------------------------------// +template +struct pair_flop_cost; + +template +struct pair_flop_cost,Index,Tensor,Tensor,std_ext::index_sequence> { + + static constexpr size_t ind0[sizeof...(Idx0)] = {Idx0... }; + static constexpr size_t ind1[sizeof...(Idx1)] = {Idx1... }; + static constexpr int nums1[sizeof...(Rest1)] = {Rest1... }; + static constexpr size_t cost_tensor0 = pack_prod::value; + static constexpr size_t remaining_cost = pack_prod::value; + static constexpr size_t value = cost_tensor0*remaining_cost; + + using resulting_tensor = typename get_resuling_tensor,Index, + Tensor,Tensor>::type; + using resulting_index = typename get_resuling_index,Index, + Tensor,Tensor>::type; +}; + + +template +struct pair_flop_cost,Index<>,Tensor,Tensor,std_ext::index_sequence> { + static constexpr size_t value = pack_prod::value; +}; + +template +struct pair_flop_cost,Index,Tensor,Tensor,std_ext::index_sequence> { + static constexpr size_t value = pack_prod::value; +}; + +template +struct pair_flop_cost,Index<>,Tensor,Tensor,std_ext::index_sequence> { + static constexpr size_t value = 1; +}; +//------------------------------------------------------------------------------------------------------------// + + + +// Cost of triplet tensor contraction (single evaluation) +//------------------------------------------------------------------------------------------------------------// +template +struct single_evaluation_triplet_cost; + +template +struct single_evaluation_triplet_cost,Index,Index, + Tensor,Tensor,Tensor> { + + using concat_tensor_01 = typename no_of_loops_to_set,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::value>::type>::type; + + using concat_index_01 = typename no_of_loops_to_set,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::value>::type>::indices; + + static constexpr size_t value = pair_flop_cost,concat_tensor_01,Tensor, + typename std_ext::make_index_sequence::type>::value; +}; +//------------------------------------------------------------------------------------------------------------// + + + + +// Cost model for triplet tensor network contraction +//------------------------------------------------------------------------------------------------------------// +template +struct triplet_flop_cost; + +template +struct triplet_flop_cost,Index,Index, + Tensor,Tensor,Tensor> { + + // Perform depth-first search + //--------------------------------------------------------------------- + // first two tensors contracted first + using resulting_tensor_0 = typename get_resuling_tensor,Index, + Tensor,Tensor>::type; + using resulting_index_0 = typename get_resuling_index,Index, + Tensor,Tensor>::type; + + static constexpr size_t flop_count_01_0 = pair_flop_cost,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_01_1 = pair_flop_cost,resulting_tensor_0,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_01 = flop_count_01_0 + flop_count_01_1; + + + // first and last tensors contracted first + using resulting_tensor_1 = typename get_resuling_tensor,Index, + Tensor,Tensor>::type; + using resulting_index_1 = typename get_resuling_index,Index, + Tensor,Tensor>::type; + + static constexpr size_t flop_count_02_0 = pair_flop_cost,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_02_1 = pair_flop_cost,resulting_tensor_1,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_02 = flop_count_02_0 + flop_count_02_1; + + + // second and last tensors contracted first + using resulting_tensor_2 = typename get_resuling_tensor,Index, + Tensor,Tensor>::type; + using resulting_index_2 = typename get_resuling_index,Index, + Tensor,Tensor>::type; + + static constexpr size_t flop_count_12_0 = pair_flop_cost,Index,Tensor,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_12_1 = pair_flop_cost,resulting_tensor_2,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_12 = flop_count_12_0 + flop_count_12_1; + + static constexpr size_t flop_count_012 = single_evaluation_triplet_cost,Index,Index, + Tensor,Tensor,Tensor>::value; + + static constexpr size_t min_cost = meta_min::value; + static constexpr int which_variant = meta_argmin::value; + + // this is the overall resulting tensor and index from overall triplet contraction + using resulting_tensor = typename std::conditional< + which_variant==0, + typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type + >::type + >::type; + + using resulting_index = typename std::conditional< + which_variant==0, + typename get_resuling_index,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type + >::type + >::type; + +}; +//------------------------------------------------------------------------------------------------------------// + + + + + +// Cost model for quartet tensor network contraction +//------------------------------------------------------------------------------------------------------------// +template +struct quartet_flop_cost; + +template +struct quartet_flop_cost,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor> { + + // Perform depth-first search + //--------------------------------------------------------------------- + // first three tensors contracted first + using triplet_cost_012 = triplet_flop_cost,Index,Index, + Tensor,Tensor,Tensor>; + + using resulting_tensor_0 = typename triplet_cost_012::resulting_tensor; + using resulting_index_0 = typename triplet_cost_012::resulting_index; + + static constexpr size_t flop_count_012 = triplet_cost_012::min_cost; + static constexpr size_t flop_count_012_3 = pair_flop_cost,resulting_tensor_0,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_0 = flop_count_012 + flop_count_012_3; + + + // first, second and last tensors contracted first + using triplet_cost_013 = triplet_flop_cost,Index,Index, + Tensor,Tensor,Tensor>; + + using resulting_tensor_1 = typename triplet_cost_013::resulting_tensor; + using resulting_index_1 = typename triplet_cost_013::resulting_index; + + static constexpr size_t flop_count_013 = triplet_cost_013::min_cost; + static constexpr size_t flop_count_013_2 = pair_flop_cost,resulting_tensor_1,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_1 = flop_count_013 + flop_count_013_2; + + + // first, third and last tensors contracted first + using triplet_cost_023 = triplet_flop_cost,Index,Index, + Tensor,Tensor,Tensor>; + + using resulting_tensor_2 = typename triplet_cost_023::resulting_tensor; + using resulting_index_2 = typename triplet_cost_023::resulting_index; + + static constexpr size_t flop_count_023 = triplet_cost_023::min_cost; + static constexpr size_t flop_count_023_1 = pair_flop_cost,resulting_tensor_2,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_2 = flop_count_023 + flop_count_023_1; + + + // last three tensors contracted first + using triplet_cost_123 = triplet_flop_cost,Index,Index, + Tensor,Tensor,Tensor>; + + using resulting_tensor_3 = typename triplet_cost_123::resulting_tensor; + using resulting_index_3 = typename triplet_cost_123::resulting_index; + + static constexpr size_t flop_count_123 = triplet_cost_123::min_cost; + static constexpr size_t flop_count_123_0 = pair_flop_cost,resulting_tensor_3,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_3 = flop_count_123 + flop_count_123_0; + + static constexpr size_t min_cost = meta_min::value; + static constexpr int which_variant = meta_argmin::value; + + + // this is the overall resulting tensor and index from overall quartet contraction + using resulting_tensor = typename std::conditional< + which_variant==0, + typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename std::conditional< + which_variant==2, + typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type, + typename get_resuling_tensor,resulting_index_3,Tensor,resulting_tensor_3>::type + >::type + >::type + >::type; + + using resulting_index = typename std::conditional< + which_variant==0, + typename get_resuling_index,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename std::conditional< + which_variant==2, + typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type, + typename get_resuling_index,resulting_index_3,Tensor,resulting_tensor_3>::type + >::type + >::type + >::type; + +}; +//------------------------------------------------------------------------------------------------------------// + + + + + + +// Cost model for quintet tensor contraction +//------------------------------------------------------------------------------------------------------------// +template +struct quintet_flop_cost; + +template +struct quintet_flop_cost,Index,Index,Index, Index, + Tensor,Tensor,Tensor,Tensor,Tensor> { + + // Perform depth-first search + //--------------------------------------------------------------------- + // first four tensors contracted first + using quartet_cost_0123 = quartet_flop_cost,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_0 = typename quartet_cost_0123::resulting_tensor; + using resulting_index_0 = typename quartet_cost_0123::resulting_index; + + static constexpr size_t flop_count_0123 = quartet_cost_0123::min_cost; + static constexpr size_t flop_count_0123_4 = pair_flop_cost,resulting_tensor_0,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_0 = flop_count_0123 + flop_count_0123_4; + + + // 1st, 2nd, 3rd, 5th tensors contracted first + using quartet_cost_0124 = quartet_flop_cost,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_1 = typename quartet_cost_0124::resulting_tensor; + using resulting_index_1 = typename quartet_cost_0124::resulting_index; + + static constexpr size_t flop_count_0124 = quartet_cost_0124::min_cost; + static constexpr size_t flop_count_0124_3 = pair_flop_cost,resulting_tensor_1,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_1 = flop_count_0124 + flop_count_0124_3; + + + // 1st, 2nd, 4th, 5th tensors contracted first + using quartet_cost_0134 = quartet_flop_cost,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_2 = typename quartet_cost_0134::resulting_tensor; + using resulting_index_2 = typename quartet_cost_0134::resulting_index; + + static constexpr size_t flop_count_0134 = quartet_cost_0134::min_cost; + static constexpr size_t flop_count_0134_2 = pair_flop_cost,resulting_tensor_2,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_2 = flop_count_0134 + flop_count_0134_2; + + + // 1st, 3rd, 4th, 5th tensors contracted first + using quartet_cost_0234 = quartet_flop_cost,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_3 = typename quartet_cost_0234::resulting_tensor; + using resulting_index_3 = typename quartet_cost_0234::resulting_index; + + static constexpr size_t flop_count_0234 = quartet_cost_0234::min_cost; + static constexpr size_t flop_count_0234_1 = pair_flop_cost,resulting_tensor_3,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_3 = flop_count_0234 + flop_count_0234_1; + + + // last four tensors contracted first + using quartet_cost_1234 = quartet_flop_cost,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_4 = typename quartet_cost_1234::resulting_tensor; + using resulting_index_4 = typename quartet_cost_1234::resulting_index; + + static constexpr size_t flop_count_1234 = quartet_cost_1234::min_cost; + static constexpr size_t flop_count_1234_0 = pair_flop_cost,resulting_tensor_4,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_4 = flop_count_1234 + flop_count_1234_0; + + + static constexpr size_t min_cost = meta_min::value; + static constexpr int which_variant = meta_argmin::value; + + + // this is the overall resulting tensor and index from overall quintet contraction + using resulting_tensor = typename std::conditional< + which_variant==0, + typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename std::conditional< + which_variant==2, + typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type, + typename std::conditional< + which_variant==3, + typename get_resuling_tensor,resulting_index_3,Tensor,resulting_tensor_3>::type, + typename get_resuling_tensor,resulting_index_4,Tensor,resulting_tensor_4>::type + >::type + >::type + >::type + >::type; + + using resulting_index = typename std::conditional< + which_variant==0, + typename get_resuling_index,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename std::conditional< + which_variant==2, + typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type, + typename std::conditional< + which_variant==3, + typename get_resuling_index,resulting_index_3,Tensor,resulting_tensor_3>::type, + typename get_resuling_index,resulting_index_4,Tensor,resulting_tensor_4>::type + >::type + >::type + >::type + >::type; + +}; +//------------------------------------------------------------------------------------------------------------// + + + + +// Cost model for sixtet tensor contraction +//------------------------------------------------------------------------------------------------------------// +template +struct sixtet_flop_cost; + +template +struct sixtet_flop_cost,Index,Index,Index, Index, Index, + Tensor,Tensor,Tensor,Tensor,Tensor,Tensor> { + + // Perform depth-first search + //--------------------------------------------------------------------- + // first 5 tensors contracted first + using quintet_cost_01234 = quintet_flop_cost,Index,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_0 = typename quintet_cost_01234::resulting_tensor; + using resulting_index_0 = typename quintet_cost_01234::resulting_index; + + static constexpr size_t flop_count_01234 = quintet_cost_01234::min_cost; + static constexpr size_t flop_count_01234_5 = pair_flop_cost,resulting_tensor_0,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_0 = flop_count_01234 + flop_count_01234_5; + + + // 1st, 2nd, 3rd, 4th, 6th tensors contracted first + using quintet_cost_01235 = quintet_flop_cost,Index,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_1 = typename quintet_cost_01235::resulting_tensor; + using resulting_index_1 = typename quintet_cost_01235::resulting_index; + + static constexpr size_t flop_count_01235 = quintet_cost_01235::min_cost; + static constexpr size_t flop_count_01235_4 = pair_flop_cost,resulting_tensor_1,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_1 = flop_count_01235 + flop_count_01235_4; + + + // 1st, 2nd, 3rd, 5th, 6th tensors contracted first + using quintet_cost_01245 = quintet_flop_cost,Index,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_2 = typename quintet_cost_01245::resulting_tensor; + using resulting_index_2 = typename quintet_cost_01245::resulting_index; + + static constexpr size_t flop_count_01245 = quintet_cost_01235::min_cost; + static constexpr size_t flop_count_01245_3 = pair_flop_cost,resulting_tensor_2,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_2 = flop_count_01245 + flop_count_01245_3; + + + // 1st, 2nd, 4th, 5th, 6th tensors contracted first + using quintet_cost_01345 = quintet_flop_cost,Index,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_3 = typename quintet_cost_01345::resulting_tensor; + using resulting_index_3 = typename quintet_cost_01345::resulting_index; + + static constexpr size_t flop_count_01345 = quintet_cost_01235::min_cost; + static constexpr size_t flop_count_01345_2 = pair_flop_cost,resulting_tensor_3,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_3 = flop_count_01345 + flop_count_01345_2; + + + // 1st, 3rd, 4th, 5th, 6th tensors contracted first + using quintet_cost_02345 = quintet_flop_cost,Index,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_4 = typename quintet_cost_02345::resulting_tensor; + using resulting_index_4 = typename quintet_cost_02345::resulting_index; + + static constexpr size_t flop_count_02345 = quintet_cost_01235::min_cost; + static constexpr size_t flop_count_02345_1 = pair_flop_cost,resulting_tensor_4,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_4 = flop_count_02345 + flop_count_02345_1; + + + // last 5 tensors contracted first + using quintet_cost_12345 = quintet_flop_cost,Index,Index,Index,Index, + Tensor,Tensor,Tensor,Tensor,Tensor>; + + using resulting_tensor_5 = typename quintet_cost_12345::resulting_tensor; + using resulting_index_5 = typename quintet_cost_12345::resulting_index; + + static constexpr size_t flop_count_12345 = quintet_cost_01235::min_cost; + static constexpr size_t flop_count_12345_0 = pair_flop_cost,resulting_tensor_5,Tensor, + typename std_ext::make_index_sequence::type>::value; + + static constexpr size_t flop_count_5 = flop_count_12345 + flop_count_12345_0; + + + static constexpr size_t min_cost = meta_min::value; + static constexpr int which_variant = meta_argmin::value; + + + // this is the overall resulting tensor and index from overall sextet contraction + using resulting_tensor = typename std::conditional< + which_variant==0, + typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename std::conditional< + which_variant==2, + typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type, + typename std::conditional< + which_variant==3, + typename get_resuling_tensor,resulting_index_3,Tensor,resulting_tensor_3>::type, + typename std::conditional< + which_variant==4, + typename get_resuling_tensor,resulting_index_4,Tensor,resulting_tensor_4>::type, + typename get_resuling_tensor,resulting_index_5,Tensor,resulting_tensor_5>::type + >::type + >::type + >::type + >::type + >::type; + + using resulting_index = typename std::conditional< + which_variant==0, + typename get_resuling_index,resulting_tensor_0,Tensor>::type, + typename std::conditional< + which_variant==1, + typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, + typename std::conditional< + which_variant==2, + typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type, + typename std::conditional< + which_variant==3, + typename get_resuling_index,resulting_index_3,Tensor,resulting_tensor_3>::type, + typename std::conditional< + which_variant==4, + typename get_resuling_index,resulting_index_4,Tensor,resulting_tensor_4>::type, + typename get_resuling_index,resulting_index_5,Tensor,resulting_tensor_5>::type + >::type + >::type + >::type + >::type + >::type; + +}; +//------------------------------------------------------------------------------------------------------------// + + + + + +// einsum helper to extract the resulting index and the resulting tensor +//------------------------------------------------------------------------------------------------------------// +template +struct einsum_helper; + +template +struct einsum_helper> { + using resulting_index = typename contraction_impl, + typename std_ext::make_index_sequence::type>::indices; + using resulting_tensor = typename contraction_impl, + typename std_ext::make_index_sequence::type>::type; +}; + +template +struct einsum_helper { + using resulting_index = typename get_resuling_index::type; + using resulting_tensor = typename get_resuling_tensor::type; +}; + +template +struct einsum_helper { + using cost_model = triplet_flop_cost; + using resulting_index = typename cost_model::resulting_index; + using resulting_tensor = typename cost_model::resulting_tensor; +}; + +template +struct einsum_helper { + using cost_model = quartet_flop_cost; + using resulting_index = typename cost_model::resulting_index; + using resulting_tensor = typename cost_model::resulting_tensor; +}; + +template +struct einsum_helper { + using cost_model = quintet_flop_cost; + using resulting_index = typename cost_model::resulting_index; + using resulting_tensor = typename cost_model::resulting_tensor; +}; + +template +struct einsum_helper { + using cost_model = sixtet_flop_cost; + using resulting_index = typename cost_model::resulting_index; + using resulting_tensor = typename cost_model::resulting_tensor; +}; +//------------------------------------------------------------------------------------------------------------// + + +} // end of namespace Fastor + + +#endif // FASTOR_DONT_PERFORM_OP_MIN + + +#endif // OPMIN_META_H diff --git a/noarch/include/Fastor/meta/tensor_meta.h b/noarch/include/Fastor/meta/tensor_meta.h new file mode 100644 index 00000000..5f86c674 --- /dev/null +++ b/noarch/include/Fastor/meta/tensor_meta.h @@ -0,0 +1,209 @@ +#ifndef TENSOR_META_H +#define TENSOR_META_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" + +namespace Fastor { + +//----------------------------------------------------------------------------------------------------------// +// UpLoType +namespace UpLoType { +struct General {}; +struct Lower {}; +struct UniLower {}; +struct StrictlyLower {}; +struct Upper {}; +struct UniUpper {}; +struct StrictlyUpper {}; +struct Diagonal {}; +struct BiDiagonal {}; +struct TriDiagonal {}; +struct BlockDiagonal {}; +struct Symmetric {}; +struct SymmetricPositiveDefinite {}; +} +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template +struct is_unique : std::integral_constant {}; + +template +struct is_unique : std::integral_constant::value> {}; + +template +struct no_of_unique : std::integral_constant {}; + +template +struct no_of_unique : std::integral_constant::value + no_of_unique::value> {}; + + + +//----------------------------------------------------------------------------------------------------------// +// Note that Intel's ICC 2017 does not support make_index_sequence and the following +// version also seems faster than c++14's built-in (for clang) on Linux systems +namespace std_ext // back port to c++11 +{ + template + struct index_sequence + { + using type = index_sequence; + using value_type = size_t; + static constexpr std::size_t size() { return sizeof...(Ints); } + }; + + // -------------------------------------------------------------- + + template + struct _merge_and_renumber; + + template + struct _merge_and_renumber, index_sequence> + : index_sequence + { }; + + // -------------------------------------------------------------- + + template + struct make_index_sequence + : _merge_and_renumber::type, + typename make_index_sequence::type> + { }; + + template<> struct make_index_sequence<0> : index_sequence<> { }; + template<> struct make_index_sequence<1> : index_sequence<0> { }; +} +//----------------------------------------------------------------------------------------------------------// + + + + +//----------------------------------------------------------------------------------------------------------// +template struct typelist { }; +template +using typelist_c = typelist...>; + +template struct concat; +template <> struct concat<> { using type = typelist<>; }; +template struct concat> { using type = typelist; }; +template +struct concat, typelist, Args...> : concat, Args...> { }; + +template struct filter_out; +template struct filter_out> + : concat< conditional_t_, typelist<>, typelist>...> { }; + +template +using filter_out_t = typename filter_out::type; + + + +template +struct uniq; + +template +using uniq_t = typename uniq::type; + +template <> +struct uniq> { + using type = typelist<>; +}; + +template +struct uniq> + : concat, uniq_t>>> +{ }; + +template +using size_t_ = std::integral_constant; + +template struct length; +template using length_t = typename length::type; +template +struct length> +: size_t_ +{ }; + +template +using no_of_uniques = length_t...>>>; +//----------------------------------------------------------------------------------------------------------// + + +//////////////// +template class Z> +struct quote_c { + template + using apply = Z; +}; + +template +struct apply_typelist; + +template +using apply_typelist_t = typename apply_typelist::type; + + +template +struct apply_typelist> { + using type = typename MFC::template apply; +}; +//----------------------------------------------------------------------------------------------------------// + + + +// Check if indices appear more than twice [for Einstein summation] +//----------------------------------------------------------------------------------------------------------// +namespace useless { + +template +constexpr const T& ct_max(T const& t1, T const& t2) { + return t1 < t2 ? t2 : t1; +} + + +template +struct count__; + +template +struct count__ { + static constexpr size_t value = std::integral_constant::value; +}; + +template +struct count__ { + static constexpr size_t value = std::integral_constant::value>::value; +}; + +template +struct count__ { + static constexpr size_t value = count__::value; +}; + +template +struct max_count; + +template <> +struct max_count<> { + static constexpr size_t value = std::integral_constant::value; +}; + +template +struct max_count { + static constexpr size_t value = std::integral_constant::value, + max_count::value)>::value; +}; + +} // useless + +template +struct no_more_than_two { + static constexpr size_t value = std::integral_constant::value <= 2>::value; +}; +//----------------------------------------------------------------------------------------------------------// + + +} + +#endif // TENSOR_META_H diff --git a/noarch/include/Fastor/simd_math/simd_math.h b/noarch/include/Fastor/simd_math/simd_math.h new file mode 100644 index 00000000..9ac07e04 --- /dev/null +++ b/noarch/include/Fastor/simd_math/simd_math.h @@ -0,0 +1,549 @@ +#ifndef SIMD_MATH_H +#define SIMD_MATH_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include + +// SHUT GCC6 -Wignored-attributes WARNINGS +#ifdef __GNUC__ +#if __GNUC__==6 +#pragma GCC diagnostic ignored "-Wignored-attributes" +#endif +#endif + +namespace Fastor { + +// minimum +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::min(((T*)&a)[i],((T*)&b)[i]); } + return out; +} +template +FASTOR_INLINE SIMDVector min(const SIMDVector &a, T b) { + return min(a,SIMDVector(b)); +} +template +FASTOR_INLINE SIMDVector min(T a, const SIMDVector &b) { + return min(SIMDVector(a),b); +} +#ifdef FASTOR_SSE2_IMPL +#ifdef FASTOR_SSE4_1_IMPL +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm_min_epi32(a.value,b.value); +} +#endif +#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm_min_epi64(a.value,b.value); +} +#endif +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm_min_ps(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm_min_pd(a.value,b.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +#ifdef FASTOR_AVX2_IMPL +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm256_min_epi32(a.value,b.value); +} +#endif +#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm256_min_epi64(a.value,b.value); +} +#endif +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm256_min_ps(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm256_min_pd(a.value,b.value); +} +#endif +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm512_min_epi32(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm512_min_epi64(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm512_min_ps(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { + return _mm512_min_pd(a.value,b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// maximum +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::max(((T*)&a)[i],((T*)&b)[i]); } + return out; +} +template +FASTOR_INLINE SIMDVector max(const SIMDVector &a, T b) { + return max(a,SIMDVector(b)); +} +template +FASTOR_INLINE SIMDVector max(T a, const SIMDVector &b) { + return max(SIMDVector(a),b); +} +#ifdef FASTOR_SSE2_IMPL +#ifdef FASTOR_SSE4_1_IMPL +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm_max_epi32(a.value,b.value); +} +#endif +#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm_max_epi64(a.value,b.value); +} +#endif +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm_max_ps(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm_max_pd(a.value,b.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +#ifdef FASTOR_AVX2_IMPL +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm256_max_epi32(a.value,b.value); +} +#endif +#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm256_max_epi64(a.value,b.value); +} +#endif +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm256_max_ps(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm256_max_pd(a.value,b.value); +} +#endif +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm512_max_epi32(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm512_max_epi64(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm512_max_ps(a.value,b.value); +} +template<> +FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { + return _mm512_max_pd(a.value,b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// ceil +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::ceil(((T*)&a)[i]);} + return out; +} +#ifdef FASTOR_SSE4_1_IMPL +template<> +FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { + return _mm_ceil_ps(a.value); +} +template<> +FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { + return _mm_ceil_pd(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { + return _mm256_ceil_ps(a.value); +} +template<> +FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { + return _mm256_ceil_pd(a.value); +} +#endif +// Part of SVML +// #ifdef FASTOR_AVX512F_IMPL +// template<> +// FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { +// return _mm512_ceil_ps(a.value); +// } +// template<> +// FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { +// return _mm512_ceil_pd(a.value); +// } +// #endif +//----------------------------------------------------------------------------------------------------------// + + +// round +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector round(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::round(((T*)&a)[i]);} + return out; +} +#ifdef FASTOR_SSE4_1_IMPL +template<> +FASTOR_INLINE SIMDVector round(const SIMDVector &a) { + return _mm_round_ps(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); +} +template<> +FASTOR_INLINE SIMDVector round(const SIMDVector &a) { + return _mm_round_pd(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector round(const SIMDVector &a) { + return _mm256_round_ps(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); +} +template<> +FASTOR_INLINE SIMDVector round(const SIMDVector &a) { + return _mm256_round_pd(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// floor +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::floor(((T*)&a)[i]);} + return out; +} +#ifdef FASTOR_SSE4_1_IMPL +template<> +FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { + return _mm_floor_ps(a.value); +} +template<> +FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { + return _mm_floor_pd(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { + return _mm256_floor_ps(a.value); +} +template<> +FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { + return _mm256_floor_pd(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + + +// remaining math functions from STL +//----------------------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------------------// + +template +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::exp(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::exp2(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::expm1(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log10(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log2(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log1p(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i = 0; i < SIMDVector::Size; i++) { ((T*)&out)[i] = std::pow(((T*)&a)[i], ((T*)&b)[i]);} + return out; +} +template +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, T b) { + return pow(a, SIMDVector(b)); +} +template +FASTOR_INLINE SIMDVector pow(T a, const SIMDVector &b) { + return pow(SIMDVector(a),b); +} + +template +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::cbrt(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::sin(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::cos(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::tan(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::asin(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::acos(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::atan(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i = 0; i < SIMDVector::Size; i++) { ((T*)&out)[i] = std::atan2(((T*)&a)[i], ((T*)&b)[i]);} + return out; +} +template +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, T b) { + return atan2(a, SIMDVector(b)); +} +template +FASTOR_INLINE SIMDVector atan2(T a, const SIMDVector &b) { + return atan2(SIMDVector(a),b); +} + +template +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::sinh(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::cosh(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::tanh(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::asinh(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::acosh(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::atanh(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::erf(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::tgamma(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::lgamma(((T*)&a)[i]);} + return out; +} + +template +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i = 0; i < SIMDVector::Size; i++) { ((T*)&out)[i] = std::hypot(((T*)&a)[i], ((T*)&b)[i]);} + return out; +} +template +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, T b) { + return hypot(a, SIMDVector(b)); +} +template +FASTOR_INLINE SIMDVector hypot(T a, const SIMDVector &b) { + return hypot(SIMDVector(a),b); +} + +template +FASTOR_INLINE SIMDVector trunc(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::trunc(((T*)&a)[i]);} + return out; +} +//----------------------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------------------// + + + +// Boolean arithmetic +// ! or not +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector::Size>> operator!(const SIMDVector &a) { + SIMDVector::Size>> out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = !(((T*)&a)[i]); } + return out; +} +//----------------------------------------------------------------------------------------------------------// + + +// isinf/nan/finite +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector::Size>> isinf(const SIMDVector &a) { + SIMDVector::Size>> out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = std::isinf(((T*)&a)[i]); } + return out; +} + +template +FASTOR_INLINE SIMDVector::Size>> isnan(const SIMDVector &a) { + SIMDVector::Size>> out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = std::isnan(((T*)&a)[i]); } + return out; +} + +template +FASTOR_INLINE SIMDVector::Size>> isfinite(const SIMDVector &a) { + SIMDVector::Size>> out; + for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = std::isfinite(((T*)&a)[i]); } + return out; +} +//----------------------------------------------------------------------------------------------------------// + + +} // end of namespace Fastor + + +// Include all backends +#include "Fastor/simd_math/sleef_backend.h" + +#endif // SIMD_MATH_H diff --git a/noarch/include/Fastor/simd_math/sleef_backend.h b/noarch/include/Fastor/simd_math/sleef_backend.h new file mode 100644 index 00000000..87378652 --- /dev/null +++ b/noarch/include/Fastor/simd_math/sleef_backend.h @@ -0,0 +1,7 @@ +#ifndef SLEEF_BACKEND_H +#define SLEEF_BACKEND_H + +#include "Fastor/simd_math/sleef_backend_u10.h" +#include "Fastor/simd_math/sleef_backend_u35.h" + +#endif // SLEEF_BACKEND_H diff --git a/noarch/include/Fastor/simd_math/sleef_backend_u10.h b/noarch/include/Fastor/simd_math/sleef_backend_u10.h new file mode 100644 index 00000000..98f68009 --- /dev/null +++ b/noarch/include/Fastor/simd_math/sleef_backend_u10.h @@ -0,0 +1,932 @@ +#ifndef SLEEF_BACKEND_U10_H +#define SLEEF_BACKEND_U10_H + +#if defined(FASTOR_USE_SLEEF_U10) || defined(FASTOR_USE_SLEEF) + +#include + +namespace Fastor { + +// exp +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// exp2 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2f4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2d2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2f8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2d4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2f16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2d8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// expm1 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1f4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1d2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1f8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1d4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1f16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1d8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log10 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10f4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10d2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10f8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10d4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10f16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10d8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log2 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2f4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2d2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2f8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2d4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2f16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2d8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log1p +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// pow - other variants are automatically taken care off through the generic overloads +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powf4_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powd2_u10(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powf8_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powd4_u10(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powf16_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powd8_u10(a.value, b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// cbrt +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// sin +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sinf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sind2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sinf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sind4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sinf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sind8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// cos +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// tan +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tanf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tand2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tanf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tand4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tanf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tand8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// asin +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asinf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asind2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asinf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asind4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asinf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asind8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// acos +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// atan +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atanf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atand2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atanf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atand4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atanf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atand8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// atan2 - other variants are automatically taken care off through the generic overloads +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2f4_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2d2_u10(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2f8_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2d4_u10(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2f16_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2d8_u10(a.value, b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// sinh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// cosh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// tanh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// asinh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// acosh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// atanh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// erf +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erff4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erfd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erff8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erfd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erff16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erfd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// tgamma +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammaf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammad2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammaf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammad4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammaf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammad8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// lgamma +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammaf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammad2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammaf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammad4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammaf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammad8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// hypot - other variants are automatically taken care off through the generic overloads +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf4_u05(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd2_u05(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX2_IMPL +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf8_u05avx2(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd4_u05avx2(a.value, b.value); +} +#elif defined(FASTOR_AVX_IMPL) +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf8_u05avx(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd4_u05avx(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf16_u05avx512f(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd8_u05avx512f(a.value, b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + +} // end of namespace Fastor + +#endif // FASTOR_USE_SLEEF_U10 + +#endif // SLEEF_BACKEND_U10_H diff --git a/noarch/include/Fastor/simd_math/sleef_backend_u35.h b/noarch/include/Fastor/simd_math/sleef_backend_u35.h new file mode 100644 index 00000000..d337e01c --- /dev/null +++ b/noarch/include/Fastor/simd_math/sleef_backend_u35.h @@ -0,0 +1,933 @@ +#ifndef SLEEF_BACKEND_U35_H +#define SLEEF_BACKEND_U35_H + +// Not all functions have u35 variant implemented, so falling back to u10 in such cases +#ifdef FASTOR_USE_SLEEF_U35 + +#include + +namespace Fastor { + +// exp +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { + return Sleef_expd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// exp2 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2f4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2d2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2f8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2d4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2f16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { + return Sleef_exp2d8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// expm1 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1f4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1d2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1f8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1d4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1f16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { + return Sleef_expm1d8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logd2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logd4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector log(const SIMDVector &a) { + return Sleef_logd8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log10 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10f4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10d2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10f8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10d4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10f16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { + return Sleef_log10d8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log2 +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2f4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2d2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2f8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2d4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2f16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { + return Sleef_log2d8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// log1p +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { + return Sleef_log1pd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// pow - other variants are automatically taken care off through the generic overloads +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powf4_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powd2_u10(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powf8_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powd4_u10(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powf16_u10(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { + return Sleef_powd8_u10(a.value, b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// cbrt +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtd2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtd4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { + return Sleef_cbrtd8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// sin +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sinf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sind2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sinf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sind4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sinf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { + return Sleef_sind8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// cos +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosd2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosd4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { + return Sleef_cosd8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// tan +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tanf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tand2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tanf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tand4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tanf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { + return Sleef_tand8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// asin +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asinf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asind2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asinf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asind4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asinf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { + return Sleef_asind8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// acos +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosd2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosd4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { + return Sleef_acosd8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// atan +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atanf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atand2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atanf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atand4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atanf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { + return Sleef_atand8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// atan2 - other variants are automatically taken care off through the generic overloads +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2f4_u35(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2d2_u35(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2f8_u35(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2d4_u35(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2f16_u35(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { + return Sleef_atan2d8_u35(a.value, b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// sinh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhd2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhd4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { + return Sleef_sinhd8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// cosh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshd2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshd4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { + return Sleef_coshd8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// tanh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhf4_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhd2_u35(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhf8_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhd4_u35(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhf16_u35(a.value); +} +template<> +FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { + return Sleef_tanhd8_u35(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// asinh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { + return Sleef_asinhd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// acosh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { + return Sleef_acoshd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// atanh +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { + return Sleef_atanhd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// erf +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erff4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erfd2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erff8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erfd4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erff16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { + return Sleef_erfd8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// tgamma +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammaf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammad2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammaf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammad4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammaf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { + return Sleef_tgammad8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// lgamma +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammaf4_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammad2_u10(a.value); +} +#endif +#ifdef FASTOR_AVX_IMPL +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammaf8_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammad4_u10(a.value); +} +#endif +#ifdef FASTOR_AVX512_IMPL +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammaf16_u10(a.value); +} +template<> +FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { + return Sleef_lgammad8_u10(a.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + + +// hypot - other variants are automatically taken care off through the generic overloads +//----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf4_u05(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd2_u05(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX2_IMPL +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf8_u05avx2(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd4_u05avx2(a.value, b.value); +} +#elif defined(FASTOR_AVX_IMPL) +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf8_u05avx(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd4_u05avx(a.value, b.value); +} +#endif +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotf16_u05avx512f(a.value, b.value); +} +template<> +FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { + return Sleef_hypotd8_u05avx512f(a.value, b.value); +} +#endif +//----------------------------------------------------------------------------------------------------------// + +} // end of namespace Fastor + +#endif // FASTOR_USE_SLEEF_U35 + +#endif // FASTOR_USE_SLEEF_U35 diff --git a/noarch/include/Fastor/simd_vector/SIMDVector.h b/noarch/include/Fastor/simd_vector/SIMDVector.h new file mode 100644 index 00000000..a4b8e53d --- /dev/null +++ b/noarch/include/Fastor/simd_vector/SIMDVector.h @@ -0,0 +1,16 @@ +#ifndef SIMDVECTOR_H +#define SIMDVECTOR_H + +#include "Fastor/simd_vector/simd_vector_base.h" +#include "Fastor/simd_vector/simd_vector_scalar.h" +#include "Fastor/simd_vector/simd_vector_float.h" +#include "Fastor/simd_vector/simd_vector_double.h" +#include "Fastor/simd_vector/simd_vector_int32.h" +#include "Fastor/simd_vector/simd_vector_int64.h" +#include "Fastor/simd_vector/simd_vector_complex_scalar.h" +#include "Fastor/simd_vector/simd_vector_complex_float.h" +#include "Fastor/simd_vector/simd_vector_complex_double.h" +#include "Fastor/simd_vector/simd_vector_common.h" + +#endif // SIMDVECTOR_H + diff --git a/noarch/include/Fastor/simd_vector/extintrin.h b/noarch/include/Fastor/simd_vector/extintrin.h new file mode 100644 index 00000000..0a19c73f --- /dev/null +++ b/noarch/include/Fastor/simd_vector/extintrin.h @@ -0,0 +1,1255 @@ +#ifndef EXT_INTRIN_H +#define EXT_INTRIN_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include + + +namespace Fastor { + +// Macros for immediate construction +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +#define ZEROPS (_mm_set1_ps(0.f)) +#define ZEROPD (_mm_set1_pd(0.0)) +#define MZEROPS (_mm_set1_ps(-0.f)) +#define MZEROPD (_mm_set1_pd(-0.0)) +#define ONEPS (_mm_set1_ps(1.f)) +#define ONEPD (_mm_set1_pd(1.0)) +#define HALFPS (_mm_set1_ps(0.5f)) +#define HALFPD (_mm_set1_pd(0.5)) +#define TWOPS (_mm_set1_ps(2.0f)) +#define TOWPD (_mm_set1_pd(2.0)) +#endif +#ifdef FASTOR_AVX_IMPL +#define VZEROPS (_mm256_set1_ps(0.f)) +#define VZEROPD (_mm256_set1_pd(0.0)) +#define MVZEROPS (_mm256_set1_ps(-0.f)) +#define MVZEROPD (_mm256_set1_pd(-0.0)) +#define VONEPS (_mm256_set1_ps(1.f)) +#define VONEPD (_mm256_set1_pd(1.0)) +#define VHALFPS (_mm256_set1_ps(0.5f)) +#define VHALFPD (_mm256_set1_pd(0.5)) +#define VTWOPS (_mm256_set1_ps(2.0f)) +#define VTOWPD (_mm256_set1_pd(2.0)) +#endif +//----------------------------------------------------------------------------------------------------------------// + + +// Mask load the 3 lower parts +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE __m128 _mm_loadl3_ps(const float *arr) { +#ifdef FASTOR_HAS_AVX512_MASKS + return _mm_mask_load_ps(ZEROPS, (__mmask8)0x07, arr); +#elif defined(FASTOR_AVX_IMPL) + __m128i mask = _mm_set_epi32(0,-1,-1,-1); + return _mm_maskload_ps(arr,(__m128i) mask); +#else + __m128i xy = _mm_loadl_epi64((const __m128i*)arr); + __m128 z = _mm_load_ss(&arr[2]); + return _mm_movelh_ps(_mm_castsi128_ps(xy), z); +#endif +} + +FASTOR_INLINE __m128 _mm_loadul3_ps(const float *arr) { +#ifdef FASTOR_HAS_AVX512_MASKS + return _mm_mask_loadu_ps(ZEROPS, (__mmask8)0x07, arr); +#elif defined(FASTOR_AVX_IMPL) + // AVX maskloads apparently have no alignment requirement + __m128i mask = _mm_set_epi32(0,-1,-1,-1); + return _mm_maskload_ps(arr,(__m128i) mask); +#else + __m128 x = _mm_load_ss( arr ); + __m128 y = _mm_load_ss(&arr[1]); + __m128 z = _mm_load_ss(&arr[2]); + __m128 xy = _mm_movelh_ps(x, y); + return _mm_shuffle_ps(xy, z, _MM_SHUFFLE(2, 0, 2, 0)); +#endif +} +#endif + +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE __m256d _mm256_loadl3_pd(const double *arr) { +#ifdef FASTOR_HAS_AVX512_MASKS + return _mm256_mask_load_pd(VZEROPD, (__mmask8)0x07, arr); +#else + __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); + return _mm256_maskload_pd(arr,(__m256i) mask); +#endif +} + +FASTOR_INLINE __m256d _mm256_loadul3_pd(const double *arr) { +#ifdef FASTOR_HAS_AVX512_MASKS + return _mm256_mask_loadu_pd(VZEROPD, (__mmask8)0x07, arr); +#else + // AVX maskloads apparently have no alignment requirement + __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); + return _mm256_maskload_pd(arr,(__m256i) mask); + // __m128d xy = _mm_loadu_pd(arr); + // __m128d z = _mm_load_sd(&arr[2]); + // __m256d vec = _mm256_castpd128_pd256(xy); + // return _mm256_insertf128_pd(vec, z,0x1); +#endif +} +#endif + +// Mask store the 3 lower parts +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE void _mm_storel3_ps(float *arr, __m128 value) { +#ifdef FASTOR_HAS_AVX512_MASKS + _mm_mask_store_ps(arr, (__mmask8)0x07, value); +#elif defined(FASTOR_AVX_IMPL) + __m128i mask = _mm_set_epi32(0,-1,-1,-1); + _mm_maskstore_ps(arr, (__m128i)mask, value); +#else + _mm_storel_pi((__m64*)arr, value); + _mm_store_ss(&arr[2],_mm_shuffle_ps(value,value,0x2)); +#endif +} + +FASTOR_INLINE void _mm_storeul3_ps(float *arr, __m128 value) { +#ifdef FASTOR_HAS_AVX512_MASKS + _mm_mask_storeu_ps(arr, (__mmask8)0x07, value); +#elif defined(FASTOR_AVX_IMPL) + __m128i mask = _mm_set_epi32(0,-1,-1,-1); + _mm_maskstore_ps(arr, (__m128i)mask, value); +#else + _mm_storel_pi((__m64*)arr, value); + _mm_store_ss(&arr[2],_mm_shuffle_ps(value,value,0x2)); +#endif +} +#endif + +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE void _mm256_storel3_pd(double *arr, __m256d value) { +#ifdef FASTOR_HAS_AVX512_MASKS + _mm256_mask_store_pd(arr, (__mmask8)0x07, value); +#else + __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); + _mm256_maskstore_pd(arr, (__m256i)mask, value); +#endif +} + +FASTOR_INLINE void _mm256_storeul3_pd(double *arr, __m256d value) { +#ifdef FASTOR_HAS_AVX512_MASKS + _mm256_mask_storeu_pd(arr, (__mmask8)0x07, value); +#else + // AVX maskloads apparently have no alignment requirement + __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); + _mm256_maskstore_pd(arr, (__m256i)mask, value); + // _mm_storeu_pd(arr , _mm256_castpd256_pd128(value) ); + // _mm_store_sd (arr+2, _mm256_extractf128_pd (value,0x1)); +#endif +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + + +//! Horizontal summation/multiplication of registers +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE int _mm_sum_epi32(__m128i a) { + // W/O HADD: IVY 5 - HW 5 - SKY 6 + // __m128i c = _mm_hadd_epi32(a,a); // SSSE3 one extra op + __m128i c = _mm_add_epi32(a,_mm_shuffle_epi32(a,_MM_SHUFFLE(2,3,0,1))); + __m128i d = _mm_add_epi32(c,_mm_shuffle_epi32(c,_MM_SHUFFLE(0,1,2,3))); + return _mm_cvtsi128_si32(d); +} + +FASTOR_INLINE int _mm_prod_epi32(__m128i a) { + // IVY 13 - HW 13 - SKY 12 + __m128i c = _mm_mul_epu32(a,_mm_shuffle_epi32(a,_MM_SHUFFLE(2,3,0,1))); + __m128i d = _mm_mul_epu32(c,_mm_shuffle_epi32(c,_MM_SHUFFLE(2,2,2,2))); + return _mm_cvtsi128_si32(d); +} +#endif + +#ifdef FASTOR_USE_HADD +#ifdef FASTOR_SSSE3_IMPL +FASTOR_INLINE float _mm_sum_ps(__m128 a) { + // 10 OPS + float sum32; + __m128 sum = _mm_hadd_ps(a, a); + _mm_store_ss(&sum32,_mm_hadd_ps(sum, sum)); + return sum32; +} +FASTOR_INLINE double _mm_sum_pd(__m128d a) { + // 5 OPS + double sum64; + _mm_store_sd(&sum64,_mm_hadd_pd(a, a)); + return sum64; +} +#endif +#else +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE float _mm_sum_ps(__m128 a) { + // 8 OPS +#ifdef FASTOR_SSE3_IMPL + __m128 shuf = _mm_movehdup_ps(a); +#else + __m128 shuf = _mm_shuffle_ps(a,a, _MM_SHUFFLE(3,3,1,1)); +#endif + __m128 sums = _mm_add_ps(a, shuf); + shuf = _mm_movehl_ps(shuf, sums); + sums = _mm_add_ss(sums, shuf); + return _mm_cvtss_f32(sums); +} +FASTOR_INLINE double _mm_sum_pd(__m128d a) { + // 4 OPS + __m128 shuftmp= _mm_movehl_ps(ZEROPS, _mm_castpd_ps(a)); + __m128d shuf = _mm_castps_pd(shuftmp); + return _mm_cvtsd_f64(_mm_add_sd(a, shuf)); +} + +FASTOR_INLINE float _mm_prod_ps(__m128 a) { + // 12 OPS +#ifdef FASTOR_SSE3_IMPL + __m128 shuf = _mm_movehdup_ps(a); +#else + __m128 shuf = _mm_shuffle_ps(a,a, _MM_SHUFFLE(3,3,1,1)); +#endif + __m128 prods = _mm_mul_ps(a, shuf); + shuf = _mm_movehl_ps(shuf, prods); + prods = _mm_mul_ss(prods, shuf); + return _mm_cvtss_f32(prods); +} +FASTOR_INLINE double _mm_prod_pd(__m128d a) { + // 6 OPS + __m128 shuftmp= _mm_movehl_ps(ZEROPS, _mm_castpd_ps(a)); + __m128d shuf = _mm_castps_pd(shuftmp); + return _mm_cvtsd_f64(_mm_mul_sd(a, shuf)); +} +#endif +#endif + +#ifdef FASTOR_AVX_IMPL + +FASTOR_INLINE float _mm256_sum_ps(__m256 a) { +#ifdef FASTOR_USE_HADD + // IVY 14 OPS - HW 16 OPS + __m256 sum = _mm256_hadd_ps(a, a); + sum = _mm256_hadd_ps(sum, sum); + __m128 result = _mm_add_ps(_mm256_castps256_ps128(sum),_mm256_extractf128_ps(sum, 0x1)); + return _mm_cvtss_f32(result); +#else + // IVY 14 OPS + return _mm_sum_ps(_mm_add_ps(_mm256_castps256_ps128(a),_mm256_extractf128_ps(a,0x1))); +#endif +} +FASTOR_INLINE double _mm256_sum_pd(__m256d a) { +#ifdef FASTOR_USE_HADD + // IVY 9 OPS - HW - 11 OPS + __m256d sum = _mm256_hadd_pd(a, a); +#else + // IVY 8 OPS - HW 10 OPS - SKY 11 OPS (BUT 2 PARALLEL ADDS SO POTENTIALLY 7OPS) + __m256d sum = _mm256_add_pd(a, _mm256_shuffle_pd(a,a,0x5)); +#endif + __m128d result = _mm_add_sd(_mm256_castpd256_pd128(sum),_mm256_extractf128_pd(sum, 0x1)); + return _mm_cvtsd_f64(result); +} + + +FASTOR_INLINE float _mm256_prod_ps(__m256 a) { + // ~ IVY 30 OPS - HW 32 OPS + return _mm_prod_ps(_mm256_castps256_ps128(a))*_mm_prod_ps(_mm256_extractf128_ps(a, 0x1)); +} +FASTOR_INLINE double _mm256_prod_pd(__m256d a) { + // IVY 12 OPS - HW - 14 OPS + __m256d sum = _mm256_mul_pd(a, _mm256_shuffle_pd(a,a,0x5)); + __m128d sum_high = _mm256_extractf128_pd(sum, 0x1); + __m128d result = _mm_mul_sd(sum_high, _mm256_castpd256_pd128(sum)); + return _mm_cvtsd_f64(result); +} + +#endif +//----------------------------------------------------------------------------------------------------------------// + + + +//! Reversing a register +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE __m128 _mm_reverse_ps(__m128 a) { + // 1OP + return _mm_shuffle_ps(a,a,_MM_SHUFFLE(0,1,2,3)); +} +FASTOR_INLINE __m128d _mm_reverse_pd(__m128d a) { + // 1OP + return _mm_shuffle_pd(a,a,0x1); +} +FASTOR_INLINE __m128i _mm_reverse_epi32(__m128i v) { + // 1 OP + return _mm_shuffle_epi32(v, 0x1b); +} +FASTOR_INLINE __m128i _mm_reverse_epi64(__m128i v) { + // 1 OP + return _mm_castpd_si128(_mm_reverse_pd(_mm_castsi128_pd(v))); +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE __m256 _mm256_reverse_ps(__m256 a) { + // IVY 2OPS / HW 4OPS + __m256 r1 = _mm256_permute2f128_ps(a,a,0x1); + return _mm256_shuffle_ps(r1,r1,27); +} +FASTOR_INLINE __m256d _mm256_reverse_pd(__m256d a) { + // IVY 2OPS / HW 4OPS + __m256d r1 = _mm256_permute2f128_pd(a,a,0x1); + return _mm256_shuffle_pd(r1,r1,5); +} +FASTOR_INLINE __m256i _mm256_reverse_epi32(__m256i v) { + // IVY 2OPS / HW 4OPS + return _mm256_castps_si256(_mm256_reverse_ps(_mm256_castsi256_ps(v))); + /* + // 8 OPS + __m128i lo = _mm_shuffle_epi32(_mm256_castsi256_si128(_a)); + __m128i hi = _mm_shuffle_epi32(_mm256_extractf128_si256(_a,1)); + __m256i out = _mm256_castsi128_si256(lo); + out = _mm256_insertf128_si256(out,hi,1); + return out; + */ +} +FASTOR_INLINE __m256i _mm256_reverse_epi64(__m256i v) { + // IVY 2OPS / HW 4OPS + return _mm256_castpd_si256(_mm256_reverse_pd(_mm256_castsi256_pd(v))); +} +#endif +#ifdef FASTOR_AVX512F_IMPL +FASTOR_INLINE __m512 _mm512_reverse_ps(__m512 a) { + // SKY/ICE 3 OPS + const __m512i reverse_mask = _mm512_set_epi32(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); + return _mm512_permutexvar_ps(reverse_mask, a); +} +FASTOR_INLINE __m512d _mm512_reverse_pd(__m512d a) { + // SKY/ICE 3 OPS + const __m512i reverse_mask = _mm512_set_epi64(0,1,2,3,4,5,6,7); + return _mm512_permutexvar_pd(reverse_mask, a); +} +FASTOR_INLINE __m512i _mm512_reverse_epi32(__m512i a) { + // SKY/ICE 3 OPS + const __m512i reverse_mask = _mm512_set_epi32(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); + return _mm512_permutexvar_epi32(reverse_mask, a); +} +FASTOR_INLINE __m512i _mm512_reverse_epi64(__m512i a) { + // SKY/ICE 3 OPS + const __m512i reverse_mask = _mm512_set_epi64(0,1,2,3,4,5,6,7); + return _mm512_permutexvar_epi64(reverse_mask, a); +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + + +//! Bit shifting - for extracting values and so on +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE __m128 _mm_shift1_ps(__m128 a) { + // 1OP + return _mm_castsi128_ps(_mm_slli_si128(_mm_castps_si128(a), 4)); +} +FASTOR_INLINE __m128 _mm_shift2_ps(__m128 a) { + // 1OP + return _mm_shuffle_ps(ZEROPS, a, 0x40); +} +FASTOR_INLINE __m128 _mm_shift3_ps(__m128 a) { + // 2OPS + __m128 shift2 = _mm_shuffle_ps(ZEROPS, a, 0x40); + return _mm_shuffle_ps(ZEROPS,shift2,_MM_SHUFFLE(2,0,2,0)); +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE __m256 _mm256_shift1_ps(__m256 a) { + // IVY 3OPS / HW 5OPS + __m256 r1 = _mm256_permute_ps(a,_MM_SHUFFLE(2,1,0,3)); + __m256 r2 = _mm256_permute2f128_ps(r1,r1,41); + return _mm256_blend_ps(r1,r2,0x11); +} +FASTOR_INLINE __m256 _mm256_shift2_ps(__m256 a) { + // IVY 3OPS / HW 5OPS + __m256 r1 = _mm256_permute_ps(a,_MM_SHUFFLE(1,0,3,2)); + __m256 r2 = _mm256_permute2f128_ps(r1,r1,41); + return _mm256_blend_ps(r1,r2,0x33); +} +FASTOR_INLINE __m256 _mm256_shift3_ps(__m256 a) { + // IVY 1OPS / HW 3OPS + return _mm256_permute2f128_ps(a,a,41); +} +FASTOR_INLINE __m256 _mm256_shift4_ps(__m256 a) { + // IVY 1OPS / HW 3OPS + return _mm256_permute2f128_ps(a,a,42); +} +FASTOR_INLINE __m256 _mm256_shift5_ps(__m256 a) { + // IVY 2 OPS - HW 4 OPS + __m128 r1 = _mm_shift1_ps(_mm256_castps256_ps128(a)); + return _mm256_insertf128_ps(VZEROPS,r1,0x1); +} +FASTOR_INLINE __m256 _mm256_shift6_ps(__m256 a) { + // IVY 2 OPS - HW 4 OPS + __m128 r1 = _mm_shift2_ps(_mm256_castps256_ps128(a)); + return _mm256_insertf128_ps(VZEROPS,r1,0x1); +} +FASTOR_INLINE __m256 _mm256_shift7_ps(__m256 a) { + // IVY 2 OPS - HW 4 OPS + __m128 r1 = _mm_shift3_ps(_mm256_castps256_ps128(a)); + return _mm256_insertf128_ps(VZEROPS,r1,0x1); +} +#endif +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE __m128d _mm_shift1_pd(__m128d a) { + // 1OP + return _mm_shuffle_pd(ZEROPD,a,0x1); +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE __m256d _mm256_shift1_pd(__m256d a) { + // IVY - 4 OPS / HW - 8 OPS + __m128d r1 = _mm256_castpd256_pd128(a); + __m128d r2 = _mm256_extractf128_pd(a,0x1); + __m128d r3 = _mm_shuffle_pd(r1,r2,0x1); + __m256d r4 = _mm256_castpd128_pd256(_mm_shift1_pd(r1)); + return _mm256_insertf128_pd(r4,r3,0x1); +} +FASTOR_INLINE __m256d _mm256_shift2_pd(__m256d a) { + // IVY - 1OP / HW - 3OPS + return _mm256_permute2f128_pd(a,a,8); +} +FASTOR_INLINE __m256d _mm256_shift3_pd(__m256d a) { + // IVY - 2OPS / HW - 4OPS + __m256d r1 = _mm256_castpd128_pd256(_mm_shift1_pd(_mm256_castpd256_pd128(a))); + return _mm256_permute2f128_pd(r1,r1,0x1); +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + +// Negation +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +// Change sign of a register - all one cycle +FASTOR_INLINE __m128 _mm_neg_ps(__m128 a) { + return _mm_xor_ps(a, MZEROPS); +} +FASTOR_INLINE __m128d _mm_neg_pd(__m128d a) { + return _mm_xor_pd(a, MZEROPD); +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE __m256 _mm256_neg_ps(__m256 a) { + return _mm256_xor_ps(a, MVZEROPS); +} +FASTOR_INLINE __m256d _mm256_neg_pd(__m256d a) { + return _mm256_xor_pd(a, MVZEROPD); +} +#endif +#ifdef FASTOR_AVX512F_IMPL +FASTOR_INLINE __m512 _mm512_neg_ps(__m512 a) { +#ifdef FASTOR_AVX512DQ_IMPL + return _mm512_xor_ps(a, _mm512_set1_ps(-0.f)); +#else + return _mm512_sub_ps(_mm512_set1_ps(0.f),a); +#endif +} +FASTOR_INLINE __m512d _mm512_neg_pd(__m512d a) { +#ifdef FASTOR_AVX512DQ_IMPL + return _mm512_xor_pd(a, _mm512_set1_pd(-0.0)); +#else + return _mm512_sub_pd(_mm512_set1_pd(0.0),a); +#endif +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + +// Absolute values +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +// Absolute value of a register - all one cycle +FASTOR_INLINE __m128 _mm_abs_ps(__m128 x) { + static const __m128 sign_mask = _mm_set1_ps(-0.f); // -0.f = 1 << 31 + return _mm_andnot_ps(sign_mask, x); +} +FASTOR_INLINE __m128d _mm_abs_pd(__m128d x) { + static const __m128d sign_mask = _mm_set1_pd(-0.); // -0. = 1 << 63 + return _mm_andnot_pd(sign_mask, x); // !sign_mask & x +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE __m256 _mm256_abs_ps(__m256 x) { + static const __m256 sign_mask = _mm256_set1_ps(-0.f); // -0.f = 1 << 31 + return _mm256_andnot_ps(sign_mask, x); +} +FASTOR_INLINE __m256d _mm256_abs_pd(__m256d x) { + static const __m256d sign_mask = _mm256_set1_pd(-0.); // -0. = 1 << 63 + return _mm256_andnot_pd(sign_mask, x); // !sign_mask & x +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + +// Horizontal max +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +// maximum value in a register - horizontal max +FASTOR_INLINE float _mm_hmax_ps(__m128 a) { + // 8OPS + __m128 max0 = _mm_max_ps(a,_mm_reverse_ps(a)); + __m128 tmp = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); + return _mm_cvtss_f32(_mm_max_ps(max0,tmp)); +} +FASTOR_INLINE double _mm_hmax_pd(__m128d a) { + // 4OPS + return _mm_cvtsd_f64(_mm_max_pd(a,_mm_reverse_pd(a))); +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE float _mm256_hmax_ps(__m256 a) { + // IVY 18OPS / HW 24 OPS + __m128 lo = _mm256_castps256_ps128(a); + __m128 max0 = _mm_max_ps(lo,_mm_reverse_ps(lo)); + __m128 tmp0 = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); + __m128 max_lo = _mm_max_ps(max0,tmp0); + + __m128 hi = _mm256_extractf128_ps(a,0x1); + __m128 max1 = _mm_max_ps(hi,_mm_reverse_ps(hi)); + __m128 tmp1 = _mm_shuffle_ps(max1,max1,_MM_SHUFFLE(0,0,0,1)); + __m128 max_hi = _mm_max_ps(max1,tmp1); + + return _mm_cvtss_f32(_mm_max_ps(max_lo,max_hi)); +} +FASTOR_INLINE double _mm256_hmax_pd(__m256d a) { + // IVY 9OPS / HW 11 OPS + __m256d max0 = _mm256_max_pd(a,_mm256_reverse_pd(a)); + __m256d tmp = _mm256_shuffle_pd(max0,max0,_MM_SHUFFLE(0,0,0,1)); + return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_max_pd(max0,tmp))); +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + +// Horizontal min +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE float _mm_hmin_ps(__m128 a) { + // 8OPS + __m128 max0 = _mm_min_ps(a,_mm_reverse_ps(a)); + __m128 tmp = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); + return _mm_cvtss_f32(_mm_min_ps(max0,tmp)); +} +FASTOR_INLINE double _mm_hmin_pd(__m128d a) { + // 4OPS + return _mm_cvtsd_f64(_mm_min_pd(a,_mm_reverse_pd(a))); +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE float _mm256_hmin_ps(__m256 a) { + // IVY 18OPS / HW 24 OPS + __m128 lo = _mm256_castps256_ps128(a); + __m128 max0 = _mm_min_ps(lo,_mm_reverse_ps(lo)); + __m128 tmp0 = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); + __m128 max_lo = _mm_min_ps(max0,tmp0); + + __m128 hi = _mm256_extractf128_ps(a,0x1); + __m128 max1 = _mm_min_ps(hi,_mm_reverse_ps(hi)); + __m128 tmp1 = _mm_shuffle_ps(max1,max1,_MM_SHUFFLE(0,0,0,1)); + __m128 max_hi = _mm_min_ps(max1,tmp1); + + return _mm_cvtss_f32(_mm_min_ps(max_lo,max_hi)); +} +FASTOR_INLINE double _mm256_hmin_pd(__m256d a) { + // IVY 9OPS / HW 11 OPS + __m256d max0 = _mm256_min_pd(a,_mm256_reverse_pd(a)); + __m256d tmp = _mm256_shuffle_pd(max0,max0,_MM_SHUFFLE(0,0,0,1)); + return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_min_pd(max0,tmp))); +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + +// Indexing a register +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE float _mm_get0_ps(__m128 a) { + // NO OP + return _mm_cvtss_f32(a); +} +FASTOR_INLINE float _mm_get1_ps(__m128 a) { + // 1 OP + return _mm_cvtss_f32(_mm_shuffle_ps(a,a,_MM_SHUFFLE(0,0,0,1))); +} +FASTOR_INLINE float _mm_get2_ps(__m128 a) { + // 1 OP + return _mm_cvtss_f32(_mm_shuffle_ps(a,a,_MM_SHUFFLE(0,0,0,2))); +} +FASTOR_INLINE float _mm_get3_ps(__m128 a) { + // 1 OP + return _mm_cvtss_f32(_mm_shuffle_ps(a,a,_MM_SHUFFLE(0,0,0,3))); +} +FASTOR_INLINE double _mm_get0_pd(__m128d a) { + // NO OP + return _mm_cvtsd_f64(a); +} +FASTOR_INLINE double _mm_get1_pd(__m128d a) { + // 1 OP + return _mm_cvtsd_f64(_mm_shuffle_pd(a,a,_MM_SHUFFLE2(0,1))); +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE float _mm256_get0_ps(__m256 a) { + // NO OP + return _mm_cvtss_f32(_mm256_castps256_ps128(a)); +} +FASTOR_INLINE float _mm256_get1_ps(__m256 a) { + // 1 OP + __m128 lower = _mm256_castps256_ps128(a); + return _mm_cvtss_f32(_mm_shuffle_ps(lower,lower,_MM_SHUFFLE(0,0,0,1))); +} +FASTOR_INLINE float _mm256_get2_ps(__m256 a) { + // 1 OP + __m128 lower = _mm256_castps256_ps128(a); + return _mm_cvtss_f32(_mm_shuffle_ps(lower,lower,_MM_SHUFFLE(0,0,0,2))); +} +FASTOR_INLINE float _mm256_get3_ps(__m256 a) { + // NO OP + __m128 lower = _mm256_castps256_ps128(a); + return _mm_cvtss_f32(_mm_shuffle_ps(lower,lower,_MM_SHUFFLE(0,0,0,3))); +} +FASTOR_INLINE float _mm256_get4_ps(__m256 a) { + // IVY 1OP / HW 3OPS + return _mm_cvtss_f32(_mm256_extractf128_ps(a,0x1)); +} +FASTOR_INLINE float _mm256_get5_ps(__m256 a) { + // IVY 2OPS/ HW 4OPS + __m128 higher = _mm256_extractf128_ps(a,0x1); + return _mm_cvtss_f32(_mm_shuffle_ps(higher,higher,_MM_SHUFFLE(0,0,0,1))); +} +FASTOR_INLINE float _mm256_get6_ps(__m256 a) { + // IVY 2OPS/ HW 4OPS + __m128 higher = _mm256_extractf128_ps(a,0x1); + return _mm_cvtss_f32(_mm_shuffle_ps(higher,higher,_MM_SHUFFLE(0,0,0,2))); +} +FASTOR_INLINE float _mm256_get7_ps(__m256 a) { + // IVY 2OPS/ HW 4OPS + __m128 higher = _mm256_extractf128_ps(a,0x1); + return _mm_cvtss_f32(_mm_shuffle_ps(higher,higher,_MM_SHUFFLE(0,0,0,3))); +} +FASTOR_INLINE double _mm256_get0_pd(__m256d a) { + // NO OP + return _mm_cvtsd_f64(_mm256_castpd256_pd128(a)); +} +FASTOR_INLINE double _mm256_get1_pd(__m256d a) { + // 1 OP + __m128d lower = _mm256_castpd256_pd128(a); + return _mm_cvtsd_f64(_mm_shuffle_pd(lower,lower,_MM_SHUFFLE2(0,1))); +} +FASTOR_INLINE double _mm256_get2_pd(__m256d a) { + // IVY 1OP / HW 3OPS + return _mm_cvtsd_f64(_mm256_extractf128_pd(a,0x1)); +} +FASTOR_INLINE double _mm256_get3_pd(__m256d a) { + // IVY 2OPS / HW 4OPS + __m128d higher = _mm256_extractf128_pd(a,0x1); + return _mm_cvtsd_f64(_mm_shuffle_pd(higher,higher,_MM_SHUFFLE2(0,1))); +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + + +// Integral arithmetics that are not available pre AVX2 +//----------------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE __m128i _mm_mul_epi32x(__m128i a, __m128i b) +{ +#ifdef FASTOR_SSE4_1_IMPL + return _mm_mullo_epi32(a, b); +#else // SSE2 + __m128i a13 = _mm_shuffle_epi32(a, 0xF5); // (-,a3,-,a1) + __m128i b13 = _mm_shuffle_epi32(b, 0xF5); // (-,b3,-,b1) + __m128i prod02 = _mm_mul_epu32(a, b); // (-,a2*b2,-,a0*b0) + __m128i prod13 = _mm_mul_epu32(a13, b13); // (-,a3*b3,-,a1*b1) + __m128i prod01 = _mm_unpacklo_epi32(prod02, prod13); // (-,-,a1*b1,a0*b0) + __m128i prod23 = _mm_unpackhi_epi32(prod02, prod13); // (-,-,a3*b3,a2*b2) + return _mm_unpacklo_epi64(prod01, prod23); // (ab3,ab2,ab1,ab0) +#endif + + +} +#endif + +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE __m128i _mm_mul_epi64(__m128i _a, __m128i _b) { + __m128i out; + for (FASTOR_INDEX i=0; i<2; i++) { + ((int64_t*)&out)[i] = (((int64_t*)&_a)[i])*(((int64_t*)&_b)[i]); + } + return out; +} +#endif + +#ifdef FASTOR_AVX_IMPL +// #ifndef FASTOR_AVX2_IMPL +FASTOR_INLINE __m256i _mm256_add_epi32x(__m256i _a, __m256i _b) { + __m128i low_a = _mm256_castsi256_si128(_a); + __m128i high_a = _mm256_extractf128_si256(_a,1); + __m128i low_b = _mm256_castsi256_si128(_b); + __m128i high_b = _mm256_extractf128_si256(_b,1); + __m128i low = _mm_add_epi32(low_a,low_b); + __m128i high = _mm_add_epi32(high_a,high_b); + __m256i out = _mm256_castsi128_si256(low); + out = _mm256_insertf128_si256(out,high,1); + return out; +} + +FASTOR_INLINE __m256i _mm256_sub_epi32x(__m256i _a, __m256i _b) { + __m128i low_a = _mm256_castsi256_si128(_a); + __m128i high_a = _mm256_extractf128_si256(_a,1); + __m128i low_b = _mm256_castsi256_si128(_b); + __m128i high_b = _mm256_extractf128_si256(_b,1); + __m128i low = _mm_sub_epi32(low_a,low_b); + __m128i high = _mm_sub_epi32(high_a,high_b); + __m256i out = _mm256_castsi128_si256(low); + out = _mm256_insertf128_si256(out,high,1); + return out; +} + +FASTOR_INLINE __m256i _mm256_mul_epi32x(__m256i _a, __m256i _b) { + __m128i low_a = _mm256_castsi256_si128(_a); + __m128i high_a = _mm256_extractf128_si256(_a,0x1); + __m128i low_b = _mm256_castsi256_si128(_b); + __m128i high_b = _mm256_extractf128_si256(_b,0x1); + __m128i low = _mm_mul_epi32x(low_a,low_b); + __m128i high = _mm_mul_epi32x(high_a,high_b); + __m256i out = _mm256_castsi128_si256(low); + out = _mm256_insertf128_si256(out,high,0x1); + return out; +} + +// 64bit +FASTOR_INLINE __m256i _mm256_add_epi64x(__m256i _a, __m256i _b) { + __m128i low_a = _mm256_castsi256_si128(_a); + __m128i high_a = _mm256_extractf128_si256(_a,1); + __m128i low_b = _mm256_castsi256_si128(_b); + __m128i high_b = _mm256_extractf128_si256(_b,1); + __m128i low = _mm_add_epi64(low_a,low_b); + __m128i high = _mm_add_epi64(high_a,high_b); + __m256i out = _mm256_castsi128_si256(low); + out = _mm256_insertf128_si256(out,high,1); + return out; +} + +FASTOR_INLINE __m256i _mm256_sub_epi64x(__m256i _a, __m256i _b) { + __m128i low_a = _mm256_castsi256_si128(_a); + __m128i high_a = _mm256_extractf128_si256(_a,1); + __m128i low_b = _mm256_castsi256_si128(_b); + __m128i high_b = _mm256_extractf128_si256(_b,1); + __m128i low = _mm_sub_epi64(low_a,low_b); + __m128i high = _mm_sub_epi64(high_a,high_b); + __m256i out = _mm256_castsi128_si256(low); + out = _mm256_insertf128_si256(out,high,1); + return out; +} +// #else +// Note that these instruction work on alternating bytes +// FASTOR_INLINE __m256i _mm256_add_epi32x(__m256i _a, __m256i _b) { +// return _mm256_add_epi32(_a,_b); +// } +// FASTOR_INLINE __m256i _mm256_sub_epi32x(__m256i _a, __m256i _b) { +// return _mm256_sub_epi32(_a,_b); +// } +// FASTOR_INLINE __m256i _mm256_mul_epi32x(__m256i _a, __m256i _b) { +// return _mm256_mul_epi32(_a,_b); +// } +// FASTOR_INLINE __m256i _mm256_add_epi64x(__m256i _a, __m256i _b) { +// return _mm256_add_epi64(_a,_b); +// } +// FASTOR_INLINE __m256i _mm256_sub_epi64x(__m256i _a, __m256i _b) { +// return _mm256_sub_epi64(_a,_b); +// } +// #endif + +FASTOR_INLINE __m256i _mm256_div_epi32x(__m256i _a, __m256i _b) { + // YIELDS INCORRECT + int *a_data = (int*) &_a; + int *b_data = (int*) &_b; + FASTOR_ARCH_ALIGN int out_data[8]; + for (int i=0; i<8; ++i) + out_data[i] = a_data[i]/b_data[i]; + __m256i out = _mm256_setzero_si256(); + _mm256_store_si256((__m256i*)out_data,out); + return out; +} + +FASTOR_INLINE __m256i _mm256_mul_epi64x(__m256i _a, __m256i _b) { + __m128i low_a = _mm256_castsi256_si128(_a); + __m128i high_a = _mm256_extractf128_si256(_a,0x1); + __m128i low_b = _mm256_castsi256_si128(_b); + __m128i high_b = _mm256_extractf128_si256(_b,0x1); + __m128i low = _mm_mul_epi64(low_a,low_b); + __m128i high = _mm_mul_epi64(high_a,high_b); + __m256i out = _mm256_castsi128_si256(low); + out = _mm256_insertf128_si256(out,high,0x1); + return out; +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + + +//! Some further auxilary functions C++ only +//----------------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------------// +#if defined(__cplusplus) +#ifdef FASTOR_SSE2_IMPL +static FASTOR_INLINE __m128d _add_pd(__m128d a) { + // IVY 4 OPS + __m128 shuftmp= _mm_movehl_ps(ZEROPS, _mm_castpd_ps(a)); + __m128d shuf = _mm_castps_pd(shuftmp); + return _mm_add_sd(a, shuf); +} +#endif + +#ifdef FASTOR_USE_HADD // hadd is beneficial here and the flag is used in the opposite way +#ifdef FASTOR_AVX_IMPL +static FASTOR_INLINE __m128d _add_pd(__m256d a) { + // IVY 12 OPS / HW 14 OPS + __m128d sum_low = _add_pd(_mm256_castpd256_pd128(a)); + __m128d sum_high = _add_pd(_mm256_extractf128_pd(sum_low, 0x1)); + return _mm_add_pd(sum_high, sum_low); +} +#endif +#else +#ifdef FASTOR_AVX_IMPL +static FASTOR_INLINE __m128d _add_pd(__m256d a) { + // IVY 9 OPS / HW 11 OPS + __m256d sum_low = _mm256_hadd_pd(a, a); + __m128d sum_high = _mm256_extractf128_pd(sum_low, 0x1); + return _mm_add_pd(sum_high, _mm256_castpd256_pd128(sum_low)); +} +#endif +#endif +#ifdef FASTOR_SSE3_IMPL +FASTOR_INLINE __m128 _add_ps(__m128 a) { + // 8 OPS + __m128 shuf = _mm_movehdup_ps(a); // line up elements 3,1 with 2,0 + __m128 sums = _mm_add_ps(a, shuf); + shuf = _mm_movehl_ps(shuf, sums); // high half -> low half + sums = _mm_add_ss(sums, shuf); + return sums; +} +#endif +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE __m128 _add_ps(__m256 a) { + // IVY 20 OPS / HW 22 OPS + __m128 sum_low = _add_ps(_mm256_castps256_ps128(a)); + __m128 sum_high = _add_ps(_mm256_extractf128_ps(a,0x1)); + return _mm_add_ss(sum_low,sum_high); +} +#endif + +// horizontal add_sub +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE __m128 _addsub_ps(__m128 a) { + // 8 OPS + // If a = [a0 a1 a2 a3] this function returns (a1+a3)-(a0+a2) + // Note that only the first element of __m128 corresponds to this + __m128 shuf = _mm_shuffle_ps(a,a,_MM_SHUFFLE(1,0,3,2)); + __m128 sums = _mm_add_ps(a, shuf); + shuf = _mm_shuffle_ps(sums,sums,_MM_SHUFFLE(2,3,0,1)); + return _mm_sub_ps(shuf, sums); +} + +FASTOR_INLINE __m128 _mulsub_ps(__m128 a) { + // 10 OPS + // If a = [a0 a1 a2 a3] this function returns (a1*a3)-(a0+a2) + // Note that only the first element of __m128 corresponds to this + __m128 shuf = _mm_shuffle_ps(a,a,_MM_SHUFFLE(1,0,3,2)); + __m128 muls = _mm_mul_ps(a, shuf); + shuf = _mm_shuffle_ps(muls,muls,_MM_SHUFFLE(2,3,0,1)); + return _mm_sub_ps(shuf, muls); +} + +FASTOR_INLINE __m128d _hsub_pd(__m128d a) { + // horizontal sub, returns a[0] - a[1] + // 4 OPS + return _mm_sub_sd(a,_mm_shuffle_pd(a,a,0x1)); +} +#endif + +#ifdef FASTOR_AVX_IMPL +// Similar to SSE4 _mm_dp_pd for dot product +FASTOR_INLINE __m128d _mm256_dp_pd(__m256d __X, __m256d __Y) { + return _add_pd(_mm256_mul_pd(__X, __Y)); +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + +// Arrangments for std::complex after load / before store +//----------------------------------------------------------------------------------------------------------------// +// complex single +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE void arrange_from_load(__m128& value_r, __m128& value_i, __m128 lo, __m128 hi) { + value_r = _mm_shuffle_ps(lo, hi, _MM_SHUFFLE(2, 0, 2, 0)); + value_i = _mm_shuffle_ps(lo, hi, _MM_SHUFFLE(3, 1, 3, 1)); +} + +FASTOR_INLINE void arrange_for_store(__m128 &lo, __m128 &hi, __m128 value_r, __m128 value_i) { + lo = _mm_unpacklo_ps(value_r,value_i); + hi = _mm_unpackhi_ps(value_r,value_i); +} +#endif + +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE void arrange_from_load(__m256& value_r, __m256& value_i, __m256 lo, __m256 hi) { +#ifdef FASTOR_AVX2_IMPL + value_r = _mm256_castpd_ps(_mm256_permute4x64_pd( + _mm256_castps_pd(_mm256_shuffle_ps(lo, hi, _MM_SHUFFLE(2, 0, 2, 0))),_MM_SHUFFLE(3, 1, 2, 0))); + value_i = _mm256_castpd_ps(_mm256_permute4x64_pd( + _mm256_castps_pd(_mm256_shuffle_ps(lo, hi, _MM_SHUFFLE(3, 1, 3, 1))),_MM_SHUFFLE(3, 1, 2, 0))); +#else + __m128 tmp0 = _mm256_castps256_ps128(lo); + __m128 tmp1 = _mm256_extractf128_ps(lo, 0x1); + __m128 tmp2 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(2, 0, 2, 0)); + __m128 tmp3 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(3, 1, 3, 1)); + value_r = _mm256_castps128_ps256(tmp2); + value_i = _mm256_castps128_ps256(tmp3); + tmp0 = _mm256_castps256_ps128(hi); + tmp1 = _mm256_extractf128_ps(hi, 0x1); + tmp2 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(2, 0, 2, 0)); + tmp3 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(3, 1, 3, 1)); + value_r = _mm256_insertf128_ps(value_r, tmp2, 0x1); + value_i = _mm256_insertf128_ps(value_i, tmp3, 0x1); +#endif +} + +FASTOR_INLINE void arrange_for_store(__m256 &lo, __m256 &hi, __m256 value_r, __m256 value_i) { + __m256 tmp0 = _mm256_unpacklo_ps(value_r, value_i); + __m256 tmp1 = _mm256_unpackhi_ps(value_r, value_i); + lo = _mm256_permute2f128_ps(tmp1, tmp0, 0x2); + hi = _mm256_permute2f128_ps(tmp0, tmp1, 0x1); + hi = _mm256_insertf128_ps(hi,_mm256_extractf128_ps(tmp1,0x1),0x1); +} +#endif + +#ifdef FASTOR_AVX512F_IMPL +FASTOR_INLINE void arrange_from_load(__m512& value_r, __m512& value_i, __m512 lo, __m512 hi) { + // Define to help with immediate construction - most likely + // the compiler will emit [vmovdqa32] + #define FT_LOAD512SIDXR _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30) + #define FT_LOAD512SIDXI _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31) + value_r = _mm512_permutex2var_ps(lo, FT_LOAD512SIDXR, hi); + value_i = _mm512_permutex2var_ps(lo, FT_LOAD512SIDXI, hi); +} + +FASTOR_INLINE void arrange_for_store(__m512 &lo, __m512 &hi, __m512 value_r, __m512 value_i) { + // Define to help with immediate construction - most likely + // the compiler will emit [vmovdqa32] + #define FT_STORE512SIDXR _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23) + #define FT_STORE512SIDXI _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31) + lo = _mm512_permutex2var_ps(value_r, FT_STORE512SIDXR, value_i); + hi = _mm512_permutex2var_ps(value_r, FT_STORE512SIDXI, value_i); +} +#endif + +// complex double +#ifdef FASTOR_SSE2_IMPL +FASTOR_INLINE void arrange_from_load(__m128d& value_r, __m128d& value_i, __m128d lo, __m128d hi) { + value_r = _mm_shuffle_pd(lo, hi, _MM_SHUFFLE2(0, 0)); + value_i = _mm_shuffle_pd(lo, hi, _MM_SHUFFLE2(1, 1)); +} + +FASTOR_INLINE void arrange_for_store(__m128d &lo, __m128d &hi, __m128d value_r, __m128d value_i) { + lo = _mm_shuffle_pd(value_r, value_i, _MM_SHUFFLE2(0, 0)); + hi = _mm_shuffle_pd(value_r, value_i, _MM_SHUFFLE2(1, 1)); +} +#endif + +#ifdef FASTOR_AVX_IMPL +FASTOR_INLINE void arrange_from_load(__m256d& value_r, __m256d& value_i, __m256d lo, __m256d hi) { +#ifdef FASTOR_AVX2_IMPL + value_r = _mm256_permute4x64_pd(_mm256_unpacklo_pd(lo, hi), _MM_SHUFFLE(3, 1, 2, 0)); + value_i = _mm256_permute4x64_pd(_mm256_unpackhi_pd(lo, hi), _MM_SHUFFLE(3, 1, 2, 0)); +#else + __m256d tmp0 = _mm256_unpacklo_pd(lo, hi); + __m256d tmp1 = _mm256_unpackhi_pd(lo, hi); + __m128d tmp2 = _mm256_castpd256_pd128(tmp0); + __m128d tmp3 = _mm256_extractf128_pd (tmp0,0x1); + value_r = _mm256_castpd128_pd256( _mm_unpacklo_pd(tmp2,tmp3)); + value_r = _mm256_insertf128_pd (value_r,_mm_unpackhi_pd(tmp2,tmp3),0x1); + tmp2 = _mm256_castpd256_pd128(tmp1); + tmp3 = _mm256_extractf128_pd (tmp1,0x1); + value_i = _mm256_castpd128_pd256( _mm_unpacklo_pd(tmp2,tmp3)); + value_i = _mm256_insertf128_pd (value_i,_mm_unpackhi_pd(tmp2,tmp3),0x1); +#endif +} + +FASTOR_INLINE void arrange_for_store(__m256d &lo, __m256d &hi, __m256d value_r, __m256d value_i) { + __m256d tmp0 = _mm256_unpacklo_pd(value_r, value_i); + __m256d tmp1 = _mm256_unpackhi_pd(value_r, value_i); + lo = _mm256_permute2f128_pd(tmp1, tmp0, 0x2); + hi = _mm256_permute2f128_pd(tmp0, tmp1, 0x1); + hi = _mm256_insertf128_pd(hi,_mm256_extractf128_pd(tmp1,0x1),0x1); +} +#endif + +#ifdef FASTOR_AVX512F_IMPL +FASTOR_INLINE void arrange_from_load(__m512d& value_r, __m512d& value_i, __m512d lo, __m512d hi) { + // Define to help with immediate construction - most likely + // the compiler will emit [vmovdqa64] + #define FT_LOAD512DIDXR _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14) + #define FT_LOAD512DIDXI _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15) + value_r = _mm512_permutex2var_pd(lo, FT_LOAD512DIDXR, hi); + value_i = _mm512_permutex2var_pd(lo, FT_LOAD512DIDXI, hi); +} + +FASTOR_INLINE void arrange_for_store(__m512d &lo, __m512d &hi, __m512d value_r, __m512d value_i) { + // Define to help with immediate construction - most likely + // the compiler will emit [vmovdqa64] + #define FT_STORE512DIDXR _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11) + #define FT_STORE512DIDXI _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15) + lo = _mm512_permutex2var_pd(value_r, FT_STORE512DIDXR, value_i); + hi = _mm512_permutex2var_pd(value_r, FT_STORE512DIDXI, value_i); +} +#endif +//----------------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------------// +// Additional math functions for scalars -> the name sqrts is to remove ambiguity with libm sqrt +template,bool> = false> +FASTOR_INLINE T sqrts(T a) {return std::sqrt(a);} +#ifdef FASTOR_SSE2_IMPL +template<> +FASTOR_INLINE float sqrts(float a) {return _mm_cvtss_f32(_mm_sqrt_ps(_mm_set1_ps(a)));} +template<> +FASTOR_INLINE double sqrts(double a) {return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_set1_pd(a)));} +#endif + +//----------------------------------------------------------------------------------------------------------------// +#endif +//----------------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------------// + + + +//----------------------------------------------------------------------------------------------------------------// +// helper functions for going from array to mask and vice-versa +// used when AVX512 masking is available +template = false> +FASTOR_INLINE uint8_t array_to_mask(const int (&b)[N]) +{ + uint8_t c = 0; + for (int i=0; i < N; ++i) { + if (b[i] == -1) { + c |= 1 << (N - i - 1); + } + } + return c; +} +template = false> +FASTOR_INLINE uint16_t array_to_mask(const int (&b)[N]) +{ + uint16_t c = 0; + for (int i=0; i < N; ++i) { + if (b[i] == -1) { + c |= 1 << (N - i - 1); + } + } + return c; +} +template = false> +FASTOR_INLINE uint32_t array_to_mask(const int (&b)[N]) +{ + uint32_t c = 0; + for (int i=0; i < N; ++i) { + if (b[i] == -1) { + c |= 1 << (N - i - 1); + } + } + return c; +} +template = false> +FASTOR_INLINE uint64_t array_to_mask(const int (&b)[N]) +{ + uint64_t c = 0; + for (int i=0; i < N; ++i) { + if (b[i] == -1) { + c |= 1 << (N - i - 1); + } + } + return c; +} + + +template = false> +FASTOR_INLINE void mask_to_array(uint8_t c, int (&b)[N]) +{ + for (int i=0; i < N; ++i) + b[i] = (c & (1 << (N - i -1))) != 0; + + // set bits need to be -1 not 1 + for (int i=0; i < N; ++i) + b[i] *= -1; +} +template = false> +FASTOR_INLINE void mask_to_array(uint16_t c, int (&b)[N]) +{ + for (int i=0; i < N; ++i) + b[i] = (c & (1 << (N - i -1))) != 0; + + // set bits need to be -1 not 1 + for (int i=0; i < N; ++i) + b[i] *= -1; +} +template = false> +FASTOR_INLINE void mask_to_array(uint32_t c, int (&b)[N]) +{ + for (int i=0; i < N; ++i) + b[i] = (c & (1 << (N - i -1))) != 0; + + // set bits need to be -1 not 1 + for (int i=0; i < N; ++i) + b[i] *= -1; +} +template = false> +FASTOR_INLINE void mask_to_array(uint64_t c, int (&b)[N]) +{ + for (int i=0; i < N; ++i) + b[i] = (c & (1 << (N - i -1))) != 0; + + // set bits need to be -1 not 1 + for (int i=0; i < N; ++i) + b[i] *= -1; +} +//----------------------------------------------------------------------------------------------------------------// + + + +// Splitting a mask in to two - used for masking complex SIMD vectors +//----------------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------------// +/* Moves "from" bit of c1 to "to" bit of c2 */ +FASTOR_INLINE uint8_t move_bit(uint8_t c1, int from, uint8_t c2, int to) +{ + int bit; + /* Get the source bit as 0/1 value */ + bit = (c1 >> from) & 1; + /* clear destination bit */ + c2 &= ~(1 << to); + /* set destination bit */ + return (uint8_t)(c2 | (bit << to)); +} + +/* Moves "from" bit of c1 to "to" bit of c2 */ +FASTOR_INLINE uint16_t move_bit(uint16_t c1, int from, uint16_t c2, int to) +{ + int bit; + /* Get the source bit as 0/1 value */ + bit = (c1 >> from) & 1; + /* clear destination bit */ + c2 &= ~(1 << to); + /* set destination bit */ + return (uint16_t)(c2 | (bit << to)); +} +//----------------------------------------------------------------------------------------------------------------// + +// Splitting a mask into 2 parts when only 2 bits of the incoming mask are set +template = false> +FASTOR_INLINE void split_mask(uint8_t mask, uint8_t& mask0, uint8_t& mask1) { + mask0 = 0; /* clear all bits */ + mask0 = move_bit(mask, 0, mask0, 0); + mask0 = move_bit(mask, 0, mask0, 1); + + mask1 = 0; /* clear all bits */ + mask1 = move_bit(mask, 1, mask1, 0); + mask1 = move_bit(mask, 1, mask1, 1); +} + +// Splitting a mask into 2 parts when only 4 bits of the incoming mask are set +template = false> +FASTOR_INLINE void split_mask(uint8_t mask, uint8_t& mask0, uint8_t& mask1) { + mask0 = 0; /* clear all bits */ + mask0 = move_bit(mask, 0, mask0, 0); + mask0 = move_bit(mask, 0, mask0, 1); + mask0 = move_bit(mask, 1, mask0, 2); + mask0 = move_bit(mask, 1, mask0, 3); + + mask1 = 0; /* clear all bits */ + mask1 = move_bit(mask, 2, mask1, 0); + mask1 = move_bit(mask, 2, mask1, 1); + mask1 = move_bit(mask, 3, mask1, 2); + mask1 = move_bit(mask, 3, mask1, 3); +} + +// Splitting a mask into 2 parts when all 8 bits of the incoming mask are set +template = false> +FASTOR_INLINE void split_mask(uint8_t mask, uint8_t& mask0, uint8_t& mask1) { + mask0 = 0; /* clear all bits */ + mask0 = move_bit(mask, 0, mask0, 0); + mask0 = move_bit(mask, 0, mask0, 1); + mask0 = move_bit(mask, 1, mask0, 2); + mask0 = move_bit(mask, 1, mask0, 3); + mask0 = move_bit(mask, 2, mask0, 4); + mask0 = move_bit(mask, 2, mask0, 5); + mask0 = move_bit(mask, 3, mask0, 6); + mask0 = move_bit(mask, 3, mask0, 7); + + mask1 = 0; /* clear all bits */ + mask1 = move_bit(mask, 4, mask1, 0); + mask1 = move_bit(mask, 4, mask1, 1); + mask1 = move_bit(mask, 5, mask1, 2); + mask1 = move_bit(mask, 5, mask1, 3); + mask1 = move_bit(mask, 6, mask1, 4); + mask1 = move_bit(mask, 6, mask1, 5); + mask1 = move_bit(mask, 7, mask1, 6); + mask1 = move_bit(mask, 7, mask1, 7); +} + +// Splitting a mask into 2 parts when all 16 bits of the incoming mask are set +template = false> +FASTOR_INLINE void split_mask(uint16_t mask, uint16_t& mask0, uint16_t& mask1) { + mask0 = 0; /* clear all bits */ + mask0 = move_bit(mask, 0, mask0, 0); + mask0 = move_bit(mask, 0, mask0, 1); + mask0 = move_bit(mask, 1, mask0, 2); + mask0 = move_bit(mask, 1, mask0, 3); + mask0 = move_bit(mask, 2, mask0, 4); + mask0 = move_bit(mask, 2, mask0, 5); + mask0 = move_bit(mask, 3, mask0, 6); + mask0 = move_bit(mask, 3, mask0, 7); + mask0 = move_bit(mask, 4, mask0, 8); + mask0 = move_bit(mask, 4, mask0, 9); + mask0 = move_bit(mask, 5, mask0, 10); + mask0 = move_bit(mask, 5, mask0, 11); + mask0 = move_bit(mask, 6, mask0, 12); + mask0 = move_bit(mask, 6, mask0, 13); + mask0 = move_bit(mask, 7, mask0, 14); + mask0 = move_bit(mask, 7, mask0, 15); + + mask1 = 0; /* clear all bits */ + mask1 = move_bit(mask, 8, mask1, 0); + mask1 = move_bit(mask, 8, mask1, 1); + mask1 = move_bit(mask, 9, mask1, 2); + mask1 = move_bit(mask, 9, mask1, 3); + mask1 = move_bit(mask, 10, mask1, 4); + mask1 = move_bit(mask, 10, mask1, 5); + mask1 = move_bit(mask, 11, mask1, 6); + mask1 = move_bit(mask, 11, mask1, 7); + mask1 = move_bit(mask, 12, mask1, 8); + mask1 = move_bit(mask, 12, mask1, 9); + mask1 = move_bit(mask, 13, mask1, 10); + mask1 = move_bit(mask, 13, mask1, 11); + mask1 = move_bit(mask, 14, mask1, 12); + mask1 = move_bit(mask, 14, mask1, 13); + mask1 = move_bit(mask, 15, mask1, 14); + mask1 = move_bit(mask, 15, mask1, 15); +} +//----------------------------------------------------------------------------------------------------------------// + +} // end of namespace Fastor + + +#endif // EXT_INTRIN_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_abi.h b/noarch/include/Fastor/simd_vector/simd_vector_abi.h new file mode 100644 index 00000000..b0530455 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_abi.h @@ -0,0 +1,225 @@ +#ifndef FASTOR_SIMD_VECTOR_ABI +#define FASTOR_SIMD_VECTOR_ABI + +#include "Fastor/meta/meta.h" +#include "Fastor/config/config.h" +#include +#include + +namespace Fastor { + +namespace simd_abi { + +struct scalar {}; +struct sse {}; +struct avx {}; +struct avx512 {}; +struct mic {}; +template struct fixed_size {}; + +#ifndef FASTOR_DONT_VECTORISE +#if defined(FASTOR_AVX512_IMPL) +using native = simd_abi::avx512; +#elif defined(FASTOR_AVX_IMPL) +using native = simd_abi::avx; +#elif defined(FASTOR_SSE2_IMPL) +using native = simd_abi::sse; +#else +using native = simd_abi::scalar; +#endif +#else +using native = simd_abi::scalar; +#endif + +} + +// Definition of DEFAULT_ABI is here +//--------------------------------------------------------------------------------------------------------------// +#define DEFAULT_ABI simd_abi::native +//--------------------------------------------------------------------------------------------------------------// + + +//--------------------------------------------------------------------------------------------------------------// +namespace internal { + +template +struct get_simd_vector_size; +template class __svec, typename T, typename ABI> +struct get_simd_vector_size<__svec> { + static constexpr size_t bitsize = std::is_same::value + ? FASTOR_AVX512_BITSIZE : (std::is_same::value + ? FASTOR_AVX_BITSIZE : (std::is_same::value + ? FASTOR_SSE_BITSIZE : sizeof(T)*8)); + + // Size should be at least 1UL + static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL; +}; +// Specialisation for fixed size simd vectors +template class __svec, typename T, size_t N> +struct get_simd_vector_size<__svec > > { + static constexpr size_t bitsize = N*8UL; + static constexpr size_t value = N; +}; +// Specialisation for complex simd vectors +template class __svec, typename ABI> +struct get_simd_vector_size<__svec,ABI>> { + using T = float; + static constexpr size_t bitsize = std::is_same::value + ? FASTOR_AVX512_BITSIZE : (std::is_same::value + ? FASTOR_AVX_BITSIZE : (std::is_same::value + ? FASTOR_SSE_BITSIZE : sizeof(T)*8)); + + // Size should be at least 1UL + static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL; +}; +template class __svec, typename ABI> +struct get_simd_vector_size<__svec,ABI>> { + using T = double; + static constexpr size_t bitsize = std::is_same::value + ? FASTOR_AVX512_BITSIZE : (std::is_same::value + ? FASTOR_AVX_BITSIZE : (std::is_same::value + ? FASTOR_SSE_BITSIZE : sizeof(T)*8)); + + // Size should be at least 1UL + static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL; +}; + + +template +struct get_half_simd_type; +template class __svec, typename T, typename ABI> +struct get_half_simd_type<__svec> { + // If not a half of simd we give back the actual incoming type + using actual_type = __svec; + using type = typename std::conditional< std::is_same::value, __svec, + typename std::conditional< std::is_same::value, __svec, actual_type>::type + >::type; +}; + + +template +struct get_quarter_simd_type; +template class __svec, typename T, typename ABI> +struct get_quarter_simd_type<__svec> { + // If not a quarter of simd we give back the actual incoming type + using actual_type = __svec; + using type = typename std::conditional< std::is_same::value, __svec, actual_type>::type; +}; + + +template +struct is_exact_multiple_of_smaller_simd; +template class __svec, typename T, typename ABI, size_t N> +struct is_exact_multiple_of_smaller_simd<__svec,N> { + // If not a fraction of simd we give back the actual incoming type + using actual_type = __svec; + // if N is half simd which=2, if it is a 1/4th which=4, else which=1 + static constexpr int which = get_simd_vector_size<__svec>::value / N == 2UL ? 2UL + : (get_simd_vector_size<__svec>::value / N == 4UL ? 4UL : 1UL); + static constexpr bool value = which != 1UL && !std::is_same::value ? true : false; + + static constexpr bool is_half_of_avx512 = std::is_same::value && which==2UL; + static constexpr bool is_half_of_avx = std::is_same::value && which==2UL; + static constexpr bool is_4th_of_avx512 = std::is_same::value && which==4UL; + // static constexpr bool is_4th_of_avx = std::is_same::value && which==4UL; + + static constexpr bool is_half_type = is_half_of_avx512 || is_half_of_avx; + static constexpr bool is_quarter_type = is_4th_of_avx512; + + using half_type = typename std::conditional< is_half_of_avx512, __svec, + typename std::conditional< is_half_of_avx, __svec, actual_type>::type + >::type; + using quarter_type = typename std::conditional< is_4th_of_avx512, __svec, actual_type>::type; + + using type = typename std::conditional::type + >::type; +}; + +template +struct choose_best_simd_type; +template class __svec, typename TT, typename ABI, size_t N> +struct choose_best_simd_type<__svec,N> { + using T = remove_cv_ref_t; + using actual_type = __svec; + static constexpr size_t _vec_size = get_simd_vector_size<__svec>::value; + // For exact fractions simd gets proper speed up for instance for matmul + using exact_multiple_t = typename is_exact_multiple_of_smaller_simd<__svec,N>::type; + static constexpr bool is_exact_multiple = is_exact_multiple_of_smaller_simd<__svec,N>::value; + +#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) + using size_based_type = typename std::conditional::type; +#else + using size_based_type = typename std::conditional::value, typename get_half_simd_type<__svec>::type, actual_type + >::type + >::type; +#endif + + // using size_based_type = typename std::conditional,N>::type, + // typename std::conditional::value, typename get_quarter_simd_type<__svec>::type, + // typename std::conditional::value, typename get_half_simd_type<__svec>::type, actual_type + // >::type + // >::type + // >::type; + + // // For other fractions masking might be a better idea than, hence this special logic for remainder using is_less. + // // For no special logic use the above case + // using size_based_type = typename std::conditional,N>::type, + // typename std::conditional::value && is_greater<_vec_size % N,1UL>::value, + // typename get_quarter_simd_type<__svec>::type, + // typename std::conditional::value && is_greater<_vec_size % N,1UL>::value, + // typename get_half_simd_type<__svec>::type, actual_type + // >::type + // >::type + // >::type; + + using type = typename std::conditional< std::is_same::value || + std::is_same::value || + std::is_same>::value || + std::is_same>::value || + std::is_same::value || + std::is_same::value, + size_based_type, + __svec + >::type; +}; + +} // end of namesapce internal + + +// This is specifically for matmul and other backend tensor kernels that need +// to choose/switch between best simd types for utmost performance +template +using choose_best_simd_t = typename internal::choose_best_simd_type<__svec,N>::type; +//--------------------------------------------------------------------------------------------------------------// + + +// For switching between complex and non-complex implementation of simd we need different return types +//--------------------------------------------------------------------------------------------------------------// +template +struct get_simd_cmplx_value_type; +template +struct get_simd_cmplx_value_type > > { + using type = T; +}; +template +struct get_simd_cmplx_value_type > > { + using type = typename T::value_type; +}; + +template +struct simd_cmplx_value_type; +template class __svec, typename T, typename ABI> +struct simd_cmplx_value_type< __svec > { + using type = typename get_simd_cmplx_value_type::type; +}; +template +using simd_cmplx_value_t = typename simd_cmplx_value_type::type; +//--------------------------------------------------------------------------------------------------------------// + + + +} // end of namesapce Fastor + +#endif // FASTOR_SIMD_VECTOR_ABI diff --git a/noarch/include/Fastor/simd_vector/simd_vector_base.h b/noarch/include/Fastor/simd_vector/simd_vector_base.h new file mode 100644 index 00000000..bf0325a8 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_base.h @@ -0,0 +1,391 @@ +#ifndef SIMD_VECTOR_BASE_H +#define SIMD_VECTOR_BASE_H + +#include "Fastor/meta/meta.h" +#include "Fastor/config/config.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/simd_vector_abi.h" + +#include +#include + + +namespace Fastor { + +/* The default SIMDVector class that falls back to scalar implementation +* if SIMD types are not available or if vectorisation is disallowed +*/ +//--------------------------------------------------------------------------------------------------------------------// +template +struct SIMDVector { + using T = remove_cv_ref_t; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + using vector_type = SIMDVector; + using value_type = T[Size]; + using scalar_value_type = T; + using abi_type = ABI; + + FASTOR_INLINE SIMDVector() : value{} {} + FASTOR_INLINE SIMDVector(T num) { std::fill(value, value+Size, num); } + FASTOR_INLINE SIMDVector(const SIMDVector &a) { std::copy(a.value,a.value+a.Size,value); } + FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) { std::copy(data,data+Size,value); unused(Aligned); } + + FASTOR_INLINE SIMDVector operator=(T num) { std::fill(value, value+Size, num); return *this;} + FASTOR_INLINE SIMDVector operator=(const SIMDVector &a) { std::copy(a.value,a.value+a.Size,value); return *this; }; + + FASTOR_INLINE void load(const T *data, bool Aligned=true ) { std::copy(data,data+Size,value); unused(Aligned);} + FASTOR_INLINE void store(T *data, bool Aligned=true ) const { std::copy(value,value+Size,data); unused(Aligned);} + + FASTOR_INLINE void aligned_load(const T *data) { std::copy(data,data+Size,value); } + FASTOR_INLINE void aligned_store(T *data) const { std::copy(value,value+Size,data);} + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + std::fill(value, value+Size, 0); + for (FASTOR_INDEX i=0; i,bool> = false> + FASTOR_INLINE SIMDVector,ABI> real() const { + simd_cmplx_value_t arr[Size]; + for (FASTOR_INDEX i=0; i,ABI> out(arr,false); + return out; + } + template,bool> = false> + FASTOR_INLINE SIMDVector,ABI> imag() const { + simd_cmplx_value_t arr[Size]; + for (FASTOR_INDEX i=0; i,ABI> out(arr,false); + return out; + } + + FASTOR_INLINE void set(T num) { + for (FASTOR_INDEX i=0; i + FASTOR_INLINE void set(U first, Args ... args) { + static_assert(sizeof...(args)+1==Size,"CANNOT SET VECTOR WITH SPECIFIED NUMBER OF VALUES DUE TO ABI CONSIDERATION"); + T arr[Size] = {first,args...}; + std::reverse_copy(arr, arr+Size, value); + } + FASTOR_INLINE void set_sequential(T num0) { + for (FASTOR_INDEX i=0; i &a) { + for (FASTOR_INDEX i=0; i &a) { + for (FASTOR_INDEX i=0; i &a) { + for (FASTOR_INDEX i=0; i &a) { + for (FASTOR_INDEX i=0; i shift(FASTOR_INDEX i) { + SIMDVector out; + std::fill(out.value,out.value+out.Size,static_cast(0)); + std::copy(value,value+Size, out.value+i); + return out; + } + FASTOR_INLINE T sum() { + T quan = 0; + for (FASTOR_INDEX i=0; i reverse() { + SIMDVector out; + std::copy(value,value+Size,out.value); + std::reverse(out.value,out.value+Size); + return out; + } + // This is for comatibility with complex simd vectors + template,bool> = false> + FASTOR_INLINE SIMDVector,ABI> magnitude() { + simd_cmplx_value_t arr[Size]; + for (FASTOR_INDEX i=0; i,ABI> out(arr,false); + return out; + } + template,bool> = false> + FASTOR_INLINE SIMDVector,ABI> norm() { + simd_cmplx_value_t arr[Size]; + for (FASTOR_INDEX i=0; i,ABI> out(arr,false); + return out; + } + FASTOR_INLINE T minimum() { + T quan = 0; + for (FASTOR_INDEX i=0; iquan) + quan = value[i]; + return quan; + } + + FASTOR_INLINE T dot(const SIMDVector &other) { + T quan = 0; + for (FASTOR_INDEX i=0; i + FASTOR_INLINE SIMDVector cast() { + SIMDVector out; + for (FASTOR_INDEX i=0; i(value[i]); + } + return out; + } + +#ifdef FASTOR_ZERO_INITIALISE + FASTOR_ARCH_ALIGN T value[Size] = {}; +#else + FASTOR_ARCH_ALIGN T value[Size]; +#endif +}; + +template +constexpr FASTOR_INDEX SIMDVector::Size; + +template +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + os << "["; + for (FASTOR_INDEX i=0; i +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] + b.value[i]; + return out; +} +template +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, T b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] + b; + return out; +} +template +FASTOR_INLINE SIMDVector operator+(T a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a + b.value[i]; + return out; +} +template +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +template +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] - b.value[i]; + return out; +} +template +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, T b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] - b; + return out; +} +template +FASTOR_INLINE SIMDVector operator-(T a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a - b.value[i]; + return out; +} +template +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = -b.value[i]; + return out; +} + +template +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] * b.value[i]; + return out; +} +template +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, T b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] * b; + return out; +} +template +FASTOR_INLINE SIMDVector operator*(T a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a * b.value[i]; + return out; +} + +template +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] / b.value[i]; + return out; +} +template +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, T b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a.value[i] / b; + return out; +} +template +FASTOR_INLINE SIMDVector operator/(T a, const SIMDVector &b) { + SIMDVector out; + for (FASTOR_INDEX i=0; i::Size; ++i) + out.value[i] = a / b.value[i]; + return out; +} + +template +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + SIMDVector out; + for (FASTOR_INDEX i=0; i,bool> = false> +FASTOR_INLINE SIMDVector conj(const SIMDVector &a) { + T arr[SIMDVector::Size]; + for (FASTOR_INDEX i=0UL; i::Size; ++i) { + arr[i] = std::conj(a[i]); + } + return SIMDVector(arr,false); +} +template,bool> = false> +FASTOR_INLINE SIMDVector arg(const SIMDVector &a) { + T arr[SIMDVector::Size]; + for (FASTOR_INDEX i=0UL; i::Size; ++i) { + arr[i] = std::arg(a[i]); + } + return SIMDVector(arr,false); +} + +} // end of namespace Fastor + +#endif // SIMD_VECTOR_H + diff --git a/noarch/include/Fastor/simd_vector/simd_vector_common.h b/noarch/include/Fastor/simd_vector/simd_vector_common.h new file mode 100644 index 00000000..908c6772 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_common.h @@ -0,0 +1,1177 @@ +#ifndef SIMD_VECTOR_COMMON_H +#define SIMD_VECTOR_COMMON_H + +#include "Fastor/meta/meta.h" +#include "Fastor/simd_vector/simd_vector_base.h" +#include "Fastor/simd_vector/simd_vector_scalar.h" +#include "Fastor/simd_vector/simd_vector_float.h" +#include "Fastor/simd_vector/simd_vector_double.h" +#include "Fastor/simd_vector/simd_vector_int32.h" +#include "Fastor/simd_vector/simd_vector_int64.h" +#include "Fastor/simd_vector/simd_vector_complex_scalar.h" +#include "Fastor/simd_vector/simd_vector_complex_float.h" +#include "Fastor/simd_vector/simd_vector_complex_double.h" + + +namespace Fastor { + +/* Common functions for all SIMDVector types +*/ + +// This is for generic use in tensor expressions that need a uniform simd type +// between all of them +//----------------------------------------------------------------------------------------------------------// +template +struct choose_best_simd_vector { + using T = remove_cv_ref_t; + using type = typename std::conditional< std::is_same::value || + std::is_same::value || + std::is_same>::value || + std::is_same>::value || + std::is_same::value || + std::is_same::value, + SIMDVector, + SIMDVector + >::type; +}; + +// helper function +template +using choose_best_simd_vector_t = typename choose_best_simd_vector::type; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +// Get architecture native size/width of the SIMDVector - for instance for [T=double and AVX512] returns 8 +template +static constexpr size_t native_simd_size_v = internal::get_simd_vector_size>::value; +// Get Fastor supported size/width of the SIMDVector +template +static constexpr size_t simd_size_v = internal::get_simd_vector_size>::value; +//----------------------------------------------------------------------------------------------------------// + + +// Broadcasting vectorisation on general strides [gather operations] +//---------------------------------------------------------------------------------------------------------------- +// 1 word scalar +template>::bitsize==8,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int ) { + vec.set(data[idx]); +} +// 1 word in a 2 - for compatibility +template>::bitsize==16,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+general_stride],data[idx]); +} +// 1 word in a 4 - for compatibility +template>::bitsize==32,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} +// 1 word in a 8 - for compatibility +template>::bitsize==64,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+7*general_stride],data[idx+6*general_stride], + data[idx+5*general_stride],data[idx+4*general_stride], + data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} +// 1 word - sse +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+15*general_stride],data[idx+14*general_stride], + data[idx+13*general_stride],data[idx+12*general_stride], + data[idx+11*general_stride],data[idx+10*general_stride], + data[idx+9*general_stride],data[idx+8*general_stride], + data[idx+7*general_stride],data[idx+6*general_stride], + data[idx+5*general_stride],data[idx+4*general_stride], + data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} +// 1 word avx +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+31*general_stride],data[idx+30*general_stride], + data[idx+29*general_stride],data[idx+28*general_stride], + data[idx+27*general_stride],data[idx+26*general_stride], + data[idx+25*general_stride],data[idx+24*general_stride], + data[idx+23*general_stride],data[idx+22*general_stride], + data[idx+21*general_stride],data[idx+20*general_stride], + data[idx+19*general_stride],data[idx+18*general_stride], + data[idx+17*general_stride],data[idx+16*general_stride], + data[idx+15*general_stride],data[idx+14*general_stride], + data[idx+13*general_stride],data[idx+12*general_stride], + data[idx+11*general_stride],data[idx+10*general_stride], + data[idx+9 *general_stride],data[idx+8 *general_stride], + data[idx+7 *general_stride],data[idx+6 *general_stride], + data[idx+5 *general_stride],data[idx+4 *general_stride], + data[idx+3 *general_stride],data[idx+2 *general_stride], + data[idx+general_stride] ,data[idx]); +} +// 1 word avx512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+63*general_stride],data[idx+62*general_stride], + data[idx+61*general_stride],data[idx+60*general_stride], + data[idx+59*general_stride],data[idx+58*general_stride], + data[idx+57*general_stride],data[idx+56*general_stride], + data[idx+55*general_stride],data[idx+54*general_stride], + data[idx+53*general_stride],data[idx+52*general_stride], + data[idx+51*general_stride],data[idx+50*general_stride], + data[idx+49*general_stride],data[idx+48*general_stride], + data[idx+47*general_stride],data[idx+46*general_stride], + data[idx+45*general_stride],data[idx+44*general_stride], + data[idx+43*general_stride],data[idx+42*general_stride], + data[idx+41*general_stride],data[idx+40*general_stride], + data[idx+39*general_stride],data[idx+38*general_stride], + data[idx+37*general_stride],data[idx+36*general_stride], + data[idx+35*general_stride],data[idx+34*general_stride], + data[idx+33*general_stride],data[idx+32*general_stride], + data[idx+31*general_stride],data[idx+30*general_stride], + data[idx+29*general_stride],data[idx+28*general_stride], + data[idx+27*general_stride],data[idx+26*general_stride], + data[idx+25*general_stride],data[idx+24*general_stride], + data[idx+23*general_stride],data[idx+22*general_stride], + data[idx+21*general_stride],data[idx+20*general_stride], + data[idx+19*general_stride],data[idx+18*general_stride], + data[idx+17*general_stride],data[idx+16*general_stride], + data[idx+15*general_stride],data[idx+14*general_stride], + data[idx+13*general_stride],data[idx+12*general_stride], + data[idx+11*general_stride],data[idx+10*general_stride], + data[idx+9 *general_stride],data[idx+8 *general_stride], + data[idx+7 *general_stride],data[idx+6 *general_stride], + data[idx+5 *general_stride],data[idx+4 *general_stride], + data[idx+3 *general_stride],data[idx+2 *general_stride], + data[idx+general_stride] ,data[idx]); +} + +// 4 word scalar +template>::bitsize==32,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int ) { + vec.set(data[idx]); +} +// 4 word in an 8 - for compatibility +template>::bitsize==64,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+general_stride],data[idx]); +} +// 4 word SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+3*general_stride],data[idx+2*general_stride],data[idx+general_stride],data[idx]); +} +// 4 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+7*general_stride],data[idx+6*general_stride], + data[idx+5*general_stride],data[idx+4*general_stride], + data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} +// 4 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+15*general_stride],data[idx+14*general_stride], + data[idx+13*general_stride],data[idx+12*general_stride], + data[idx+11*general_stride],data[idx+10*general_stride], + data[idx+9*general_stride],data[idx+8*general_stride], + data[idx+7*general_stride],data[idx+6*general_stride], + data[idx+5*general_stride],data[idx+4*general_stride], + data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} + +// 8 word scalar +template>::bitsize==64,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int ) { + vec.set(data[idx]); +} +// 8 word SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+general_stride],data[idx]); +} +// 8 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} +// 8 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+7*general_stride],data[idx+6*general_stride], + data[idx+5*general_stride],data[idx+4*general_stride], + data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} + +// // 16 word scalar +// template>::bitsize==64,bool>::type=0> +// FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { +// vec.set(data[idx]); +// } +// 16 word scalar/SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+general_stride],data[idx]); +} +// 16 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} +// 16 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { + vec.set(data[idx+7*general_stride],data[idx+6*general_stride], + data[idx+5*general_stride],data[idx+4*general_stride], + data[idx+3*general_stride],data[idx+2*general_stride], + data[idx+general_stride],data[idx]); +} +//---------------------------------------------------------------------------------------------------------------- + + +// [Gather operations], when strides are not constant (i.e totally random) +//---------------------------------------------------------------------------------------------------------------- +// 4 word scalar +template>::bitsize==32,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { + vec.set(data[a[0]]); +} +// 4 word in an 8 - for compatibility +template>::bitsize==64,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { + vec.set(data[a[1]],data[a[0]]); +} +// 4 word SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[3]],data[a[2]],data[a[1]],data[a[0]]); +} +// 4 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[7]],data[a[6]],data[a[5]],data[a[4]], + data[a[3]],data[a[2]],data[a[1]],data[a[0]]); +} +// 4 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[15]],data[a[14]],data[a[13]],data[a[12]], + data[a[11]],data[a[10]],data[a[9]],data[a[8]], + data[a[7]],data[a[6]],data[a[5]],data[a[4]], + data[a[3]],data[a[2]],data[a[1]],data[a[0]]); +} + +// 8 word scalar +template>::bitsize==64,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { + vec.set(data[a[0]]); +} +// 8 word SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[1]],data[a[0]]); +} +// 8 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[3]],data[a[2]],data[a[1]],data[a[0]]); +} +// 8 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[7]],data[a[6]],data[a[5]],data[a[4]], + data[a[3]],data[a[2]],data[a[1]],data[a[0]]); +} + +// 16 word scalar/SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { + vec.set(data[a[1]],data[a[0]]); +} +// 16 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[3]],data[a[2]],data[a[1]],data[a[0]]); +} +// 16 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { + vec.set(data[a[7]],data[a[6]],data[a[5]],data[a[4]], + data[a[3]],data[a[2]],data[a[1]],data[a[0]]); +} +//---------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------- + + + + +// Scatter operations +//---------------------------------------------------------------------------------------------------------------- +// 4 word scalar +template>::bitsize==32,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int ) { + data[idx] = vec.value; +} +// 4 word in an 8 - for compatibility +template>::bitsize==64,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; +} +// 4 word SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; + data[idx+2*general_stride] = vec[2]; + data[idx+3*general_stride] = vec[3]; +} +// 4 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; + data[idx+2*general_stride] = vec[2]; + data[idx+3*general_stride] = vec[3]; + data[idx+4*general_stride] = vec[4]; + data[idx+5*general_stride] = vec[5]; + data[idx+6*general_stride] = vec[6]; + data[idx+7*general_stride] = vec[7]; +} +// 4 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; + data[idx+2*general_stride] = vec[2]; + data[idx+3*general_stride] = vec[3]; + data[idx+4*general_stride] = vec[4]; + data[idx+5*general_stride] = vec[5]; + data[idx+6*general_stride] = vec[6]; + data[idx+7*general_stride] = vec[7]; + data[idx+8*general_stride] = vec[8]; + data[idx+9*general_stride] = vec[9]; + data[idx+10*general_stride] = vec[10]; + data[idx+11*general_stride] = vec[11]; + data[idx+12*general_stride] = vec[12]; + data[idx+13*general_stride] = vec[13]; + data[idx+14*general_stride] = vec[14]; + data[idx+15*general_stride] = vec[15]; +} + +// 8 word scalar +template>::bitsize==64,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int ) { + data[idx] = vec.value; +} +// 8 word SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; +} +// 8 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; + data[idx+2*general_stride] = vec[2]; + data[idx+3*general_stride] = vec[3]; +} +// 8 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; + data[idx+2*general_stride] = vec[2]; + data[idx+3*general_stride] = vec[3]; + data[idx+4*general_stride] = vec[4]; + data[idx+5*general_stride] = vec[5]; + data[idx+6*general_stride] = vec[6]; + data[idx+7*general_stride] = vec[7]; +} + +// 16 word scalar/SSE +template>::bitsize==128,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int ) { + data[idx] = vec.value; +} +// 16 word AVX +template>::bitsize==256,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; +} +// 16 word AVX 512 +template>::bitsize==512,bool>::type=0> +FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { + data[idx] = vec[0]; + data[idx+general_stride] = vec[1]; + data[idx+2*general_stride] = vec[2]; + data[idx+3*general_stride] = vec[3]; +} +//---------------------------------------------------------------------------------------------------------------- + + + + + + + + + +// Mask load and store operations implemented as free functions till SIMDVector finds a proper +// companion masked vector. Note that these require at least AVX even the SSE variants +//---------------------------------------------------------------------------------------------------------------- +template +FASTOR_INLINE V +maskload(const typename V::scalar_value_type * FASTOR_RESTRICT a, const int (&maska)[V::Size]) { + // masked array is reversed like other intrinsics + typename V::scalar_value_type val_out[V::Size] = {}; // zero out the rest + for (FASTOR_INDEX i=0; i +FASTOR_INLINE SIMDVector,simd_abi::avx> +maskload,simd_abi::avx>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[4]) { + // Split the mask in to a higher and lower part - we need two masks for this + __m256i mask0 = _mm256_set_epi64x(maska[2],maska[2],maska[3],maska[3]); + __m256d lo = _mm256_maskload_pd(reinterpret_cast(a ), (__m256i) mask0); + __m256i mask1 = _mm256_set_epi64x(maska[0],maska[0],maska[1],maska[1]); + __m256d hi = _mm256_maskload_pd(reinterpret_cast(a+2), (__m256i) mask1); + __m256d value_r, value_i; + arrange_from_load(value_r, value_i, lo, hi); + return SIMDVector,simd_abi::avx>(value_r,value_i); +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx> +maskload,simd_abi::avx>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[8]) { + // Split the mask in to a higher and lower part - we need two masks for this + __m256i mask0 = _mm256_set_epi32(maska[4],maska[4],maska[5],maska[5],maska[6],maska[6],maska[7],maska[7]); + __m256 lo = _mm256_maskload_ps(reinterpret_cast(a ), (__m256i) mask0); + __m256i mask1 = _mm256_set_epi32(maska[0],maska[0],maska[1],maska[1],maska[2],maska[2],maska[3],maska[3]); + __m256 hi = _mm256_maskload_ps(reinterpret_cast(a+4), (__m256i) mask1); + __m256 value_r, value_i; + arrange_from_load(value_r, value_i, lo, hi); + return SIMDVector,simd_abi::avx>(value_r,value_i); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const double * FASTOR_RESTRICT a, const int (&maska)[4]) { + __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); + return _mm256_maskload_pd(a,(__m256i) mask); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const Int64 * FASTOR_RESTRICT a, const int (&maska)[4]) { + __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); + return _mm256_maskload_epi64(reinterpret_cast(a),(__m256i) mask); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const float * FASTOR_RESTRICT a, const int (&maska)[8]) { + __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); + return _mm256_maskload_ps(a,(__m256i) mask); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const int * FASTOR_RESTRICT a, const int (&maska)[8]) { + __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); + return _mm256_maskload_epi32(a,(__m256i) mask); +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> +maskload,simd_abi::sse>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[2]) { + // Split the mask in to a higher and lower part - we need two masks for this + __m128i mask0 = _mm_set_epi64x(maska[1],maska[1]); + __m128d lo = _mm_maskload_pd(reinterpret_cast(a ), (__m128i) mask0); + __m128i mask1 = _mm_set_epi64x(maska[0],maska[0]); + __m128d hi = _mm_maskload_pd(reinterpret_cast(a+1), (__m128i) mask1); + __m128d value_r, value_i; + arrange_from_load(value_r, value_i, lo, hi); + return SIMDVector,simd_abi::sse>(value_r,value_i); +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> +maskload,simd_abi::sse>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[4]) { + // Split the mask in to a higher and lower part - we need two masks for this + __m128i mask0 = _mm_set_epi32(maska[2],maska[2],maska[3],maska[3]); + __m128 lo = _mm_maskload_ps(reinterpret_cast(a ), (__m128i) mask0); + __m128i mask1 = _mm_set_epi32(maska[0],maska[0],maska[1],maska[1]); + __m128 hi = _mm_maskload_ps(reinterpret_cast(a+2), (__m128i) mask1); + __m128 value_r, value_i; + arrange_from_load(value_r, value_i, lo, hi); + return SIMDVector,simd_abi::sse>(value_r,value_i); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const double * FASTOR_RESTRICT a, const int (&maska)[2]) { + __m128i mask = _mm_set_epi64x(maska[0],maska[1]); + return _mm_maskload_pd(a,(__m128i) mask); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const Int64 * FASTOR_RESTRICT a, const int (&maska)[2]) { + __m128i mask = _mm_set_epi64x(maska[0],maska[1]); + return _mm_maskload_epi64(reinterpret_cast(a),(__m128i) mask); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const float * FASTOR_RESTRICT a, const int (&maska)[4]) { + __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); + return _mm_maskload_ps(a,(__m128i) mask); +} +template<> +FASTOR_INLINE SIMDVector +maskload>(const int * FASTOR_RESTRICT a, const int (&maska)[4]) { + __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); + return _mm_maskload_epi32(a,(__m128i) mask); +} +#endif // FASTOR_AVX2_IMPL + + +template +FASTOR_INLINE +void maskstore(typename V::scalar_value_type * FASTOR_RESTRICT a, const int (&maska)[V::Size], V& v) { + for (FASTOR_INDEX i=0; i +FASTOR_INLINE +void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector,simd_abi::avx> &v) { + // Split the mask in to a higher and lower part - we need two masks for this + __m256i mask0 = _mm256_set_epi64x(maska[2],maska[2],maska[3],maska[3]); + __m256i mask1 = _mm256_set_epi64x(maska[0],maska[0],maska[1],maska[1]); + __m256d lo, hi; + arrange_for_store(lo, hi, v.value_r, v.value_i); + _mm256_maskstore_pd(reinterpret_cast(a ), (__m256i) mask0, lo); + _mm256_maskstore_pd(reinterpret_cast(a+2), (__m256i) mask1, hi); +} +template<> +FASTOR_INLINE +void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[8], SIMDVector,simd_abi::avx> &v) { + // Split the mask in to a higher and lower part - we need two masks for this + __m256i mask0 = _mm256_set_epi32(maska[4],maska[4],maska[5],maska[5],maska[6],maska[6],maska[7],maska[7]); + __m256i mask1 = _mm256_set_epi32(maska[0],maska[0],maska[1],maska[1],maska[2],maska[2],maska[3],maska[3]); + __m256 lo, hi; + arrange_for_store(lo, hi, v.value_r, v.value_i); + _mm256_maskstore_ps(reinterpret_cast(a ), (__m256i) mask0, lo); + _mm256_maskstore_ps(reinterpret_cast(a+4), (__m256i) mask1, hi); +} +template<> +FASTOR_INLINE +void maskstore(double * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { + __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); + _mm256_maskstore_pd(a,(__m256i) mask, v.value); +} +template<> +FASTOR_INLINE +void maskstore(Int64 * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { + __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); + _mm256_maskstore_epi64(reinterpret_cast(a),(__m256i) mask, v.value); +} +template<> +FASTOR_INLINE +void maskstore(float * FASTOR_RESTRICT a, const int (&maska)[8], SIMDVector &v) { + __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); + _mm256_maskstore_ps(a,(__m256i) mask, v.value); +} +template<> +FASTOR_INLINE +void maskstore(int * FASTOR_RESTRICT a, const int (&maska)[8], SIMDVector &v) { + __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); + _mm256_maskstore_epi32(a,(__m256i) mask, v.value); +} +template<> +FASTOR_INLINE +void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[2], SIMDVector,simd_abi::sse> &v) { + // Split the mask in to a higher and lower part - we need two masks for this + __m128i mask0 = _mm_set_epi64x(maska[1],maska[1]); + __m128i mask1 = _mm_set_epi64x(maska[0],maska[0]); + __m128d lo, hi; + arrange_for_store(lo, hi, v.value_r, v.value_i); + _mm_maskstore_pd(reinterpret_cast(a ), (__m128i) mask0, lo); + _mm_maskstore_pd(reinterpret_cast(a+1), (__m128i) mask1, hi); +} +template<> +FASTOR_INLINE +void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector,simd_abi::sse> &v) { + // Split the mask in to a higher and lower part - we need two masks for this + __m128i mask0 = _mm_set_epi32(maska[2],maska[2],maska[3],maska[3]); + __m128i mask1 = _mm_set_epi32(maska[0],maska[0],maska[1],maska[1]); + __m128 lo, hi; + arrange_for_store(lo, hi, v.value_r, v.value_i); + _mm_maskstore_ps(reinterpret_cast(a ), (__m128i) mask0, lo); + _mm_maskstore_ps(reinterpret_cast(a+2), (__m128i) mask1, hi); +} +template<> +FASTOR_INLINE +void maskstore(double * FASTOR_RESTRICT a, const int (&maska)[2], SIMDVector &v) { + __m128i mask = _mm_set_epi64x(maska[0],maska[1]); + _mm_maskstore_pd(a,(__m128i) mask, v.value); +} +template<> +FASTOR_INLINE +void maskstore(Int64 * FASTOR_RESTRICT a, const int (&maska)[2], SIMDVector &v) { + __m128i mask = _mm_set_epi64x(maska[0],maska[1]); + _mm_maskstore_epi64(reinterpret_cast(a),(__m128i) mask, v.value); +} +template<> +FASTOR_INLINE +void maskstore(float * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { + __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); + _mm_maskstore_ps(a,(__m128i) mask, v.value); +} +template<> +FASTOR_INLINE +void maskstore(int * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { + __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); + _mm_maskstore_epi32(a,(__m128i) mask, v.value); +} +#endif // FASTOR_AVX2_IMPL +//---------------------------------------------------------------------------------------------------------------- + + + + + + + +// FMAs +//---------------------------------------------------------------------------------------------------------------- +template +FASTOR_INLINE SIMDVector fmadd(const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + return a*b+c; +} +template +FASTOR_INLINE SIMDVector fmsub(const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + return a*b-c; +} +template +FASTOR_INLINE SIMDVector fnmadd(const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + return c-a*b; +} + +#ifdef FASTOR_FMA_IMPL +// fmadd +template<> +FASTOR_INLINE SIMDVector fmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm_fmadd_ps(a.value,b.value,c.value); + return out; +} +template<> +FASTOR_INLINE SIMDVector fmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm256_fmadd_ps(a.value,b.value,c.value); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector fmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm512_fmadd_ps(a.value,b.value,c.value); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector fmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm_fmadd_pd(a.value,b.value,c.value); + return out; +} +template<> +FASTOR_INLINE SIMDVector fmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm256_fmadd_pd(a.value,b.value,c.value); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector fmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm512_fmadd_pd(a.value,b.value,c.value); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> fmadd,simd_abi::sse>( + const SIMDVector,simd_abi::sse> &a, + const SIMDVector,simd_abi::sse> &b, + const SIMDVector,simd_abi::sse> &c) { + SIMDVector,simd_abi::sse> out; + // ar*br - ai*bi + cr + out.value_r = _mm_fnmadd_ps(a.value_i,b.value_i,_mm_fmadd_ps(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm_fmadd_ps (a.value_i,b.value_r,_mm_fmadd_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx> fmadd,simd_abi::avx>( + const SIMDVector,simd_abi::avx> &a, + const SIMDVector,simd_abi::avx> &b, + const SIMDVector,simd_abi::avx> &c) { + SIMDVector,simd_abi::avx> out; + // ar*br - ai*bi + cr + out.value_r = _mm256_fnmadd_ps(a.value_i,b.value_i,_mm256_fmadd_ps(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm256_fmadd_ps (a.value_i,b.value_r,_mm256_fmadd_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx512> fmadd,simd_abi::avx512>( + const SIMDVector,simd_abi::avx512> &a, + const SIMDVector,simd_abi::avx512> &b, + const SIMDVector,simd_abi::avx512> &c) { + SIMDVector,simd_abi::avx512> out; + // ar*br - ai*bi + cr + out.value_r = _mm512_fnmadd_ps(a.value_i,b.value_i,_mm512_fmadd_ps(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm512_fmadd_ps (a.value_i,b.value_r,_mm512_fmadd_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> fmadd,simd_abi::sse>( + const SIMDVector,simd_abi::sse> &a, + const SIMDVector,simd_abi::sse> &b, + const SIMDVector,simd_abi::sse> &c) { + SIMDVector,simd_abi::sse> out; + // ar*br - ai*bi + cr + out.value_r = _mm_fnmadd_pd(a.value_i,b.value_i,_mm_fmadd_pd(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm_fmadd_pd (a.value_i,b.value_r,_mm_fmadd_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx> fmadd,simd_abi::avx>( + const SIMDVector,simd_abi::avx> &a, + const SIMDVector,simd_abi::avx> &b, + const SIMDVector,simd_abi::avx> &c) { + SIMDVector,simd_abi::avx> out; + // ar*br - ai*bi + cr + out.value_r = _mm256_fnmadd_pd(a.value_i,b.value_i,_mm256_fmadd_pd(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm256_fmadd_pd (a.value_i,b.value_r,_mm256_fmadd_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx512> fmadd,simd_abi::avx512>( + const SIMDVector,simd_abi::avx512> &a, + const SIMDVector,simd_abi::avx512> &b, + const SIMDVector,simd_abi::avx512> &c) { + SIMDVector,simd_abi::avx512> out; + // ar*br - ai*bi + cr + out.value_r = _mm512_fnmadd_pd(a.value_i,b.value_i,_mm512_fmadd_pd(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm512_fmadd_pd (a.value_i,b.value_r,_mm512_fmadd_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +#endif + +// fmsub +template<> +FASTOR_INLINE SIMDVector fmsub( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm_fmsub_ps(a.value,b.value,c.value); + return out; +} +template<> +FASTOR_INLINE SIMDVector fmsub( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm256_fmsub_ps(a.value,b.value,c.value); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector fmsub( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm512_fmsub_ps(a.value,b.value,c.value); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector fmsub( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm_fmsub_pd(a.value,b.value,c.value); + return out; +} +template<> +FASTOR_INLINE SIMDVector fmsub( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm256_fmsub_pd(a.value,b.value,c.value); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector fmsub( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm512_fmsub_pd(a.value,b.value,c.value); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> fmsub,simd_abi::sse>( + const SIMDVector,simd_abi::sse> &a, + const SIMDVector,simd_abi::sse> &b, + const SIMDVector,simd_abi::sse> &c) { + SIMDVector,simd_abi::sse> out; + // ar*br - ai*bi - cr + out.value_r = _mm_fnmadd_ps(a.value_i,b.value_i,_mm_fmsub_ps(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br - ci + out.value_i = _mm_fmadd_ps (a.value_i,b.value_r,_mm_fmsub_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx> fmsub,simd_abi::avx>( + const SIMDVector,simd_abi::avx> &a, + const SIMDVector,simd_abi::avx> &b, + const SIMDVector,simd_abi::avx> &c) { + SIMDVector,simd_abi::avx> out; + // ar*br - ai*bi + cr + out.value_r = _mm256_fnmadd_ps(a.value_i,b.value_i,_mm256_fmsub_ps(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm256_fmadd_ps (a.value_i,b.value_r,_mm256_fmsub_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx512> fmsub,simd_abi::avx512>( + const SIMDVector,simd_abi::avx512> &a, + const SIMDVector,simd_abi::avx512> &b, + const SIMDVector,simd_abi::avx512> &c) { + SIMDVector,simd_abi::avx512> out; + // ar*br - ai*bi + cr + out.value_r = _mm512_fnmadd_ps(a.value_i,b.value_i,_mm512_fmsub_ps(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm512_fmadd_ps (a.value_i,b.value_r,_mm512_fmsub_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> fmsub,simd_abi::sse>( + const SIMDVector,simd_abi::sse> &a, + const SIMDVector,simd_abi::sse> &b, + const SIMDVector,simd_abi::sse> &c) { + SIMDVector,simd_abi::sse> out; + // ar*br - ai*bi - cr + out.value_r = _mm_fnmadd_pd(a.value_i,b.value_i,_mm_fmsub_pd(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br - ci + out.value_i = _mm_fmadd_pd (a.value_i,b.value_r,_mm_fmsub_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx> fmsub,simd_abi::avx>( + const SIMDVector,simd_abi::avx> &a, + const SIMDVector,simd_abi::avx> &b, + const SIMDVector,simd_abi::avx> &c) { + SIMDVector,simd_abi::avx> out; + // ar*br - ai*bi + cr + out.value_r = _mm256_fnmadd_pd(a.value_i,b.value_i,_mm256_fmsub_pd(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm256_fmadd_pd (a.value_i,b.value_r,_mm256_fmsub_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx512> fmsub,simd_abi::avx512>( + const SIMDVector,simd_abi::avx512> &a, + const SIMDVector,simd_abi::avx512> &b, + const SIMDVector,simd_abi::avx512> &c) { + SIMDVector,simd_abi::avx512> out; + // ar*br - ai*bi + cr + out.value_r = _mm512_fnmadd_pd(a.value_i,b.value_i,_mm512_fmsub_pd(a.value_r,b.value_r,c.value_r)); + // ar*bi + ai*br + ci + out.value_i = _mm512_fmadd_pd (a.value_i,b.value_r,_mm512_fmsub_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +#endif + +// fnmadd +template<> +FASTOR_INLINE SIMDVector fnmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm_fnmadd_ps(a.value,b.value,c.value); + return out; +} +template<> +FASTOR_INLINE SIMDVector fnmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm256_fnmadd_ps(a.value,b.value,c.value); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector fnmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm512_fnmadd_ps(a.value,b.value,c.value); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector fnmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm_fnmadd_pd(a.value,b.value,c.value); + return out; +} +template<> +FASTOR_INLINE SIMDVector fnmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm256_fnmadd_pd(a.value,b.value,c.value); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector fnmadd( + const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { + SIMDVector out; + out.value = _mm512_fnmadd_pd(a.value,b.value,c.value); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> fnmadd,simd_abi::sse>( + const SIMDVector,simd_abi::sse> &a, + const SIMDVector,simd_abi::sse> &b, + const SIMDVector,simd_abi::sse> &c) { + SIMDVector,simd_abi::sse> out; + // -ar*br + ai*bi + cr + out.value_r = _mm_fmadd_ps (a.value_i,b.value_i,_mm_fnmadd_ps(a.value_r,b.value_r,c.value_r)); + // -ar*bi - ai*br + ci + out.value_i = _mm_fnmadd_ps(a.value_i,b.value_r,_mm_fnmadd_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx> fnmadd,simd_abi::avx>( + const SIMDVector,simd_abi::avx> &a, + const SIMDVector,simd_abi::avx> &b, + const SIMDVector,simd_abi::avx> &c) { + SIMDVector,simd_abi::avx> out; + // -ar*br + ai*bi + cr + out.value_r = _mm256_fmadd_ps (a.value_i,b.value_i,_mm256_fnmadd_ps(a.value_r,b.value_r,c.value_r)); + // -ar*bi - ai*br + ci + out.value_i = _mm256_fnmadd_ps(a.value_i,b.value_r,_mm256_fnmadd_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx512> fnmadd,simd_abi::avx512>( + const SIMDVector,simd_abi::avx512> &a, + const SIMDVector,simd_abi::avx512> &b, + const SIMDVector,simd_abi::avx512> &c) { + SIMDVector,simd_abi::avx512> out; + // -ar*br + ai*bi + cr + out.value_r = _mm512_fmadd_ps (a.value_i,b.value_i,_mm512_fnmadd_ps(a.value_r,b.value_r,c.value_r)); + // -ar*bi - ai*br + ci + out.value_i = _mm512_fnmadd_ps(a.value_i,b.value_r,_mm512_fnmadd_ps(a.value_r,b.value_i,c.value_i)); + return out; +} +#endif +template<> +FASTOR_INLINE SIMDVector,simd_abi::sse> fnmadd,simd_abi::sse>( + const SIMDVector,simd_abi::sse> &a, + const SIMDVector,simd_abi::sse> &b, + const SIMDVector,simd_abi::sse> &c) { + SIMDVector,simd_abi::sse> out; + // -ar*br + ai*bi + cr + out.value_r = _mm_fmadd_pd (a.value_i,b.value_i,_mm_fnmadd_pd(a.value_r,b.value_r,c.value_r)); + // -ar*bi - ai*br + ci + out.value_i = _mm_fnmadd_pd(a.value_i,b.value_r,_mm_fnmadd_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx> fnmadd,simd_abi::avx>( + const SIMDVector,simd_abi::avx> &a, + const SIMDVector,simd_abi::avx> &b, + const SIMDVector,simd_abi::avx> &c) { + SIMDVector,simd_abi::avx> out; + // -ar*br + ai*bi + cr + out.value_r = _mm256_fmadd_pd (a.value_i,b.value_i,_mm256_fnmadd_pd(a.value_r,b.value_r,c.value_r)); + // -ar*bi - ai*br + ci + out.value_i = _mm256_fnmadd_pd(a.value_i,b.value_r,_mm256_fnmadd_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +#ifdef FASTOR_AVX512F_IMPL +template<> +FASTOR_INLINE SIMDVector,simd_abi::avx512> fnmadd,simd_abi::avx512>( + const SIMDVector,simd_abi::avx512> &a, + const SIMDVector,simd_abi::avx512> &b, + const SIMDVector,simd_abi::avx512> &c) { + SIMDVector,simd_abi::avx512> out; + // -ar*br + ai*bi + cr + out.value_r = _mm512_fmadd_pd (a.value_i,b.value_i,_mm512_fnmadd_pd(a.value_r,b.value_r,c.value_r)); + // -ar*bi - ai*br + ci + out.value_i = _mm512_fnmadd_pd(a.value_i,b.value_r,_mm512_fnmadd_pd(a.value_r,b.value_i,c.value_i)); + return out; +} +#endif + +#endif +//---------------------------------------------------------------------------------------------------------------- + + + + +// Binary comparison ops +//----------------------------------------------------------------------------------------------------------------// +#define FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(OP) \ +template \ +FASTOR_INLINE SIMDVector::Size>> operator OP(const SIMDVector &a, const SIMDVector &b) { \ + constexpr FASTOR_INDEX Size = SIMDVector::Size;\ + FASTOR_ARCH_ALIGN T val_a[Size];\ + a.store(val_a);\ + FASTOR_ARCH_ALIGN T val_b[Size];\ + b.store(val_b);\ + SIMDVector::Size>> out;\ + FASTOR_ARCH_ALIGN bool val_out[Size];\ + out.store(val_out);\ + for (FASTOR_INDEX i=0; i) +FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(<) +FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(>=) +FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(<=) +FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(&&) +FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(||) + + +#define FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(OP) \ +template \ +FASTOR_INLINE SIMDVector::Size>> operator OP(const SIMDVector &a, U b) { \ + constexpr FASTOR_INDEX Size = SIMDVector::Size;\ + FASTOR_ARCH_ALIGN T val_a[Size];\ + a.store(val_a);\ + SIMDVector::Size>> out;\ + FASTOR_ARCH_ALIGN bool val_out[Size];\ + out.store(val_out);\ + for (FASTOR_INDEX i=0; i) +FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(<) +FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(>=) +FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(<=) +FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(&&) +FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(||) + + +#define FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(OP) \ +template \ +FASTOR_INLINE SIMDVector::Size>> operator OP(U a, const SIMDVector &b) { \ + constexpr FASTOR_INDEX Size = SIMDVector::Size;\ + FASTOR_ARCH_ALIGN T val_b[Size];\ + b.store(val_b);\ + SIMDVector::Size>> out;\ + FASTOR_ARCH_ALIGN bool val_out[Size];\ + out.store(val_out);\ + for (FASTOR_INDEX i=0; i) +FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(<) +FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(>=) +FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(<=) +FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(&&) +FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(||) +//----------------------------------------------------------------------------------------------------------------// + + +} // end of namespace Fastor + +#endif // SIMD_VECTOR_COMMON_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_complex_double.h b/noarch/include/Fastor/simd_vector/simd_vector_complex_double.h new file mode 100644 index 00000000..59e8b88c --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_complex_double.h @@ -0,0 +1,1611 @@ +#ifndef SIMD_VECTOR_COMPLEX_DOUBLE_H +#define SIMD_VECTOR_COMPLEX_DOUBLE_H + +#include "Fastor/util/extended_algorithms.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/simd_vector_base.h" +#include "Fastor/simd_vector/simd_vector_double.h" +#include +#include + +namespace Fastor { + + +// AVX512 VERSION +//------------------------------------------------------------------------------------------------------------ + +#ifdef FASTOR_AVX512F_IMPL + +template <> +struct SIMDVector, simd_abi::avx512> { + using vector_type = SIMDVector, simd_abi::avx512>; + using value_type = __m512d; + using scalar_value_type = std::complex; + using abi_type = simd_abi::avx512; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value_r(_mm512_setzero_pd()), value_i(_mm512_setzero_pd()) {} + FASTOR_INLINE SIMDVector(std::complex num) { + value_r = _mm512_set1_pd(num.real()); + value_i = _mm512_set1_pd(num.imag()); + } + FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} + FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + + FASTOR_INLINE SIMDVector,simd_abi::avx512> operator=(std::complex num) { + value_r = _mm512_set1_pd(num.real()); + value_i = _mm512_set1_pd(num.imag()); + return *this; + } + + FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { + if (Aligned) + complex_aligned_store(data); + else + complex_unaligned_store(data); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { + if (!Aligned) + complex_mask_unaligned_load(data,mask); + else + complex_mask_aligned_load(data,mask); + } + FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { + if (!Aligned) + complex_mask_unaligned_store(data,mask); + else + complex_mask_aligned_store(data,mask); + } + + FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { + return scalar_value_type(reinterpret_cast(&value_r)[i],reinterpret_cast(&value_i)[i]); + } + + FASTOR_INLINE SIMDVector real() const { + return value_r; + } + FASTOR_INLINE SIMDVector imag() const { + return value_i; + } + + FASTOR_INLINE void set(std::complex num) { + value_r = _mm512_set1_pd(num.real()); + value_i = _mm512_set1_pd(num.imag()); + } + FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, + scalar_value_type num2, scalar_value_type num3, + scalar_value_type num4, scalar_value_type num5, + scalar_value_type num6, scalar_value_type num7) { + const scalar_value_type tmp[Size] = {num0,num1,num2,num3,num4,num5,num6,num7}; + complex_unaligned_load(tmp); + } + + // In-place operators + template,bool> = false> + FASTOR_INLINE void operator+=(U num) { + value_r = _mm512_add_pd(value_r,_mm512_set1_pd(num)); + } + FASTOR_INLINE void operator+=(scalar_value_type num) { + *this += vector_type(num); + } + FASTOR_INLINE void operator+=(const vector_type &a) { + value_r = _mm512_add_pd(value_r,a.value_r); + value_i = _mm512_add_pd(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + value_r = _mm512_sub_pd(value_r,_mm512_set1_pd(num)); + } + FASTOR_INLINE void operator-=(scalar_value_type num) { + *this -= vector_type(num); + } + FASTOR_INLINE void operator-=(const vector_type &a) { + value_r = _mm512_sub_pd(value_r,a.value_r); + value_i = _mm512_sub_pd(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + __m512d val = _mm512_set1_pd(num); + value_r = _mm512_mul_pd(value_r,val); + value_i = _mm512_mul_pd(value_i,val); + } + FASTOR_INLINE void operator*=(scalar_value_type num) { + *this *= vector_type(num); + } + FASTOR_INLINE void operator*=(const vector_type &a) { +#ifdef FASTOR_FMA_IMPL + __m512d tmp = _mm512_fmsub_pd(value_r,a.value_r,_mm512_mul_pd(value_i,a.value_i)); + value_i = _mm512_fmadd_pd(value_r,a.value_i,_mm512_mul_pd(value_i,a.value_r)); +#else + __m512d tmp = _mm512_sub_pd(_mm512_mul_pd(value_r,a.value_r),_mm512_mul_pd(value_i,a.value_i)); + value_i = _mm512_add_pd(_mm512_mul_pd(value_r,a.value_i),_mm512_mul_pd(value_i,a.value_r)); +#endif + value_r = tmp; + } + + template,bool> = false> + FASTOR_INLINE void operator/=(U num) { + __m512d val = _mm512_set1_pd(num); + value_r = _mm512_div_pd(value_r,val); + value_i = _mm512_div_pd(value_i,val); + } + FASTOR_INLINE void operator/=(scalar_value_type num) { + *this /= vector_type(num); + } + FASTOR_INLINE void operator/=(const vector_type &a) { + __m512d tmp = value_r; +#ifdef FASTOR_FMA_IMPL + value_r = _mm512_fmadd_pd(value_r , a.value_r, _mm512_mul_pd(value_i,a.value_i)); + value_i = _mm512_fmsub_pd(value_i , a.value_r, _mm512_mul_pd(tmp,a.value_i)); + __m512d den = _mm512_fmadd_pd(a.value_r, a.value_r, _mm512_mul_pd(a.value_i,a.value_i)); +#else + value_r = _mm512_add_pd(_mm512_mul_pd(value_r , a.value_r), _mm512_mul_pd(value_i,a.value_i)); + value_i = _mm512_sub_pd(_mm512_mul_pd(value_i , a.value_r), _mm512_mul_pd(tmp,a.value_i)); + __m512d den = _mm512_add_pd(_mm512_mul_pd(a.value_r, a.value_r), _mm512_mul_pd(a.value_i,a.value_i)); +#endif + value_r = _mm512_div_pd(value_r,den); + value_i = _mm512_div_pd(value_i,den); + } + // end of in-place operators + + FASTOR_INLINE scalar_value_type sum() const { +#ifdef FASTOR_HAS_AVX512_REDUCE_ADD + return scalar_value_type(_mm512_reduce_add_pd(value_r),_mm512_reduce_add_pd(value_i)); +#else + __m256d lor = _mm512_castpd512_pd256(value_r); + __m256d hir = _mm512_extractf64x4_pd(value_r,0x1); + __m256d loi = _mm512_castpd512_pd256(value_i); + __m256d hii = _mm512_extractf64x4_pd(value_i,0x1); + return scalar_value_type(_mm256_sum_pd(_mm256_add_pd(lor,hir)), _mm256_sum_pd(_mm256_add_pd(loi,hii))); +#endif + } + FASTOR_INLINE scalar_value_type product() const { + vector_type tmp(*this); + return tmp[0]*tmp[1]*tmp[2]*tmp[3]*tmp[4]*tmp[5]*tmp[6]*tmp[7]; + } + FASTOR_INLINE vector_type reverse() const { + vector_type out; + out.value_r = _mm512_reverse_pd(value_r); + out.value_i = _mm512_reverse_pd(value_i); + return out; + } + /* Actual magnitude - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector magnitude() const { +#ifdef FASTOR_FMA_IMPL + return _mm512_sqrt_pd(_mm512_fmadd_pd(value_r,value_r,_mm512_mul_pd(value_i,value_i))); +#else + return _mm512_sqrt_pd(_mm512_add_pd(_mm512_mul_pd(value_r,value_r),_mm512_mul_pd(value_i,value_i))); +#endif + } + /* STL compliant squared norm - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector norm() const { +#ifdef FASTOR_FMA_IMPL + return _mm512_fmadd_pd(value_r,value_r,_mm512_mul_pd(value_i,value_i)); +#else + return _mm512_add_pd(_mm512_mul_pd(value_r,value_r),_mm512_mul_pd(value_i,value_i)); +#endif + } + // Magnitude based minimum + FASTOR_INLINE scalar_value_type minimum() const; + // Magnitude based maximum + FASTOR_INLINE scalar_value_type maximum() const; + + FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { + vector_type out(*this); + out *= other; + return out.sum(); + } + + value_type value_r; + value_type value_i; + +protected: + FASTOR_INLINE void complex_aligned_load(const std::complex *data) { + __m512d lo = _mm512_load_pd(reinterpret_cast(data )); + __m512d hi = _mm512_load_pd(reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); + } + FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { + __m512d lo = _mm512_loadu_pd(reinterpret_cast(data )); + __m512d hi = _mm512_loadu_pd(reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); + } + + FASTOR_INLINE void complex_aligned_store(std::complex *data) const { + __m512d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm512_store_pd(reinterpret_cast(data ), lo); + _mm512_store_pd(reinterpret_cast(data+4), hi); + } + FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { + __m512d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm512_storeu_pd(reinterpret_cast(data ), lo); + _mm512_storeu_pd(reinterpret_cast(data+4), hi); + } + + FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { +#ifdef FASTOR_HAS_AVX512_MASKS + __m512d lo, hi; + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + lo = _mm512_mask_load_pd(lo, mask0, reinterpret_cast(data )); + hi = _mm512_mask_load_pd(hi, mask1, reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm512_setzero_pd(); + value_i = _mm512_setzero_pd(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + lo = _mm512_mask_loadu_pd(lo, mask0, reinterpret_cast(data )); + hi = _mm512_mask_loadu_pd(hi, mask1, reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm512_setzero_pd(); + value_i = _mm512_setzero_pd(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + _mm512_mask_store_pd(reinterpret_cast(data ), mask0, lo); + _mm512_mask_store_pd(reinterpret_cast(data+4), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { +#ifdef FASTOR_HAS_AVX512_MASKS + __m512d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + _mm512_mask_storeu_pd(reinterpret_cast(data ), mask0, lo); + _mm512_mask_storeu_pd(reinterpret_cast(data+4), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx512> a) { + // ICC crashes without a copy + const __m512d vr = a.value_r; + const __m512d vi = a.value_i; + const double* value_r = reinterpret_cast(&vr); + const double* value_i = reinterpret_cast(&vi); + os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " + << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " + << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " + << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j, " + << value_r[4] << signum_string(value_i[4]) << std::abs(value_i[4]) << "j, " + << value_r[5] << signum_string(value_i[5]) << std::abs(value_i[5]) << "j, " + << value_r[6] << signum_string(value_i[6]) << std::abs(value_i[6]) << "j, " + << value_r[7] << signum_string(value_i[7]) << std::abs(value_i[7]) << "j" << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; + out.value_r = _mm512_add_pd(a.value_r,b.value_r); + out.value_i = _mm512_add_pd(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a + SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) + b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out(a); + out.value_r = _mm512_add_pd(a.value_r,_mm512_set1_pd(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(U a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out(b); + out.value_r = _mm512_add_pd(_mm512_set1_pd(a), b.value_r); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &b) { + return b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; + out.value_r = _mm512_sub_pd(a.value_r,b.value_r); + out.value_i = _mm512_sub_pd(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a - SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) - b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out(a); + out.value_r = _mm512_sub_pd(a.value_r,_mm512_set1_pd(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(U a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(std::complex(a,0)) - b; +} +/* This is negation and not complex conjugate */ +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out; + out.value_r = _mm512_neg_pd(a.value_r); + out.value_i = _mm512_neg_pd(a.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm512_fmsub_pd(a.value_r,b.value_r,_mm512_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm512_fmadd_pd(a.value_r,b.value_i,_mm512_mul_pd(a.value_i,b.value_r)); +#else + out.value_r = _mm512_sub_pd(_mm512_mul_pd(a.value_r,b.value_r),_mm512_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm512_add_pd(_mm512_mul_pd(a.value_r,b.value_i),_mm512_mul_pd(a.value_i,b.value_r)); +#endif + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a * SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) * b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out; + __m512d val = _mm512_set1_pd(b); + out.value_r = _mm512_mul_pd(a.value_r,val); + out.value_i = _mm512_mul_pd(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(U a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; + __m512d val = _mm512_set1_pd(a); + out.value_r = _mm512_mul_pd(val,b.value_r); + out.value_i = _mm512_mul_pd(val,b.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm512_fmadd_pd(a.value_r,b.value_r,_mm512_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm512_fmsub_pd(a.value_i,b.value_r,_mm512_mul_pd(a.value_r,b.value_i)); + __m512d den = _mm512_fmadd_pd(b.value_r,b.value_r,_mm512_mul_pd(b.value_i,b.value_i)); +#else + out.value_r = _mm512_add_pd(_mm512_mul_pd(a.value_r,b.value_r),_mm512_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm512_sub_pd(_mm512_mul_pd(a.value_i,b.value_r),_mm512_mul_pd(a.value_r,b.value_i)); + __m512d den = _mm512_add_pd(_mm512_mul_pd(b.value_r,b.value_r),_mm512_mul_pd(b.value_i,b.value_i)); +#endif + out.value_r = _mm512_div_pd(out.value_r,den); + out.value_i = _mm512_div_pd(out.value_i,den); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a / SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) / b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out; + __m512d val = _mm512_set1_pd(b); + out.value_r = _mm512_div_pd(a.value_r,val); + out.value_i = _mm512_div_pd(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(U a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(std::complex(a,0)) / b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +rcp(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out; +#ifdef FASTOR_FMA_IMPL + __m512d den = _mm512_fmadd_pd(a.value_r,a.value_r,_mm512_mul_pd(a.value_i,a.value_i)); +#else + __m512d den = _mm512_add_pd(_mm512_mul_pd(a.value_r,a.value_r),_mm512_mul_pd(a.value_i,a.value_i)); +#endif + out.value_r = _mm512_div_pd(out.value_r,den); + out.value_i = _mm512_neg_pd(_mm512_div_pd(out.value_i,den)); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +sqrt(const SIMDVector,simd_abi::avx512> &a) = delete; + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +rsqrt(const SIMDVector,simd_abi::avx512> &a) = delete; + +/* This intentionally return a complex vector */ +FASTOR_INLINE SIMDVector,simd_abi::avx512> +abs(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out; + out.value_r = a.magnitude().value; + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +conj(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out(a); + out.value_i = _mm512_neg_pd(out.value_i); + return out; +} + +/* Argument or phase angle */ +FASTOR_INLINE SIMDVector,simd_abi::avx512> +arg(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out(a); + for (FASTOR_INDEX i=0UL; i<8UL; ++i) { + ((double*)&out.value_r)[i] = std::atan2(((double*)&a.value_i)[i],((double*)&a.value_r)[i]); + } + return out; +} +//------------------------------------------------------------------------------------------------------------ + +#endif + + + +// AVX VERSION +//------------------------------------------------------------------------------------------------------------ + +#ifdef FASTOR_AVX_IMPL + +template <> +struct SIMDVector, simd_abi::avx> { + using vector_type = SIMDVector, simd_abi::avx>; + using value_type = __m256d; + using scalar_value_type = std::complex; + using abi_type = simd_abi::avx; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value_r(_mm256_setzero_pd()), value_i(_mm256_setzero_pd()) {} + FASTOR_INLINE SIMDVector(std::complex num) { + value_r = _mm256_set1_pd(num.real()); + value_i = _mm256_set1_pd(num.imag()); + } + FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} + FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + + FASTOR_INLINE SIMDVector,simd_abi::avx> operator=(std::complex num) { + value_r = _mm256_set1_pd(num.real()); + value_i = _mm256_set1_pd(num.imag()); + return *this; + } + + FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { + if (Aligned) + complex_aligned_store(data); + else + complex_unaligned_store(data); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { + if (!Aligned) + complex_mask_unaligned_load(data,mask); + else + complex_mask_aligned_load(data,mask); + } + FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { + if (!Aligned) + complex_mask_unaligned_store(data,mask); + else + complex_mask_aligned_store(data,mask); + } + + FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { + if (i == 0) { return scalar_value_type(_mm256_get0_pd(value_r), _mm256_get0_pd(value_i)); } + else if (i == 1) { return scalar_value_type(_mm256_get1_pd(value_r), _mm256_get1_pd(value_i)); } + else if (i == 2) { return scalar_value_type(_mm256_get2_pd(value_r), _mm256_get2_pd(value_i)); } + else { return scalar_value_type(_mm256_get3_pd(value_r), _mm256_get3_pd(value_i)); } + } + + FASTOR_INLINE SIMDVector real() const { + return value_r; + } + FASTOR_INLINE SIMDVector imag() const { + return value_i; + } + + FASTOR_INLINE void set(std::complex num) { + value_r = _mm256_set1_pd(num.real()); + value_i = _mm256_set1_pd(num.imag()); + } + FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, + scalar_value_type num2, scalar_value_type num3) { + const scalar_value_type tmp[Size] = {num0,num1,num2,num3}; + complex_unaligned_load(tmp); + } + + // In-place operators + template,bool> = false> + FASTOR_INLINE void operator+=(U num) { + value_r = _mm256_add_pd(value_r,_mm256_set1_pd(num)); + } + FASTOR_INLINE void operator+=(scalar_value_type num) { + *this += vector_type(num); + } + FASTOR_INLINE void operator+=(const vector_type &a) { + value_r = _mm256_add_pd(value_r,a.value_r); + value_i = _mm256_add_pd(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + value_r = _mm256_sub_pd(value_r,_mm256_set1_pd(num)); + } + FASTOR_INLINE void operator-=(scalar_value_type num) { + *this -= vector_type(num); + } + FASTOR_INLINE void operator-=(const vector_type &a) { + value_r = _mm256_sub_pd(value_r,a.value_r); + value_i = _mm256_sub_pd(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + __m256d val = _mm256_set1_pd(num); + value_r = _mm256_mul_pd(value_r,val); + value_i = _mm256_mul_pd(value_i,val); + } + FASTOR_INLINE void operator*=(scalar_value_type num) { + *this *= vector_type(num); + } + FASTOR_INLINE void operator*=(const vector_type &a) { +#ifdef FASTOR_FMA_IMPL + __m256d tmp = _mm256_fmsub_pd(value_r,a.value_r,_mm256_mul_pd(value_i,a.value_i)); + value_i = _mm256_fmadd_pd(value_r,a.value_i,_mm256_mul_pd(value_i,a.value_r)); +#else + __m256d tmp = _mm256_sub_pd(_mm256_mul_pd(value_r,a.value_r),_mm256_mul_pd(value_i,a.value_i)); + value_i = _mm256_add_pd(_mm256_mul_pd(value_r,a.value_i),_mm256_mul_pd(value_i,a.value_r)); +#endif + value_r = tmp; + } + + template,bool> = false> + FASTOR_INLINE void operator/=(U num) { + __m256d val = _mm256_set1_pd(num); + value_r = _mm256_div_pd(value_r,val); + value_i = _mm256_div_pd(value_i,val); + } + FASTOR_INLINE void operator/=(scalar_value_type num) { + *this /= vector_type(num); + } + FASTOR_INLINE void operator/=(const vector_type &a) { + __m256d tmp = value_r; +#ifdef FASTOR_FMA_IMPL + value_r = _mm256_fmadd_pd(value_r , a.value_r, _mm256_mul_pd(value_i,a.value_i)); + value_i = _mm256_fmsub_pd(value_i , a.value_r, _mm256_mul_pd(tmp,a.value_i)); + __m256d den = _mm256_fmadd_pd(a.value_r, a.value_r, _mm256_mul_pd(a.value_i,a.value_i)); +#else + value_r = _mm256_add_pd(_mm256_mul_pd(value_r , a.value_r), _mm256_mul_pd(value_i,a.value_i)); + value_i = _mm256_sub_pd(_mm256_mul_pd(value_i , a.value_r), _mm256_mul_pd(tmp,a.value_i)); + __m256d den = _mm256_add_pd(_mm256_mul_pd(a.value_r, a.value_r), _mm256_mul_pd(a.value_i,a.value_i)); +#endif + value_r = _mm256_div_pd(value_r,den); + value_i = _mm256_div_pd(value_i,den); + } + // end of in-place operators + + FASTOR_INLINE scalar_value_type sum() const { + return scalar_value_type(_mm256_sum_pd(value_r),_mm256_sum_pd(value_i)); + } + FASTOR_INLINE scalar_value_type product() const { + vector_type tmp(*this); + return tmp[0]*tmp[1]*tmp[2]*tmp[3]; + } + FASTOR_INLINE vector_type reverse() const { + vector_type out; + out.value_r = _mm256_reverse_pd(value_r); + out.value_i = _mm256_reverse_pd(value_i); + return out; + } + /* Actual magnitude - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector magnitude() const { +#ifdef FASTOR_FMA_IMPL + return _mm256_sqrt_pd(_mm256_fmadd_pd(value_r,value_r,_mm256_mul_pd(value_i,value_i))); +#else + return _mm256_sqrt_pd(_mm256_add_pd(_mm256_mul_pd(value_r,value_r),_mm256_mul_pd(value_i,value_i))); +#endif + } + /* STL compliant squared norm - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector norm() const { +#ifdef FASTOR_FMA_IMPL + return _mm256_fmadd_pd(value_r,value_r,_mm256_mul_pd(value_i,value_i)); +#else + return _mm256_add_pd(_mm256_mul_pd(value_r,value_r),_mm256_mul_pd(value_i,value_i)); +#endif + } + // Magnitude based minimum + FASTOR_INLINE scalar_value_type minimum() const; + // Magnitude based maximum + FASTOR_INLINE scalar_value_type maximum() const; + + FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { + vector_type out(*this); + out *= other; + return out.sum(); + } + + value_type value_r; + value_type value_i; + +protected: + FASTOR_INLINE void complex_aligned_load(const std::complex *data) { + __m256d lo = _mm256_load_pd(reinterpret_cast(data )); + __m256d hi = _mm256_load_pd(reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); + } + FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { + __m256d lo = _mm256_loadu_pd(reinterpret_cast(data )); + __m256d hi = _mm256_loadu_pd(reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); + } + + FASTOR_INLINE void complex_aligned_store(std::complex *data) const { + __m256d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm256_store_pd(reinterpret_cast(data ), lo); + _mm256_store_pd(reinterpret_cast(data+2), hi); + } + FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { + __m256d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm256_storeu_pd(reinterpret_cast(data ), lo); + _mm256_storeu_pd(reinterpret_cast(data+2), hi); + } + + FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { +#ifdef FASTOR_HAS_AVX512_MASKS + __m256d lo, hi; + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + lo = _mm256_mask_load_pd(lo, mask0, reinterpret_cast(data )); + hi = _mm256_mask_load_pd(hi, mask1, reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm256_setzero_pd(); + value_i = _mm256_setzero_pd(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + lo = _mm256_mask_loadu_pd(lo, mask0, reinterpret_cast(data )); + hi = _mm256_mask_loadu_pd(hi, mask1, reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm256_setzero_pd(); + value_i = _mm256_setzero_pd(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + _mm256_mask_store_pd(reinterpret_cast(data ), mask0, lo); + _mm256_mask_store_pd(reinterpret_cast(data+2), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { +#ifdef FASTOR_HAS_AVX512_MASKS + __m256d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + _mm256_mask_storeu_pd(reinterpret_cast(data ), mask0, lo); + _mm256_mask_storeu_pd(reinterpret_cast(data+2), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx> a) { + // ICC crashes without a copy + const __m256d vr = a.value_r; + const __m256d vi = a.value_i; + const double* value_r = reinterpret_cast(&vr); + const double* value_i = reinterpret_cast(&vi); + os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " + << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " + << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " + << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j" << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; + out.value_r = _mm256_add_pd(a.value_r,b.value_r); + out.value_i = _mm256_add_pd(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a + SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) + b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out(a); + out.value_r = _mm256_add_pd(a.value_r,_mm256_set1_pd(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(U a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out(b); + out.value_r = _mm256_add_pd(_mm256_set1_pd(a), b.value_r); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &b) { + return b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; + out.value_r = _mm256_sub_pd(a.value_r,b.value_r); + out.value_i = _mm256_sub_pd(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a - SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) - b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out(a); + out.value_r = _mm256_sub_pd(a.value_r,_mm256_set1_pd(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(U a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(std::complex(a,0)) - b; +} +/* This is negation and not complex conjugate */ +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out; + out.value_r = _mm256_neg_pd(a.value_r); + out.value_i = _mm256_neg_pd(a.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm256_fmsub_pd(a.value_r,b.value_r,_mm256_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm256_fmadd_pd(a.value_r,b.value_i,_mm256_mul_pd(a.value_i,b.value_r)); +#else + out.value_r = _mm256_sub_pd(_mm256_mul_pd(a.value_r,b.value_r),_mm256_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm256_add_pd(_mm256_mul_pd(a.value_r,b.value_i),_mm256_mul_pd(a.value_i,b.value_r)); +#endif + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a * SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) * b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out; + __m256d val = _mm256_set1_pd(b); + out.value_r = _mm256_mul_pd(a.value_r,val); + out.value_i = _mm256_mul_pd(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(U a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; + __m256d val = _mm256_set1_pd(a); + out.value_r = _mm256_mul_pd(val,b.value_r); + out.value_i = _mm256_mul_pd(val,b.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm256_fmadd_pd(a.value_r,b.value_r,_mm256_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm256_fmsub_pd(a.value_i,b.value_r,_mm256_mul_pd(a.value_r,b.value_i)); + __m256d den = _mm256_fmadd_pd(b.value_r,b.value_r,_mm256_mul_pd(b.value_i,b.value_i)); +#else + out.value_r = _mm256_add_pd(_mm256_mul_pd(a.value_r,b.value_r),_mm256_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm256_sub_pd(_mm256_mul_pd(a.value_i,b.value_r),_mm256_mul_pd(a.value_r,b.value_i)); + __m256d den = _mm256_add_pd(_mm256_mul_pd(b.value_r,b.value_r),_mm256_mul_pd(b.value_i,b.value_i)); +#endif + out.value_r = _mm256_div_pd(out.value_r,den); + out.value_i = _mm256_div_pd(out.value_i,den); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a / SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) / b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out; + __m256d val = _mm256_set1_pd(b); + out.value_r = _mm256_div_pd(a.value_r,val); + out.value_i = _mm256_div_pd(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(U a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(std::complex(a,0)) / b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +rcp(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out; +#ifdef FASTOR_FMA_IMPL + __m256d den = _mm256_fmadd_pd(a.value_r,a.value_r,_mm256_mul_pd(a.value_i,a.value_i)); +#else + __m256d den = _mm256_add_pd(_mm256_mul_pd(a.value_r,a.value_r),_mm256_mul_pd(a.value_i,a.value_i)); +#endif + out.value_r = _mm256_div_pd(out.value_r,den); + out.value_i = _mm256_neg_pd(_mm256_div_pd(out.value_i,den)); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +sqrt(const SIMDVector,simd_abi::avx> &a) = delete; + +FASTOR_INLINE SIMDVector,simd_abi::avx> +rsqrt(const SIMDVector,simd_abi::avx> &a) = delete; + +/* This intentionally return a complex vector */ +FASTOR_INLINE SIMDVector,simd_abi::avx> +abs(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out; + out.value_r = a.magnitude().value; + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +conj(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out(a); + out.value_i = _mm256_neg_pd(out.value_i); + return out; +} + +/* Argument or phase angle */ +FASTOR_INLINE SIMDVector,simd_abi::avx> +arg(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out(a); + for (FASTOR_INDEX i=0UL; i<4UL; ++i) { + ((double*)&out.value_r)[i] = std::atan2(((double*)&a.value_i)[i],((double*)&a.value_r)[i]); + } + return out; +} +//------------------------------------------------------------------------------------------------------------ + +#endif + + + + +// SSE VERSION +//------------------------------------------------------------------------------------------------------------ + +#ifdef FASTOR_SSE2_IMPL + +template <> +struct SIMDVector, simd_abi::sse> { + using vector_type = SIMDVector, simd_abi::sse>; + using value_type = __m128d; + using scalar_value_type = std::complex; + using abi_type = simd_abi::sse; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value_r(_mm_setzero_pd()), value_i(_mm_setzero_pd()) {} + FASTOR_INLINE SIMDVector(std::complex num) { + value_r = _mm_set1_pd(num.real()); + value_i = _mm_set1_pd(num.imag()); + } + FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} + FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + + FASTOR_INLINE SIMDVector,simd_abi::sse> operator=(std::complex num) { + value_r = _mm_set1_pd(num.real()); + value_i = _mm_set1_pd(num.imag()); + return *this; + } + + FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { + if (Aligned) + complex_aligned_store(data); + else + complex_unaligned_store(data); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { + if (!Aligned) + complex_mask_unaligned_load(data,mask); + else + complex_mask_aligned_load(data,mask); + } + FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { + if (!Aligned) + complex_mask_unaligned_store(data,mask); + else + complex_mask_aligned_store(data,mask); + } + + FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { + if (i == 0) { return scalar_value_type(_mm_get0_pd(value_r), _mm_get0_pd(value_i)); } + else { return scalar_value_type(_mm_get1_pd(value_r), _mm_get1_pd(value_i)); } + } + + FASTOR_INLINE SIMDVector real() const { + return value_r; + } + FASTOR_INLINE SIMDVector imag() const { + return value_i; + } + + FASTOR_INLINE void set(std::complex num) { + value_r = _mm_set1_pd(num.real()); + value_i = _mm_set1_pd(num.imag()); + } + FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1) { + const scalar_value_type tmp[Size] = {num0,num1}; + complex_unaligned_load(tmp); + } + + // In-place operators + template,bool> = false> + FASTOR_INLINE void operator+=(U num) { + value_r = _mm_add_pd(value_r,_mm_set1_pd(num)); + } + FASTOR_INLINE void operator+=(scalar_value_type num) { + *this += vector_type(num); + } + FASTOR_INLINE void operator+=(const vector_type &a) { + value_r = _mm_add_pd(value_r,a.value_r); + value_i = _mm_add_pd(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + value_r = _mm_sub_pd(value_r,_mm_set1_pd(num)); + } + FASTOR_INLINE void operator-=(scalar_value_type num) { + *this -= vector_type(num); + } + FASTOR_INLINE void operator-=(const vector_type &a) { + value_r = _mm_sub_pd(value_r,a.value_r); + value_i = _mm_sub_pd(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + __m128d val = _mm_set1_pd(num); + value_r = _mm_mul_pd(value_r,val); + value_i = _mm_mul_pd(value_i,val); + } + FASTOR_INLINE void operator*=(scalar_value_type num) { + *this *= vector_type(num); + } + FASTOR_INLINE void operator*=(const vector_type &a) { +#ifdef FASTOR_FMA_IMPL + __m128d tmp = _mm_fmsub_pd(value_r,a.value_r,_mm_mul_pd(value_i,a.value_i)); + value_i = _mm_fmadd_pd(value_r,a.value_i,_mm_mul_pd(value_i,a.value_r)); +#else + __m128d tmp = _mm_sub_pd(_mm_mul_pd(value_r,a.value_r),_mm_mul_pd(value_i,a.value_i)); + value_i = _mm_add_pd(_mm_mul_pd(value_r,a.value_i),_mm_mul_pd(value_i,a.value_r)); +#endif + value_r = tmp; + } + + template,bool> = false> + FASTOR_INLINE void operator/=(U num) { + __m128d val = _mm_set1_pd(num); + value_r = _mm_div_pd(value_r,val); + value_i = _mm_div_pd(value_i,val); + } + FASTOR_INLINE void operator/=(scalar_value_type num) { + *this /= vector_type(num); + } + FASTOR_INLINE void operator/=(const vector_type &a) { + __m128d tmp = value_r; +#ifdef FASTOR_FMA_IMPL + value_r = _mm_fmadd_pd(value_r , a.value_r, _mm_mul_pd(value_i,a.value_i)); + value_i = _mm_fmsub_pd(value_i , a.value_r, _mm_mul_pd(tmp,a.value_i)); + __m128d den = _mm_fmadd_pd(a.value_r, a.value_r, _mm_mul_pd(a.value_i,a.value_i)); +#else + value_r = _mm_add_pd(_mm_mul_pd(value_r , a.value_r), _mm_mul_pd(value_i,a.value_i)); + value_i = _mm_sub_pd(_mm_mul_pd(value_i , a.value_r), _mm_mul_pd(tmp,a.value_i)); + __m128d den = _mm_add_pd(_mm_mul_pd(a.value_r, a.value_r), _mm_mul_pd(a.value_i,a.value_i)); +#endif + value_r = _mm_div_pd(value_r,den); + value_i = _mm_div_pd(value_i,den); + } + // end of in-place operators + + FASTOR_INLINE scalar_value_type sum() const { + return scalar_value_type(_mm_sum_pd(value_r),_mm_sum_pd(value_i)); + } + FASTOR_INLINE scalar_value_type product() const { + vector_type tmp(*this); + // Multiply vector of complex numbers + tmp *= tmp.reverse(); + // Alternatively if relied on the compiler + // return tmp[0]*tmp[1]; + return tmp[0]; + } + FASTOR_INLINE vector_type reverse() const { + vector_type out; + out.value_r = _mm_reverse_pd(value_r); + out.value_i = _mm_reverse_pd(value_i); + return out; + } + /* Actual magnitude - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector magnitude() const { +#ifdef FASTOR_FMA_IMPL + return _mm_sqrt_pd(_mm_fmadd_pd(value_r,value_r,_mm_mul_pd(value_i,value_i))); +#else + return _mm_sqrt_pd(_mm_add_pd(_mm_mul_pd(value_r,value_r),_mm_mul_pd(value_i,value_i))); +#endif + } + /* STL compliant squared norm - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector norm() const { +#ifdef FASTOR_FMA_IMPL + return _mm_fmadd_pd(value_r,value_r,_mm_mul_pd(value_i,value_i)); +#else + return _mm_add_pd(_mm_mul_pd(value_r,value_r),_mm_mul_pd(value_i,value_i)); +#endif + } + // Magnitude based minimum + FASTOR_INLINE scalar_value_type minimum() const { + vector_type out; + SIMDVector tmp(this->norm()); + if ( _mm_get0_pd(tmp.value) < _mm_get1_pd(tmp.value) ) { + return scalar_value_type(_mm_get0_pd(value_r),_mm_get0_pd(value_i)); + } + else { + return scalar_value_type(_mm_get1_pd(value_r),_mm_get1_pd(value_i)); + } + } + // Magnitude based maximum + FASTOR_INLINE scalar_value_type maximum() const { + vector_type out; + SIMDVector tmp(this->norm()); + if ( _mm_get0_pd(tmp.value) > _mm_get1_pd(tmp.value) ) { + return scalar_value_type(_mm_get0_pd(value_r),_mm_get0_pd(value_i)); + } + else { + return scalar_value_type(_mm_get1_pd(value_r),_mm_get1_pd(value_i)); + } + } + + FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { + vector_type out(*this); + out *= other; + return out.sum(); + } + + value_type value_r; + value_type value_i; + +protected: + FASTOR_INLINE void complex_aligned_load(const std::complex *data) { + __m128d lo = _mm_load_pd(reinterpret_cast(data )); + __m128d hi = _mm_load_pd(reinterpret_cast(data+1)); + arrange_from_load(value_r, value_i, lo, hi); + } + FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { + __m128d lo = _mm_loadu_pd(reinterpret_cast(data )); + __m128d hi = _mm_loadu_pd(reinterpret_cast(data+1)); + arrange_from_load(value_r, value_i, lo, hi); + } + + FASTOR_INLINE void complex_aligned_store(std::complex *data) const { + __m128d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm_store_pd(reinterpret_cast(data ),lo); + _mm_store_pd(reinterpret_cast(data+1),hi); + } + FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { + __m128d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm_storeu_pd(reinterpret_cast(data ),lo); + _mm_storeu_pd(reinterpret_cast(data+1),hi); + } + + FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { +#ifdef FASTOR_HAS_AVX512_MASKS + __m128d lo, hi; + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + lo = _mm_mask_load_pd(lo, mask0, reinterpret_cast(data )); + hi = _mm_mask_load_pd(hi, mask1, reinterpret_cast(data+1)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm_setzero_pd(); + value_i = _mm_setzero_pd(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + lo = _mm_mask_loadu_pd(lo, mask0, reinterpret_cast(data )); + hi = _mm_mask_loadu_pd(hi, mask1, reinterpret_cast(data+1)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm_setzero_pd(); + value_i = _mm_setzero_pd(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + _mm_mask_store_pd(reinterpret_cast(data ), mask0, lo); + _mm_mask_store_pd(reinterpret_cast(data+1), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { +#ifdef FASTOR_HAS_AVX512_MASKS + __m128d lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + _mm_mask_storeu_pd(reinterpret_cast(data ), mask0, lo); + _mm_mask_storeu_pd(reinterpret_cast(data+1), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::sse> a) { + // ICC crashes without a copy + const __m128d vr = a.value_r; + const __m128d vi = a.value_i; + const double* value_r = reinterpret_cast(&vr); + const double* value_i = reinterpret_cast(&vi); + os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " + << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j" << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; + out.value_r = _mm_add_pd(a.value_r,b.value_r); + out.value_i = _mm_add_pd(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a + SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) + b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out(a); + out.value_r = _mm_add_pd(a.value_r,_mm_set1_pd(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(U a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out(b); + out.value_r = _mm_add_pd(_mm_set1_pd(a), b.value_r); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &b) { + return b; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; + out.value_r = _mm_sub_pd(a.value_r,b.value_r); + out.value_i = _mm_sub_pd(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a - SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) - b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out(a); + out.value_r = _mm_sub_pd(a.value_r,_mm_set1_pd(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(U a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(std::complex(a,0)) - b; +} +/* This is negation and not complex conjugate */ +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out; + out.value_r = _mm_neg_pd(a.value_r); + out.value_i = _mm_neg_pd(a.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm_fmsub_pd(a.value_r,b.value_r,_mm_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm_fmadd_pd(a.value_r,b.value_i,_mm_mul_pd(a.value_i,b.value_r)); +#else + out.value_r = _mm_sub_pd(_mm_mul_pd(a.value_r,b.value_r),_mm_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm_add_pd(_mm_mul_pd(a.value_r,b.value_i),_mm_mul_pd(a.value_i,b.value_r)); +#endif + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a * SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) * b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out; + __m128d val = _mm_set1_pd(b); + out.value_r = _mm_mul_pd(a.value_r,val); + out.value_i = _mm_mul_pd(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(U a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; + __m128d val = _mm_set1_pd(a); + out.value_r = _mm_mul_pd(val,b.value_r); + out.value_i = _mm_mul_pd(val,b.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm_fmadd_pd(a.value_r,b.value_r,_mm_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm_fmsub_pd(a.value_i,b.value_r,_mm_mul_pd(a.value_r,b.value_i)); + __m128d den = _mm_fmadd_pd(b.value_r,b.value_r,_mm_mul_pd(b.value_i,b.value_i)); +#else + out.value_r = _mm_add_pd(_mm_mul_pd(a.value_r,b.value_r),_mm_mul_pd(a.value_i,b.value_i)); + out.value_i = _mm_sub_pd(_mm_mul_pd(a.value_i,b.value_r),_mm_mul_pd(a.value_r,b.value_i)); + __m128d den = _mm_add_pd(_mm_mul_pd(b.value_r,b.value_r),_mm_mul_pd(b.value_i,b.value_i)); +#endif + out.value_r = _mm_div_pd(out.value_r,den); + out.value_i = _mm_div_pd(out.value_i,den); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a / SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) / b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out; + __m128d val = _mm_set1_pd(b); + out.value_r = _mm_div_pd(a.value_r,val); + out.value_i = _mm_div_pd(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(U a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(std::complex(a,0)) / b; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +rcp(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out; +#ifdef FASTOR_FMA_IMPL + __m128d den = _mm_fmadd_pd(a.value_r,a.value_r,_mm_mul_pd(a.value_i,a.value_i)); +#else + __m128d den = _mm_add_pd(_mm_mul_pd(a.value_r,a.value_r),_mm_mul_pd(a.value_i,a.value_i)); +#endif + out.value_r = _mm_div_pd(out.value_r,den); + out.value_i = _mm_neg_pd(_mm_div_pd(out.value_i,den)); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +sqrt(const SIMDVector,simd_abi::sse> &a) = delete; + +FASTOR_INLINE SIMDVector,simd_abi::sse> +rsqrt(const SIMDVector,simd_abi::sse> &a) = delete; + +/* This intentionally return a complex vector */ +FASTOR_INLINE SIMDVector,simd_abi::sse> +abs(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out; + out.value_r = a.magnitude().value; + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +conj(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out(a); + out.value_i = _mm_neg_pd(out.value_i); + return out; +} + +/* Argument or phase angle */ +FASTOR_INLINE SIMDVector,simd_abi::sse> +arg(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out(a); + for (FASTOR_INDEX i=0UL; i<2UL; ++i) { + ((double*)&out.value_r)[i] = std::atan2(((double*)&a.value_i)[i],((double*)&a.value_r)[i]); + } + return out; +} +//------------------------------------------------------------------------------------------------------------ + +#endif + + +} // end of namespace Fastor + +#endif // SIMD_VECTOR_COMPLEX_DOUBLE_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_complex_float.h b/noarch/include/Fastor/simd_vector/simd_vector_complex_float.h new file mode 100644 index 00000000..cfe0a6c6 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_complex_float.h @@ -0,0 +1,1619 @@ +#ifndef SIMD_VECTOR_COMPLEX_FLOAT_H +#define SIMD_VECTOR_COMPLEX_FLOAT_H + +#include "Fastor/util/extended_algorithms.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/simd_vector_base.h" +#include "Fastor/simd_vector/simd_vector_float.h" +#include +#include + +namespace Fastor { + + +// AVX512 VERSION +//------------------------------------------------------------------------------------------------------------ + +#ifdef FASTOR_AVX512F_IMPL + +template <> +struct SIMDVector, simd_abi::avx512> { + using vector_type = SIMDVector, simd_abi::avx512>; + using value_type = __m512; + using scalar_value_type = std::complex; + using abi_type = simd_abi::avx512; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value_r(_mm512_setzero_ps()), value_i(_mm512_setzero_ps()) {} + FASTOR_INLINE SIMDVector(std::complex num) { + value_r = _mm512_set1_ps(num.real()); + value_i = _mm512_set1_ps(num.imag()); + } + FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} + FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + + FASTOR_INLINE SIMDVector,simd_abi::avx512> operator=(std::complex num) { + value_r = _mm512_set1_ps(num.real()); + value_i = _mm512_set1_ps(num.imag()); + return *this; + } + + FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { + if (Aligned) + complex_aligned_store(data); + else + complex_unaligned_store(data); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *data, uint16_t mask, bool Aligned=false) { + if (!Aligned) + complex_mask_unaligned_load(data,mask); + else + complex_mask_aligned_load(data,mask); + } + FASTOR_INLINE void mask_store(scalar_value_type *data, uint16_t mask, bool Aligned=false) const { + if (!Aligned) + complex_mask_unaligned_store(data,mask); + else + complex_mask_aligned_store(data,mask); + } + + FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { + return scalar_value_type(reinterpret_cast(&value_r)[i],reinterpret_cast(&value_i)[i]); + } + + FASTOR_INLINE SIMDVector real() const { + return value_r; + } + FASTOR_INLINE SIMDVector imag() const { + return value_i; + } + + FASTOR_INLINE void set(std::complex num) { + value_r = _mm512_set1_ps(num.real()); + value_i = _mm512_set1_ps(num.imag()); + } + FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, + scalar_value_type num2, scalar_value_type num3, + scalar_value_type num4, scalar_value_type num5, + scalar_value_type num6, scalar_value_type num7, + scalar_value_type num8, scalar_value_type num9, + scalar_value_type num10, scalar_value_type num11, + scalar_value_type num12, scalar_value_type num13, + scalar_value_type num14, scalar_value_type num15) { + const scalar_value_type tmp[Size] = {num0,num1,num2,num3,num4,num5,num6,num7, + num8,num9,num10,num11,num12,num13,num14,num15}; + complex_unaligned_load(tmp); + } + + // In-place operators + template,bool> = false> + FASTOR_INLINE void operator+=(U num) { + value_r = _mm512_add_ps(value_r,_mm512_set1_ps(num)); + } + FASTOR_INLINE void operator+=(scalar_value_type num) { + *this += vector_type(num); + } + FASTOR_INLINE void operator+=(const vector_type &a) { + value_r = _mm512_add_ps(value_r,a.value_r); + value_i = _mm512_add_ps(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + value_r = _mm512_sub_ps(value_r,_mm512_set1_ps(num)); + } + FASTOR_INLINE void operator-=(scalar_value_type num) { + *this -= vector_type(num); + } + FASTOR_INLINE void operator-=(const vector_type &a) { + value_r = _mm512_sub_ps(value_r,a.value_r); + value_i = _mm512_sub_ps(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + __m512 val = _mm512_set1_ps(num); + value_r = _mm512_mul_ps(value_r,val); + value_i = _mm512_mul_ps(value_i,val); + } + FASTOR_INLINE void operator*=(scalar_value_type num) { + *this *= vector_type(num); + } + FASTOR_INLINE void operator*=(const vector_type &a) { +#ifdef FASTOR_FMA_IMPL + __m512 tmp = _mm512_fmsub_ps(value_r,a.value_r,_mm512_mul_ps(value_i,a.value_i)); + value_i = _mm512_fmadd_ps(value_r,a.value_i,_mm512_mul_ps(value_i,a.value_r)); +#else + __m512 tmp = _mm512_sub_ps(_mm512_mul_ps(value_r,a.value_r),_mm512_mul_ps(value_i,a.value_i)); + value_i = _mm512_add_ps(_mm512_mul_ps(value_r,a.value_i),_mm512_mul_ps(value_i,a.value_r)); +#endif + value_r = tmp; + } + + template,bool> = false> + FASTOR_INLINE void operator/=(U num) { + __m512 val = _mm512_set1_ps(num); + value_r = _mm512_div_ps(value_r,val); + value_i = _mm512_div_ps(value_i,val); + } + FASTOR_INLINE void operator/=(scalar_value_type num) { + *this /= vector_type(num); + } + FASTOR_INLINE void operator/=(const vector_type &a) { + __m512 tmp = value_r; +#ifdef FASTOR_FMA_IMPL + value_r = _mm512_fmadd_ps(value_r , a.value_r, _mm512_mul_ps(value_i,a.value_i)); + value_i = _mm512_fmsub_ps(value_i , a.value_r, _mm512_mul_ps(tmp,a.value_i)); + __m512 den = _mm512_fmadd_ps(a.value_r, a.value_r, _mm512_mul_ps(a.value_i,a.value_i)); +#else + value_r = _mm512_add_ps(_mm512_mul_ps(value_r , a.value_r), _mm512_mul_ps(value_i,a.value_i)); + value_i = _mm512_sub_ps(_mm512_mul_ps(value_i , a.value_r), _mm512_mul_ps(tmp,a.value_i)); + __m512 den = _mm512_add_ps(_mm512_mul_ps(a.value_r, a.value_r), _mm512_mul_ps(a.value_i,a.value_i)); +#endif + value_r = _mm512_div_ps(value_r,den); + value_i = _mm512_div_ps(value_i,den); + } + // end of in-place operators + + FASTOR_INLINE scalar_value_type sum() const { +#ifdef FASTOR_HAS_AVX512_REDUCE_ADD + return scalar_value_type(_mm512_reduce_add_ps(value_r),_mm512_reduce_add_ps(value_i)); +#else + __m256 lor = _mm512_castps512_ps256(value_r); + __m256 hir = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value_r),0x1)); + __m256 loi = _mm512_castps512_ps256(value_i); + __m256 hii = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value_i),0x1)); + return scalar_value_type(_mm256_sum_ps(_mm256_add_ps(lor,hir)), _mm256_sum_ps(_mm256_add_ps(loi,hii))); +#endif + } + FASTOR_INLINE scalar_value_type product() const { + vector_type tmp(*this); + scalar_value_type out(tmp[0]); + for (FASTOR_INDEX i=1; i magnitude() const { +#ifdef FASTOR_FMA_IMPL + return _mm512_sqrt_ps(_mm512_fmadd_ps(value_r,value_r,_mm512_mul_ps(value_i,value_i))); +#else + return _mm512_sqrt_ps(_mm512_add_ps(_mm512_mul_ps(value_r,value_r),_mm512_mul_ps(value_i,value_i))); +#endif + } + /* STL compliant squared norm - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector norm() const { +#ifdef FASTOR_FMA_IMPL + return _mm512_fmadd_ps(value_r,value_r,_mm512_mul_ps(value_i,value_i)); +#else + return _mm512_add_ps(_mm512_mul_ps(value_r,value_r),_mm512_mul_ps(value_i,value_i)); +#endif + } + // Magnitude based minimum + FASTOR_INLINE scalar_value_type minimum() const; + // Magnitude based maximum + FASTOR_INLINE scalar_value_type maximum() const; + + FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { + vector_type out(*this); + out *= other; + return out.sum(); + } + + value_type value_r; + value_type value_i; + +protected: + FASTOR_INLINE void complex_aligned_load(const std::complex *data) { + __m512 lo = _mm512_load_ps(reinterpret_cast(data )); + __m512 hi = _mm512_load_ps(reinterpret_cast(data+8)); + arrange_from_load(value_r, value_i, lo, hi); + } + FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { + __m512 lo = _mm512_loadu_ps(reinterpret_cast(data )); + __m512 hi = _mm512_loadu_ps(reinterpret_cast(data+8)); + arrange_from_load(value_r, value_i, lo, hi); + } + + FASTOR_INLINE void complex_aligned_store(std::complex *data) const { + __m512 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm512_store_ps(reinterpret_cast(data ), lo); + _mm512_store_ps(reinterpret_cast(data+8), hi); + } + FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { + __m512 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm512_storeu_ps(reinterpret_cast(data ), lo); + _mm512_storeu_ps(reinterpret_cast(data+8), hi); + } + + FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint16_t mask) { +#ifdef FASTOR_HAS_AVX512_MASKS + __m512 lo, hi; + uint16_t mask0, mask1; + split_mask(mask, mask0, mask1); + lo = _mm512_mask_load_ps(lo, mask0, reinterpret_cast(data )); + hi = _mm512_mask_load_ps(hi, mask1, reinterpret_cast(data+8)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm512_setzero_ps(); + value_i = _mm512_setzero_ps(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + lo = _mm512_mask_loadu_ps(lo, mask0, reinterpret_cast(data )); + hi = _mm512_mask_loadu_ps(hi, mask1, reinterpret_cast(data+8)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm512_setzero_ps(); + value_i = _mm512_setzero_ps(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + _mm512_mask_store_ps(reinterpret_cast(data ), mask0, lo); + _mm512_mask_store_ps(reinterpret_cast(data+8), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint16_t mask) const { +#ifdef FASTOR_HAS_AVX512_MASKS + __m512 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + uint16_t mask0, mask1; + split_mask(mask, mask0, mask1); + _mm512_mask_storeu_ps(reinterpret_cast(data ), mask0, lo); + _mm512_mask_storeu_ps(reinterpret_cast(data+8), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx512> a) { + // ICC crashes without a copy + const __m512 vr = a.value_r; + const __m512 vi = a.value_i; + const float* value_r = reinterpret_cast(&vr); + const float* value_i = reinterpret_cast(&vi); + os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " + << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " + << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " + << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j, " + << value_r[4] << signum_string(value_i[4]) << std::abs(value_i[4]) << "j, " + << value_r[5] << signum_string(value_i[5]) << std::abs(value_i[5]) << "j, " + << value_r[6] << signum_string(value_i[6]) << std::abs(value_i[6]) << "j, " + << value_r[7] << signum_string(value_i[7]) << std::abs(value_i[7]) << "j, " + << value_r[8] << signum_string(value_i[8]) << std::abs(value_i[8]) << "j, " + << value_r[9] << signum_string(value_i[9]) << std::abs(value_i[9]) << "j, " + << value_r[10] << signum_string(value_i[10]) << std::abs(value_i[10]) << "j, " + << value_r[11] << signum_string(value_i[11]) << std::abs(value_i[11]) << "j, " + << value_r[12] << signum_string(value_i[12]) << std::abs(value_i[12]) << "j, " + << value_r[13] << signum_string(value_i[13]) << std::abs(value_i[13]) << "j, " + << value_r[14] << signum_string(value_i[14]) << std::abs(value_i[14]) << "j, " + << value_r[15] << signum_string(value_i[15]) << std::abs(value_i[15]) << "j" << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; + out.value_r = _mm512_add_ps(a.value_r,b.value_r); + out.value_i = _mm512_add_ps(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a + SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) + b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out(a); + out.value_r = _mm512_add_ps(a.value_r,_mm512_set1_ps(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(U a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out(b); + out.value_r = _mm512_add_ps(_mm512_set1_ps(a), b.value_r); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator+(const SIMDVector,simd_abi::avx512> &b) { + return b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; + out.value_r = _mm512_sub_ps(a.value_r,b.value_r); + out.value_i = _mm512_sub_ps(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a - SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) - b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out(a); + out.value_r = _mm512_sub_ps(a.value_r,_mm512_set1_ps(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(U a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(std::complex(a,0)) - b; +} +/* This is negation and not complex conjugate */ +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator-(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out; + out.value_r = _mm512_neg_ps(a.value_r); + out.value_i = _mm512_neg_ps(a.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm512_fmsub_ps(a.value_r,b.value_r,_mm512_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm512_fmadd_ps(a.value_r,b.value_i,_mm512_mul_ps(a.value_i,b.value_r)); +#else + out.value_r = _mm512_sub_ps(_mm512_mul_ps(a.value_r,b.value_r),_mm512_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm512_add_ps(_mm512_mul_ps(a.value_r,b.value_i),_mm512_mul_ps(a.value_i,b.value_r)); +#endif + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a * SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) * b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out; + __m512 val = _mm512_set1_ps(b); + out.value_r = _mm512_mul_ps(a.value_r,val); + out.value_i = _mm512_mul_ps(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator*(U a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; + __m512 val = _mm512_set1_ps(a); + out.value_r = _mm512_mul_ps(val,b.value_r); + out.value_i = _mm512_mul_ps(val,b.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { + SIMDVector,simd_abi::avx512> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm512_fmadd_ps(a.value_r,b.value_r,_mm512_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm512_fmsub_ps(a.value_i,b.value_r,_mm512_mul_ps(a.value_r,b.value_i)); + __m512 den = _mm512_fmadd_ps(b.value_r,b.value_r,_mm512_mul_ps(b.value_i,b.value_i)); +#else + out.value_r = _mm512_add_ps(_mm512_mul_ps(a.value_r,b.value_r),_mm512_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm512_sub_ps(_mm512_mul_ps(a.value_i,b.value_r),_mm512_mul_ps(a.value_r,b.value_i)); + __m512 den = _mm512_add_ps(_mm512_mul_ps(b.value_r,b.value_r),_mm512_mul_ps(b.value_i,b.value_i)); +#endif + out.value_r = _mm512_div_ps(out.value_r,den); + out.value_i = _mm512_div_ps(out.value_i,den); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(const SIMDVector,simd_abi::avx512> &a, std::complex b) { + return a / SIMDVector,simd_abi::avx512>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(std::complex a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(a) / b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(const SIMDVector,simd_abi::avx512> &a, U b) { + SIMDVector,simd_abi::avx512> out; + __m512 val = _mm512_set1_ps(b); + out.value_r = _mm512_div_ps(a.value_r,val); + out.value_i = _mm512_div_ps(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx512> +operator/(U a, const SIMDVector,simd_abi::avx512> &b) { + return SIMDVector,simd_abi::avx512>(std::complex(a,0)) / b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +rcp(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out; +#ifdef FASTOR_FMA_IMPL + __m512 den = _mm512_fmadd_ps(a.value_r,a.value_r,_mm512_mul_ps(a.value_i,a.value_i)); +#else + __m512 den = _mm512_add_ps(_mm512_mul_ps(a.value_r,a.value_r),_mm512_mul_ps(a.value_i,a.value_i)); +#endif + out.value_r = _mm512_div_ps(out.value_r,den); + out.value_i = _mm512_neg_ps(_mm512_div_ps(out.value_i,den)); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +sqrt(const SIMDVector,simd_abi::avx512> &a) = delete; + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +rsqrt(const SIMDVector,simd_abi::avx512> &a) = delete; + +/* This intentionally return a complex vector */ +FASTOR_INLINE SIMDVector,simd_abi::avx512> +abs(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out; + out.value_r = a.magnitude().value; + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx512> +conj(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out(a); + out.value_i = _mm512_neg_ps(out.value_i); + return out; +} + +/* Argument or phase angle */ +FASTOR_INLINE SIMDVector,simd_abi::avx512> +arg(const SIMDVector,simd_abi::avx512> &a) { + SIMDVector,simd_abi::avx512> out(a); + for (FASTOR_INDEX i=0UL; i<16UL; ++i) { + ((float*)&out.value_r)[i] = std::atan2(((float*)&a.value_i)[i],((float*)&a.value_r)[i]); + } + return out; +} +//------------------------------------------------------------------------------------------------------------ + +#endif + + + +// AVX VERSION +//------------------------------------------------------------------------------------------------------------ + +#ifdef FASTOR_AVX_IMPL + +template <> +struct SIMDVector, simd_abi::avx> { + using vector_type = SIMDVector, simd_abi::avx>; + using value_type = __m256; + using scalar_value_type = std::complex; + using abi_type = simd_abi::avx; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value_r(_mm256_setzero_ps()), value_i(_mm256_setzero_ps()) {} + FASTOR_INLINE SIMDVector(std::complex num) { + value_r = _mm256_set1_ps(num.real()); + value_i = _mm256_set1_ps(num.imag()); + } + FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} + FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + + FASTOR_INLINE SIMDVector,simd_abi::avx> operator=(std::complex num) { + value_r = _mm256_set1_ps(num.real()); + value_i = _mm256_set1_ps(num.imag()); + return *this; + } + + FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { + if (Aligned) + complex_aligned_store(data); + else + complex_unaligned_store(data); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { + if (!Aligned) + complex_mask_unaligned_load(data,mask); + else + complex_mask_aligned_load(data,mask); + } + FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { + if (!Aligned) + complex_mask_unaligned_store(data,mask); + else + complex_mask_aligned_store(data,mask); + } + + FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { + if (i == 0) { return scalar_value_type(_mm256_get0_ps(value_r), _mm256_get0_ps(value_i)); } + else if (i == 1) { return scalar_value_type(_mm256_get1_ps(value_r), _mm256_get1_ps(value_i)); } + else if (i == 2) { return scalar_value_type(_mm256_get2_ps(value_r), _mm256_get2_ps(value_i)); } + else if (i == 3) { return scalar_value_type(_mm256_get3_ps(value_r), _mm256_get3_ps(value_i)); } + else if (i == 4) { return scalar_value_type(_mm256_get4_ps(value_r), _mm256_get4_ps(value_i)); } + else if (i == 5) { return scalar_value_type(_mm256_get5_ps(value_r), _mm256_get5_ps(value_i)); } + else if (i == 6) { return scalar_value_type(_mm256_get6_ps(value_r), _mm256_get6_ps(value_i)); } + else { return scalar_value_type(_mm256_get7_ps(value_r), _mm256_get7_ps(value_i)); } + } + + FASTOR_INLINE SIMDVector real() const { + return value_r; + } + FASTOR_INLINE SIMDVector imag() const { + return value_i; + } + + FASTOR_INLINE void set(std::complex num) { + value_r = _mm256_set1_ps(num.real()); + value_i = _mm256_set1_ps(num.imag()); + } + FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, + scalar_value_type num2, scalar_value_type num3, + scalar_value_type num4, scalar_value_type num5, + scalar_value_type num6, scalar_value_type num7) { + const scalar_value_type tmp[Size] = {num0,num1,num2,num3,num4,num5,num6,num7}; + complex_unaligned_load(tmp); + } + + // In-place operators + template,bool> = false> + FASTOR_INLINE void operator+=(U num) { + value_r = _mm256_add_ps(value_r,_mm256_set1_ps(num)); + } + FASTOR_INLINE void operator+=(scalar_value_type num) { + *this += vector_type(num); + } + FASTOR_INLINE void operator+=(const vector_type &a) { + value_r = _mm256_add_ps(value_r,a.value_r); + value_i = _mm256_add_ps(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + value_r = _mm256_sub_ps(value_r,_mm256_set1_ps(num)); + } + FASTOR_INLINE void operator-=(scalar_value_type num) { + *this -= vector_type(num); + } + FASTOR_INLINE void operator-=(const vector_type &a) { + value_r = _mm256_sub_ps(value_r,a.value_r); + value_i = _mm256_sub_ps(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + __m256 val = _mm256_set1_ps(num); + value_r = _mm256_mul_ps(value_r,val); + value_i = _mm256_mul_ps(value_i,val); + } + FASTOR_INLINE void operator*=(scalar_value_type num) { + *this *= vector_type(num); + } + FASTOR_INLINE void operator*=(const vector_type &a) { +#ifdef FASTOR_FMA_IMPL + __m256 tmp = _mm256_fmsub_ps(value_r,a.value_r,_mm256_mul_ps(value_i,a.value_i)); + value_i = _mm256_fmadd_ps(value_r,a.value_i,_mm256_mul_ps(value_i,a.value_r)); +#else + __m256 tmp = _mm256_sub_ps(_mm256_mul_ps(value_r,a.value_r),_mm256_mul_ps(value_i,a.value_i)); + value_i = _mm256_add_ps(_mm256_mul_ps(value_r,a.value_i),_mm256_mul_ps(value_i,a.value_r)); +#endif + value_r = tmp; + } + + template,bool> = false> + FASTOR_INLINE void operator/=(U num) { + __m256 val = _mm256_set1_ps(num); + value_r = _mm256_div_ps(value_r,val); + value_i = _mm256_div_ps(value_i,val); + } + FASTOR_INLINE void operator/=(scalar_value_type num) { + *this /= vector_type(num); + } + FASTOR_INLINE void operator/=(const vector_type &a) { + __m256 tmp = value_r; +#ifdef FASTOR_FMA_IMPL + value_r = _mm256_fmadd_ps(value_r , a.value_r, _mm256_mul_ps(value_i,a.value_i)); + value_i = _mm256_fmsub_ps(value_i , a.value_r, _mm256_mul_ps(tmp,a.value_i)); + __m256 den = _mm256_fmadd_ps(a.value_r, a.value_r, _mm256_mul_ps(a.value_i,a.value_i)); +#else + value_r = _mm256_add_ps(_mm256_mul_ps(value_r , a.value_r), _mm256_mul_ps(value_i,a.value_i)); + value_i = _mm256_sub_ps(_mm256_mul_ps(value_i , a.value_r), _mm256_mul_ps(tmp,a.value_i)); + __m256 den = _mm256_add_ps(_mm256_mul_ps(a.value_r, a.value_r), _mm256_mul_ps(a.value_i,a.value_i)); +#endif + value_r = _mm256_div_ps(value_r,den); + value_i = _mm256_div_ps(value_i,den); + } + // end of in-place operators + + FASTOR_INLINE scalar_value_type sum() const { + return scalar_value_type(_mm256_sum_ps(value_r),_mm256_sum_ps(value_i)); + } + FASTOR_INLINE scalar_value_type product() const { + vector_type tmp(*this); + return tmp[0]*tmp[1]*tmp[2]*tmp[3]*tmp[4]*tmp[5]*tmp[6]*tmp[7]; + } + FASTOR_INLINE vector_type reverse() const { + vector_type out; + out.value_r = _mm256_reverse_ps(value_r); + out.value_i = _mm256_reverse_ps(value_i); + return out; + } + /* Actual magnitude - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector magnitude() const { +#ifdef FASTOR_FMA_IMPL + return _mm256_sqrt_ps(_mm256_fmadd_ps(value_r,value_r,_mm256_mul_ps(value_i,value_i))); +#else + return _mm256_sqrt_ps(_mm256_add_ps(_mm256_mul_ps(value_r,value_r),_mm256_mul_ps(value_i,value_i))); +#endif + } + /* STL compliant squared norm - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector norm() const { +#ifdef FASTOR_FMA_IMPL + return _mm256_fmadd_ps(value_r,value_r,_mm256_mul_ps(value_i,value_i)); +#else + return _mm256_add_ps(_mm256_mul_ps(value_r,value_r),_mm256_mul_ps(value_i,value_i)); +#endif + } + // Magnitude based minimum + FASTOR_INLINE scalar_value_type minimum() const; + // Magnitude based maximum + FASTOR_INLINE scalar_value_type maximum() const; + + FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { + vector_type out(*this); + out *= other; + return out.sum(); + } + + value_type value_r; + value_type value_i; + +protected: + FASTOR_INLINE void complex_aligned_load(const std::complex *data) { + __m256 lo = _mm256_load_ps(reinterpret_cast(data )); + __m256 hi = _mm256_load_ps(reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); + } + FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { + __m256 lo = _mm256_loadu_ps(reinterpret_cast(data )); + __m256 hi = _mm256_loadu_ps(reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); + } + + FASTOR_INLINE void complex_aligned_store(std::complex *data) const { + __m256 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm256_store_ps(reinterpret_cast(data ), lo); + _mm256_store_ps(reinterpret_cast(data+4), hi); + } + FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { + __m256 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm256_storeu_ps(reinterpret_cast(data ), lo); + _mm256_storeu_ps(reinterpret_cast(data+4), hi); + } + + FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { +#ifdef FASTOR_HAS_AVX512_MASKS + __m256 lo, hi; + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + lo = _mm256_mask_load_ps(lo, mask0, reinterpret_cast(data )); + hi = _mm256_mask_load_ps(hi, mask1, reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm256_setzero_ps(); + value_i = _mm256_setzero_ps(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + lo = _mm256_mask_loadu_ps(lo, mask0, reinterpret_cast(data )); + hi = _mm256_mask_loadu_ps(hi, mask1, reinterpret_cast(data+4)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm256_setzero_ps(); + value_i = _mm256_setzero_ps(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + _mm256_mask_store_ps(reinterpret_cast(data ), mask0, lo); + _mm256_mask_store_ps(reinterpret_cast(data+4), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { +#ifdef FASTOR_HAS_AVX512_MASKS + __m256 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + _mm256_mask_storeu_ps(reinterpret_cast(data ), mask0, lo); + _mm256_mask_storeu_ps(reinterpret_cast(data+4), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx> a) { + // ICC crashes without a copy + const __m256 vr = a.value_r; + const __m256 vi = a.value_i; + const float* value_r = reinterpret_cast(&vr); + const float* value_i = reinterpret_cast(&vi); + os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " + << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " + << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " + << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j, " + << value_r[4] << signum_string(value_i[4]) << std::abs(value_i[4]) << "j, " + << value_r[5] << signum_string(value_i[5]) << std::abs(value_i[5]) << "j, " + << value_r[6] << signum_string(value_i[6]) << std::abs(value_i[6]) << "j, " + << value_r[7] << signum_string(value_i[7]) << std::abs(value_i[7]) << "j" << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; + out.value_r = _mm256_add_ps(a.value_r,b.value_r); + out.value_i = _mm256_add_ps(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a + SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) + b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out(a); + out.value_r = _mm256_add_ps(a.value_r,_mm256_set1_ps(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(U a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out(b); + out.value_r = _mm256_add_ps(_mm256_set1_ps(a), b.value_r); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator+(const SIMDVector,simd_abi::avx> &b) { + return b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; + out.value_r = _mm256_sub_ps(a.value_r,b.value_r); + out.value_i = _mm256_sub_ps(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a - SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) - b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out(a); + out.value_r = _mm256_sub_ps(a.value_r,_mm256_set1_ps(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(U a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(std::complex(a,0)) - b; +} +/* This is negation and not complex conjugate */ +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator-(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out; + out.value_r = _mm256_neg_ps(a.value_r); + out.value_i = _mm256_neg_ps(a.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm256_fmsub_ps(a.value_r,b.value_r,_mm256_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm256_fmadd_ps(a.value_r,b.value_i,_mm256_mul_ps(a.value_i,b.value_r)); +#else + out.value_r = _mm256_sub_ps(_mm256_mul_ps(a.value_r,b.value_r),_mm256_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm256_add_ps(_mm256_mul_ps(a.value_r,b.value_i),_mm256_mul_ps(a.value_i,b.value_r)); +#endif + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a * SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) * b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out; + __m256 val = _mm256_set1_ps(b); + out.value_r = _mm256_mul_ps(a.value_r,val); + out.value_i = _mm256_mul_ps(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator*(U a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; + __m256 val = _mm256_set1_ps(a); + out.value_r = _mm256_mul_ps(val,b.value_r); + out.value_i = _mm256_mul_ps(val,b.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { + SIMDVector,simd_abi::avx> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm256_fmadd_ps(a.value_r,b.value_r,_mm256_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm256_fmsub_ps(a.value_i,b.value_r,_mm256_mul_ps(a.value_r,b.value_i)); + __m256 den = _mm256_fmadd_ps(b.value_r,b.value_r,_mm256_mul_ps(b.value_i,b.value_i)); +#else + out.value_r = _mm256_add_ps(_mm256_mul_ps(a.value_r,b.value_r),_mm256_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm256_sub_ps(_mm256_mul_ps(a.value_i,b.value_r),_mm256_mul_ps(a.value_r,b.value_i)); + __m256 den = _mm256_add_ps(_mm256_mul_ps(b.value_r,b.value_r),_mm256_mul_ps(b.value_i,b.value_i)); +#endif + out.value_r = _mm256_div_ps(out.value_r,den); + out.value_i = _mm256_div_ps(out.value_i,den); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(const SIMDVector,simd_abi::avx> &a, std::complex b) { + return a / SIMDVector,simd_abi::avx>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(std::complex a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(a) / b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(const SIMDVector,simd_abi::avx> &a, U b) { + SIMDVector,simd_abi::avx> out; + __m256 val = _mm256_set1_ps(b); + out.value_r = _mm256_div_ps(a.value_r,val); + out.value_i = _mm256_div_ps(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::avx> +operator/(U a, const SIMDVector,simd_abi::avx> &b) { + return SIMDVector,simd_abi::avx>(std::complex(a,0)) / b; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +rcp(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out; +#ifdef FASTOR_FMA_IMPL + __m256 den = _mm256_fmadd_ps(a.value_r,a.value_r,_mm256_mul_ps(a.value_i,a.value_i)); +#else + __m256 den = _mm256_add_ps(_mm256_mul_ps(a.value_r,a.value_r),_mm256_mul_ps(a.value_i,a.value_i)); +#endif + out.value_r = _mm256_div_ps(out.value_r,den); + out.value_i = _mm256_neg_ps(_mm256_div_ps(out.value_i,den)); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +sqrt(const SIMDVector,simd_abi::avx> &a) = delete; + +FASTOR_INLINE SIMDVector,simd_abi::avx> +rsqrt(const SIMDVector,simd_abi::avx> &a) = delete; + +/* This intentionally return a complex vector */ +FASTOR_INLINE SIMDVector,simd_abi::avx> +abs(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out; + out.value_r = a.magnitude().value; + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::avx> +conj(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out(a); + out.value_i = _mm256_neg_ps(out.value_i); + return out; +} + +/* Argument or phase angle */ +FASTOR_INLINE SIMDVector,simd_abi::avx> +arg(const SIMDVector,simd_abi::avx> &a) { + SIMDVector,simd_abi::avx> out(a); + for (FASTOR_INDEX i=0UL; i<8UL; ++i) { + ((float*)&out.value_r)[i] = std::atan2(((float*)&a.value_i)[i],((float*)&a.value_r)[i]); + } + return out; +} +//------------------------------------------------------------------------------------------------------------ + +#endif + + + + +// SSE VERSION +//------------------------------------------------------------------------------------------------------------ + +#ifdef FASTOR_SSE2_IMPL + +template <> +struct SIMDVector, simd_abi::sse> { + using vector_type = SIMDVector, simd_abi::sse>; + using value_type = __m128; + using scalar_value_type = std::complex; + using abi_type = simd_abi::sse; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value_r(_mm_setzero_ps()), value_i(_mm_setzero_ps()) {} + FASTOR_INLINE SIMDVector(std::complex num) { + value_r = _mm_set1_ps(num.real()); + value_i = _mm_set1_ps(num.imag()); + } + FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} + FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + + FASTOR_INLINE SIMDVector,simd_abi::sse> operator=(std::complex num) { + value_r = _mm_set1_ps(num.real()); + value_i = _mm_set1_ps(num.imag()); + return *this; + } + + FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { + if (Aligned) + complex_aligned_load(data); + else + complex_unaligned_load(data); + } + FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { + if (Aligned) + complex_aligned_store(data); + else + complex_unaligned_store(data); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { + if (!Aligned) + complex_mask_unaligned_load(data,mask); + else + complex_mask_aligned_load(data,mask); + } + FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { + if (!Aligned) + complex_mask_unaligned_store(data,mask); + else + complex_mask_aligned_store(data,mask); + } + + FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { + if (i == 0) { return scalar_value_type(_mm_get0_ps(value_r), _mm_get0_ps(value_i)); } + else if (i == 1) { return scalar_value_type(_mm_get1_ps(value_r), _mm_get1_ps(value_i)); } + else if (i == 2) { return scalar_value_type(_mm_get2_ps(value_r), _mm_get2_ps(value_i)); } + else { return scalar_value_type(_mm_get3_ps(value_r), _mm_get3_ps(value_i)); } + } + + FASTOR_INLINE SIMDVector real() const { + return value_r; + } + FASTOR_INLINE SIMDVector imag() const { + return value_i; + } + + FASTOR_INLINE void set(std::complex num) { + value_r = _mm_set1_ps(num.real()); + value_i = _mm_set1_ps(num.imag()); + } + FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, + scalar_value_type num2, scalar_value_type num3) { + const scalar_value_type tmp[Size] = {num0,num1,num2,num3}; + complex_unaligned_load(tmp); + } + + // In-place operators + template,bool> = false> + FASTOR_INLINE void operator+=(U num) { + value_r = _mm_add_ps(value_r,_mm_set1_ps(num)); + } + FASTOR_INLINE void operator+=(scalar_value_type num) { + *this += vector_type(num); + } + FASTOR_INLINE void operator+=(const vector_type &a) { + value_r = _mm_add_ps(value_r,a.value_r); + value_i = _mm_add_ps(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + value_r = _mm_sub_ps(value_r,_mm_set1_ps(num)); + } + FASTOR_INLINE void operator-=(scalar_value_type num) { + *this -= vector_type(num); + } + FASTOR_INLINE void operator-=(const vector_type &a) { + value_r = _mm_sub_ps(value_r,a.value_r); + value_i = _mm_sub_ps(value_i,a.value_i); + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + __m128 val = _mm_set1_ps(num); + value_r = _mm_mul_ps(value_r,val); + value_i = _mm_mul_ps(value_i,val); + } + FASTOR_INLINE void operator*=(scalar_value_type num) { + *this *= vector_type(num); + } + FASTOR_INLINE void operator*=(const vector_type &a) { +#ifdef FASTOR_FMA_IMPL + __m128 tmp = _mm_fmsub_ps(value_r,a.value_r,_mm_mul_ps(value_i,a.value_i)); + value_i = _mm_fmadd_ps(value_r,a.value_i,_mm_mul_ps(value_i,a.value_r)); +#else + __m128 tmp = _mm_sub_ps(_mm_mul_ps(value_r,a.value_r),_mm_mul_ps(value_i,a.value_i)); + value_i = _mm_add_ps(_mm_mul_ps(value_r,a.value_i),_mm_mul_ps(value_i,a.value_r)); +#endif + value_r = tmp; + } + + template,bool> = false> + FASTOR_INLINE void operator/=(U num) { + __m128 val = _mm_set1_ps(num); + value_r = _mm_div_ps(value_r,val); + value_i = _mm_div_ps(value_i,val); + } + FASTOR_INLINE void operator/=(scalar_value_type num) { + *this /= vector_type(num); + } + FASTOR_INLINE void operator/=(const vector_type &a) { + __m128 tmp = value_r; +#ifdef FASTOR_FMA_IMPL + value_r = _mm_fmadd_ps(value_r , a.value_r, _mm_mul_ps(value_i,a.value_i)); + value_i = _mm_fmsub_ps(value_i , a.value_r, _mm_mul_ps(tmp,a.value_i)); + __m128 den = _mm_fmadd_ps(a.value_r, a.value_r, _mm_mul_ps(a.value_i,a.value_i)); +#else + value_r = _mm_add_ps(_mm_mul_ps(value_r , a.value_r), _mm_mul_ps(value_i,a.value_i)); + value_i = _mm_sub_ps(_mm_mul_ps(value_i , a.value_r), _mm_mul_ps(tmp,a.value_i)); + __m128 den = _mm_add_ps(_mm_mul_ps(a.value_r, a.value_r), _mm_mul_ps(a.value_i,a.value_i)); +#endif + value_r = _mm_div_ps(value_r,den); + value_i = _mm_div_ps(value_i,den); + } + // end of in-place operators + + FASTOR_INLINE scalar_value_type sum() const { + return scalar_value_type(_mm_sum_ps(value_r),_mm_sum_ps(value_i)); + } + FASTOR_INLINE scalar_value_type product() const { + vector_type tmp(*this); + return tmp[0]*tmp[1]*tmp[2]*tmp[3]; + } + FASTOR_INLINE vector_type reverse() const { + vector_type out; + out.value_r = _mm_reverse_ps(value_r); + out.value_i = _mm_reverse_ps(value_i); + return out; + } + /* Actual magnitude - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector magnitude() const { +#ifdef FASTOR_FMA_IMPL + return _mm_sqrt_ps(_mm_fmadd_ps(value_r,value_r,_mm_mul_ps(value_i,value_i))); +#else + return _mm_sqrt_ps(_mm_add_ps(_mm_mul_ps(value_r,value_r),_mm_mul_ps(value_i,value_i))); +#endif + } + /* STL compliant squared norm - Note that this is a vertical operation */ + FASTOR_INLINE SIMDVector norm() const { +#ifdef FASTOR_FMA_IMPL + return _mm_fmadd_ps(value_r,value_r,_mm_mul_ps(value_i,value_i)); +#else + return _mm_add_ps(_mm_mul_ps(value_r,value_r),_mm_mul_ps(value_i,value_i)); +#endif + } + // Magnitude based minimum + FASTOR_INLINE scalar_value_type minimum() const; + // Magnitude based maximum + FASTOR_INLINE scalar_value_type maximum() const; + + FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { + vector_type out(*this); + out *= other; + return out.sum(); + } + + value_type value_r; + value_type value_i; + +protected: + FASTOR_INLINE void complex_aligned_load(const std::complex *data) { + __m128 lo = _mm_load_ps(reinterpret_cast(data )); + __m128 hi = _mm_load_ps(reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); + } + FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { + __m128 lo = _mm_loadu_ps(reinterpret_cast(data )); + __m128 hi = _mm_loadu_ps(reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); + } + + FASTOR_INLINE void complex_aligned_store(std::complex *data) const { + __m128 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm_store_ps(reinterpret_cast(data ),lo); + _mm_store_ps(reinterpret_cast(data+2),hi); + } + FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { + __m128 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + _mm_storeu_ps(reinterpret_cast(data ),lo); + _mm_storeu_ps(reinterpret_cast(data+2),hi); + } + + FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { +#ifdef FASTOR_HAS_AVX512_MASKS + __m128 lo, hi; + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + lo = _mm_mask_load_ps(lo, mask0, reinterpret_cast(data )); + hi = _mm_mask_load_ps(hi, mask1, reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm_setzero_ps(); + value_i = _mm_setzero_ps(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + lo = _mm_mask_loadu_ps(lo, mask0, reinterpret_cast(data )); + hi = _mm_mask_loadu_ps(hi, mask1, reinterpret_cast(data+2)); + arrange_from_load(value_r, value_i, lo, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + value_r = _mm_setzero_ps(); + value_i = _mm_setzero_ps(); + for (FASTOR_INDEX i=0; i(mask, mask0, mask1); + _mm_mask_store_ps(reinterpret_cast(data ), mask0, lo); + _mm_mask_store_ps(reinterpret_cast(data+2), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { +#ifdef FASTOR_HAS_AVX512_MASKS + __m128 lo, hi; + arrange_for_store(lo, hi, value_r, value_i); + uint8_t mask0, mask1; + split_mask(mask, mask0, mask1); + _mm_mask_storeu_ps(reinterpret_cast(data ), mask0, lo); + _mm_mask_storeu_ps(reinterpret_cast(data+2), mask1, hi); +#else + int maska[Size]; + mask_to_array(mask,maska); + for (FASTOR_INDEX i=0; i(_real,_imag); + } + else { + data[Size - i - 1] = std::complex(0,0); + } + } +#endif + } + +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::sse> a) { + // ICC crashes without a copy + const __m128 vr = a.value_r; + const __m128 vi = a.value_i; + const float* value_r = reinterpret_cast(&vr); + const float* value_i = reinterpret_cast(&vi); + os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " + << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " + << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " + << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j" << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; + out.value_r = _mm_add_ps(a.value_r,b.value_r); + out.value_i = _mm_add_ps(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a + SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) + b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out(a); + out.value_r = _mm_add_ps(a.value_r,_mm_set1_ps(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(U a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out(b); + out.value_r = _mm_add_ps(_mm_set1_ps(a), b.value_r); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator+(const SIMDVector,simd_abi::sse> &b) { + return b; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; + out.value_r = _mm_sub_ps(a.value_r,b.value_r); + out.value_i = _mm_sub_ps(a.value_i,b.value_i); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a - SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) - b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out(a); + out.value_r = _mm_sub_ps(a.value_r,_mm_set1_ps(b)); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(U a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(std::complex(a,0)) - b; +} +/* This is negation and not complex conjugate */ +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator-(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out; + out.value_r = _mm_neg_ps(a.value_r); + out.value_i = _mm_neg_ps(a.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm_fmsub_ps(a.value_r,b.value_r,_mm_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm_fmadd_ps(a.value_r,b.value_i,_mm_mul_ps(a.value_i,b.value_r)); +#else + out.value_r = _mm_sub_ps(_mm_mul_ps(a.value_r,b.value_r),_mm_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm_add_ps(_mm_mul_ps(a.value_r,b.value_i),_mm_mul_ps(a.value_i,b.value_r)); +#endif + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a * SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) * b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out; + __m128 val = _mm_set1_ps(b); + out.value_r = _mm_mul_ps(a.value_r,val); + out.value_i = _mm_mul_ps(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator*(U a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; + __m128 val = _mm_set1_ps(a); + out.value_r = _mm_mul_ps(val,b.value_r); + out.value_i = _mm_mul_ps(val,b.value_i); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { + SIMDVector,simd_abi::sse> out; +#ifdef FASTOR_FMA_IMPL + out.value_r = _mm_fmadd_ps(a.value_r,b.value_r,_mm_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm_fmsub_ps(a.value_i,b.value_r,_mm_mul_ps(a.value_r,b.value_i)); + __m128 den = _mm_fmadd_ps(b.value_r,b.value_r,_mm_mul_ps(b.value_i,b.value_i)); +#else + out.value_r = _mm_add_ps(_mm_mul_ps(a.value_r,b.value_r),_mm_mul_ps(a.value_i,b.value_i)); + out.value_i = _mm_sub_ps(_mm_mul_ps(a.value_i,b.value_r),_mm_mul_ps(a.value_r,b.value_i)); + __m128 den = _mm_add_ps(_mm_mul_ps(b.value_r,b.value_r),_mm_mul_ps(b.value_i,b.value_i)); +#endif + out.value_r = _mm_div_ps(out.value_r,den); + out.value_i = _mm_div_ps(out.value_i,den); + return out; +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(const SIMDVector,simd_abi::sse> &a, std::complex b) { + return a / SIMDVector,simd_abi::sse>(b); +} +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(std::complex a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(a) / b; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(const SIMDVector,simd_abi::sse> &a, U b) { + SIMDVector,simd_abi::sse> out; + __m128 val = _mm_set1_ps(b); + out.value_r = _mm_div_ps(a.value_r,val); + out.value_i = _mm_div_ps(a.value_i,val); + return out; +} +template,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::sse> +operator/(U a, const SIMDVector,simd_abi::sse> &b) { + return SIMDVector,simd_abi::sse>(std::complex(a,0)) / b; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +rcp(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out; +#ifdef FASTOR_FMA_IMPL + __m128 den = _mm_fmadd_ps(a.value_r,a.value_r,_mm_mul_ps(a.value_i,a.value_i)); +#else + __m128 den = _mm_add_ps(_mm_mul_ps(a.value_r,a.value_r),_mm_mul_ps(a.value_i,a.value_i)); +#endif + out.value_r = _mm_div_ps(out.value_r,den); + out.value_i = _mm_neg_ps(_mm_div_ps(out.value_i,den)); + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +sqrt(const SIMDVector,simd_abi::sse> &a) = delete; + +FASTOR_INLINE SIMDVector,simd_abi::sse> +rsqrt(const SIMDVector,simd_abi::sse> &a) = delete; + +/* This intentionally return a complex vector */ +FASTOR_INLINE SIMDVector,simd_abi::sse> +abs(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out; + out.value_r = a.magnitude().value; + return out; +} + +FASTOR_INLINE SIMDVector,simd_abi::sse> +conj(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out(a); + out.value_i = _mm_neg_ps(out.value_i); + return out; +} + +/* Argument or phase angle */ +FASTOR_INLINE SIMDVector,simd_abi::sse> +arg(const SIMDVector,simd_abi::sse> &a) { + SIMDVector,simd_abi::sse> out(a); + for (FASTOR_INDEX i=0UL; i<4UL; ++i) { + ((float*)&out.value_r)[i] = std::atan2(((float*)&a.value_i)[i],((float*)&a.value_r)[i]); + } + return out; +} +//------------------------------------------------------------------------------------------------------------ + +#endif + + +} // end of namespace Fastor + +#endif // SIMD_VECTOR_COMPLEX_FLOAT_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h b/noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h new file mode 100644 index 00000000..c6fcc996 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h @@ -0,0 +1,378 @@ +#ifndef SIMD_VECTOR_COMPLEX_SCALAR +#define SIMD_VECTOR_COMPLEX_SCALAR + +#include "Fastor/util/extended_algorithms.h" +#include "Fastor/simd_vector/extintrin.h" +#include "Fastor/simd_vector/simd_vector_base.h" +#include "Fastor/simd_vector/simd_vector_double.h" +#include +#include + + +namespace Fastor { + + +// SCALAR IMPLEMENTATION OF SIMDVECTOR FOR COMPLEX +//------------------------------------------------------------------------------------------------------------ + +template +struct SIMDVector, simd_abi::scalar> { + using vector_type = SIMDVector, simd_abi::scalar>; + using value_type = T; + using scalar_value_type = std::complex; + using abi_type = simd_abi::scalar; + static constexpr FASTOR_INDEX Size = 1; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;} + + FASTOR_INLINE SIMDVector() : value_r(0), value_i(0) {} + FASTOR_INLINE SIMDVector(scalar_value_type num) { + value_r = num.real(); + value_i = num.imag(); + } + FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} + FASTOR_INLINE SIMDVector(const scalar_value_type *data, bool Aligned=true) { + value_r = (*data).real(); + value_i = (*data).imag(); + } + + FASTOR_INLINE SIMDVector operator=(std::complex num) { + value_r = num.real(); + value_i = num.imag(); + return *this; + } + + FASTOR_INLINE void load(const scalar_value_type *data, bool ) { + value_r = (*data).real(); + value_i = (*data).imag(); + } + FASTOR_INLINE void store(scalar_value_type *data, bool ) const { + data[0] = scalar_value_type(value_r,value_i); + } + + FASTOR_INLINE void aligned_load(const T *data) { + value_r = (*data).real(); + value_i = (*data).imag(); + } + FASTOR_INLINE void aligned_store(T *data) const { + data[0] = scalar_value_type(value_r,value_i); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool ) { + if (mask != 0x0) { + value_r = (*data).real(); + value_i = (*data).imag(); + } + } + FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool) const { + if (mask != 0x0) { + data[0] = scalar_value_type(value_r,value_i); + } + } + + FASTOR_INLINE T operator[](FASTOR_INDEX) const {return scalar_value_type(value_r,value_i);} + + FASTOR_INLINE SIMDVector real() const { + return value_r; + } + FASTOR_INLINE SIMDVector imag() const { + return value_i; + } + + FASTOR_INLINE void set(scalar_value_type num) { + value_r = num.real(); + value_i = num.imag(); + } + + FASTOR_INLINE void set_sequential(scalar_value_type num) { + value_r = num.real(); + value_i = num.imag(); + } + + // In-place operators + template,bool> = false> + FASTOR_INLINE void operator+=(U num) { + value_r += num; + } + FASTOR_INLINE void operator+=(scalar_value_type num) { + value_r += num.real(); + value_i += num.imag(); + } + FASTOR_INLINE void operator+=(const vector_type &a) { + value_r += a.value_r; + value_i += a.value_i; + } + + template,bool> = false> + FASTOR_INLINE void operator-=(U num) { + value_r -= num; + } + FASTOR_INLINE void operator-=(scalar_value_type num) { + value_r -= num.real(); + value_i -= num.imag(); + } + FASTOR_INLINE void operator-=(const vector_type &a) { + value_r -= a.value_r; + value_i -= a.value_i; + } + + template,bool> = false> + FASTOR_INLINE void operator*=(U num) { + value_r *= num; + value_i *= num; + } + FASTOR_INLINE void operator*=(scalar_value_type num) { + scalar_value_type tmp(value_r, value_i); + tmp *= num; + value_r = tmp.real(); + value_i = tmp.imag(); + } + FASTOR_INLINE void operator*=(const vector_type &a) { + scalar_value_type tmp0(value_r, value_i); + scalar_value_type tmp1(a.value_r, a.value_i); + tmp0 *= tmp1; + value_r = tmp0.real(); + value_i = tmp0.imag(); + } + + template,bool> = false> + FASTOR_INLINE void operator/=(U num) { + value_r /= num; + value_i /= num; + } + FASTOR_INLINE void operator/=(scalar_value_type num) { + scalar_value_type tmp(value_r, value_i); + tmp /= num; + value_r = tmp.real(); + value_i = tmp.imag(); + } + FASTOR_INLINE void operator/=(const vector_type &a) { + scalar_value_type tmp0(value_r, value_i); + scalar_value_type tmp1(a.value_r, a.value_i); + tmp0 /= tmp1; + value_r = tmp0.real(); + value_i = tmp0.imag(); + } + // end of in-place operators + + FASTOR_INLINE scalar_value_type sum() const {return scalar_value_type(value_r, value_i);} + FASTOR_INLINE scalar_value_type product() const {return scalar_value_type(value_r, value_i);} + FASTOR_INLINE vector_type reverse() const { return scalar_value_type(value_r, value_i); } + FASTOR_INLINE SIMDVector magnitude() const { return std::abs(scalar_value_type(value_r, value_i));} + FASTOR_INLINE SIMDVector norm() const { return std::norm(scalar_value_type(value_r, value_i));} + FASTOR_INLINE scalar_value_type minimum() const {return scalar_value_type(value_r, value_i);} + FASTOR_INLINE scalar_value_type maximum() const {return scalar_value_type(value_r, value_i);} + FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { + return vector_type(value_r, value_i)*vector_type(other.value_r, other.value_i); + } + + value_type value_r; + value_type value_i; +}; + +template +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::scalar> a) { + os << "[" << a.value_r << signum_string(a.value_i) << std::abs(a.value_i) << "j]\n"; + return os; +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator+(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out += b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator+(const SIMDVector,simd_abi::scalar> &a, std::complex b) { + SIMDVector,simd_abi::scalar> out(a); + out += b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator+(std::complex a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out += b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator+(const SIMDVector,simd_abi::scalar> &a, U b) { + SIMDVector,simd_abi::scalar> out(a); + out += b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator+(U a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(std::complex(a,0)); + out += b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator+(const SIMDVector,simd_abi::scalar> &b) { + return b; +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator-(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out -= b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator-(const SIMDVector,simd_abi::scalar> &a, std::complex b) { + SIMDVector,simd_abi::scalar> out(a); + out -= b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator-(std::complex a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out -= b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator-(const SIMDVector,simd_abi::scalar> &a, U b) { + SIMDVector,simd_abi::scalar> out(a); + out -= b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator-(U a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(std::complex(a,0)); + out -= b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator-(const SIMDVector,simd_abi::scalar> &b) { + return SIMDVector,simd_abi::scalar>(0,0) - b; +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator*(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out *= b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator*(const SIMDVector,simd_abi::scalar> &a, std::complex b) { + SIMDVector,simd_abi::scalar> out(a); + out *= b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator*(std::complex a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out *= b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator*(const SIMDVector,simd_abi::scalar> &a, U b) { + SIMDVector,simd_abi::scalar> out(a); + out *= b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator*(U a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(std::complex(a,0)); + out *= b; + return out; +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator/(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out /= b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator/(const SIMDVector,simd_abi::scalar> &a, std::complex b) { + SIMDVector,simd_abi::scalar> out(a); + out /= b; + return out; +} +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator/(std::complex a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(a); + out /= b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator/(const SIMDVector,simd_abi::scalar> &a, U b) { + SIMDVector,simd_abi::scalar> out(a); + out /= b; + return out; +} +template ,bool> = false> +FASTOR_INLINE SIMDVector,simd_abi::scalar> +operator/(U a, const SIMDVector,simd_abi::scalar> &b) { + SIMDVector,simd_abi::scalar> out(std::complex(a,0)); + out /= b; + return out; +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +rcp(const SIMDVector,simd_abi::scalar> &a) { + SIMDVector,simd_abi::scalar> out(std::complex(1,0)); + out /= a; + return out; +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +sqrt(const SIMDVector,simd_abi::scalar> &a) { + std::complex out(a.value_r,a.value_i); + return std::sqrt(out); +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +rsqrt(const SIMDVector,simd_abi::scalar> &a) { + return rcp(sqrt(a)); +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +abs(const SIMDVector,simd_abi::scalar> &a) { + SIMDVector,simd_abi::scalar> out; + out.value_r = a.magnitude().value; + return out; +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +conj(const SIMDVector,simd_abi::scalar> &a) { + std::complex out(a.value_r,a.value_i); + return std::conj(out); +} + +template +FASTOR_INLINE SIMDVector,simd_abi::scalar> +arg(const SIMDVector,simd_abi::scalar> &a) { + std::complex out(a.value_r,a.value_i); + return SIMDVector,simd_abi::scalar>(std::arg(out),0); +} +//------------------------------------------------------------------------------------------------------------ + +} // end of namespace Fastor + +#endif // SIMD_VECTOR_COMPLEX_SCALAR diff --git a/noarch/include/Fastor/simd_vector/simd_vector_double.h b/noarch/include/Fastor/simd_vector/simd_vector_double.h new file mode 100644 index 00000000..55543d9a --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_double.h @@ -0,0 +1,914 @@ +#ifndef SIMD_VECTOR_DOUBLE_H +#define SIMD_VECTOR_DOUBLE_H + +#include "Fastor/simd_vector/simd_vector_base.h" + +namespace Fastor { + +// AVX512 VERSION +//-------------------------------------------------------------------------------------------------- +#ifdef FASTOR_AVX512_IMPL + +template <> +struct SIMDVector { + using value_type = __m512d; + using scalar_value_type = double; + using abi_type = simd_abi::avx512; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm512_setzero_pd()) {} + FASTOR_INLINE SIMDVector(double num) : value(_mm512_set1_pd(num)) {} + FASTOR_INLINE SIMDVector(__m512d regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_pd(data); + else + value = _mm512_loadu_pd(data); + } + + FASTOR_INLINE SIMDVector operator=(double num) { + value = _mm512_set1_pd(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m512d regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const double *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_pd(data); + else + value = _mm512_loadu_pd(data); + } + FASTOR_INLINE void store(double *data, bool Aligned=true) const { + if (Aligned) + _mm512_store_pd(data,value); + else + _mm512_storeu_pd(data,value); + } + + FASTOR_INLINE void aligned_load(const double *data) { + value =_mm512_load_pd(data); + } + FASTOR_INLINE void aligned_store(double *data) const { + _mm512_store_pd(data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm512_mask_loadu_pd(value, mask, a); + else + value = _mm512_mask_load_pd(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm512_setzero_pd(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(double num) { + value = _mm512_set1_pd(num); + } + FASTOR_INLINE void set(double num0, double num1, double num2, double num3, double num4, double num5, double num6, double num7) { + value = _mm512_set_pd(num0,num1,num2,num3,num4,num5,num6,num7); + } + FASTOR_INLINE void set_sequential(double num0) { + value = _mm512_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0,num0+4.0,num0+5.0,num0+6.0,num0+7.0); + } + FASTOR_INLINE void broadcast(const double *data) { + // value = _mm512_broadcast_sd(data); + } + + // In-place operators + FASTOR_INLINE void operator+=(double num) { + value = _mm512_add_pd(value,_mm512_set1_pd(num)); + } + FASTOR_INLINE void operator+=(__m512d regi) { + value = _mm512_add_pd(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm512_add_pd(value,a.value); + } + + FASTOR_INLINE void operator-=(double num) { + value = _mm512_sub_pd(value,_mm512_set1_pd(num)); + } + FASTOR_INLINE void operator-=(__m512d regi) { + value = _mm512_sub_pd(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm512_sub_pd(value,a.value); + } + + FASTOR_INLINE void operator*=(double num) { + value = _mm512_mul_pd(value,_mm512_set1_pd(num)); + } + FASTOR_INLINE void operator*=(__m512d regi) { + value = _mm512_mul_pd(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm512_mul_pd(value,a.value); + } + + FASTOR_INLINE void operator/=(double num) { + value = _mm512_div_pd(value,_mm512_set1_pd(num)); + } + FASTOR_INLINE void operator/=(__m512d regi) { + value = _mm512_div_pd(value,regi); + } + FASTOR_INLINE void operator/=(const SIMDVector &a) { + value = _mm512_div_pd(value,a.value); + } + // end of in-place operators + + // FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { + // SIMDVector out; + // if (i==1) + // out.value = _mm512_shift1_pd(value); + // else if (i==2) + // out.value = _mm512_shift2_pd(value); + // else if (i==3) + // out.value = _mm512_shift3_pd(value); + // return out; + // } + FASTOR_INLINE double sum() { +#ifdef FASTOR_HAS_AVX512_REDUCE_ADD + return _mm512_reduce_add_pd(value); +#else + __m256d low = _mm512_castpd512_pd256(value); + __m256d high = _mm512_extractf64x4_pd(value,0x1); + return _mm256_sum_pd(_mm256_add_pd(low,high)); +#endif + } + FASTOR_INLINE double product() { + __m256d low = _mm512_castpd512_pd256(value); + __m256d high = _mm512_extractf64x4_pd(value,1); + return _mm256_prod_pd(_mm256_mul_pd(low,high)); + } + FASTOR_INLINE SIMDVector reverse() { + return _mm512_reverse_pd(value); + } + // FASTOR_INLINE double minimum() {return _mm512_hmin_pd(value);} + // FASTOR_INLINE double maximum() {return _mm512_hmax_pd(value);} + + FASTOR_INLINE double dot(const SIMDVector &other) { + __m512d res = _mm512_mul_pd(value,other.value); + __m256d low = _mm512_castpd512_pd256(res); + __m256d high = _mm512_extractf64x4_pd(res,1); + return _mm256_sum_pd(_mm256_add_pd(low,high)); + } + + __m512d value; +}; + + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + // ICC crashes without a copy + const __m512d v = a.value; + const double* value = reinterpret_cast(&v); + os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] + << " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm512_add_pd(a.value,_mm512_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_pd(_mm512_set1_pd(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm512_sub_pd(a.value,_mm512_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_pd(_mm512_set1_pd(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + return _mm512_neg_pd(b.value); +} + +FASTOR_INLINE SIMDVector operator*( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_mul_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm512_mul_pd(a.value,_mm512_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_mul_pd(_mm512_set1_pd(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_div_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm512_div_pd(a.value,_mm512_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator/(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_div_pd(_mm512_set1_pd(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + SIMDVector out; + out.value = _mm512_rcp14_pd(a.value); + return out; +} + +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm512_sqrt_pd(a.value); + return out; +} + +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm512_rsqrt14_pd(a.value); + return out; +} + +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + SIMDVector out; +#ifdef FASTOR_HAS_AVX512_ABS + out.value = _mm512_abs_pd(a.value); +#else + for (FASTOR_INDEX i=0UL; i<8UL; ++i) { + ((double*)&out.value)[i] = std::abs(((double*)&a.value)[i]); + } +#endif + return out; +} + +#endif + + + +// AVX VERSION +//-------------------------------------------------------------------------------------------------- +#ifdef FASTOR_AVX_IMPL + +template <> +struct SIMDVector { + using value_type = __m256d; + using scalar_value_type = double; + using abi_type = simd_abi::avx; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm256_setzero_pd()) {} + FASTOR_INLINE SIMDVector(double num) : value(_mm256_set1_pd(num)) {} + FASTOR_INLINE SIMDVector(__m256d regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_pd(data); + else + value = _mm256_loadu_pd(data); + } + + FASTOR_INLINE SIMDVector operator=(double num) { + value = _mm256_set1_pd(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m256d regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const double *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_pd(data); + else + value = _mm256_loadu_pd(data); + } + FASTOR_INLINE void store(double *data, bool Aligned=true) const { + if (Aligned) + _mm256_store_pd(data,value); + else + _mm256_storeu_pd(data,value); + } + + FASTOR_INLINE void aligned_load(const double *data) { + value =_mm256_load_pd(data); + } + FASTOR_INLINE void aligned_store(double *data) const { + _mm256_store_pd(data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm256_mask_loadu_pd(value, mask, a); + else + value = _mm256_mask_load_pd(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm256_setzero_pd(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(double num) { + value = _mm256_set1_pd(num); + } + FASTOR_INLINE void set(double num0, double num1, double num2, double num3) { + value = _mm256_set_pd(num0,num1,num2,num3); + } + FASTOR_INLINE void set_sequential(double num0) { + value = _mm256_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0); + } + FASTOR_INLINE void broadcast(const double *data) { + value = _mm256_broadcast_sd(data); + } + + // In-place operators + FASTOR_INLINE void operator+=(double num) { + value = _mm256_add_pd(value,_mm256_set1_pd(num)); + } + FASTOR_INLINE void operator+=(__m256d regi) { + value = _mm256_add_pd(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm256_add_pd(value,a.value); + } + + FASTOR_INLINE void operator-=(double num) { + value = _mm256_sub_pd(value,_mm256_set1_pd(num)); + } + FASTOR_INLINE void operator-=(__m256d regi) { + value = _mm256_sub_pd(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm256_sub_pd(value,a.value); + } + + FASTOR_INLINE void operator*=(double num) { + value = _mm256_mul_pd(value,_mm256_set1_pd(num)); + } + FASTOR_INLINE void operator*=(__m256d regi) { + value = _mm256_mul_pd(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm256_mul_pd(value,a.value); + } + + FASTOR_INLINE void operator/=(double num) { + value = _mm256_div_pd(value,_mm256_set1_pd(num)); + } + FASTOR_INLINE void operator/=(__m256d regi) { + value = _mm256_div_pd(value,regi); + } + FASTOR_INLINE void operator/=(const SIMDVector &a) { + value = _mm256_div_pd(value,a.value); + } + // end of in-place operators + + FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { + SIMDVector out; + if (i==1) + out.value = _mm256_shift1_pd(value); + else if (i==2) + out.value = _mm256_shift2_pd(value); + else if (i==3) + out.value = _mm256_shift3_pd(value); + return out; + } + FASTOR_INLINE double sum() {return _mm256_sum_pd(value);} + FASTOR_INLINE double product() {return _mm256_prod_pd(value);} + FASTOR_INLINE SIMDVector reverse() { + SIMDVector out; + out.value = _mm256_reverse_pd(value); + return out; + } + FASTOR_INLINE double minimum() {return _mm256_hmin_pd(value);} + FASTOR_INLINE double maximum() {return _mm256_hmax_pd(value);} + + FASTOR_INLINE double dot(const SIMDVector &other) { + return _mm_cvtsd_f64(_mm256_dp_pd(value,other.value)); + } + + __m256d value; +}; + + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + // ICC crashes without a copy + const __m256d v = a.value; + const double* value = reinterpret_cast(&v); + os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm256_add_pd(a.value,_mm256_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_pd(_mm256_set1_pd(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm256_sub_pd(a.value,_mm256_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_pd(_mm256_set1_pd(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_neg_pd(b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm256_mul_pd(a.value,_mm256_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_pd(_mm256_set1_pd(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_div_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm256_div_pd(a.value,_mm256_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator/(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_div_pd(_mm256_set1_pd(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + SIMDVector out; + // This is very inaccurate for double precision + out.value = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value))); + return out; + + // // For making it more accurate using Newton Raphson use this + // __m128d xmm0 = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value))); + // xmm0 = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0))); + // out.value = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0))); + // return out; + +} + +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm256_sqrt_pd(a.value); + return out; +} + +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + SIMDVector out; + // This is very inaccurate for double precision + out.value = _mm256_cvtps_pd(_mm_rsqrt_ps(_mm256_cvtpd_ps(a.value))); + return out; +} + +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + SIMDVector out; + out.value = _mm256_abs_pd(a.value); + return out; +} + +#endif + + + + + + + +// SSE VERSION +//------------------------------------------------------------------------------------------------------------ + +#ifdef FASTOR_SSE2_IMPL + +template <> +struct SIMDVector { + using value_type = __m128d; + using scalar_value_type = double; + using abi_type = simd_abi::sse; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm_setzero_pd()) {} + FASTOR_INLINE SIMDVector(double num) : value(_mm_set1_pd(num)) {} + FASTOR_INLINE SIMDVector(__m128d regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_pd(data); + else + value = _mm_loadu_pd(data); + } + + FASTOR_INLINE SIMDVector operator=(double num) { + value = _mm_set1_pd(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m128d regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const double *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_pd(data); + else + value = _mm_loadu_pd(data); + } + FASTOR_INLINE void store(double *data, bool Aligned=true) const { + if (Aligned) + _mm_store_pd(data,value); + else + _mm_storeu_pd(data,value); + } + + FASTOR_INLINE void aligned_load(const double *data) { + value =_mm_load_pd(data); + } + FASTOR_INLINE void aligned_store(double *data) const { + _mm_store_pd(data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm_mask_loadu_pd(value, mask, a); + else + value = _mm_mask_load_pd(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm_setzero_pd(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(double num) { + value = _mm_set1_pd(num); + } + FASTOR_INLINE void set(double num0, double num1) { + value = _mm_set_pd(num0,num1); + } + FASTOR_INLINE void set_sequential(double num0) { + value = _mm_setr_pd(num0,num0+1.0); + } + FASTOR_INLINE void broadcast(const double *data) { + value = _mm_load1_pd(data); + } + + // In-place operators + FASTOR_INLINE void operator+=(double num) { + value = _mm_add_pd(value,_mm_set1_pd(num)); + } + FASTOR_INLINE void operator+=(__m128d regi) { + value = _mm_add_pd(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm_add_pd(value,a.value); + } + + FASTOR_INLINE void operator-=(double num) { + value = _mm_sub_pd(value,_mm_set1_pd(num)); + } + FASTOR_INLINE void operator-=(__m128d regi) { + value = _mm_sub_pd(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm_sub_pd(value,a.value); + } + + FASTOR_INLINE void operator*=(double num) { + value = _mm_mul_pd(value,_mm_set1_pd(num)); + } + FASTOR_INLINE void operator*=(__m128d regi) { + value = _mm_mul_pd(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm_mul_pd(value,a.value); + } + + FASTOR_INLINE void operator/=(double num) { + value = _mm_div_pd(value,_mm_set1_pd(num)); + } + FASTOR_INLINE void operator/=(__m128d regi) { + value = _mm_div_pd(value,regi); + } + FASTOR_INLINE void operator/=(const SIMDVector &a) { + value = _mm_div_pd(value,a.value); + } + // end of in-place operators + + FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { + SIMDVector out; + FASTOR_ASSERT(i==1,"INCORRECT SHIFT INDEX"); + out.value = _mm_shift1_pd(value); + return out; + } + FASTOR_INLINE double sum() {return _mm_sum_pd(value);} + FASTOR_INLINE double product() {return _mm_prod_pd(value);} + FASTOR_INLINE SIMDVector reverse() { + SIMDVector out; + out.value = _mm_reverse_pd(value); + return out; + } + FASTOR_INLINE double minimum() {return _mm_hmin_pd(value);} + FASTOR_INLINE double maximum() {return _mm_hmax_pd(value);} + + FASTOR_INLINE double dot(const SIMDVector &other) { +#ifdef FASTOR_SSE4_1_IMPL + return _mm_cvtsd_f64(_mm_dp_pd(value,other.value,0xff)); +#else + return _mm_sum_pd(_mm_mul_pd(value,other.value)); +#endif + } + + __m128d value; +}; + + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + // ICC crashes without a copy + const __m128d v = a.value; + const double* value = reinterpret_cast(&v); + os << "[" << value[0] << " " << value[1] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_add_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm_add_pd(a.value,_mm_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_add_pd(_mm_set1_pd(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_sub_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm_sub_pd(a.value,_mm_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_sub_pd(_mm_set1_pd(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + out.value = _mm_neg_pd(b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_mul_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm_mul_pd(a.value,_mm_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_mul_pd(_mm_set1_pd(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_div_pd(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, double b) { + SIMDVector out; + out.value = _mm_div_pd(a.value,_mm_set1_pd(b)); + return out; +} +FASTOR_INLINE SIMDVector operator/(double a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_div_pd(_mm_set1_pd(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + SIMDVector out; + // This is very inaccurate for double precision + out.value = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value))); + return out; + + /* + // For making it more accurate using Newton Raphson use this + __m128d xmm0 = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value))); + xmm0 = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0))); + out.value = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0))); + return out; + */ +} + +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm_sqrt_pd(a.value); + return out; +} + +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + SIMDVector out; + // This is very inaccurate for double precision + out.value = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(a.value))); + return out; +} + +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + SIMDVector out; + out.value = _mm_abs_pd(a.value); + return out; +} + + +#endif + + +} // end of namespace Fastor + +#endif // // SIMD_VECTOR_DOUBLE_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_float.h b/noarch/include/Fastor/simd_vector/simd_vector_float.h new file mode 100644 index 00000000..ce78cfb8 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_float.h @@ -0,0 +1,918 @@ +#ifndef SIMD_VECTOR_FLOAT_H +#define SIMD_VECTOR_FLOAT_H + +#include "Fastor/simd_vector/simd_vector_base.h" + +namespace Fastor { + + +// AVX512 VERSION +//-------------------------------------------------------------------------------------------------- + +#ifdef FASTOR_AVX512_IMPL + +template <> +struct SIMDVector { + using value_type = __m512; + using scalar_value_type = float; + using abi_type = simd_abi::avx512; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm512_setzero_ps()) {} + FASTOR_INLINE SIMDVector(float num) : value(_mm512_set1_ps(num)) {} + FASTOR_INLINE SIMDVector(__m512 regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_ps(data); + else + value = _mm512_loadu_ps(data); + } + + FASTOR_INLINE SIMDVector operator=(float num) { + value = _mm512_set1_ps(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m512 regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const float *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_ps(data); + else + value = _mm512_loadu_ps(data); + } + FASTOR_INLINE void store(float *data, bool Aligned=true) const { + if (Aligned) + _mm512_store_ps(data,value); + else + _mm512_storeu_ps(data,value); + } + + FASTOR_INLINE void aligned_load(const float *data) { + value =_mm512_load_ps(data); + } + FASTOR_INLINE void aligned_store(float *data) const { + _mm512_store_ps(data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint16_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm512_mask_loadu_ps(value, mask, a); + else + value = _mm512_mask_load_ps(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm512_setzero_ps(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(float num) { + value = _mm512_set1_ps(num); + } + FASTOR_INLINE void set(float num0, float num1, float num2, float num3, + float num4, float num5, float num6, float num7, + float num8, float num9, float num10, float num11, + float num12, float num13, float num14, float num15) { + value = _mm512_set_ps(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15); + } + FASTOR_INLINE void set_sequential(float num0) { + value = _mm512_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f, + num0+8.f,num0+9.f,num0+10.f,num0+11.f,num0+12.f,num0+13.f,num0+14.f,num0+15.f); + } + FASTOR_INLINE void broadcast(const float *data) { + // value = _mm512_broadcast_ss(data); + } + + // In-place operators + FASTOR_INLINE void operator+=(float num) { + value = _mm512_add_ps(value,_mm512_set1_ps(num)); + } + FASTOR_INLINE void operator+=(__m512 regi) { + value = _mm512_add_ps(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm512_add_ps(value,a.value); + } + + FASTOR_INLINE void operator-=(float num) { + value = _mm512_sub_ps(value,_mm512_set1_ps(num)); + } + FASTOR_INLINE void operator-=(__m512 regi) { + value = _mm512_sub_ps(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm512_sub_ps(value,a.value); + } + + FASTOR_INLINE void operator*=(float num) { + value = _mm512_mul_ps(value,_mm512_set1_ps(num)); + } + FASTOR_INLINE void operator*=(__m512 regi) { + value = _mm512_mul_ps(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm512_mul_ps(value,a.value); + } + + FASTOR_INLINE void operator/=(float num) { + value = _mm512_div_ps(value,_mm512_set1_ps(num)); + } + FASTOR_INLINE void operator/=(__m512 regi) { + value = _mm512_div_ps(value,regi); + } + FASTOR_INLINE void operator/=(const SIMDVector &a) { + value = _mm512_div_ps(value,a.value); + } + // end of in-place operators + + FASTOR_INLINE float sum() { +#ifdef FASTOR_HAS_AVX512_REDUCE_ADD + return _mm512_reduce_add_ps(value); +#else + __m256 low = _mm512_castps512_ps256(value); + __m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1)); + return _mm256_sum_ps(_mm256_add_ps(low,high)); +#endif + } + FASTOR_INLINE float product() { + __m256 low = _mm512_castps512_ps256(value); + __m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1)); + return _mm256_prod_ps(_mm256_mul_ps(low,high)); + } + FASTOR_INLINE SIMDVector reverse() { + return _mm512_reverse_ps(value); + } + // FASTOR_INLINE float minimum() {return _mm512_hmin_ps(value);} + // FASTOR_INLINE float maximum() {return _mm512_hmax_ps(value);} + + FASTOR_INLINE float dot(const SIMDVector &other) { + __m512 res = _mm512_mul_ps(value,other.value); + __m256 low = _mm512_castps512_ps256(res); + __m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(res),1)); + return _mm256_sum_ps(_mm256_add_ps(low,high)); + } + + __m512 value; +}; + + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + // ICC crashes without a copy + const __m512 v = a.value; + const float* value = reinterpret_cast(&v); + os << "[" + << value[0] << " " << value[1] << " " + << value[2] << " " << value[3] << " " + << value[4] << " " << value[5] << " " + << value[6] << " " << value[7] << " " + << value[8] << " " << value[9] << " " + << value[10] << " " << value[11] << " " + << value[12] << " " << value[13] << " " + << value[14] << " " << value[15] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm512_add_ps(a.value,_mm512_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_ps(_mm512_set1_ps(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm512_sub_ps(a.value,_mm512_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_ps(_mm512_set1_ps(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + return _mm512_neg_ps(b.value); +} + +FASTOR_INLINE SIMDVector operator*( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_mul_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm512_mul_ps(a.value,_mm512_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_mul_ps(_mm512_set1_ps(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/( + const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_div_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm512_div_ps(a.value,_mm512_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator/(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_div_ps(_mm512_set1_ps(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + SIMDVector out; + out.value = _mm512_rcp14_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm512_sqrt_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm512_rsqrt14_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + SIMDVector out; +#ifdef FASTOR_HAS_AVX512_ABS + out.value = _mm512_abs_ps(a.value); +#else + for (FASTOR_INDEX i=0UL; i<16UL; ++i) { + ((float*)&out.value)[i] = std::abs(((float*)&a.value)[i]); + } +#endif + return out; +} + + +#endif + + + + +// AVX VERSION +//-------------------------------------------------------------------------------------------------- + +#ifdef FASTOR_AVX_IMPL + +template <> +struct SIMDVector { + using value_type = __m256; + using scalar_value_type = float; + using abi_type = simd_abi::avx; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm256_setzero_ps()) {} + FASTOR_INLINE SIMDVector(float num) : value(_mm256_set1_ps(num)) {} + FASTOR_INLINE SIMDVector(__m256 regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_ps(data); + else + value = _mm256_loadu_ps(data); + } + + FASTOR_INLINE SIMDVector operator=(float num) { + value = _mm256_set1_ps(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m256 regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const float *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_ps(data); + else + value = _mm256_loadu_ps(data); + } + FASTOR_INLINE void store(float *data, bool Aligned=true) const { + if (Aligned) + _mm256_store_ps(data,value); + else + _mm256_storeu_ps(data,value); + } + + FASTOR_INLINE void aligned_load(const float *data) { + value =_mm256_load_ps(data); + } + FASTOR_INLINE void aligned_store(float *data) const { + _mm256_store_ps(data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm256_mask_loadu_ps(value, mask, a); + else + value = _mm256_mask_load_ps(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm256_setzero_ps(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(float num) { + value = _mm256_set1_ps(num); + } + FASTOR_INLINE void set(float num0, float num1, float num2, float num3, + float num4, float num5, float num6, float num7) { + value = _mm256_set_ps(num0,num1,num2,num3,num4,num5,num6,num7); + } + FASTOR_INLINE void set_sequential(float num0) { + value = _mm256_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f); + } + FASTOR_INLINE void broadcast(const float *data) { + value = _mm256_broadcast_ss(data); + } + + // In-place operators + FASTOR_INLINE void operator+=(float num) { + value = _mm256_add_ps(value,_mm256_set1_ps(num)); + } + FASTOR_INLINE void operator+=(__m256 regi) { + value = _mm256_add_ps(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm256_add_ps(value,a.value); + } + + FASTOR_INLINE void operator-=(float num) { + value = _mm256_sub_ps(value,_mm256_set1_ps(num)); + } + FASTOR_INLINE void operator-=(__m256 regi) { + value = _mm256_sub_ps(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm256_sub_ps(value,a.value); + } + + FASTOR_INLINE void operator*=(float num) { + value = _mm256_mul_ps(value,_mm256_set1_ps(num)); + } + FASTOR_INLINE void operator*=(__m256 regi) { + value = _mm256_mul_ps(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm256_mul_ps(value,a.value); + } + + FASTOR_INLINE void operator/=(float num) { + value = _mm256_div_ps(value,_mm256_set1_ps(num)); + } + FASTOR_INLINE void operator/=(__m256 regi) { + value = _mm256_div_ps(value,regi); + } + FASTOR_INLINE void operator/=(const SIMDVector &a) { + value = _mm256_div_ps(value,a.value); + } + // end of in-place operators + + FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { + SIMDVector out; + if (i==1) + out.value = _mm256_shift1_ps(value); + else if (i==2) + out.value = _mm256_shift2_ps(value); + else if (i==3) + out.value = _mm256_shift3_ps(value); + else if (i==4) + out.value = _mm256_shift4_ps(value); + else if (i==5) + out.value = _mm256_shift5_ps(value); + else if (i==6) + out.value = _mm256_shift6_ps(value); + else if (i==7) + out.value = _mm256_shift7_ps(value); + return out; + } + FASTOR_INLINE float sum() {return _mm256_sum_ps(value);} + FASTOR_INLINE float product() {return _mm256_prod_ps(value);} + FASTOR_INLINE SIMDVector reverse() { + SIMDVector out; + out.value = _mm256_reverse_ps(value); + return out; + } + FASTOR_INLINE float minimum() {return _mm256_hmin_ps(value);} + FASTOR_INLINE float maximum() {return _mm256_hmax_ps(value);} + + FASTOR_INLINE float dot(const SIMDVector &other) { + __m256 tmp = _mm256_dp_ps(value,other.value,0xff); + return _mm256_get0_ps(tmp)+_mm256_get4_ps(tmp); + } + + __m256 value; +}; + + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + // ICC crashes without a copy + const __m256 v = a.value; + const float* value = reinterpret_cast(&v); + os << "[" << value[0] << " " << value[1] << " " + << value[2] << " " << value[3] << " " + << value[4] << " " << value[5] << " " + << value[6] << " " << value[7] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm256_add_ps(a.value,_mm256_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_ps(_mm256_set1_ps(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm256_sub_ps(a.value,_mm256_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_ps(_mm256_set1_ps(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_neg_ps(b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm256_mul_ps(a.value,_mm256_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_ps(_mm256_set1_ps(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_div_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm256_div_ps(a.value,_mm256_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator/(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_div_ps(_mm256_set1_ps(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + SIMDVector out; + out.value = _mm256_rcp_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm256_sqrt_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm256_rsqrt_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + SIMDVector out; + out.value = _mm256_abs_ps(a.value); + return out; +} + + +#endif + + + + + + +// SSE VERSION +//-------------------------------------------------------------------------------------------------- + +#ifdef FASTOR_SSE2_IMPL + +template <> +struct SIMDVector { + using value_type = __m128; + using scalar_value_type = float; + using abi_type = simd_abi::sse; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm_setzero_ps()) {} + FASTOR_INLINE SIMDVector(float num) : value(_mm_set1_ps(num)) {} + FASTOR_INLINE SIMDVector(__m128 regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_ps(data); + else + value = _mm_loadu_ps(data); + } + + FASTOR_INLINE SIMDVector operator=(float num) { + value = _mm_set1_ps(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m128 regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const float *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_ps(data); + else + value = _mm_loadu_ps(data); + } + FASTOR_INLINE void store(float *data, bool Aligned=true) const { + if (Aligned) + _mm_store_ps(data,value); + else + _mm_storeu_ps(data,value); + } + + FASTOR_INLINE void aligned_load(const float *data) { + value =_mm_load_ps(data); + } + FASTOR_INLINE void aligned_store(float *data) const { + _mm_store_ps(data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm_mask_loadu_ps(value, mask, a); + else + value = _mm_mask_load_ps(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm_setzero_ps(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(float num) { + value = _mm_set1_ps(num); + } + FASTOR_INLINE void set(float num0, float num1, float num2, float num3) { + value = _mm_set_ps(num0,num1,num2,num3); + } + FASTOR_INLINE void set_sequential(float num0) { + value = _mm_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f); + } + FASTOR_INLINE void broadcast(const float *data) { + value = _mm_load1_ps(data); + } + + // In-place operators + FASTOR_INLINE void operator+=(float num) { + value = _mm_add_ps(value,_mm_set1_ps(num)); + } + FASTOR_INLINE void operator+=(__m128 regi) { + value = _mm_add_ps(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm_add_ps(value,a.value); + } + + FASTOR_INLINE void operator-=(float num) { + value = _mm_sub_ps(value,_mm_set1_ps(num)); + } + FASTOR_INLINE void operator-=(__m128 regi) { + value = _mm_sub_ps(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm_sub_ps(value,a.value); + } + + FASTOR_INLINE void operator*=(float num) { + value = _mm_mul_ps(value,_mm_set1_ps(num)); + } + FASTOR_INLINE void operator*=(__m128 regi) { + value = _mm_mul_ps(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm_mul_ps(value,a.value); + } + + FASTOR_INLINE void operator/=(float num) { + value = _mm_div_ps(value,_mm_set1_ps(num)); + } + FASTOR_INLINE void operator/=(__m128 regi) { + value = _mm_div_ps(value,regi); + } + FASTOR_INLINE void operator/=(const SIMDVector &a) { + value = _mm_div_ps(value,a.value); + } + // end of in-place operators + + FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { + SIMDVector out; + if (i==1) + out.value = _mm_shift1_ps(value); + else if (i==2) + out.value = _mm_shift2_ps(value); + else if (i==3) + out.value = _mm_shift3_ps(value); + return out; + } + FASTOR_INLINE float sum() {return _mm_sum_ps(value);} + FASTOR_INLINE float product() {return _mm_prod_ps(value);} + FASTOR_INLINE SIMDVector reverse() { + SIMDVector out; + out.value = _mm_reverse_ps(value); + return out; + } + FASTOR_INLINE float minimum() {return _mm_hmin_ps(value);} + FASTOR_INLINE float maximum() {return _mm_hmax_ps(value);} + + FASTOR_INLINE float dot(const SIMDVector &other) { +#ifdef FASTOR_SSE4_1_IMPL + return _mm_cvtss_f32(_mm_dp_ps(value,other.value,0xff)); +#else + return _mm_sum_ps(_mm_mul_ps(value,other.value)); +#endif + } + + __m128 value; +}; + + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + // ICC crashes without a copy + const __m128 v = a.value; + const float* value = reinterpret_cast(&v); + os << "[" << value[0] << " " << value[1] << " " + << value[2] << " " << value[3] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_add_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm_add_ps(a.value,_mm_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_add_ps(_mm_set1_ps(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_sub_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm_sub_ps(a.value,_mm_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_sub_ps(_mm_set1_ps(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + out.value = _mm_neg_ps(b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_mul_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm_mul_ps(a.value,_mm_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_mul_ps(_mm_set1_ps(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_div_ps(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, float b) { + SIMDVector out; + out.value = _mm_div_ps(a.value,_mm_set1_ps(b)); + return out; +} +FASTOR_INLINE SIMDVector operator/(float a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_div_ps(_mm_set1_ps(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + SIMDVector out; + out.value = _mm_rcp_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm_sqrt_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + SIMDVector out; + out.value = _mm_rsqrt_ps(a.value); + return out; +} + +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + SIMDVector out; + out.value = _mm_abs_ps(a.value); + return out; +} + + +#endif + + +} // end of namespace Fastor + + +#endif // SIMD_VECTOR_FLOAT_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_int32.h b/noarch/include/Fastor/simd_vector/simd_vector_int32.h new file mode 100644 index 00000000..0521defa --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_int32.h @@ -0,0 +1,988 @@ +#ifndef SIMD_VECTOR_INT_H +#define SIMD_VECTOR_INT_H + +#include "Fastor/simd_vector/simd_vector_base.h" +#include + +namespace Fastor { + + +// AVX512 VERSION +//----------------------------------------------------------------------------------------------- + +#ifdef FASTOR_AVX512F_IMPL + +template<> +struct SIMDVector { + using value_type = __m512i; + using scalar_value_type = int32_t; + using abi_type = simd_abi::avx512; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm512_setzero_si512()) {} + FASTOR_INLINE SIMDVector(int32_t num) : value(_mm512_set1_epi32(num)) {} + FASTOR_INLINE SIMDVector(__m512i regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_si512((__m512i*)data); + else + value = _mm512_loadu_si512((__m512i*)data); + } + + FASTOR_INLINE SIMDVector operator=(int32_t num) { + value = _mm512_set1_epi32(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m512i regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_si512((__m512i*)data); + else + value = _mm512_loadu_si512((__m512i*)data); + } + FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const { + if (Aligned) + _mm512_store_si512((__m512i*)data,value); + else + _mm512_storeu_si512((__m512i*)data,value); + } + + FASTOR_INLINE void aligned_load(const int32_t *data) { + value =_mm512_load_si512((__m512i*)data); + } + FASTOR_INLINE void aligned_store(int32_t *data) const { + _mm512_store_si512((__m512i*)data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm512_mask_loadu_epi32(value, mask, a); + else + value = _mm512_mask_load_epi32(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm512_setzero_si512(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(int32_t num) { + value = _mm512_set1_epi32(num); + } + FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7, + int32_t num8, int32_t num9, int32_t num10, int32_t num11, int32_t num12, int32_t num13, int32_t num14, int32_t num15) { + value = _mm512_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15); + } + FASTOR_INLINE void set_sequential(int32_t num0) { + value = _mm512_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7, + num0+8,num0+9,num0+10,num0+11,num0+12,num0+13,num0+14,num0+15); + } + + // In-place operators + FASTOR_INLINE void operator+=(int32_t num) { + value = _mm512_add_epi32(value,_mm512_set1_epi32(num)); + + } + FASTOR_INLINE void operator+=(__m512i regi) { + value = _mm512_add_epi32(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm512_add_epi32(value,a.value); + } + + FASTOR_INLINE void operator-=(int32_t num) { + value = _mm512_sub_epi32(value,_mm512_set1_epi32(num)); + } + FASTOR_INLINE void operator-=(__m512i regi) { + value = _mm512_sub_epi32(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm512_sub_epi32(value,a.value); + } + + FASTOR_INLINE void operator*=(int32_t num) { + value = _mm512_mullo_epi32(value,_mm512_set1_epi32(num)); + } + FASTOR_INLINE void operator*=(__m512i regi) { + value = _mm512_mullo_epi32(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm512_mullo_epi32(value,a.value); + } + + FASTOR_INLINE void operator/=(int32_t num) { +#ifdef FASTOR_INTEL + value = _mm512_div_epi32(value,_mm512_set1_epi32(num)); +#else + int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value); + for (FASTOR_INDEX i=0; i &a) { +#ifdef FASTOR_INTEL + value = _mm512_div_epi32(value,a.value); +#else + int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value); + int32_t val_a[Size]; _mm512_storeu_si512((__m512i*)val_a, a.value); + for (FASTOR_INDEX i=0; iquan) + quan = vals[i]; + return quan; + } + FASTOR_INLINE SIMDVector reverse() { + return _mm512_reverse_epi32(value); + } + + FASTOR_INLINE int32_t sum() { +#ifdef FASTOR_HAS_AVX512_REDUCE_ADD + return _mm512_reduce_add_epi32(value); +#else + int32_t vals[Size]; _mm512_storeu_si512((__m512i*)vals, value); + int32_t quan = 0; + for (FASTOR_INDEX i=0; i &other) { +#ifdef FASTOR_HAS_AVX512_REDUCE_ADD + return _mm512_reduce_add_epi32(_mm512_mullo_epi32(value,other.value)); +#else + return SIMDVector(_mm512_mullo_epi32(value,other.value)).sum(); +#endif + } + + __m512i value; +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + const int32_t *value = (int32_t*) &a.value; + os << "[" + << value[0] << " " << value[1] << " " + << value[2] << " " << value[3] << " " + << value[4] << " " << value[5] << " " + << value[6] << " " << value[7] << " " + << value[8] << " " << value[9] << " " + << value[10] << " " << value[11] << " " + << value[12] << " " << value[13] << " " + << value[14] << " " << value[15] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_epi32(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm512_add_epi32(a.value,_mm512_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_epi32(_mm512_set1_epi32(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_epi32(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm512_sub_epi32(a.value,_mm512_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_epi32(_mm512_set1_epi32(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + return _mm512_castps_si512(_mm512_neg_ps(_mm512_castsi512_ps(b.value))); +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_mullo_epi32(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm512_mullo_epi32(a.value,_mm512_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_mullo_epi32(_mm512_set1_epi32(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; +#ifdef FASTOR_INTEL + out.value = _mm512_div_epi32(a.value,b.value); +#else + int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); + int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); + int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); + for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int32_t b) { + SIMDVector out; +#ifdef FASTOR_INTEL + out.value = _mm512_div_epi32(a.value,_mm512_set1_epi32(b)); +#else + int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); + int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); + for (FASTOR_INDEX i=0; i operator/(int32_t a, const SIMDVector &b) { + SIMDVector out; +#ifdef FASTOR_INTEL + out.value = _mm512_div_epi32(_mm512_set1_epi32(a),b.value); +#else + int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); + int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); + for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { + SIMDVector out; +#ifdef FASTOR_HAS_AVX512_ABS + out.value = _mm512_abs_epi32(a.value); +#else + for (FASTOR_INDEX i=0UL; i<16UL; ++i) { + ((int32_t*)&out.value)[i] = std::abs(((int32_t*)&a.value)[i]); + } +#endif + return out; +} + + +#endif + + +// AVX VERSION +//----------------------------------------------------------------------------------------------- + +#ifdef FASTOR_AVX2_IMPL + +template<> +struct SIMDVector { + using value_type = __m256i; + using scalar_value_type = int32_t; + using abi_type = simd_abi::avx; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm256_setzero_si256()) {} + FASTOR_INLINE SIMDVector(int32_t num) : value(_mm256_set1_epi32(num)) {} + FASTOR_INLINE SIMDVector(__m256i regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_si256((__m256i*)data); + else + value = _mm256_loadu_si256((__m256i*)data); + } + + FASTOR_INLINE SIMDVector operator=(int32_t num) { + value = _mm256_set1_epi32(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m256i regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_si256((__m256i*)data); + else + value = _mm256_loadu_si256((__m256i*)data); + } + FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const { + if (Aligned) + _mm256_store_si256((__m256i*)data,value); + else + _mm256_storeu_si256((__m256i*)data,value); + } + + FASTOR_INLINE void aligned_load(const int32_t *data) { + value =_mm256_load_si256((__m256i*)data); + } + FASTOR_INLINE void aligned_store(int32_t *data) const { + _mm256_store_si256((__m256i*)data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm256_mask_loadu_epi32(value, mask, a); + else + value = _mm256_mask_load_epi32(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm256_setzero_si256(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(int32_t num) { + value = _mm256_set1_epi32(num); + } + FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7) { + value = _mm256_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7); + } + FASTOR_INLINE void set_sequential(int32_t num0) { + value = _mm256_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7); + } + + // In-place operators + FASTOR_INLINE void operator+=(int32_t num) { + value = _mm256_add_epi32x(value,_mm256_set1_epi32(num)); + + } + FASTOR_INLINE void operator+=(__m256i regi) { + value = _mm256_add_epi32x(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm256_add_epi32x(value,a.value); + } + + FASTOR_INLINE void operator-=(int32_t num) { + value = _mm256_sub_epi32x(value,_mm256_set1_epi32(num)); + } + FASTOR_INLINE void operator-=(__m256i regi) { + value = _mm256_sub_epi32x(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm256_sub_epi32x(value,a.value); + } + + FASTOR_INLINE void operator*=(int32_t num) { + value = _mm256_mul_epi32x(value,_mm256_set1_epi32(num)); + } + FASTOR_INLINE void operator*=(__m256i regi) { + value = _mm256_mul_epi32x(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm256_mul_epi32x(value,a.value); + } + + FASTOR_INLINE void operator/=(int32_t num) { + int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); + for (FASTOR_INDEX i=0; i &a) { + int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); + int32_t val_a[Size]; _mm256_storeu_si256((__m256i*)val_a, a.value); + for (FASTOR_INDEX i=0; iquan) + quan = vals[i]; + return quan; + } + FASTOR_INLINE SIMDVector reverse() { + SIMDVector out; + out.value = _mm256_reverse_epi32(value); + return out; + } + + FASTOR_INLINE int32_t sum() { + int32_t vals[Size]; _mm256_storeu_si256((__m256i*)vals, value); + int32_t quan = 0; + for (FASTOR_INDEX i=0; i &other) { + int32_t vals0[Size]; _mm256_storeu_si256((__m256i*)vals0, value); + int32_t vals1[Size]; _mm256_storeu_si256((__m256i*)vals1, other.value); + int32_t quan = 0; + for (FASTOR_INDEX i=0; i a) { + const int32_t *value = (int32_t*) &a.value; + os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] + << " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_epi32x(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm256_add_epi32x(a.value,_mm256_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_epi32x(_mm256_set1_epi32(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_epi32x(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm256_sub_epi32x(a.value,_mm256_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_epi32x(_mm256_set1_epi32(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + return _mm256_castps_si256(_mm256_neg_ps(_mm256_castsi256_ps(b.value))); +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_epi32x(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm256_mul_epi32x(a.value,_mm256_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_epi32x(_mm256_set1_epi32(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); + int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); + int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); + for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int32_t b) { + SIMDVector out; + int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); + int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); + for (FASTOR_INDEX i=0; i operator/(int32_t a, const SIMDVector &b) { + SIMDVector out; + int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); + int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); + for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { + SIMDVector out; +#ifdef __AVX2__ + out.value = _mm256_abs_epi32(a.value); +#else + // THIS IS ALSO AVX2 VERSION! + // __m128i lo = _mm_abs_epi32(_mm256_castsi256_si128(a.value)); + // __m128i hi = _mm_abs_epi32(_mm256_extracti128_si256(a.value,0x1)); + // out.value = _mm256_castsi128_si256(lo); + // out.value = _mm256_insertf128_si256(out.value,hi,0x1); + + int32_t *value = (int32_t*) &a.value; + for (int32_t i=0; i<8; ++i) { + value[i] = std::abs(value[i]); + } +#endif + return out; +} + + +#endif + + +// SSE VERSION +//----------------------------------------------------------------------------------------------- + +#ifdef FASTOR_SSE2_IMPL + +template<> +struct SIMDVector { + using value_type = __m128i; + using scalar_value_type = int32_t; + using abi_type = simd_abi::sse; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm_setzero_si128()) {} + FASTOR_INLINE SIMDVector(int32_t num) : value(_mm_set1_epi32(num)) {} + FASTOR_INLINE SIMDVector(__m128i regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_si128((__m128i*)data); + else + value = _mm_loadu_si128((__m128i*)data); + } + + FASTOR_INLINE SIMDVector operator=(int32_t num) { + value = _mm_set1_epi32(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m128i regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_si128((__m128i*)data); + else + value = _mm_loadu_si128((__m128i*)data); + } + FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const { + if (Aligned) + _mm_store_si128((__m128i*)data,value); + else + _mm_storeu_si128((__m128i*)data,value); + } + + FASTOR_INLINE void aligned_load(const int32_t *data) { + value =_mm_load_si128((__m128i*)data); + } + FASTOR_INLINE void aligned_store(int32_t *data) const { + _mm_store_si128((__m128i*)data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm_mask_loadu_epi32(value, mask, a); + else + value = _mm_mask_load_epi32(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm_setzero_si128(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(int32_t num) { + value = _mm_set1_epi32(num); + } + FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3) { + value = _mm_set_epi32(num0,num1,num2,num3); + } + FASTOR_INLINE void set_sequential(int32_t num0) { + value = _mm_setr_epi32(num0,num0+1,num0+2,num0+3); + } + + // In-place operators + FASTOR_INLINE void operator+=(int32_t num) { + value = _mm_add_epi32(value,_mm_set1_epi32(num)); + + } + FASTOR_INLINE void operator+=(__m128i regi) { + value = _mm_add_epi32(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm_add_epi32(value,a.value); + } + + FASTOR_INLINE void operator-=(int32_t num) { + value = _mm_sub_epi32(value,_mm_set1_epi32(num)); + } + FASTOR_INLINE void operator-=(__m128i regi) { + value = _mm_sub_epi32(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm_sub_epi32(value,a.value); + } + + FASTOR_INLINE void operator*=(int32_t num) { + value = _mm_mul_epi32x(value,_mm_set1_epi32(num)); + } + FASTOR_INLINE void operator*=(__m128i regi) { + value = _mm_mul_epi32x(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm_mul_epi32x(value,a.value); + } + + FASTOR_INLINE void operator/=(int32_t num) { + int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value); + for (FASTOR_INDEX i=0; i &a) { + int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value); + int32_t val_a[Size]; _mm_storeu_si128((__m128i*)val_a, a.value); + for (FASTOR_INDEX i=0; iquan) + quan = vals[i]; + return quan; + } + FASTOR_INLINE SIMDVector reverse() { + return _mm_reverse_epi32(value); + } + + FASTOR_INLINE int32_t sum() {return _mm_sum_epi32(value);} + FASTOR_INLINE int32_t product() {return _mm_prod_epi32(value);} + + FASTOR_INLINE int32_t dot(const SIMDVector &other) { + return _mm_sum_epi32(_mm_mul_epi32x(value,other.value)); + } + + __m128i value; +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + const int32_t *value = (int32_t*) &a.value; + os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_add_epi32(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm_add_epi32(a.value,_mm_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_add_epi32(_mm_set1_epi32(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_sub_epi32(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm_sub_epi32(a.value,_mm_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_sub_epi32(_mm_set1_epi32(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + return _mm_castps_si128(_mm_neg_ps(_mm_castsi128_ps(b.value))); +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_mul_epi32x(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int32_t b) { + SIMDVector out; + out.value = _mm_mul_epi32x(a.value,_mm_set1_epi32(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(int32_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_mul_epi32x(_mm_set1_epi32(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); + int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); + int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); + for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int32_t b) { + SIMDVector out; + int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); + int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); + for (FASTOR_INDEX i=0; i operator/(int32_t a, const SIMDVector &b) { + SIMDVector out; + int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); + int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); + for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { + SIMDVector out; +#ifdef FASTOR_SSSE3_IMPL + out.value = _mm_abs_epi32(a.value); +#else // SSE2 + __m128i sign = _mm_srai_epi32(a.value, 31); + __m128i inv = _mm_xor_si128(a.value, sign); + out.value = _mm_sub_epi32(inv, sign); +#endif + return out; +} + + +#endif + + +} // end of namespace Fastor + + +#endif // SIMD_VECTOR_INT_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_int64.h b/noarch/include/Fastor/simd_vector/simd_vector_int64.h new file mode 100644 index 00000000..f76e9468 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_int64.h @@ -0,0 +1,1040 @@ +#ifndef SIMD_VECTOR_INT64_H +#define SIMD_VECTOR_INT64_H + +#include "Fastor/simd_vector/simd_vector_base.h" +#include + +namespace Fastor { + + +// AVX512 VERSION +//----------------------------------------------------------------------------------------------- + +#ifdef FASTOR_AVX512F_IMPL + +template<> +struct SIMDVector { + using value_type = __m512i; + using scalar_value_type = int64_t; + using abi_type = simd_abi::avx512; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm512_setzero_si512()) {} + FASTOR_INLINE SIMDVector(int64_t num) : value(_mm512_set1_epi64(num)) {} + FASTOR_INLINE SIMDVector(__m512i regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const int64_t *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_si512((__m512i*)data); + else + value = _mm512_loadu_si512((__m512i*)data); + } + + FASTOR_INLINE SIMDVector operator=(int64_t num) { + value = _mm512_set1_epi64(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m512i regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const int64_t *data, bool Aligned=true) { + if (Aligned) + value =_mm512_load_si512((__m512i*)data); + else + value = _mm512_loadu_si512((__m512i*)data); + } + FASTOR_INLINE void store(int64_t *data, bool Aligned=true) const { + if (Aligned) + _mm512_store_si512((__m512i*)data,value); + else + _mm512_storeu_si512((__m512i*)data,value); + } + + FASTOR_INLINE void aligned_load(const int64_t *data) { + value =_mm512_load_si512((__m512i*)data); + } + FASTOR_INLINE void aligned_store(int64_t *data) const { + _mm512_store_si512((__m512i*)data,value); + } + + FASTOR_INLINE int64_t operator[](FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + FASTOR_INLINE int64_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm512_mask_loadu_epi64(value, mask, a); + else + value = _mm512_mask_load_epi64(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm512_setzero_si512(); + for (FASTOR_INDEX i=0; i &a) { + value = _mm512_add_epi64(value,a.value); + } + + FASTOR_INLINE void operator-=(int64_t num) { + value = _mm512_sub_epi64(value,_mm512_set1_epi64(num)); + } + FASTOR_INLINE void operator-=(__m512i regi) { + value = _mm512_sub_epi64(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm512_sub_epi64(value,a.value); + } + + FASTOR_INLINE void operator*=(int64_t num) { +#ifdef FASTOR_AVX512DQ_IMPL + value = _mm512_mullo_epi64(value,_mm512_set1_epi64(num)); +#else + for (FASTOR_INDEX i=0; i &a) { +#ifdef FASTOR_AVX512DQ_IMPL + value = _mm512_mullo_epi64(value,a.value); +#else + for (FASTOR_INDEX i=0; i &a) { +#ifdef FASTOR_INTEL + value = _mm512_div_epi64(value,a.value); +#else + int64_t val[Size]; _mm512_storeu_si512((__m512i*)val, value); + int64_t val_a[Size]; _mm512_storeu_si512((__m512i*)val_a, a.value); + for (FASTOR_INDEX i=0; i(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; iquan) + quan = vals[i]; + return quan; + } + FASTOR_INLINE SIMDVector reverse() { + return _mm512_reverse_epi64(value); + } + + FASTOR_INLINE int64_t sum() { +#ifdef FASTOR_HAS_AVX512_REDUCE_ADD + return _mm512_reduce_add_epi64(value); +#else + const int64_t *vals = reinterpret_cast(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i(&value); + int64_t quan = 1; + for (FASTOR_INDEX i=0; i &other) { + return (*this * other).sum(); + } + + __m512i value; +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + const int64_t *value = reinterpret_cast(&a.value); + os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] + << " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_epi64(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int64_t b) { + SIMDVector out; + out.value = _mm512_add_epi64(a.value,_mm512_set1_epi64(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(int64_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_add_epi64(_mm512_set1_epi64(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_epi64(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int64_t b) { + SIMDVector out; + out.value = _mm512_sub_epi64(a.value,_mm512_set1_epi64(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(int64_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm512_sub_epi64(_mm512_set1_epi64(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + return _mm512_castps_si512(_mm512_neg_ps(_mm512_castsi512_ps(b.value))); +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; +#ifdef FASTOR_AVX512DQ_IMPL + out.value = _mm512_mullo_epi64(a.value,b.value); +#else + for (FASTOR_INDEX i=0; i operator*(const SIMDVector &a, int64_t b) { + SIMDVector out; +#ifdef FASTOR_AVX512DQ_IMPL + out.value = _mm512_mullo_epi64(a.value,_mm512_set1_epi64(b)); +#else + for (FASTOR_INDEX i=0; i operator*(int64_t a, const SIMDVector &b) { + SIMDVector out; +#ifdef FASTOR_AVX512DQ_IMPL + out.value = _mm512_mullo_epi64(_mm512_set1_epi64(a),b.value); +#else + for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; +#ifdef FASTOR_INTEL + out.value = _mm512_div_epi64(a.value,b.value); +#else + int64_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); + int64_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); + int64_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); + for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int64_t b) { + SIMDVector out; +#ifdef FASTOR_INTEL + out.value = _mm512_div_epi64(a.value,_mm512_set1_epi64(b)); +#else + int64_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); + int64_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); + for (FASTOR_INDEX i=0; i operator/(int64_t a, const SIMDVector &b) { + SIMDVector out; +#ifdef FASTOR_INTEL + out.value = _mm512_div_epi64(_mm512_set1_epi64(a),b.value); +#else + int64_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); + int64_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); + for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { + SIMDVector out; +#ifdef FASTOR_HAS_AVX512_ABS + out.value = _mm512_abs_epi64(a.value); +#else + for (FASTOR_INDEX i=0UL; i<8UL; ++i) { + ((int64_t*)&out.value)[i] = std::abs(((int64_t*)&a.value)[i]); + } +#endif + return out; +} + +#endif + + + + +// AVX VERSION +//----------------------------------------------------------------------------------------------- + +#ifdef FASTOR_AVX2_IMPL + +template<> +struct SIMDVector { + using value_type = __m256i; + using scalar_value_type = int64_t; + using abi_type = simd_abi::avx; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm256_setzero_si256()) {} + FASTOR_INLINE SIMDVector(int64_t num) : value(_mm256_set1_epi64x(num)) {} + FASTOR_INLINE SIMDVector(__m256i regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const int64_t *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_si256((__m256i*)data); + else + value = _mm256_loadu_si256((__m256i*)data); + } + + FASTOR_INLINE SIMDVector operator=(int64_t num) { + value = _mm256_set1_epi64x(num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m256i regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const int64_t *data, bool Aligned=true) { + if (Aligned) + value =_mm256_load_si256((__m256i*)data); + else + value = _mm256_loadu_si256((__m256i*)data); + } + FASTOR_INLINE void store(int64_t *data, bool Aligned=true) const { + if (Aligned) + _mm256_store_si256((__m256i*)data,value); + else + _mm256_storeu_si256((__m256i*)data,value); + } + + FASTOR_INLINE void aligned_load(const int64_t *data) { + value =_mm256_load_si256((__m256i*)data); + } + FASTOR_INLINE void aligned_store(int64_t *data) const { + _mm256_store_si256((__m256i*)data,value); + } + + FASTOR_INLINE int64_t operator[](FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + FASTOR_INLINE int64_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm256_mask_loadu_epi64(value, mask, a); + else + value = _mm256_mask_load_epi64(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm256_setzero_si256(); + for (FASTOR_INDEX i=0; i &a) { + value = _mm256_add_epi64x(value,a.value); + } + + FASTOR_INLINE void operator-=(int64_t num) { + value = _mm256_sub_epi64x(value,_mm256_set1_epi64x(num)); + } + FASTOR_INLINE void operator-=(__m256i regi) { + value = _mm256_sub_epi64x(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm256_sub_epi64x(value,a.value); + } + + FASTOR_INLINE void operator*=(int64_t num) { + value = _mm256_mul_epi64x(value,_mm256_set1_epi64x(num)); + } + FASTOR_INLINE void operator*=(__m256i regi) { + value = _mm256_mul_epi64x(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm256_mul_epi64x(value,a.value); + } + + FASTOR_INLINE void operator/=(int64_t num) { + int64_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); + for (FASTOR_INDEX i=0; i &a) { + int64_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); + int64_t val_a[Size]; _mm256_storeu_si256((__m256i*)val_a, a.value); + for (FASTOR_INDEX i=0; i(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; iquan) + quan = vals[i]; + return quan; + } + FASTOR_INLINE SIMDVector reverse() { + // Reversing a 64 bit vector seems really expensive regardless + // of which of the following methods being used + // SIMDVector out(_mm256_set_epi64x(value[0],value[1],value[2],value[3])); + // SIMDVector out; out.set(value[3],value[2],value[1],value[0]); + // return out; + SIMDVector out; + out.value = _mm256_reverse_epi64(value); + return out; + } + + FASTOR_INLINE int64_t sum() { + const int64_t *vals = reinterpret_cast(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i(&value); + int64_t quan = 1; + for (FASTOR_INDEX i=0; i &other) { + const int64_t *vals0 = reinterpret_cast(&value); + const int64_t *vals1 = reinterpret_cast(&other.value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i a) { + const int64_t *value = reinterpret_cast(&a.value); + os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_epi64x(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int64_t b) { + SIMDVector out; + out.value = _mm256_add_epi64x(a.value,_mm256_set1_epi64x(b)); + return out; +} +FASTOR_INLINE SIMDVector operator+(int64_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_add_epi64x(_mm256_set1_epi64x(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_epi64x(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int64_t b) { + SIMDVector out; + out.value = _mm256_sub_epi64x(a.value,_mm256_set1_epi64x(b)); + return out; +} +FASTOR_INLINE SIMDVector operator-(int64_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_sub_epi64x(_mm256_set1_epi64x(a),b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_castpd_si256(_mm256_neg_pd(_mm256_castsi256_pd(b.value))); + return out; +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_epi64x(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int64_t b) { + SIMDVector out; + out.value = _mm256_mul_epi64x(a.value,_mm256_set1_epi64x(b)); + return out; +} +FASTOR_INLINE SIMDVector operator*(int64_t a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm256_mul_epi64x(_mm256_set1_epi64x(a),b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + int64_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); + int64_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); + int64_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); + for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int64_t b) { + SIMDVector out; + int64_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); + int64_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); + for (FASTOR_INDEX i=0; i operator/(int64_t a, const SIMDVector &b) { + SIMDVector out; + int64_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); + int64_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); + for (FASTOR_INDEX i=0; i +struct SIMDVector { + using value_type = __m128i; + using scalar_value_type = int64_t; + using abi_type = simd_abi::sse; + static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} + + FASTOR_INLINE SIMDVector() : value(_mm_setzero_si128()) {} + FASTOR_INLINE SIMDVector(int64_t num) { + value = _mm_set_epi64x(num,num); + } + FASTOR_INLINE SIMDVector(__m128i regi) : value(regi) {} + FASTOR_INLINE SIMDVector(const int64_t *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_si128((__m128i*)data); + else + value = _mm_loadu_si128((__m128i*)data); + } + + FASTOR_INLINE SIMDVector operator=(int64_t num) { + value = _mm_set_epi64x(num,num); + return *this; + } + FASTOR_INLINE SIMDVector operator=(__m128i regi) { + value = regi; + return *this; + } + + FASTOR_INLINE void load(const int64_t *data, bool Aligned=true) { + if (Aligned) + value =_mm_load_si128((__m128i*)data); + else + value = _mm_loadu_si128((__m128i*)data); + } + FASTOR_INLINE void store(int64_t *data, bool Aligned=true) const { + if (Aligned) + _mm_store_si128((__m128i*)data,value); + else + _mm_storeu_si128((__m128i*)data,value); + } + + FASTOR_INLINE void aligned_load(const int64_t *data) { + value =_mm_load_si128((__m128i*)data); + } + FASTOR_INLINE void aligned_store(int64_t *data) const { + _mm_store_si128((__m128i*)data,value); + } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { +#ifdef FASTOR_HAS_AVX512_MASKS + if (!Aligned) + value = _mm_mask_loadu_epi64(value, mask, a); + else + value = _mm_mask_load_epi64(value, mask, a); +#else + // perhaps very inefficient but they never get used + int maska[Size]; + mask_to_array(mask,maska); + value = _mm_setzero_si128(); + for (FASTOR_INDEX i=0; i(&value)[i];} + FASTOR_INLINE int64_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} + + FASTOR_INLINE void set(int64_t num) { + value = _mm_set_epi64x(num,num); + } + FASTOR_INLINE void set(int64_t num0, int64_t num1) { + value = _mm_set_epi64x(num0,num1); + } + FASTOR_INLINE void set_sequential(int64_t num0) { + value = _mm_set_epi64x(num0+1,num0); + } + + // In-place operators + FASTOR_INLINE void operator+=(int64_t num) { + auto numb = _mm_set_epi64x(num,num); + value = _mm_add_epi64(value,numb); + } + FASTOR_INLINE void operator+=(__m128i regi) { + value = _mm_add_epi64(value,regi); + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value = _mm_add_epi64(value,a.value); + } + + FASTOR_INLINE void operator-=(int64_t num) { + auto numb = _mm_set_epi64x(num,num); + value = _mm_sub_epi64(value,numb); + } + FASTOR_INLINE void operator-=(__m128i regi) { + value = _mm_sub_epi64(value,regi); + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value = _mm_sub_epi64(value,a.value); + } + + FASTOR_INLINE void operator*=(int64_t num) { + auto numb = _mm_set_epi64x(num,num); + value = _mm_mul_epi64(value,numb); + } + FASTOR_INLINE void operator*=(__m128i regi) { + value = _mm_mul_epi64(value,regi); + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value = _mm_mul_epi64(value,a.value); + } + + FASTOR_INLINE void operator/=(int64_t num) { + int64_t val[Size]; _mm_storeu_si128((__m128i*)val, value); + for (FASTOR_INDEX i=0; i &a) { + int64_t val[Size]; _mm_storeu_si128((__m128i*)val, value); + int64_t val_a[Size]; _mm_storeu_si128((__m128i*)val_a, a.value); + for (FASTOR_INDEX i=0; i(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i(quan); + } + FASTOR_INLINE int64_t maximum() { + const int64_t *vals = reinterpret_cast(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; iquan) + quan = vals[i]; + return static_cast(quan); + } + FASTOR_INLINE SIMDVector reverse() { + return _mm_reverse_epi64(value); + } + + FASTOR_INLINE int64_t sum() { + const int64_t *vals = reinterpret_cast(&value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i<2; ++i) + quan += vals[i]; + return static_cast(quan); + } + FASTOR_INLINE int64_t product() { + const int64_t *vals = reinterpret_cast(&value); + int64_t quan = 1; + for (FASTOR_INDEX i=0; i &other) { + const int64_t *vals0 = reinterpret_cast(&value); + const int64_t *vals1 = reinterpret_cast(&other.value); + int64_t quan = 0; + for (FASTOR_INDEX i=0; i<2; ++i) + quan += vals0[i]*vals1[i]; + return static_cast(quan); + } + + __m128i value; +}; + +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + const int64_t *value = reinterpret_cast(&a.value); + os << "[" << value[0] << " " << value[1] << "]\n"; + return os; +} + +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_add_epi64(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int64_t b) { + SIMDVector out; + auto numb = _mm_set_epi64x(b,b); + out.value = _mm_add_epi64(a.value,numb); + return out; +} +FASTOR_INLINE SIMDVector operator+(int64_t a, const SIMDVector &b) { + SIMDVector out; + auto numb = _mm_set_epi64x(a,a); + out.value = _mm_add_epi64(numb,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_sub_epi64(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int64_t b) { + SIMDVector out; + auto numb = _mm_set_epi64x(b,b); + out.value = _mm_sub_epi64(a.value,numb); + return out; +} +FASTOR_INLINE SIMDVector operator-(int64_t a, const SIMDVector &b) { + SIMDVector out; + auto numb = _mm_set_epi64x(a,a); + out.value = _mm_sub_epi64(numb,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + out.value = _mm_castpd_si128(_mm_neg_pd(_mm_castsi128_pd(b.value))); + return out; +} + +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = _mm_mul_epi64(a.value,b.value); + return out; +} +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int64_t b) { + SIMDVector out; + auto numb = _mm_set_epi64x(b,b); + out.value = _mm_mul_epi64(a.value,numb); + return out; +} +FASTOR_INLINE SIMDVector operator*(int64_t a, const SIMDVector &b) { + SIMDVector out; + auto numb = _mm_set_epi64x(a,a); + out.value = _mm_mul_epi64(numb,b.value); + return out; +} + +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + int64_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); + int64_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); + int64_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); + for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int64_t b) { + SIMDVector out; + int64_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); + int64_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); + for (FASTOR_INDEX i=0; i operator/(int64_t a, const SIMDVector &b) { + SIMDVector out; + int64_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); + int64_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); + for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { + SIMDVector out; +#ifdef FASTOR_AVX512VL_IMPL + return _mm_abs_epi64(a.value); +#elif defined (FASTOR_SSE4_2_IMPL) + __m128i sign = _mm_cmpgt_epi64(_mm_setzero_si128(), a.value); // 0 > a + __m128i inv = _mm_xor_si128(a.value, sign); // invert bits if negative + return _mm_sub_epi64(inv, sign); // add 1 +#else // SSE2 + __m128i signh = _mm_srai_epi32(a.value, 31); // sign in high dword + __m128i sign = _mm_shuffle_epi32(signh, 0xF5); // copy sign to low dword + __m128i inv = _mm_xor_si128(a.value, sign); // invert bits if negative + return _mm_sub_epi64(inv, sign); // add 1 +#endif +} + + +#endif + + +} // end of namespace Fastor + + +#endif // SIMD_VECTOR_INT64_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_scalar.h b/noarch/include/Fastor/simd_vector/simd_vector_scalar.h new file mode 100644 index 00000000..d3861072 --- /dev/null +++ b/noarch/include/Fastor/simd_vector/simd_vector_scalar.h @@ -0,0 +1,216 @@ +#ifndef SIMD_VECTOR_T_SCALAR_H +#define SIMD_VECTOR_T_SCALAR_H + +#include "Fastor/simd_vector/simd_vector_base.h" + +namespace Fastor { + +template +struct SIMDVector { + using value_type = T; + using scalar_value_type = T; + using abi_type = simd_abi::scalar; + static constexpr FASTOR_INDEX Size = 1; + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;} + + FASTOR_INLINE SIMDVector() : value(0) {} + FASTOR_INLINE SIMDVector(T num) : value(num) {} + FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) : value(*data) {} + + FASTOR_INLINE SIMDVector operator=(T num) { + value = num; + return *this; + } + + FASTOR_INLINE void load(const T *data, bool Aligned=true) { value = *data; unused(Aligned); } + FASTOR_INLINE void store(T *data, bool Aligned=true) const { data[0] = value; unused(Aligned); } + + FASTOR_INLINE void aligned_load(const T *data) { value = *data; } + FASTOR_INLINE void aligned_store(T *data) const { data[0] = value; } + + FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool ) { + if (mask != 0x0) value = *a; + } + FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool) const { + if (mask != 0x0) a[0] = value; + } + + FASTOR_INLINE T operator[](FASTOR_INDEX) const {return value;} + FASTOR_INLINE T operator()(FASTOR_INDEX) const {return value;} + + FASTOR_INLINE void set(T num) { + value = num; + } + + FASTOR_INLINE void set_sequential(T num) { + value = num; + } + + FASTOR_INLINE void broadcast(const T *data) { + value = *data; + } + + // In-place operators + FASTOR_INLINE void operator+=(T num) { + value += num; + } + FASTOR_INLINE void operator+=(const SIMDVector &a) { + value += a.value; + } + + FASTOR_INLINE void operator-=(T num) { + value -= num; + } + FASTOR_INLINE void operator-=(const SIMDVector &a) { + value -= a.value; + } + + FASTOR_INLINE void operator*=(T num) { + value *= num; + } + FASTOR_INLINE void operator*=(const SIMDVector &a) { + value *= a.value; + } + + FASTOR_INLINE void operator/=(T num) { + value /= num; + } + FASTOR_INLINE void operator/=(const SIMDVector &a) { + value /= a.value; + } + // end of in-place operators + + FASTOR_INLINE SIMDVector shift(FASTOR_INDEX) { + return *this; + } + FASTOR_INLINE T sum() {return value;} + FASTOR_INLINE T product() {return value;} + FASTOR_INLINE SIMDVector reverse() { + return *this; + } + FASTOR_INLINE T minimum() {return value;} + FASTOR_INLINE T maximum() {return value;} + + FASTOR_INLINE T dot(const SIMDVector &other) { + return value*other.value; + } + + T value; +}; + +template +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { + os << "[" << a.value << "]\n"; + return os; +} + +template +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = a.value+b.value; + return out; +} +template +FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, T b) { + SIMDVector out; + out.value = a.value+b; + return out; +} +template +FASTOR_INLINE SIMDVector operator+(T a, const SIMDVector &b) { + SIMDVector out; + out.value = a+b.value; + return out; +} +template +FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { + return b; +} + +template +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = a.value-b.value; + return out; +} +template +FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, T b) { + SIMDVector out; + out.value = a.value-b; + return out; +} +template +FASTOR_INLINE SIMDVector operator-(T a, const SIMDVector &b) { + SIMDVector out; + out.value = a-b.value; + return out; +} +template +FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { + SIMDVector out; + out.value = -b.value; + return out; +} + +template +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = a.value*b.value; + return out; +} +template +FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, T b) { + SIMDVector out; + out.value = a.value*b; + return out; +} +template +FASTOR_INLINE SIMDVector operator*(T a, const SIMDVector &b) { + SIMDVector out; + out.value = a*b.value; + return out; +} + +template +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { + SIMDVector out; + out.value = a.value/b.value; + return out; +} +template +FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, T b) { + SIMDVector out; + out.value = a.value/b; + return out; +} +template +FASTOR_INLINE SIMDVector operator/(T a, const SIMDVector &b) { + SIMDVector out; + out.value = a/b.value; + return out; +} + +template +FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { + return T(1) / a.value; +} + +template +FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { + return std::sqrt(a.value); +} + +template +FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { + return T(1) / std::sqrt(a.value); +} + +template +FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { + return std::abs(a.value); +} + +} // end of namespace Fastor + + +#endif // SIMD_VECTOR_T_SCALAR_H diff --git a/noarch/include/Fastor/tensor/AbstractTensor.h b/noarch/include/Fastor/tensor/AbstractTensor.h new file mode 100644 index 00000000..92100812 --- /dev/null +++ b/noarch/include/Fastor/tensor/AbstractTensor.h @@ -0,0 +1,33 @@ +#ifndef TENSORBASE_H +#define TENSORBASE_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/tensor_meta.h" + + +namespace Fastor { + +template +class Tensor; +template +class TensorMap; + + +template +class AbstractTensor { +public: + constexpr FASTOR_INLINE AbstractTensor() = default; + constexpr FASTOR_INLINE const Derived& self() const {return *static_cast(this);} + FASTOR_INLINE Derived& self() {return *static_cast(this);} + + static constexpr FASTOR_INDEX Dimension = Rank; +#ifndef FASTOR_DYNAMIC_MODE + static constexpr FASTOR_INDEX size() {return Derived::Size;} +#else + FASTOR_INDEX size() const {return (*static_cast(this)).size();} +#endif +}; + +} + +#endif // TENSORBASE_H diff --git a/noarch/include/Fastor/tensor/AbstractTensorFunctions.h b/noarch/include/Fastor/tensor/AbstractTensorFunctions.h new file mode 100644 index 00000000..d928cc41 --- /dev/null +++ b/noarch/include/Fastor/tensor/AbstractTensorFunctions.h @@ -0,0 +1,393 @@ +#ifndef ABSTRACT_TENSOR_FUNCTIONS_H +#define ABSTRACT_TENSOR_FUNCTIONS_H + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/TensorTraits.h" + +#include + +namespace Fastor { + +/* The implementation of the evaluate function that evaluates any expression in to a tensor +*/ +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE const Tensor& evaluate(const Tensor &src) { + return src; +} +template +FASTOR_INLINE typename Derived::result_type evaluate(const AbstractTensor &src) { + typename Derived::result_type out(src); + return out; +} +//----------------------------------------------------------------------------------------------------------// + + +/* IO for tensor expressions */ +//----------------------------------------------------------------------------------------------------------// +template +inline std::ostream& operator<<(std::ostream &os, const AbstractTensor &src) { + using result_type = typename Expr::result_type; + result_type tmp(src); + print(tmp); + return os; +} + +template +inline void print(const AbstractTensor &src) { + using result_type = typename Expr::result_type; + result_type tmp(src); + print(tmp); +} +//----------------------------------------------------------------------------------------------------------// + + +/* These are the set of functions that work on any expression that evaluate immediately +*/ + +/* Add all the elements of the tensor in a flattened sense +*/ +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type sum(const AbstractTensor &_src) { + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out(src); + return out.sum(); +} +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type sum(const AbstractTensor &_src) { + + const Derived &src = _src.self(); + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + FASTOR_INDEX i; + T _scal=0; V _vec(_scal); + for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { + _vec += src.template eval(i); + } + for (; i < src.size(); ++i) { + _scal += src.template eval_s(i); + } + return _vec.sum() + _scal; +} + +/* Multiply all the elements of the tensor in a flattened sense +*/ +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type product(const AbstractTensor &_src) { + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out(src); + return out.product(); +} +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type product(const AbstractTensor &_src) { + + const Derived &src = _src.self(); + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + FASTOR_INDEX i; + T _scal=1; V _vec(_scal); + for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { + _vec *= src.template eval(i); + } + for (; i < src.size(); ++i) { + _scal *= src.template eval_s(i); + } + return _vec.product() * _scal; +} + +/* Get minimum element of a tensor +*/ +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type min(const AbstractTensor &_src) { + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out(src); + return min(out); +} +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type min(const AbstractTensor &_src) { + + const Derived &src = _src.self(); + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + FASTOR_INDEX i; + T _scal=std::numeric_limits::max(); V _vec(_scal); + for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { + _vec = min(src.template eval(i),_vec); + } + for (; i < src.size(); ++i) { + _scal = std::min(src.template eval_s(i),_scal); + } + return std::min(_vec.minimum(), _scal); +} + +/* Get maximum element of a tensor +*/ +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type max(const AbstractTensor &_src) { + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out(src); + return max(out); +} +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type max(const AbstractTensor &_src) { + + const Derived &src = _src.self(); + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + FASTOR_INDEX i; + T _scal=std::numeric_limits::min(); V _vec(_scal); + for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { + _vec = max(src.template eval(i),_vec); + } + for (; i < src.size(); ++i) { + _scal = std::max(src.template eval_s(i),_scal); + } + return std::max(_vec.maximum(), _scal); +} + +/* Get the lower triangular matrix from a 2D expression +*/ +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type tril(const AbstractTensor &_src, int k = 0) { + static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIL"); + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out(src); + return tril(out,k); +} +template,bool> = false> +FASTOR_INLINE typename Derived::result_type tril(const AbstractTensor &_src, int k = 0) { + static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIL"); + const Derived &src = _src.self(); + using T = typename Derived::scalar_type; + typename Derived::result_type out(0); + + int M = int(src.dimension(0)); + int N = int(src.dimension(1)); + for (int i = 0; i < M; ++i) { + int jcount = k + i < N ? k + i : N - 1; + for (int j = 0; j <= jcount; ++j) { + out(i,j) = src.template eval_s(i,j); + } + } + return out; +} + +/* Get the upper triangular matrix from a 2D expression +*/ +template,bool> = false> +FASTOR_INLINE typename Derived::scalar_type triu(const AbstractTensor &_src, int k = 0) { + static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIU"); + const Derived &src = _src.self(); + using result_type = typename Derived::result_type; + const result_type out(src); + return triu(out,k); +} +template,bool> = false> +FASTOR_INLINE typename Derived::result_type triu(const AbstractTensor &_src, int k = 0) { + static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIU"); + const Derived &src = _src.self(); + using T = typename Derived::scalar_type; + typename Derived::result_type out(0); + + int M = int(src.dimension(0)); + int N = int(src.dimension(1)); + for (int i = 0; i < M; ++i) { + int jcount = k + i < 0 ? 0 : k + i; + for (int j = jcount; j < N; ++j) { + out(i,j) = src.template eval_s(i,j); + } + } + return out; +} +//----------------------------------------------------------------------------------------------------------// + + +// Boolean functions +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// +template && requires_evaluation_v,bool> = false> +FASTOR_INLINE bool all_of(const AbstractTensor &_src) { + using result_type = typename Derived::result_type; + const result_type tmp(_src.self()); + return all_of(tmp); +} +template && !requires_evaluation_v,bool> = false> +FASTOR_INLINE bool all_of(const AbstractTensor &_src) { + const Derived &src = _src.self(); + bool val = true; + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + if (src.template eval_s(i) == false) { + val = false; + break; + } + } + return val; +} + +template && requires_evaluation_v,bool> = false> +FASTOR_INLINE bool any_of(const AbstractTensor &_src) { + using result_type = typename Derived::result_type; + const result_type tmp(_src.self()); + return any_of(tmp); +} +template && !requires_evaluation_v,bool> = false> +FASTOR_INLINE bool any_of(const AbstractTensor &_src) { + const Derived &src = _src.self(); + bool val = false; + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + if (src.template eval_s(i) == true) { + val = true; + break; + } + } + return val; +} + +template && requires_evaluation_v,bool> = false> +FASTOR_INLINE bool none_of(const AbstractTensor &_src) { + using result_type = typename Derived::result_type; + const result_type tmp(_src.self()); + return none_of(tmp); +} +template && !requires_evaluation_v,bool> = false> +FASTOR_INLINE bool none_of(const AbstractTensor &_src) { + const Derived &src = _src.self(); + bool val = false; + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + if (src.template eval_s(i) == true) { + val = true; + break; + } + } + return val; +} + + +/* Is a second order tensor expression a uniform + A tensor expression is uniform if it spans equally in all dimensions, + i.e. generalisation of square matrix to N-dimensions +*/ +template +constexpr FASTOR_INLINE bool isuniform(const AbstractTensor &_src) { + return is_tensor_uniform_v; +} + +/* Is a second order tensor expression a square matrix +*/ +template = false> +constexpr FASTOR_INLINE bool issquare(const AbstractTensor &_src) { + return is_tensor_uniform_v; +} + +/* Is a second order tensor expression orthogonal +*/ +template = false> +FASTOR_INLINE bool isorthogonal(const AbstractTensor &_src, const double Tol=PRECI_TOL) { + typename Derived::result_type tmp1(_src.self()); + typename Derived::result_type tmp2(matmul(transpose(tmp1),tmp1)); + typename Derived::result_type I; I.eye2(); + return isequal(tmp2,I,Tol); +} + +/* Is a tensor expression symmetric - for higher order tensor two axes can defining a plane + provided to determine if a tensor expression is symmertric in that plane +*/ +template,bool> = false> +FASTOR_INLINE bool issymmetric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { + if (!issquare(_src.self())) return false; + return all_of( abs(evaluate(trans(_src.self()) - _src.self())) < Tol); +} +template,bool> = false> +FASTOR_INLINE bool issymmetric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { + if (!issquare(_src.self())) return false; + // Avoid copies + const Derived& src = _src.self(); + using T = typename Derived::scalar_type; + using result_type = typename Derived::result_type; + constexpr size_t M = get_tensor_dimension_v<0,result_type>; + constexpr size_t N = get_tensor_dimension_v<1,result_type>; + + bool _issym = true; + for (size_t i=0; i(i*N+j) - src.template eval_s(j*N+i) ) > Tol ) { + _issym = false; + break; + } + } + } + return _issym; +} + +/* Is a second order tensor expression deviatoric - a 2D tensor expression is deviatoric if it is + trace free +*/ +template = false> +constexpr FASTOR_INLINE bool isdeviatoric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { + return std::abs( trace(_src.self()) ) < Tol ? true : false; +} + +/* Is a second order tensor expression volumetric - a 2D tensor expression is volumetric if 1/3*[A:I]*I = A +*/ +template = false> +constexpr FASTOR_INLINE bool isvolumetric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { + typename Derived::result_type I; I.eye2(); + typename Derived::result_type tmp = trace(_src.self()) * I; + return all_of( abs(tmp - _src.self()) < Tol); +} + +/* A second order tensor expression belongs to the special linear group if +it's determinant is +1 +*/ +template = false> +FASTOR_INLINE bool doesbelongtoSL3(const AbstractTensor &_src, const double Tol=PRECI_TOL) { + // Expression must be square + if ( !issquare(_src.self()) ) return false; + return std::abs(determinant(_src.self()) - 1) < Tol ? true : false; +} + +/* A second order tensor/expression belongs to the special orthogonal group if +it is orthogonal and it's determinant is +1 +*/ +template = false> +FASTOR_INLINE bool doesbelongtoSO3(const AbstractTensor &_src, const double Tol=PRECI_TOL) { + // Expression must be square and orthogonal + return issquare(_src.self()) && isorthogonal(_src.self()) ? true : false; +} + +/* Are two tensor expressions approximately equal +*/ +template && !requires_evaluation_v,bool> = false> +FASTOR_INLINE bool isequal( + const AbstractTensor &_src0, + const AbstractTensor &_src1, + const double Tol=PRECI_TOL) { + if ( DIMS0 != DIMS1) return false; + if ( _src0.self().size() != _src1.self().size()) return false; + return all_of( abs(_src0.self() - _src1.self()) < Tol); +} +template || requires_evaluation_v,bool> = false> +FASTOR_INLINE bool isequal( + const AbstractTensor &_src0, + const AbstractTensor &_src1, + const double Tol=PRECI_TOL) { + if ( DIMS0 != DIMS1) return false; + if ( _src0.self().size() != _src1.self().size()) return false; + return all_of( abs(evaluate(_src0.self() - _src1.self())) < Tol); +} +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// + +} // end of namespace Fastor + + +#endif // #ifndef TENSOR_FUNCTIONS_H diff --git a/noarch/include/Fastor/tensor/Aliasing.h b/noarch/include/Fastor/tensor/Aliasing.h new file mode 100644 index 00000000..8b81887a --- /dev/null +++ b/noarch/include/Fastor/tensor/Aliasing.h @@ -0,0 +1,67 @@ +#ifndef ALIASING_H_ +#define ALIASING_H_ + +#include "Fastor/tensor/ForwardDeclare.h" +#include "Fastor/tensor/Tensor.h" + +namespace Fastor { + +// template +// FASTOR_INLINE bool does_alias(const Tensor &dst, const Tensor &src) { +// return dst.data() == src.data() ? true : false; +// } +template +FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const Tensor &src) { + return dst.self().data() == src.data() ? true : false; +} +// template +// FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const AbstractTensor &src) { +// return does_alias(dst.self(),src.self()); +// } + + +#define FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(NAME)\ +template\ +FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ + return does_alias(dst.self(),src.expr().self());\ +}\ + +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Add ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sub ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Abs ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sqrt) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Exp ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Log ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sin ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Cos ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Tan ) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Asin) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Acos) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Atan) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sinh) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Cosh) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Tanh) + + +// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Det ) +// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Norm) +// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Trace) +FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Trans) + + +#define FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(NAME)\ +template\ +FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ + return does_alias(dst.self(),src.lhs().self()) || does_alias(dst.self(),src.rhs().self());\ +}\ + +FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Add) +FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Sub) +FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Mul) +FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Div) +FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(MatMul) + +} + + +#endif // ALIASING_H_ diff --git a/noarch/include/Fastor/tensor/BlockIndexing.h b/noarch/include/Fastor/tensor/BlockIndexing.h new file mode 100644 index 00000000..5513858c --- /dev/null +++ b/noarch/include/Fastor/tensor/BlockIndexing.h @@ -0,0 +1,502 @@ +#ifndef BLOCK_INDEXING_H +#define BLOCK_INDEXING_H + + +//----------------------------------------------------------------------------------------------------------// +// Block indexing +//----------------------------------------------------------------------------------------------------------// +// Calls scalar indexing so they are fully bounds checked. +template +FASTOR_INLINE Tensor::value> operator()(const iseq& idx) { + + static_assert(1==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor::value> out; + FASTOR_INDEX counter = 0; + for (FASTOR_INDEX i=F; ioperator()(i); + counter++; + } + return out; +} + +template +FASTOR_INLINE Tensor::value,range_detector::value> + operator()(iseq, iseq) { + + static_assert(2==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + + Tensor::value,range_detector::value> out; + FASTOR_INDEX counter_i = 0; + for (FASTOR_INDEX i=F0; ioperator()(i,j); + counter_j++; + } + counter_i++; + } + + return out; +} + +template +FASTOR_INLINE Tensor::value,range_detector::value,range_detector::value> + operator()(iseq, iseq, iseq) const { + static_assert(3==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor::value, + range_detector::value, + range_detector::value> out; + FASTOR_INDEX counter_i = 0; + for (FASTOR_INDEX i=F0; ioperator()(i,j,k); + counter_k++; + } + counter_j++; + } + counter_i++; + } + return out; +} + +template +FASTOR_INLINE Tensor::value, + range_detector::value, + range_detector::value, + range_detector::value> + operator ()(iseq, iseq, + iseq, iseq) { + + static_assert(4==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor::value, + range_detector::value, + range_detector::value, + range_detector::value> out; + FASTOR_INDEX counter_i = 0; + for (FASTOR_INDEX i=F0; ioperator()(i,j,k,l); + counter_l++; + } + counter_k++; + } + counter_j++; + } + counter_i++; + } + return out; +} + +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// +FASTOR_INLINE TensorViewExpr,1> operator()(seq _s) { + static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,1>(*this,_s); +} + +FASTOR_INLINE TensorViewExpr,2> operator()(seq _s0, seq _s1) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,2>(*this,_s0,_s1); +} +template +FASTOR_INLINE TensorViewExpr,2> +operator()(fseq _s0, seq _s1) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,2>(*this,_s0,_s1); +} +template +FASTOR_INLINE TensorViewExpr,2> +operator()(seq _s0, fseq _s1) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,2>(*this,_s0,_s1); +} +template::value,bool>::type=0> +FASTOR_INLINE TensorViewExpr,2> operator()(seq _s0, Int num) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,2>(*this,_s0,seq(num)); +} +template::value,bool>::type=0> +FASTOR_INLINE TensorViewExpr,2> operator()(Int num, seq _s1) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,2>(*this,seq(num),_s1); +} + +template && !is_fixed_sequence_pack_v,bool> = false> +FASTOR_INLINE TensorViewExpr,sizeof...(Seq)> operator()(Seq ... _seqs) { + static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,sizeof...(Seq)>(*this, {_seqs...}); +} + +template,bool> = false> +FASTOR_INLINE TensorFixedViewExprnD,Fseq...> operator()(Fseq... ) { + static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorFixedViewExprnD,Fseq...>(*this); +} + +// if fseq == fall - then just return a reference to the tensor +template, + get_value<1,Rest...>::value>::type>::value == get_value<1,Rest...>::value, + bool>::type =0> +FASTOR_INLINE Tensor& +operator()(fseq) { + static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return (*this); +} +// if fseq != fall - return a view +template, + get_value<1,Rest...>::value>::type>::value != get_value<1,Rest...>::value, + bool>::type =0> +FASTOR_INLINE TensorFixedViewExpr1D, + typename to_positive,pack_prod::value>::type,1> +operator()(fseq) { + static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorFixedViewExpr1D, + typename to_positive,pack_prod::value>::type,1>(*this); +} + +// if fseq == fall - then just return a reference to the tensor +template,get_value<1,Rest...>::value>::type>::value == get_value<1,Rest...>::value && + internal::fseq_range_detector,get_value<2,Rest...>::value>::type>::value == get_value<2,Rest...>::value, + bool>::type =0> +FASTOR_INLINE Tensor& +operator()(fseq, fseq) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return (*this); +} +// if fseq != fall - return a view +template,get_value<1,Rest...>::value>::type>::value != get_value<1,Rest...>::value || + internal::fseq_range_detector,get_value<2,Rest...>::value>::type>::value != get_value<2,Rest...>::value, + bool>::type =0> +FASTOR_INLINE TensorFixedViewExpr2D, + typename to_positive,get_value<1,Rest...>::value>::type, + typename to_positive,get_value<2,Rest...>::value>::type,2> +operator()(fseq, fseq) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorFixedViewExpr2D, + typename to_positive,get_value<1,Rest...>::value>::type, + typename to_positive,get_value<2,Rest...>::value>::type,2>(*this); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorViewExpr,2> operator()(fseq _s, Int num) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,2>(*this,seq(_s),seq(num)); +} +template::value,bool>::type=0> +FASTOR_INLINE TensorViewExpr,2> operator()(Int num, fseq _s) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,2>(*this,seq(num),seq(_s)); +} + +// Selecting a row and returning a TensorMap - does not seem to speed up the code +//----------------------------------------------------------------------------------------------------------// +// template::value && +// internal::fseq_range_detector, +// get_value<2,Rest...>::value>::type>::value != get_value<2,Rest...>::value,bool>::type=0> +// FASTOR_INLINE TensorViewExpr,2> operator()(Int num, fseq _s) { +// static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); +// return TensorViewExpr,2>(*this,seq(num),seq(_s)); +// } +// // Selecting a row from a 2D tensor returns a TensorMap +// template::value && +// internal::fseq_range_detector, +// get_value<2,Rest...>::value>::type>::value == get_value<2,Rest...>::value,bool>::type=0> +// FASTOR_INLINE TensorMap::value> operator()(Int num, fseq _s) { +// static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); +// constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; +// return TensorMap(&_data[num*N]); +// } +//----------------------------------------------------------------------------------------------------------// + +template::value,bool>::type=0> +FASTOR_INLINE TensorRandomViewExpr,Tensor,1> operator()(const Tensor &_it) { + static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorRandomViewExpr,Tensor,1>(*this,_it); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorRandomViewExpr,Tensor,sizeof...(Rest)> +operator()(const Tensor &_it) { + static_assert(dimension_t::value==sizeof...(IterSizes),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorRandomViewExpr,Tensor,sizeof...(Rest)>(*this,_it); +} + +template::value && std::is_integral::value,bool>::type=0> +FASTOR_INLINE TensorRandomViewExpr,Tensor,2> +operator()(const Tensor &_it0, const Tensor &_it1) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor tmp_it; + constexpr int NCols = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); +} + +template::value && std::is_integral::value,bool>::type=0> +FASTOR_INLINE TensorRandomViewExpr,Tensor,2> +operator()(const Tensor &_it0, Int1 num) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor tmp_it; + constexpr int NCols = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); +} + +template::value && std::is_integral::value,bool>::type=0> +FASTOR_INLINE TensorRandomViewExpr,Tensor,2> +operator()(Int1 num, const Tensor &_it0) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor tmp_it; + constexpr int NCols = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorRandomViewExpr,Tensor,get_value<2,Rest...>::value>::type::Size>,2> +operator()(const Tensor &_it0, fseq) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + constexpr int NCols = get_value<2,Rest...>::value; + using _seq = typename to_positive,NCols>::type; + constexpr int ColSize = _seq::Size; + Tensor tmp_it; + for (FASTOR_INDEX i = 0; i,Tensor,get_value<2,Rest...>::value>::type::Size>,2> (*this,tmp_it); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorRandomViewExpr,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> +operator()(fseq, const Tensor &_it0) { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + constexpr int NRows = get_value<1,Rest...>::value; + constexpr int NCols = get_value<2,Rest...>::value; + using _seq = typename to_positive,NRows>::type; + constexpr int RowSize = _seq::Size; + Tensor tmp_it; + for (FASTOR_INDEX i = 0; i,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> (*this,tmp_it); +} + + + +//----------------------------------------------------------------------------------------------------------// +// Filter views +FASTOR_INLINE TensorFilterViewExpr,Tensor,sizeof...(Rest)> +operator()(const Tensor &_fl) { + return TensorFilterViewExpr,Tensor,sizeof...(Rest)>(*this,_fl); +} +FASTOR_INLINE TensorFilterViewExpr,TensorMap,sizeof...(Rest)> +operator()(const TensorMap &_fl) { + return TensorFilterViewExpr,TensorMap,sizeof...(Rest)>(*this,_fl); +} +//----------------------------------------------------------------------------------------------------------// + +FASTOR_INLINE TensorConstViewExpr,1> operator()(seq _s) const { + static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstViewExpr,1>(*this,_s); +} + +FASTOR_INLINE TensorConstViewExpr,2> operator()(seq _s0, seq _s1) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstViewExpr,2>(*this,_s0,_s1); +} +template::value,bool>::type=0> +FASTOR_INLINE TensorConstViewExpr,2> operator()(seq _s0, Int num) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstViewExpr,2>(*this,_s0,seq(num)); +} +template::value,bool>::type=0> +FASTOR_INLINE TensorConstViewExpr,2> operator()(Int num, seq _s1) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstViewExpr,2>(*this,seq(num),_s1); +} + +template && !is_fixed_sequence_pack_v,bool> = false> +FASTOR_INLINE TensorConstViewExpr,sizeof...(Seq)> operator()(Seq ... _seqs) const { + static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstViewExpr,sizeof...(Seq)>(*this, {_seqs...}); +} + +template,bool> = false> +FASTOR_INLINE TensorConstFixedViewExprnD,Fseq...> operator()(Fseq... ) const { + static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstFixedViewExprnD,Fseq...>(*this); +} + +template +FASTOR_INLINE TensorConstFixedViewExpr1D, + typename to_positive,pack_prod::value>::type,1> operator()(fseq) const { + static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstFixedViewExpr1D, + typename to_positive,pack_prod::value>::type,1>(*this); +} + +template +FASTOR_INLINE TensorConstFixedViewExpr2D, + typename to_positive,get_value<1,Rest...>::value>::type, + typename to_positive,get_value<2,Rest...>::value>::type,2> +operator()(fseq, fseq) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstFixedViewExpr2D, + typename to_positive,get_value<1,Rest...>::value>::type, + typename to_positive,get_value<2,Rest...>::value>::type,2>(*this); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorConstViewExpr,2> operator()(fseq _s, Int num) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstViewExpr,2>(*this,seq(_s),seq(num)); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorConstViewExpr,2> operator()(Int num, fseq _s) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstViewExpr,2>(*this,seq(num),seq(_s)); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorConstRandomViewExpr,Tensor,1> +operator()(const Tensor &_it) const { + static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstRandomViewExpr,Tensor,1>(*this,_it); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorConstRandomViewExpr,Tensor,sizeof...(Rest)> +operator()(const Tensor &_it) const { + static_assert(dimension_t::value==sizeof...(IterSizes),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorConstRandomViewExpr,Tensor,sizeof...(Rest)>(*this,_it); +} + +template::value && std::is_integral::value,bool>::type=0> +FASTOR_INLINE TensorConstRandomViewExpr,Tensor,2> +operator()(const Tensor &_it0, const Tensor &_it1) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor tmp_it; + constexpr int NCols = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); +} + +template::value && std::is_integral::value,bool>::type=0> +FASTOR_INLINE TensorConstRandomViewExpr,Tensor,2> +operator()(const Tensor &_it0, Int1 num) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor tmp_it; + constexpr int NCols = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); +} + +template::value && std::is_integral::value,bool>::type=0> +FASTOR_INLINE TensorConstRandomViewExpr,Tensor,2> +operator()(Int1 num, const Tensor &_it0) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + Tensor tmp_it; + constexpr int NCols = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorConstRandomViewExpr,Tensor,get_value<2,Rest...>::value>::type::Size>,2> +operator()(const Tensor &_it0, fseq) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + constexpr int NCols = get_value<2,Rest...>::value; + using _seq = typename to_positive,NCols>::type; + constexpr int ColSize = _seq::Size; + Tensor tmp_it; + for (FASTOR_INDEX i = 0; i,Tensor,get_value<2,Rest...>::value>::type::Size>,2> (*this,tmp_it); +} + +template::value,bool>::type=0> +FASTOR_INLINE TensorConstRandomViewExpr,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> +operator()(fseq, const Tensor &_it0) const { + static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + constexpr int NRows = get_value<1,Rest...>::value; + constexpr int NCols = get_value<2,Rest...>::value; + using _seq = typename to_positive,NRows>::type; + constexpr int RowSize = _seq::Size; + Tensor tmp_it; + for (FASTOR_INDEX i = 0; i,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> (*this,tmp_it); +} +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// + + + + +#endif // BLOCK_INDEXING_H diff --git a/noarch/include/Fastor/tensor/ForwardDeclare.h b/noarch/include/Fastor/tensor/ForwardDeclare.h new file mode 100644 index 00000000..5293fbb9 --- /dev/null +++ b/noarch/include/Fastor/tensor/ForwardDeclare.h @@ -0,0 +1,154 @@ +#ifndef FORWARD_DECLARE_H +#define FORWARD_DECLARE_H + + +namespace Fastor { + +// FORWARD DECLARATIONS +//---------------------------------------------------------------- +template +struct BinaryAddOp; + +template +struct BinarySubOp; + +template +struct BinaryMulOp; + +template +struct BinaryDivOp; + +template +struct BinaryMatMulOp; + + +template +struct UnaryAddOp; + +template +struct UnarySubOp; + +template +struct UnaryAbsOp; + +template +struct UnarySqrtOp; + +template +struct UnaryExpOp; + +template +struct UnaryLogOp; + +template +struct UnarySinOp; + +template +struct UnaryCosOp; + +template +struct UnaryTanOp; + +template +struct UnaryAsinOp; + +template +struct UnaryAcosOp; + +template +struct UnaryAtanOp; + +template +struct UnarySinhOp; + +template +struct UnaryCoshOp; + +template +struct UnaryTanhOp; + + +template +struct UnaryTransOp; + + +template +struct TensorViewExpr; + +template +struct TensorConstViewExpr; + +template +struct TensorRandomViewExpr; + +template +struct TensorFilterViewExpr; + +template +struct TensorConstRandomViewExpr; + +template +struct TensorFixedViewExpr1D; + +template +struct TensorConstFixedViewExpr1D; + +template +struct TensorFixedViewExpr2D; + +template +struct TensorConstFixedViewExpr2D; + +template +struct TensorConstFixedViewExprnD; + +template +struct TensorFixedViewExprnD; + +template +struct TensorDiagViewExpr; + + +template +struct Index; + +template +struct nprods; + +template +struct nprods_views; + +#define FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(NAME)\ +template\ +struct Unary ##NAME ## Op;\ + +FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Not) +FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isinf) +FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isnan) +FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isfinite) + +template +struct is_unary_bool_op; + + +#define FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(NAME)\ +template\ +struct BinaryCmpOp##NAME ;\ + +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(EQ) +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(NEQ) +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(LT) +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(GT) +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(LE) +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(GE) +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(AND) +FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(OR) + +template +struct is_binary_cmp_op; +//---------------------------------------------------------------- + +} + + +#endif // FORWARD_DECLARE_H diff --git a/noarch/include/Fastor/tensor/IndexRetriever.h b/noarch/include/Fastor/tensor/IndexRetriever.h new file mode 100644 index 00000000..61a3386f --- /dev/null +++ b/noarch/include/Fastor/tensor/IndexRetriever.h @@ -0,0 +1,120 @@ +#ifndef INDEX_RETRIEVER_H +#define INDEX_RETRIEVER_H + +// Retrieving index +// Given a flat index get the flat index in to the tensor - note that for tensor class the incoming index is +// the out-going index but it may not be the same for special tensors for instance for SingleValueTensor and +// views and such the index may be different +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE U get_mem_index(U index) const { +#if FASTOR_BOUNDS_CHECK + FASTOR_ASSERT((index>=0 && index < size()), "INDEX OUT OF BOUNDS"); +#endif + return index; +} + +// Retrieving index for nD tensors +// Given a multi-dimensional index get the flat index in to the tensor +//----------------------------------------------------------------------------------------------------------// +template::value,bool>::type =0> +FASTOR_INLINE size_t get_flat_index(Args ... args) const { + constexpr size_t M = get_value<1,Rest...>::value; + const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); +#if FASTOR_BOUNDS_CHECK + FASTOR_ASSERT( ( (i>=0 && i::value,bool>::type =0> +FASTOR_INLINE size_t get_flat_index(Args ... args) const { + constexpr size_t M = get_value<1,Rest...>::value; + constexpr size_t N = get_value<2,Rest...>::value; + const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); + const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...); +#if FASTOR_BOUNDS_CHECK + FASTOR_ASSERT( ( (i>=0 && i=0 && j::value,bool>::type =0> +FASTOR_INLINE size_t get_flat_index(Args ... args) const { + constexpr size_t M = get_value<1,Rest...>::value; + constexpr size_t N = get_value<2,Rest...>::value; + constexpr size_t P = get_value<3,Rest...>::value; + const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); + const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...); + const size_t k = get_index<2>(args...) < 0 ? P + get_index<2>(args...) : get_index<2>(args...); +#if FASTOR_BOUNDS_CHECK + FASTOR_ASSERT( ( (i>=0 && i=0 && j=0 && k::value,bool>::type =0> +FASTOR_INLINE size_t get_flat_index(Args ... args) const { + constexpr size_t M = get_value<1,Rest...>::value; + constexpr size_t N = get_value<2,Rest...>::value; + constexpr size_t P = get_value<3,Rest...>::value; + constexpr size_t Q = get_value<4,Rest...>::value; + const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); + const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...); + const size_t k = get_index<2>(args...) < 0 ? P + get_index<2>(args...) : get_index<2>(args...); + const size_t l = get_index<3>(args...) < 0 ? Q + get_index<3>(args...) : get_index<3>(args...); +#if FASTOR_BOUNDS_CHECK + FASTOR_ASSERT( ( (i>=0 && i=0 && j=0 && k=0 && l=5 + && sizeof...(Args)==dimension_t::value && is_arithmetic_pack::value,bool>::type =0> +FASTOR_INLINE size_t get_flat_index(Args ... args) const { + /* The type of largs needs to be the type of incoming pack i.e. + whatever the tensor is indexed with + */ + get_nth_type<0,Args...> largs[sizeof...(Args)] = {args...}; + + constexpr std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; + constexpr size_t DimensionHolder[dimension_t::value] = {Rest...}; + + for (size_t i=0; i=0 && largs[i] &as) const { + constexpr std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; + size_t index = 0; + for (size_t i=0; i=0 && index,bool> = false > + constexpr + FASTOR_INLINE Tensor(const std::initializer_list &lst) { + static_assert(sizeof...(Rest)==1,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); +#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) + FASTOR_ASSERT(pack_prod::value==lst.size(), "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); +#endif + auto counter = 0; + for (const auto &i: lst) {_data[counter] = i; counter++;} + } + + template,bool> = false > + constexpr + FASTOR_INLINE Tensor(const std::initializer_list> &lst2d) { + static_assert(sizeof...(Rest)==2,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); +#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + auto size_ = 0; + FASTOR_ASSERT(M==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + for (auto &lst: lst2d) { + auto curr_size = lst.size(); + FASTOR_ASSERT(N==lst.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + size_ += curr_size; + } + FASTOR_ASSERT(pack_prod::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); +#endif + auto counter = 0; + for (const auto &lst1d: lst2d) { + for (const auto &i: lst1d) { + _data[counter] = T(i); + counter++; + } + } + } + + template,bool> = false > + constexpr + FASTOR_INLINE Tensor(const std::initializer_list>> &lst3d) { + static_assert(sizeof...(Rest)==3,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); +#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + constexpr FASTOR_INDEX P = get_value<3,Rest...>::value; + auto size_ = 0; + FASTOR_ASSERT(M==lst3d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + for (const auto &lst2d: lst3d) { + FASTOR_ASSERT(N==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + for (const auto &lst: lst2d) { + const auto curr_size = lst.size(); + FASTOR_ASSERT(P==lst.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + size_ += curr_size; + } + } + FASTOR_ASSERT(pack_prod::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); +#endif + auto counter = 0; + for (const auto &lst2d: lst3d) { + for (const auto &lst1d: lst2d) { + for (const auto &i: lst1d) { + _data[counter] = i; + counter++; + } + } + } + } + + template,bool> = false > + constexpr + FASTOR_INLINE Tensor(const std::initializer_list>>> &lst4d) { + static_assert(sizeof...(Rest)==4,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); +#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + constexpr FASTOR_INDEX P = get_value<3,Rest...>::value; + constexpr FASTOR_INDEX Q = get_value<4,Rest...>::value; + auto size_ = 0; + FASTOR_ASSERT(M==lst4d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + for (const auto &lst3d: lst4d) { + FASTOR_ASSERT(N==lst3d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + for (const auto &lst2d: lst3d) { + FASTOR_ASSERT(P==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + for (const auto &lst: lst2d) { + const auto curr_size = lst.size(); + FASTOR_ASSERT(Q==curr_size,"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); + size_ += curr_size; + } + } + } + FASTOR_ASSERT(pack_prod::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); +#endif + auto counter = 0; + for (const auto &lst3d: lst4d) { + for (const auto &lst2d: lst3d) { + for (const auto &lst1d: lst2d) { + for (const auto &i: lst1d) { + _data[counter] = i; + counter++; + } + } + } + } + } +//----------------------------------------------------------------------------------------------------------// + +#endif // INITIALIZER_LIST_CONSTRUCTORS_H diff --git a/noarch/include/Fastor/tensor/PODConverters.h b/noarch/include/Fastor/tensor/PODConverters.h new file mode 100644 index 00000000..dee58222 --- /dev/null +++ b/noarch/include/Fastor/tensor/PODConverters.h @@ -0,0 +1,24 @@ +#ifndef PODCONVERTERS_H +#define PODCONVERTERS_H + +FASTOR_INLINE T toscalar() const { + //! Returns a scalar + static_assert(size()==1,"ONLY TENSORS OF SIZE 1 CAN BE CONVERTED TO A SCALAR"); + return (*_data); +} + +FASTOR_INLINE std::array toarray() const { + //! Returns std::array + std::array out; + std::copy(_data,_data+size(),out.begin()); + return out; +} + +FASTOR_INLINE std::vector tovector() const { + //! Returns std::vector + std::vector out(size()); + std::copy(_data,_data+size(),out.begin()); + return out; +} + +#endif //PODCONVERTERS_H diff --git a/noarch/include/Fastor/tensor/Ranges.h b/noarch/include/Fastor/tensor/Ranges.h new file mode 100644 index 00000000..7b1e3d71 --- /dev/null +++ b/noarch/include/Fastor/tensor/Ranges.h @@ -0,0 +1,210 @@ +#ifndef RANGES_H +#define RANGES_H + +#include "Fastor/config/config.h" +#include "Fastor/meta/meta.h" +#include + +namespace Fastor { + +// range detector for fseq +//----------------------------------------------------------------------------------------------------------// +template +struct range_detector { + static constexpr int range = last - first; + static constexpr int value = range % step==0 ? range/step : range/step+1; +}; + +namespace internal { +template +struct fseq_range_detector; + +template class Seq, int first, int last, int step> +struct fseq_range_detector> { + static constexpr int range = last - first; + static constexpr int value = range % step==0 ? range/step : range/step+1; +}; +} // internal +//----------------------------------------------------------------------------------------------------------// + + + + +// Immediate sequence +//----------------------------------------------------------------------------------------------------------// +template +struct iseq { + static constexpr size_t _first = F; + static constexpr size_t _last= L; + static constexpr size_t _step = S; +}; +//----------------------------------------------------------------------------------------------------------// + + +// Fixed sequence +//----------------------------------------------------------------------------------------------------------// +template +struct fseq { + static constexpr int _first = F; + static constexpr int _last= L; + static constexpr int _step = S; + + static constexpr int Size = range_detector::value; + constexpr FASTOR_INLINE int size() const {return range_detector::value;} +}; + +static constexpr fseq<0,-1,1> fall; + +template +static constexpr fseq fix{}; + +static constexpr fseq<0 ,1 ,1> ffirst; +static constexpr fseq<-1,-1,1> flast; +//----------------------------------------------------------------------------------------------------------// + + +// Dynamic sequence +//----------------------------------------------------------------------------------------------------------// +struct seq { + + int _first; + int _last; + int _step = 1; + + constexpr FASTOR_INLINE seq(int _f, int _l, int _s=1) : _first(_f), _last(_l), _step(_s) {} + constexpr FASTOR_INLINE seq(int num) : _first(num), _last(num+1), _step(1) {} + + template + constexpr FASTOR_INLINE seq(fseq) : _first(F), _last(L), _step(S) {} + + // Do not allow construction of seq using std::initializer_list, as it happens + // implicitly. Overloading operator() with std::initializer_list should imply + // TensorRandomView, not TensorView + template + constexpr FASTOR_INLINE seq(std::initializer_list _s1) = delete; + + // Do not provide this overload as it is meaningless [iseq stands for immediate evaluation] + // template + // constexpr FASTOR_INLINE seq(iseq) : _first(F), _last(L), _step(S) {} + + FASTOR_INLINE int size() const { + int range = _last - _first; + return range % _step==0 ? range/_step : range/_step+1; + } + + constexpr FASTOR_INLINE bool operator==(seq other) const { + return (_first==other._first && _last==other._last && _step==other._step) ? true : false; + } + constexpr FASTOR_INLINE bool operator!=(seq other) const { + return (_first!=other._first || _last!=other._last || _step!=other._step) ? true : false; + } + +}; + +static constexpr int first = 0; +static constexpr int last = -1; + +// static constexpr seq all = seq(0,-1,1); +// why not this? +static constexpr fseq<0,-1,1> all; +//----------------------------------------------------------------------------------------------------------// + + +// traits +//----------------------------------------------------------------------------------------------------------// +template +struct is_fixed_sequence { + static constexpr bool value = false; +}; +template +struct is_fixed_sequence> { + static constexpr bool value = true; +}; +template +struct is_fixed_sequence> { + static constexpr bool value = true; +}; + +template +static constexpr bool is_fixed_sequence_v = is_fixed_sequence::value; + +template +struct is_fixed_sequence_pack; +template +struct is_fixed_sequence_pack { + static constexpr bool value = is_fixed_sequence::value && is_fixed_sequence_pack::value; +}; +template +struct is_fixed_sequence_pack { + static constexpr bool value = is_fixed_sequence::value; +}; + +template +static constexpr bool is_fixed_sequence_pack_v = is_fixed_sequence_pack::value; +//----------------------------------------------------------------------------------------------------------// + + + +// Transform sequence with negative indices to positive indices; +//----------------------------------------------------------------------------------------------------------// +template +struct to_positive; + +template +struct to_positive,N> { + // Same logic as seq used in the constructor of dynamic tensor views + static constexpr int _first = (L==0 && F==-1) ? N-1 : ( L < 0 && F < 0 ? F + N + 1 : F); + static constexpr int _last = (L < 0 && F >=0) ? L + N + 1 : ( (L==0 && F==-1) ? N : ( L < 0 && F < 0 ? L + N + 1 : L) ); + using type = fseq<_first,_last,S>; +}; + +template +struct to_positive,N> { + // Same logic as seq used in the constructor of dynamic tensor views + static constexpr int _first = (L==0 && F==-1) ? N-1 : ( L < 0 && F < 0 ? F + N + 1 : F); + static constexpr int _last = (L < 0 && F >=0) ? L + N + 1 : ( (L==0 && F==-1) ? N : ( L < 0 && F < 0 ? L + N + 1 : L) ); + using type = iseq<_first,_last,S>; +}; + +template +using to_positive_t = typename to_positive::type; +//----------------------------------------------------------------------------------------------------------// + + +//----------------------------------------------------------------------------------------------------------// +template +struct get_fixed_sequence_pack_dimensions; + +template class Derived, typename T, size_t ...Rest, size_t... ss, typename ... Fseqs> +struct get_fixed_sequence_pack_dimensions, std_ext::index_sequence, Fseqs...>{ + static constexpr std::array dims = { internal::fseq_range_detector>::value... }; + // using type = Derived::value...>; + using type = Derived>::value...>; + static constexpr size_t Size = pack_prod::value; +}; + +template class Derived, typename T, size_t ...Rest, size_t... ss, typename ... Fseqs> +constexpr std::array get_fixed_sequence_pack_dimensions, std_ext::index_sequence, Fseqs...>::dims; +//----------------------------------------------------------------------------------------------------------// + + + +//----------------------------------------------------------------------------------------------------------// +template +struct ravel_2d_indices; + +template +struct ravel_2d_indices> { + static constexpr size_t size_1 = range_detector::value; + static constexpr std::array idx = {(S0*(ss/size_1)*Ncol + S1*(ss%size_1) + F0*Ncol + F1)...}; +}; +template +constexpr std::array +ravel_2d_indices>::idx; +//----------------------------------------------------------------------------------------------------------// + + +} + +#endif // RANGES_H + diff --git a/noarch/include/Fastor/tensor/ScalarIndexing.h b/noarch/include/Fastor/tensor/ScalarIndexing.h new file mode 100644 index 00000000..93404bc7 --- /dev/null +++ b/noarch/include/Fastor/tensor/ScalarIndexing.h @@ -0,0 +1,39 @@ +#ifndef SCALAR_INDEXING_NONCONST_H +#define SCALAR_INDEXING_NONCONST_H + + +// Scalar indexing non-const +//----------------------------------------------------------------------------------------------------------// +template::value,bool>::type =0> +FASTOR_INLINE T& operator()(Args ... args) { + return _data[get_flat_index(args...)]; +} +template::value,bool>::type =0> +FASTOR_INLINE T& operator[](Arg arg) { + return _data[get_flat_index(arg)]; +} +//----------------------------------------------------------------------------------------------------------// + +#endif // SCALAR_INDEXING_NONCONST_H + + +#ifndef SCALAR_INDEXING_CONST_H +#define SCALAR_INDEXING_CONST_H + +// Scalar indexing const +//----------------------------------------------------------------------------------------------------------// +template::value,bool>::type =0> +constexpr FASTOR_INLINE const T& operator()(Args ... args) const { + return _data[get_flat_index(args...)]; +} +template::value,bool>::type =0> +constexpr FASTOR_INLINE const T& operator[](Arg arg) const { + return _data[get_flat_index(arg)]; +} +//----------------------------------------------------------------------------------------------------------// + +#endif // SCALAR_INDEXING_CONST_H diff --git a/noarch/include/Fastor/tensor/SpecialisedConstructors.h b/noarch/include/Fastor/tensor/SpecialisedConstructors.h new file mode 100644 index 00000000..44c28e3c --- /dev/null +++ b/noarch/include/Fastor/tensor/SpecialisedConstructors.h @@ -0,0 +1,374 @@ +#ifndef SPECIALISED_CONSTRUCTORS_H +#define SPECIALISED_CONSTRUCTORS_H + + +//----------------------------------------------------------------------------------------------------------// +template::type=0> +FASTOR_INLINE Tensor(const TensorFixedViewExpr2D,Seq0,Seq1,2>& src) { + using scalar_type_ = T; + constexpr FASTOR_INDEX Stride_ = simd_size_v; +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); + } +#endif + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); + } + for (; j < N; ++j) { + _data[i*N+j] = src.template eval_s(i,j); + } + } +} + + +template = false> +FASTOR_INLINE Tensor(const TensorFixedViewExprnD,Fseqs...>& src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); +#endif + constexpr int DimensionHolder[dimension_t::value] = {Rest...}; + std::array as = {}; + int jt, counter=0; + + if (src.is_vectorisable() || src.is_strided_vectorisable()) + { + using V = SIMDVector; + V _vec; + while(counter < size()) + { + _vec = src.template teval(as); + _vec.store(&_data[counter],false); + + counter+=V::Size; + for(jt = dimension_t::value-1; jt>=0; jt--) + { + if (jt == dimension_t::value-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt](as); + + counter++; + for(jt = dimension_t::value-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt] && !has_tensor_fixed_view_nd_v && has_tensor_fixed_view_2d_v + && DIMS==sizeof...(Rest) + && requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + // const typename Derived::result_type& tmp = evaluate(src_.self()); + FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + assign(*this,src_.self()); +} +template && !has_tensor_fixed_view_nd_v && has_tensor_fixed_view_2d_v + && DIMS==sizeof...(Rest) + && !requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + using scalar_type_ = typename scalar_type_finder::type; + constexpr FASTOR_INDEX Stride_ = simd_size_v; + const Derived &src = src_.self(); +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); + } +#endif + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); + } + for (; j (i,j); + } + } + } + else { + for (FASTOR_INDEX i = 0; i (i,j); + } + } + } +} + + +template && has_tensor_fixed_view_nd_v + && DIMS!=2 && DIMS==sizeof...(Rest) && + requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + assign(*this,src_.self()); +} +template && has_tensor_fixed_view_nd_v + && DIMS!=2 && DIMS==sizeof...(Rest) && + !requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + const Derived &src = src_.self(); +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); + } +#endif + + constexpr int DimensionHolder[dimension_t::value] = {Rest...}; + std::array as = {}; + int jt, counter=0; + + while(counter < size()) + { + _data[counter] = src.template teval_s(as); + + counter++; + for(jt = dimension_t::value-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt] = false> +FASTOR_INLINE Tensor(const TensorViewExpr,2>& src) { + using scalar_type_ = T; + constexpr FASTOR_INDEX Stride_ = simd_size_v; +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); + } +#endif + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); + } + for (; j < N; ++j) { + _data[i*N+j] = src.template eval_s(i,j); + } + } +} + + +template::value,bool> = false> +FASTOR_INLINE Tensor(const TensorViewExpr,sizeof...(Rest)>& src) { +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); + } +#endif + constexpr int DimensionHolder[dimension_t::value] = {Rest...}; + std::array as = {}; + int jt, counter=0; + + if (src.is_vectorisable() || src.is_strided_vectorisable()) + { + using V = SIMDVector; + V _vec; + while(counter < size()) + { + _vec = src.template teval(as); + _vec.store(&_data[counter],false); + + counter+=V::Size; + for(jt = dimension_t::value-1; jt>=0; jt--) + { + if (jt == dimension_t::value-1) as[jt]+=V::Size; + else as[jt] +=1; + if(as[jt](as); + + counter++; + for(jt = dimension_t::value-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt] && DIMS==2 && DIMS==sizeof...(Rest) && + requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + assign(*this,src_.self()); +} +template && DIMS==2 && DIMS==sizeof...(Rest) && + !requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + using scalar_type_ = typename scalar_type_finder::type; + constexpr FASTOR_INDEX Stride_ = simd_size_v; + const Derived &src = src_.self(); +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); + } +#endif + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { + for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); + } + for (; j < N; ++j) { + _data[i*N+j] = src.template eval_s(i,j); + } + } + } + else { + for (FASTOR_INDEX i = 0; i (i,j); + } + } + } +} + +template && DIMS!=2 && DIMS==sizeof...(Rest) && + requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); + assign(*this,src_.self()); +} +template && DIMS!=2 && DIMS==sizeof...(Rest) && + !requires_evaluation_v,bool> = false> +FASTOR_INLINE Tensor(const AbstractTensor& src_) { + // using scalar_type_ = typename scalar_type_finder::type; + // constexpr FASTOR_INDEX Stride_ = simd_size_v; + const Derived &src = src_.self(); +#ifndef NDEBUG + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); + } +#endif + + constexpr int DimensionHolder[dimension_t::value] = {Rest...}; + std::array as = {}; + int jt, counter=0; + + while(counter < size()) + { + _data[counter] = src.template teval_s(as); + + counter++; + for(jt = dimension_t::value-1; jt>=0; jt--) + { + as[jt] +=1; + if(as[jt]; + // while(counter < size()) + // { + // const FASTOR_INDEX remainder = DimensionHolder[dimension_t::value-1] - as[dimension_t::value-1]; + // if (remainder > V::Size) { + // // V _vec = src.template eval(counter); + // V _vec = src.template teval(as); + // _vec.store(&_data[counter],false); + // counter+=V::Size; + // } + // else { + // // _data[counter] = src.template eval_s(counter); + // _data[counter] = src.template teval_s(as); + // counter++; + // } + + // for(jt = dimension_t::value-1; jt>=0; jt--) + // { + // if (jt == dimension_t::value-1 && remainder > V::Size) as[jt]+=V::Size; + // else as[jt] +=1; + // if(as[jt] +#include + +namespace Fastor { + +template +class Tensor: public AbstractTensor,sizeof...(Rest)> { +public: + using scalar_type = T; + using simd_vector_type = choose_best_simd_vector_t; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor; + using dimension_t = std::integral_constant; + static constexpr FASTOR_INLINE FASTOR_INDEX rank() {return sizeof...(Rest);} + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return pack_prod::value;} + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX dim) const { +#if FASTOR_SHAPE_CHECK + FASTOR_ASSERT(dim>=0 && dim < sizeof...(Rest), "TENSOR SHAPE MISMATCH"); +#endif + constexpr FASTOR_INDEX DimensionHolder[sizeof...(Rest)] = {Rest...}; + return DimensionHolder[dim]; + } + FASTOR_INLINE Tensor& noalias() {return *this;} + + // Classic constructors + //----------------------------------------------------------------------------------------------------------// + // Default constructor + constexpr FASTOR_INLINE Tensor() = default; + + // Copy constructor + FASTOR_INLINE Tensor(const Tensor &other) { + // This constructor cannot be default + if (_data == other.data()) return; + // fast memcopy + std::copy(other.data(),other.data()+size(),_data); + }; + + // Constructor from a scalar + template,bool> = false> + constexpr FASTOR_INLINE Tensor(U num) { +#ifdef FASTOR_ZERO_INITIALISE + // This is for compile time initialisation, so it is fine + scalar_type cnum = (scalar_type)num; + FASTOR_INDEX i = 0; + for (; i::value> &arr, int layout=RowMajor) { + std::copy(arr.data(),arr.data()+pack_prod::value,_data); + if (layout == RowMajor) + return; + else + *this = tocolumnmajor(*this); + } + FASTOR_INLINE Tensor(const std::vector &arr, int layout=RowMajor) { + std::copy(arr.data(),arr.data()+pack_prod::value,_data); + if (layout == RowMajor) + return; + else + *this = tocolumnmajor(*this); + } + //----------------------------------------------------------------------------------------------------------// + + // CRTP constructors + //----------------------------------------------------------------------------------------------------------// + //----------------------------------------------------------------------------------------------------------// + // Generic AbstractTensors +#ifndef FASTOR_DISABLE_SPECIALISED_CTR + template && !has_tensor_fixed_view_2d_v && + !has_tensor_fixed_view_nd_v) || DIMS!=sizeof...(Rest),bool> = false> +#else + template +#endif + FASTOR_INLINE Tensor(const AbstractTensor& src) { + FASTOR_ASSERT(src.self().size()==size(), "TENSOR SIZE MISMATCH"); + assign(*this, src.self()); + } + //----------------------------------------------------------------------------------------------------------// + + // Specialised constructors + //----------------------------------------------------------------------------------------------------------// + #include "Fastor/tensor/SpecialisedConstructors.h" + //----------------------------------------------------------------------------------------------------------// + + // AbstractTensor and scalar in-place operators + //----------------------------------------------------------------------------------------------------------// + #include "Fastor/tensor/TensorInplaceOperators.h" + //----------------------------------------------------------------------------------------------------------// + + // Raw pointer providers + //----------------------------------------------------------------------------------------------------------// +#ifdef FASTOR_ZERO_INITIALISE + constexpr FASTOR_INLINE T* data() const { return const_cast(this->_data);} +#else + FASTOR_INLINE T* data() const { return const_cast(this->_data);} +#endif + + FASTOR_INLINE T* data() {return this->_data;} + //----------------------------------------------------------------------------------------------------------// + + // Scalar & block indexing + //----------------------------------------------------------------------------------------------------------// + #include "Fastor/tensor/IndexRetriever.h" + #include "Fastor/tensor/ScalarIndexing.h" + #include "Fastor/tensor/BlockIndexing.h" + //----------------------------------------------------------------------------------------------------------// + + // Expression templates evaluators + //----------------------------------------------------------------------------------------------------------// + #include "Fastor/tensor/TensorEvaluator.h" + //----------------------------------------------------------------------------------------------------------// + + // Tensor methods + //----------------------------------------------------------------------------------------------------------// + #include "Fastor/tensor/TensorMethods.h" + //----------------------------------------------------------------------------------------------------------// + + // Converters + //----------------------------------------------------------------------------------------------------------// + #include "Fastor/tensor/PODConverters.h" + //----------------------------------------------------------------------------------------------------------// + + // Cast method + //----------------------------------------------------------------------------------------------------------// + template + FASTOR_INLINE Tensor cast() const { + Tensor out; + U *out_data = out.data(); + for (FASTOR_INDEX i=0; i(_data[i]); + } + return out; + } + //----------------------------------------------------------------------------------------------------------// + + + //----------------------------------------------------------------------------------------------------------// +protected: + template + FASTOR_INLINE void verify_dimensions(const AbstractTensor& src_) const { + static_assert(DIMS==dimension_t::value, "TENSOR RANK MISMATCH"); +#ifndef NDEBUG + const Derived &src = src_.self(); + FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); + // Check if shape of tensors match + for (FASTOR_INDEX i=0; i::value] = {}; +#else + FASTOR_ALIGN T _data[pack_prod::value]; +#endif + //----------------------------------------------------------------------------------------------------------// +}; + + +} // end of namespace Fastor + + +#include "Fastor/tensor/TensorAssignment.h" + + +#endif // TENSOR_H + diff --git a/noarch/include/Fastor/tensor/TensorAssignment.h b/noarch/include/Fastor/tensor/TensorAssignment.h new file mode 100644 index 00000000..a21db81b --- /dev/null +++ b/noarch/include/Fastor/tensor/TensorAssignment.h @@ -0,0 +1,299 @@ +#ifndef TENSOR_ASSIGNMENT_H +#define TENSOR_ASSIGNMENT_H + +namespace Fastor { + +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE void trivial_assign(AbstractTensor &dst, const AbstractTensor &src_) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + const OtherDerived &src = src_.self(); + FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); + T* _data = dst.self().data(); + + FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { + FASTOR_INDEX i = 0; + for (; i (i).store(&_data[i], FASTOR_ALIGNED); + } + for (; i < src.size(); ++i) { + _data[i] = src.template eval_s(i); + } + } + else { + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + _data[i] = src.template eval_s(i); + } + } +} + +template +FASTOR_INLINE void trivial_assign_add(AbstractTensor &dst, const AbstractTensor &src_) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + const OtherDerived &src = src_.self(); + FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); + T* _data = dst.self().data(); + + FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { + FASTOR_INDEX i = 0; + for (; i (i); + _vec.store(&_data[i], FASTOR_ALIGNED); + } + for (; i < src.size(); ++i) { + _data[i] += src.template eval_s(i); + } + } + else { + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + _data[i] += src.template eval_s(i); + } + } +} + +template +FASTOR_INLINE void trivial_assign_sub(AbstractTensor &dst, const AbstractTensor &src_) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + const OtherDerived &src = src_.self(); + FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); + T* _data = dst.self().data(); + + FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { + FASTOR_INDEX i = 0; + for (; i (i); + _vec.store(&_data[i], FASTOR_ALIGNED); + } + for (; i < src.size(); ++i) { + _data[i] -= src.template eval_s(i); + } + } + else { + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + _data[i] -= src.template eval_s(i); + } + } +} + +template +FASTOR_INLINE void trivial_assign_mul(AbstractTensor &dst, const AbstractTensor &src_) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + const OtherDerived &src = src_.self(); + FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); + T* _data = dst.self().data(); + + FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { + FASTOR_INDEX i = 0; + for (; i (i); + _vec.store(&_data[i], FASTOR_ALIGNED); + } + for (; i < src.size(); ++i) { + _data[i] *= src.template eval_s(i); + } + } + else { + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + _data[i] *= src.template eval_s(i); + } + } +} + +template +FASTOR_INLINE void trivial_assign_div(AbstractTensor &dst, const AbstractTensor &src_) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + const OtherDerived &src = src_.self(); + FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); + T* _data = dst.self().data(); + + FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { + FASTOR_INDEX i = 0; + for (; i (i); + _vec.store(&_data[i], FASTOR_ALIGNED); + } + for (; i < src.size(); ++i) { + _data[i] /= src.template eval_s(i); + } + } + else { + for (FASTOR_INDEX i = 0; i < src.size(); ++i) { + _data[i] /= src.template eval_s(i); + } + } +} +//----------------------------------------------------------------------------------------------------------// + +//----------------------------------------------------------------------------------------------------------// +template, bool> = false> +FASTOR_INLINE void trivial_assign(AbstractTensor &dst, U num) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + T* _data = dst.self().data(); + T cnum = (T)num; + V _vec(cnum); + FASTOR_INDEX i = 0; + for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { + _vec.store(&_data[i], FASTOR_ALIGNED); + } + for (; i, bool> = false> +FASTOR_INLINE void trivial_assign_add(AbstractTensor &dst, U num) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + T* _data = dst.self().data(); + T cnum = (T)num; + V _vec(cnum); + FASTOR_INDEX i = 0; + for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { + V _vec_out(&_data[i], FASTOR_ALIGNED); + _vec_out += _vec; + _vec_out.store(&_data[i], FASTOR_ALIGNED); + } + for (; i, bool> = false> +FASTOR_INLINE void trivial_assign_sub(AbstractTensor &dst, U num) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + T* _data = dst.self().data(); + T cnum = (T)num; + V _vec(cnum); + FASTOR_INDEX i = 0; + for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { + V _vec_out(&_data[i], FASTOR_ALIGNED); + _vec_out -= _vec; + _vec_out.store(&_data[i], FASTOR_ALIGNED); + } + for (; i, bool> = false> +FASTOR_INLINE void trivial_assign_mul(AbstractTensor &dst, U num) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + T* _data = dst.self().data(); + T cnum = (T)num; + V _vec(cnum); + FASTOR_INDEX i = 0; + for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { + V _vec_out(&_data[i], FASTOR_ALIGNED); + _vec_out *= _vec; + _vec_out.store(&_data[i], FASTOR_ALIGNED); + } + for (; i && !is_integral_v_, bool> = false> +FASTOR_INLINE void trivial_assign_div(AbstractTensor &dst, U num) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + T* _data = dst.self().data(); + T cnum = T(1) / (T)num; + V _vec(cnum); + FASTOR_INDEX i = 0; + for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { + V _vec_out(&_data[i], FASTOR_ALIGNED); + _vec_out *= _vec; + _vec_out.store(&_data[i], FASTOR_ALIGNED); + } + for (; i && is_integral_v_, bool> = false> +FASTOR_INLINE void trivial_assign_div(AbstractTensor &dst, U num) { + using T = typename Derived::scalar_type; + using V = typename Derived::simd_vector_type; + T* _data = dst.self().data(); + T cnum = (T)num; + V _vec(cnum); + FASTOR_INDEX i = 0; + for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { + V _vec_out(&_data[i], FASTOR_ALIGNED); + _vec_out /= _vec; + _vec_out.store(&_data[i], FASTOR_ALIGNED); + } + for (; i +constexpr FASTOR_INLINE void assign(AbstractTensor &dst, const Tensor &src) { + if (dst.self().data()==src.data()) return; + trivial_assign(dst.self(),src); +} +template +constexpr FASTOR_INLINE void assign_add(AbstractTensor &dst, const Tensor &src) { + trivial_assign_add(dst.self(),src); +} +template +constexpr FASTOR_INLINE void assign_sub(AbstractTensor &dst, const Tensor &src) { + trivial_assign_sub(dst.self(),src); +} +template +constexpr FASTOR_INLINE void assign_mul(AbstractTensor &dst, const Tensor &src) { + trivial_assign_mul(dst.self(),src); +} +template +constexpr FASTOR_INLINE void assign_div(AbstractTensor &dst, const Tensor &src) { + trivial_assign_div(dst.self(),src); +} + +template,bool> = false> +constexpr FASTOR_INLINE void assign(AbstractTensor &dst, U num) { + trivial_assign(dst.self(),num); +} +template,bool> = false> +constexpr FASTOR_INLINE void assign_add(AbstractTensor &dst, U num) { + trivial_assign_add(dst.self(),num); +} +template,bool> = false> +constexpr FASTOR_INLINE void assign_sub(AbstractTensor &dst, U num) { + trivial_assign_sub(dst.self(),num); +} +template,bool> = false> +constexpr FASTOR_INLINE void assign_mul(AbstractTensor &dst, U num) { + trivial_assign_mul(dst.self(),num); +} +template,bool> = false> +constexpr FASTOR_INLINE void assign_div(AbstractTensor &dst, U num) { + trivial_assign_div(dst.self(),num); +} +//----------------------------------------------------------------------------------------------------------// +//----------------------------------------------------------------------------------------------------------// + +} // end of namespace Fastor + +#endif // TENSOR_ASSIGNMENT_H diff --git a/noarch/include/Fastor/tensor/TensorEvaluator.h b/noarch/include/Fastor/tensor/TensorEvaluator.h new file mode 100644 index 00000000..27bc8d1e --- /dev/null +++ b/noarch/include/Fastor/tensor/TensorEvaluator.h @@ -0,0 +1,39 @@ +#ifndef TENSOR_EVALUATOR_H +#define TENSOR_EVALUATOR_H + +// Expression templates evaluators +//----------------------------------------------------------------------------------------------------------// +template +FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { + SIMDVector _vec; + _vec.load(&_data[get_mem_index(i)],false); + return _vec; +} +template +FASTOR_INLINE T eval_s(FASTOR_INDEX i) const { + return _data[get_mem_index(i)]; +} +template +FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { + SIMDVector _vec; + _vec.load(&_data[get_flat_index(i,j)],false); + return _vec; +} +template +FASTOR_INLINE T eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { + return _data[get_flat_index(i,j)]; +} + +template +FASTOR_INLINE SIMDVector teval(const std::array &as) const { + SIMDVector _vec; + _vec.load(&_data[get_flat_index(as)],false); + return _vec; +} +template +FASTOR_INLINE T teval_s(const std::array &as) const { + return _data[get_flat_index(as)]; +} +//----------------------------------------------------------------------------------------------------------// + +#endif // end of TENSOR_EVALUATOR_H diff --git a/noarch/include/Fastor/tensor/TensorFunctions.h b/noarch/include/Fastor/tensor/TensorFunctions.h new file mode 100644 index 00000000..235d5622 --- /dev/null +++ b/noarch/include/Fastor/tensor/TensorFunctions.h @@ -0,0 +1,215 @@ +#ifndef TENSOR_FUNCTIONS_H +#define TENSOR_FUNCTIONS_H + +#include "Fastor/meta/meta.h" +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor/TensorMap.h" +#include "Fastor/tensor/TensorTraits.h" + +namespace Fastor { + +/* Turns a row-major tensor to column-major */ +template class TensorType, typename T, size_t ... Rest> +FASTOR_INLINE Tensor tocolumnmajor(const TensorType &a) { + constexpr int Dimension = sizeof...(Rest); + if (Dimension < 2) { + return a; + } + else { + Tensor out; + T *arr_out = out.data(); + const T *a_data = a.data(); + + if (Dimension == 2) { + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + for (FASTOR_INDEX i=0; i::value; + std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; + FASTOR_INDEX DimensionHolder[Dimension] = {Rest...}; + std::reverse(DimensionHolder,DimensionHolder+Dimension); + std::reverse(products_.begin(),products_.end()); + std::array as = {}; + + int jt; + FASTOR_INDEX counter=0; + while(counter < Size) + { + FASTOR_INDEX index = 0; + for (int ii=0; ii=0; jt--) + { + as[jt] +=1; + if(as[jt] class TensorType, typename T, size_t ... Rest> +FASTOR_INLINE Tensor torowmajor(const TensorType &a) { + constexpr int Dimension = sizeof...(Rest); + if (Dimension < 2) { + return a; + } + else { + Tensor out; + T *arr_out = out.data(); + const T *a_data = a.data(); + + if (Dimension == 2) { + constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + for (FASTOR_INDEX i=0; i::value; + std::array products_ = nprods_views, + typename std_ext::make_index_sequence::type>::values; + FASTOR_INDEX DimensionHolder[Dimension] = {Rest...}; + std::reverse(DimensionHolder,DimensionHolder+Dimension); + std::reverse(products_.begin(),products_.end()); + std::array as = {}; + + int jt; + FASTOR_INDEX counter=0; + while(counter < Size) + { + FASTOR_INDEX index = 0; + for (int ii=0; ii=0; jt--) + { + as[jt] +=1; + if(as[jt] class TensorType, typename T, size_t ... Rest> +FASTOR_INLINE +index_to_tensor_map_t> +squeeze(const TensorType &a) { + return index_to_tensor_map_t>(a.data()); +} + +/* reshape - reshapes a tensor to a tensor of different shape and returns a TensorMap + A TensorMap is a view in to an existing tensor so modifying the reshaped + tensor will modify the original tensor and vice versa + Note that you cannot call this function on tensor expressions as this is + a view in to a concrete tensor type holding storage + + example: + auto b = reshape(a); +*/ +template +FASTOR_INLINE TensorMap reshape(const Tensor &a) { + static_assert(pack_prod::value==pack_prod::value, "SIZE OF TENSOR SHOULD REMAIN THE SAME DURING RESHAPE"); + return TensorMap(a.data()); +} + +/* flatten - creates a flattened 1D view of a tensor and returns a TensorMap + A TensorMap is a view in to an existing tensor so modifying the reshaped + tensor will modify the original tensor and vice versa + Note that you cannot call this function on tensor expressions as this is + a view in to a concrete tensor type holding storage + + example: + auto b = flatten(a); +*/ +template +FASTOR_INLINE TensorMap::value> flatten(const Tensor &a) { + return TensorMap::value>(a.data()); +} + + + + + + + + +#if FASTOR_NIL +// Constant tensors +static FASTOR_INLINE +Tensor levi_civita_ps() { + Tensor LeCi_ps; + LeCi_ps(0,1,2) = 1.f; + LeCi_ps(1,2,0) = 1.f; + LeCi_ps(2,0,1) = 1.f; + LeCi_ps(1,0,2) = -1.f; + LeCi_ps(2,1,0) = -1.f; + LeCi_ps(0,2,1) = -1.f; + + return LeCi_ps; +} + +static FASTOR_INLINE +Tensor levi_civita_pd() { + Tensor LeCi_pd; + LeCi_pd(0,1,2) = 1.; + LeCi_pd(1,2,0) = 1.; + LeCi_pd(2,0,1) = 1.; + LeCi_pd(1,0,2) = -1.; + LeCi_pd(2,1,0) = -1.; + LeCi_pd(0,2,1) = -1.; + + return LeCi_pd; +} + +template +static FASTOR_INLINE +Tensor kronecker_delta() { + Tensor out; out.eye(); + return out; +} +#endif + +} + +#endif // TENSOR_FUNCTIONS_H + diff --git a/noarch/include/Fastor/tensor/TensorIO.h b/noarch/include/Fastor/tensor/TensorIO.h new file mode 100644 index 00000000..e58fb73b --- /dev/null +++ b/noarch/include/Fastor/tensor/TensorIO.h @@ -0,0 +1,180 @@ +#ifndef TENSOR_PRINT_H +#define TENSOR_PRINT_H + +#include "Fastor/tensor/Tensor.h" + +namespace Fastor { + + +namespace internal { + +template +using std_matrix = typename std::vector>::type; + +// Generate combinations +template +FASTOR_INLINE std::vector> index_generator() { + // Do NOT change int to size_t, comparison overflows + std::vector> idx; idx.resize(pack_prod::value); + std::array maxes = {M,N,Rest...}; + std::array a; + int i,j; + std::fill(a.begin(),a.end(),0); + + auto counter=0; + while(1) + { + std::vector current_idx; //current_idx.reserve(sizeof...(Rest)+2); + for(i = 0; i< sizeof...(Rest)+2; i++) { + current_idx.push_back(a[i]); + } + idx[counter] = current_idx; + counter++; + for(j = sizeof...(Rest)+2-1 ; j>=0 ; j--) + { + if(++a[j] +int get_row_width(const std::ostream &os, const T *a_data, size_t size) { + // compute the largest width + int width = 0; + for(size_t j = 0; j < size; ++j) + { + std::stringstream sstr; + sstr.copyfmt(os); + sstr << a_data[j]; + width = std::max(width, int(sstr.str().length())); + } + return width; +} + + +template class t_type, typename T, size_t ...Rest> +int get_row_width(const std::ostream &os, const t_type& a) { + int width = 0; + for(size_t j = 0; j < pack_prod::value; ++j) + { + std::stringstream sstr; + sstr.copyfmt(os); + sstr << a.eval_s(j); + width = std::max(width, int(sstr.str().length())); + } + return width; +} + +} // end of namespace internal + + +#define FASTOR_MAKE_OS_STREAM_TENSOR0(t_type) \ +template\ +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) {\ + IOFormat fmt = FASTOR_DEFINE_IO_FORMAT;\ + os.precision(fmt._precision);\ + os << *a.data();\ + return os;\ +}\ + +#define FASTOR_MAKE_OS_STREAM_TENSOR1(t_type) \ +template \ +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) {\ + IOFormat fmt = FASTOR_DEFINE_IO_FORMAT;\ + os.precision(fmt._precision);\ + int width = internal::get_row_width(os, a);\ + for(size_t i = 0; i < M; ++i)\ + {\ + os << fmt._rowprefix;\ + if(width) os.width(width);\ + os << a(i);\ + os << fmt._rowsuffix;\ + os << fmt._rowsep;\ + }\ + return os;\ +}\ + +#define FASTOR_MAKE_OS_STREAM_TENSOR2(t_type) \ +template \ +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) { \ + IOFormat fmt = FASTOR_DEFINE_IO_FORMAT; \ + os.precision(fmt._precision); \ + int width = internal::get_row_width(os, a); \ + for(size_t i = 0; i < M; ++i) \ + {\ + os << fmt._rowprefix;\ + if(width) os.width(width);\ + os << a(i, 0);\ + for(size_t j = 1; j < N; ++j)\ + {\ + os << fmt._colsep;\ + if(width) os.width(width);\ + os << a(i, j);\ + }\ + os << fmt._rowsuffix;\ + if( i < M - 1)\ + os << fmt._rowsep;\ + }\ + return os;\ +}\ + + +#define FASTOR_MAKE_OS_STREAM_TENSORn(t_type) \ +template=3,bool>::type=0> \ +FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) {\ + IOFormat fmt = FASTOR_DEFINE_IO_FORMAT;\ + constexpr std::array DimensionHolder = {Rest...};\ + constexpr int M = get_value::value;\ + constexpr int N = get_value::value;\ + constexpr int lastrowcol = M*N;\ + constexpr size_t prods = pack_prod::value;\ + std::vector> combs = internal::index_generator();\ + os.precision(fmt._precision);\ + int width = internal::get_row_width(os, a);\ + for (size_t dims=0; dims +FASTOR_INLINE auto& operator +=(const AbstractTensor& src_) { + assign_add(*this, src_.self()); + return *this; +} + +template +FASTOR_INLINE auto& operator -=(const AbstractTensor& src_) { + assign_sub(*this, src_.self()); + return *this; +} + +template +FASTOR_INLINE auto& operator *=(const AbstractTensor& src_) { + assign_mul(*this, src_.self()); + return *this; +} + +template +FASTOR_INLINE auto& operator /=(const AbstractTensor& src_) { + assign_div(*this, src_.self()); + return *this; +} +//---------------------------------------------------------------------------------------------// + +// Scalar overloads for in-place operators +//---------------------------------------------------------------------------------------------// +template,bool> = 0 > +FASTOR_INLINE auto& operator +=(U num) { + trivial_assign_add(*this, num); + return *this; +} + +template,bool> = 0 > +FASTOR_INLINE auto& operator -=(U num) { + trivial_assign_sub(*this, num); + return *this; +} + +template,bool> = 0 > +FASTOR_INLINE auto& operator *=(U num) { + trivial_assign_mul(*this, num); + return *this; +} + +template,bool> = 0 > +FASTOR_INLINE auto& operator /=(U num) { + trivial_assign_div(*this, num); + return *this; +} +//---------------------------------------------------------------------------------------------// + +#endif // TENSOR_INPLACE_OPERATORS_H diff --git a/noarch/include/Fastor/tensor/TensorMap.h b/noarch/include/Fastor/tensor/TensorMap.h new file mode 100644 index 00000000..5b2a452d --- /dev/null +++ b/noarch/include/Fastor/tensor/TensorMap.h @@ -0,0 +1,170 @@ +#ifndef TENSOR_MAP_H +#define TENSOR_MAP_H + +#include "Fastor/config/config.h" +#include "Fastor/backend/backend.h" +#include "Fastor/simd_vector/SIMDVector.h" +#include "Fastor/tensor/AbstractTensor.h" +#include "Fastor/tensor/Ranges.h" +#include "Fastor/tensor/ForwardDeclare.h" +#include "Fastor/expressions/linalg_ops/linalg_ops.h" +#include "Fastor/tensor/TensorIO.h" + + +namespace Fastor { + +template +class TensorMap: public AbstractTensor,sizeof...(Rest)> { +public: + using scalar_type = T; + using simd_vector_type = choose_best_simd_vector_t; + using simd_abi_type = typename simd_vector_type::abi_type; + using result_type = Tensor,Rest...>; + using dimension_t = std::integral_constant; + static constexpr FASTOR_INLINE FASTOR_INDEX rank() {return sizeof...(Rest);} + static constexpr FASTOR_INLINE FASTOR_INDEX size() {return pack_prod::value;} + FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX dim) const { +#if FASTOR_SHAPE_CHECK + FASTOR_ASSERT(dim>=0 && dim < sizeof...(Rest), "TENSOR SHAPE MISMATCH"); +#endif + const FASTOR_INDEX DimensionHolder[sizeof...(Rest)] = {Rest...}; + return DimensionHolder[dim]; + } + FASTOR_INLINE Tensor& noalias() {return *this;} + + // Constructors + //----------------------------------------------------------------------------------------------------------// + constexpr TensorMap(scalar_type* data) : _data(data) {} + template constexpr TensorMap(Tensor &a) : _data(a.data()) {} + //----------------------------------------------------------------------------------------------------------// + + // Raw pointer providers + //----------------------------------------------------------------------------------------------------------// + FASTOR_INLINE T* data() const { return const_cast(this->_data);} + FASTOR_INLINE T* data() {return this->_data;} + //----------------------------------------------------------------------------------------------------------// + + // Scalar indexing + //----------------------------------------------------------------------------------------------------------// +#undef SCALAR_INDEXING_NONCONST_H +#undef SCALAR_INDEXING_CONST_H +#undef INDEX_RETRIEVER_H + #include "Fastor/tensor/IndexRetriever.h" + #include "Fastor/tensor/ScalarIndexing.h" +#define INDEX_RETRIEVER_H +#define SCALAR_INDEXING_NONCONST_H +#define SCALAR_INDEXING_CONST_H + + // Block indexing (all variants excluding iseq) + //----------------------------------------------------------------------------------------------------------// + template && !is_fixed_sequence_pack_v,bool> = false> + FASTOR_INLINE TensorViewExpr,sizeof...(Seq)> operator()(Seq ... _seqs) { + static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorViewExpr,sizeof...(Seq)>(*this, {_seqs...}); + } + + template,bool> = false> + FASTOR_INLINE TensorFixedViewExprnD,Fseq...> operator()(Fseq... ) { + static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); + return TensorFixedViewExprnD,Fseq...>(*this); + } + + FASTOR_INLINE TensorFilterViewExpr,Tensor,sizeof...(Rest)> + operator()(const Tensor &_fl) { + return TensorFilterViewExpr,Tensor,sizeof...(Rest)>(*this,_fl); + } + FASTOR_INLINE TensorFilterViewExpr,TensorMap,sizeof...(Rest)> + operator()(const TensorMap &_fl) { + return TensorFilterViewExpr,TensorMap,sizeof...(Rest)>(*this,_fl); + } + + //----------------------------------------------------------------------------------------------------------// + + // Expression templates evaluators + //----------------------------------------------------------------------------------------------------------// +#undef TENSOR_EVALUATOR_H + #include "Fastor/tensor/TensorEvaluator.h" +#define TENSOR_EVALUATOR_H + //----------------------------------------------------------------------------------------------------------// + + // No constructor should be added + // Provide generic AbstractTensors copy constructor though + //----------------------------------------------------------------------------------------------------------// + template + FASTOR_INLINE void operator=(const AbstractTensor& src) { + FASTOR_ASSERT(src.self().size()==size(), "TENSOR SIZE MISMATCH"); + assign(*this, src.self()); + } + + // AbstractTensor and scalar in-place operators + //----------------------------------------------------------------------------------------------------------// +#undef TENSOR_INPLACE_OPERATORS_H + #include "Fastor/tensor/TensorInplaceOperators.h" +#define TENSOR_INPLACE_OPERATORS_H + //----------------------------------------------------------------------------------------------------------// + + //----------------------------------------------------------------------------------------------------------// +#undef TENSOR_METHODS_CONST_H +#undef TENSOR_METHODS_NONCONST_H + #include "Fastor/tensor/TensorMethods.h" +#define TENSOR_METHODS_CONST_H +#define TENSOR_METHODS_NONCONST_H + //----------------------------------------------------------------------------------------------------------// + + // Converters + //----------------------------------------------------------------------------------------------------------// +#undef PODCONVERTERS_H + #include "Fastor/tensor/PODConverters.h" +#define PODCONVERTERS_H + //----------------------------------------------------------------------------------------------------------// + + // Cast method + //----------------------------------------------------------------------------------------------------------// + template + FASTOR_INLINE Tensor cast() const { + Tensor out; + U *out_data = out.data(); + for (FASTOR_INDEX i=0; i(_data[i]); + } + return out; + } + //----------------------------------------------------------------------------------------------------------// + + +private: + scalar_type* _data; +}; + +FASTOR_MAKE_OS_STREAM_TENSOR0(TensorMap) +FASTOR_MAKE_OS_STREAM_TENSOR1(TensorMap) +FASTOR_MAKE_OS_STREAM_TENSOR2(TensorMap) +FASTOR_MAKE_OS_STREAM_TENSORn(TensorMap) + + +template +FASTOR_INLINE void assign(AbstractTensor &dst, const TensorMap &src) { + if (dst.self().data()==src.data()) return; + trivial_assign(dst.self(),src); +} +template +FASTOR_INLINE void assign_add(AbstractTensor &dst, const TensorMap &src) { + trivial_assign_add(dst.self(),src); +} +template +FASTOR_INLINE void assign_sub(AbstractTensor &dst, const TensorMap &src) { + trivial_assign_sub(dst.self(),src); +} +template +FASTOR_INLINE void assign_mul(AbstractTensor &dst, const TensorMap &src) { + trivial_assign_mul(dst.self(),src); +} +template +FASTOR_INLINE void assign_div(AbstractTensor &dst, const TensorMap &src) { + trivial_assign_div(dst.self(),src); +} + + +} + +#endif // TENSOR_MAP_H diff --git a/noarch/include/Fastor/tensor/TensorMethods.h b/noarch/include/Fastor/tensor/TensorMethods.h new file mode 100644 index 00000000..99ef9e8e --- /dev/null +++ b/noarch/include/Fastor/tensor/TensorMethods.h @@ -0,0 +1,167 @@ +#ifndef TENSOR_METHODS_NONCONST_H +#define TENSOR_METHODS_NONCONST_H + +FASTOR_INLINE void fill(T num0) { + FASTOR_INDEX i = 0UL; + using V = simd_vector_type; + V _vec(num0); + for (; i(num0); + // using V = SIMDVector; + // V _vec; + // FASTOR_INDEX i=0; + // for (; ifill(static_cast(1)); +} + +FASTOR_INLINE void eye2() { + // Second order identity tensor (identity matrices) + static_assert(sizeof...(Rest)==2, "CANNOT BUILD AN IDENTITY TENSOR"); + static_assert(no_of_unique::value==1, "TENSOR MUST BE UNIFORM"); + constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; + zeros(); + for (FASTOR_INDEX i=0; i=2, "CANNOT BUILD AN IDENTITY TENSOR"); + static_assert(no_of_unique::value==1, "TENSOR MUST BE UNIFORM"); + zeros(); + + constexpr int ndim = sizeof...(Rest); + constexpr std::array maxes_a = {Rest...}; + std::array products; + std::fill(products.begin(),products.end(),0); + + for (int j=ndim-1; j>0; --j) { + int num = maxes_a[ndim-1]; + for (int k=0; k(1); + } +} + +FASTOR_INLINE void random() { + //! Populate tensor with random FP numbers + for (FASTOR_INDEX i=0; i; + V vec; + FASTOR_INDEX i = 0; + for (; i< ROUND_DOWN(size(),V::Size); i+=V::Size) { + vec.load(&tmp[size() - i - V::Size],false); + vec.reverse().store(&_data[i],false); + } + for (; i< size(); ++i) { + _data[i] = tmp[size()-i-1]; + } +} + +#endif // TENSOR_METHODS_NONCONST_H + + +#ifndef TENSOR_METHODS_CONST_H +#define TENSOR_METHODS_CONST_H + +FASTOR_INLINE T sum() const { + + if ((size()==0) || (size()==1)) return _data[0]; + using V = SIMDVector; + V vec = static_cast(0); + V _vec_in; + FASTOR_INDEX i = 0; + for (; i(0); + for (; i< size(); ++i) { + scalar += _data[i]; + } + return vec.sum() + scalar; +} + +FASTOR_INLINE T product() const { + + if ((size()==0) || (size()==1)) return _data[0]; + + using V = SIMDVector; + FASTOR_INDEX i = 0; + + V vec = static_cast(1); + for (; i< ROUND_DOWN(size(),V::Size); i+=V::Size) { + vec *= V(&_data[i],false); + } + T scalar = static_cast(1); + for (; i< size(); ++i) { + scalar *= _data[i]; + } + + return vec.product()*scalar; +} + +#endif // TENSOR_METHODS_CONST_H diff --git a/noarch/include/Fastor/tensor/TensorTraits.h b/noarch/include/Fastor/tensor/TensorTraits.h new file mode 100644 index 00000000..e98aaa8d --- /dev/null +++ b/noarch/include/Fastor/tensor/TensorTraits.h @@ -0,0 +1,303 @@ +#ifndef TENSOR_POST_META_H +#define TENSOR_POST_META_H + +#include "Fastor/tensor/Tensor.h" +#include "Fastor/tensor_algebra/indicial.h" + +namespace Fastor { + + +/* Classify/specialise Tensor as primitive if needed. + This specialisation hurts the performance of some specialised + kernels like matmul/norm/LU/inv that unroll aggressively unless + Tensor is specialised to wrap T only +*/ +//--------------------------------------------------------------------------------------------------------------------// +// template +// struct is_primitive> { +// static constexpr bool value = is_primitive_v_ ? true : false; +// }; +//--------------------------------------------------------------------------------------------------------------------// + + +/* Find the underlying scalar type of an expression */ +//--------------------------------------------------------------------------------------------------------------------// +template +struct scalar_type_finder { + using type = T; +}; + +template +struct scalar_type_finder> { + using type = T; +}; +// This specific specialisation is needed to avoid ambiguity for vectors +template +struct scalar_type_finder> { + using type = T; +}; +template +struct scalar_type_finder> { + using type = T; +}; +// This specific specialisation is needed to avoid ambiguity for vectors +template +struct scalar_type_finder> { + using type = T; +}; + +template