diff --git a/noarch/include/Fastor/Fastor.h b/noarch/include/Fastor/Fastor.h deleted file mode 100644 index 1e066aaa..00000000 --- a/noarch/include/Fastor/Fastor.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef FASTOR_ALL_INCLUDE_H -#define FASTOR_ALL_INCLUDE_H - -#if defined(_MSC_VER) -// Disable MSVC warnings for macros -#pragma warning (disable: 4003) -// Disable MSVC warnings for conversion -#pragma warning (disable: 4244) -// Disable MSVC unreferenced parameters -#pragma warning (disable: 4100) -// Disable MSVC narrowing conversion -#pragma warning (disable: 4267) -#endif - -#include "config/config.h" -#include "util/util.h" -#include "simd_vector/SIMDVector.h" -#include "simd_math/simd_math.h" -#include "tensor/Tensor.h" -#include "tensor/TensorMap.h" -#include "tensor/TensorIO.h" -#include "tensor/TensorFunctions.h" -#include "tensor/AbstractTensorFunctions.h" -#include "tensor_algebra/einsum.h" -#include "tensor_algebra/network_einsum.h" -#include "tensor_algebra/einsum_explicit.h" -#include "tensor_algebra/abstract_contraction.h" -#include "expressions/expressions.h" -#include "backend/voigt.h" - -#if defined(_MSC_VER) -#pragma warning (default: 4003) -#pragma warning (default: 4244) -#pragma warning (default: 4100) -#pragma warning (default: 4267) -#endif - -#endif // FASTOR_ALL_INCLUDE_H - diff --git a/noarch/include/Fastor/backend/adjoint.h b/noarch/include/Fastor/backend/adjoint.h deleted file mode 100644 index 8a5dd1eb..00000000 --- a/noarch/include/Fastor/backend/adjoint.h +++ /dev/null @@ -1,105 +0,0 @@ -#ifndef ADJOINT_H -#define ADJOINT_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/extintrin.h" - -namespace Fastor { - -template, bool> = false> -FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); - -template, bool> = false> -FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { - *dst = *src; -} - -template, bool> = false> -FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T src0 = src[0]; - T src1 = src[1]; - T src2 = src[2]; - T src3 = src[3]; - - /* Compute adjoint: */ - dst[0] = + src3; - dst[1] = - src1; - dst[2] = - src2; - dst[3] = + src0; -} - -template, bool> = false> -FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T src0 = src[0]; - T src1 = src[1]; - T src2 = src[2]; - T src3 = src[3]; - T src4 = src[4]; - T src5 = src[5]; - T src6 = src[6]; - T src7 = src[7]; - T src8 = src[8]; - - /* Compute adjoint: */ - dst[0] = + src4 * src8 - src5 * src7; - dst[1] = - src1 * src8 + src2 * src7; - dst[2] = + src1 * src5 - src2 * src4; - dst[3] = - src3 * src8 + src5 * src6; - dst[4] = + src0 * src8 - src2 * src6; - dst[5] = - src0 * src5 + src2 * src3; - dst[6] = + src3 * src7 - src4 * src6; - dst[7] = - src0 * src7 + src1 * src6; - dst[8] = + src0 * src4 - src1 * src3; -} - -template, bool> = false> -FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - /* Compute adjoint: */ - T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2]; - T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1]; - T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1]; - - dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3; - dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3; - - T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0]; - T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0]; - - dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5; - dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5; - - t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0]; - - dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1; - dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1; - dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1; - dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1; - - t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2]; - t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1]; - t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1]; - - dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3; - dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3; - - t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0]; - t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0]; - - dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5; - dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5; - - t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0]; - - dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1; - dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1; - dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1; - dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1; -} - -} // end of namespace Fastor - -#endif // ADJOINT_H diff --git a/noarch/include/Fastor/backend/backend.h b/noarch/include/Fastor/backend/backend.h deleted file mode 100644 index c232b650..00000000 --- a/noarch/include/Fastor/backend/backend.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef BACKEND_H -#define BACKEND_H - - -#include "Fastor/backend/adjoint.h" -#include "Fastor/backend/cofactor.h" -#include "Fastor/backend/cyclic_0.h" -#include "Fastor/backend/determinant.h" -#include "Fastor/backend/doublecontract.h" -#include "Fastor/backend/dyadic.h" -#include "Fastor/backend/inner.h" -#include "Fastor/backend/inverse.h" -#include "Fastor/backend/lufact.h" -#include "Fastor/backend/lut_inverse.h" -#include "Fastor/backend/matmul/matmul.h" -#include "Fastor/backend/matmul/tmatmul.h" -#include "Fastor/backend/norm.h" -#include "Fastor/backend/outer.h" -#include "Fastor/backend/tensor_cross.h" -#include "Fastor/backend/trace.h" -#include "Fastor/backend/transpose/transpose.h" - -#endif // BACKEND_H - diff --git a/noarch/include/Fastor/backend/cofactor.h b/noarch/include/Fastor/backend/cofactor.h deleted file mode 100644 index 8bece5af..00000000 --- a/noarch/include/Fastor/backend/cofactor.h +++ /dev/null @@ -1,105 +0,0 @@ -#ifndef COFACTOR_H -#define COFACTOR_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/extintrin.h" - -namespace Fastor { - -template, bool> = false> -FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); - -template, bool> = false> -FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { - *dst = *src; -} - -template, bool> = false> -FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T src0 = src[0]; - T src1 = src[1]; - T src2 = src[2]; - T src3 = src[3]; - - /* Compute cofactor: */ - dst[0] = + src3; - dst[1] = - src2; - dst[2] = - src1; - dst[3] = + src0; -} - -template, bool> = false> -FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T src0 = src[0]; - T src1 = src[1]; - T src2 = src[2]; - T src3 = src[3]; - T src4 = src[4]; - T src5 = src[5]; - T src6 = src[6]; - T src7 = src[7]; - T src8 = src[8]; - - /* Compute cofactor: */ - dst[0] = + src4 * src8 - src5 * src7; - dst[1] = - src3 * src8 + src5 * src6; - dst[2] = + src3 * src7 - src4 * src6; - dst[3] = - src1 * src8 + src2 * src7; - dst[4] = + src0 * src8 - src2 * src6; - dst[5] = - src0 * src7 + src1 * src6; - dst[6] = + src1 * src5 - src2 * src4; - dst[7] = - src0 * src5 + src2 * src3; - dst[8] = + src0 * src4 - src1 * src3; -} - -template, bool> = false> -FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - /* Compute cofactor: */ - T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2]; - T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1]; - T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1]; - - dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3; - dst[4] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3; - - T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0]; - T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0]; - - dst[1] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5; - dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5; - - t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0]; - - dst[2] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1; - dst[6] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1; - dst[3] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1; - dst[7] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1; - - t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2]; - t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1]; - t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1]; - - dst[8] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3; - dst[12] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3; - - t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0]; - t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0]; - - dst[9] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5; - dst[13] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5; - - t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0]; - - dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1; - dst[14] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1; - dst[11] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1; - dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1; -} - -} - -#endif // COFACTOR_H diff --git a/noarch/include/Fastor/backend/cyclic_0.h b/noarch/include/Fastor/backend/cyclic_0.h deleted file mode 100644 index c0c643ff..00000000 --- a/noarch/include/Fastor/backend/cyclic_0.h +++ /dev/null @@ -1,172 +0,0 @@ -#ifndef CYCLIC_0_H -#define CYCLIC_0_H - -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" - -namespace Fastor { - - -//! Version 0 of cyclic product of two second order tensors i.e. C_ijkl = A_ik * B_jl - -template -FASTOR_HINT_INLINE void _cyclic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - for (size_t i=0; i -FASTOR_HINT_INLINE void _cyclic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - __m256d as = _mm256_load_pd(a); - __m256d bs = _mm256_load_pd(b); - - // First 2x2 block - __m256d c0 = _mm256_mul_pd(as,bs); - _mm_store_sd(out,_mm256_castpd256_pd128(c0)); - _mm_store_sd(out+1,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1)); - __m128d c0_high = _mm256_extractf128_pd(c0,0x1); - _mm_store_sd(out+3,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1)); - _mm_store_sd(out+4,_mm_shuffle_pd(c0_high,c0_high,0x1)); - // c_02 - __m128d a0 = _mm256_castpd256_pd128(as); - __m128d b0 = _mm256_castpd256_pd128(bs); - b0 = _mm_shuffle_pd(b0,b0,0x1); - __m128d c1 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0))); - _mm_store_sd(out+2,c1); - _mm_store_sd(out+6,c1); - // c_12 - __m128d a1 = _mm256_extractf128_pd(as,0x1); - __m128d b1 = _mm256_extractf128_pd(bs,0x1); - b1 = _mm_shuffle_pd(b1,b1,0x1); - __m128d c2 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1))); - _mm_store_sd(out+5,c2); - _mm_store_sd(out+7,c2); - // c_22 - __m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b1))); - _mm_store_sd(out+8,c3); -} - -template<> -FASTOR_HINT_INLINE void _cyclic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - // 34+ OPS - __m256d a_low = _mm256_load_pd(a); - __m256d a_high = _mm256_load_pd(a+4); - __m128d a_end = _mm_load_sd(a+8); - - __m256d b_low = _mm256_load_pd(b); - __m256d b_high = _mm256_load_pd(b+4); - __m128d b_end = _mm_load_sd(b+8); - - // The first 3x3 block - __m256d c0 = _mm256_mul_pd(a_low,b_low); - __m256d c1 = _mm256_mul_pd(a_high,b_high); - __m128d c2 = _mm_mul_sd(a_end,b_end); - - _mm256_store_pd(out,c0); - _mm_store_sd(out+6,_mm_set_sd(_mm256_get1_pd(c0))); - _mm256_storeu_pd(out+7,c1); -// _mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0))); - _mm_store_sd(out+12,_mm256_extractf128_pd(c0,0x1)); - _mm_store_sd(out+13,_mm_set_sd(_mm256_get1_pd(c1))); - _mm_store_sd(out+14,c2); - - // rest - // c_03 - __m128d a0 = _mm256_castpd256_pd128(a_low); - __m128d b0 = _mm256_castpd256_pd128(b_low); - b0 = _mm_shuffle_pd(b0,b0,0x1); - __m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0))); - _mm_store_sd(out+3,c3); - _mm_store_sd(out+18,c3); - // c_04 - __m128d a1 = _mm_set_pd(_mm256_get0_pd(a_low),_mm256_get2_pd(a_low)); - __m128d b1 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get0_pd(b_low)); - __m128d c4 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1))); - _mm_store_sd(out+4,c4); - _mm_store_sd(out+24,c4); - // c_05 - __m128d a2 = _mm_set_pd(_mm256_get1_pd(a_low),_mm256_get2_pd(a_low)); - __m128d b2 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get1_pd(b_low)); - __m128d c5 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b2))); - _mm_store_sd(out+5,c5); - _mm_store_sd(out+30,c5); - // c_13 - __m128d a3 = _mm_setr_pd(_mm256_get3_pd(a_low),_mm256_get0_pd(a_high)); - __m128d b3 = _mm_setr_pd(_mm256_get0_pd(b_high),_mm256_get3_pd(b_low)); - __m128d c6 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b3))); - _mm_store_sd(out+9,c6); - _mm_store_sd(out+19,c6); - // c_14 - __m128d a4 = _mm_set_pd(_mm256_get3_pd(a_low),_mm256_get1_pd(a_high)); - __m128d b4 = _mm_set_pd(_mm256_get1_pd(b_high),_mm256_get3_pd(b_low)); - __m128d c7 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b4))); - _mm_store_sd(out+10,c7); - _mm_store_sd(out+25,c7); - // c_15 - __m128d a5 = _mm256_castpd256_pd128(a_high); - __m128d b5 = _mm256_castpd256_pd128(b_high); - b5 = _mm_shuffle_pd(b5,b5,0x1); - __m128d c8 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,b5))); - _mm_store_sd(out+11,c8); - _mm_store_sd(out+31,c8); - // c_23 - __m128d a6 = _mm256_extractf128_pd(a_high,0x1); - __m128d b6 = _mm256_extractf128_pd(b_high,0x1); - b6 = _mm_shuffle_pd(b6,b6,0x1); - __m128d c9 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a6,b6))); - _mm_store_sd(out+15,c9); - _mm_store_sd(out+20,c9); - // c_24 - __m128d a7 = _mm_set_pd(_mm256_get2_pd(a_high),_mm_get0_pd(a_end)); - __m128d b7 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get2_pd(b_high)); - __m128d c10 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a7,b7))); - _mm_store_sd(out+16,c10); - _mm_store_sd(out+26,c10); - // c_25 - __m128d a8 = _mm_set_pd(_mm256_get3_pd(a_high),_mm_get0_pd(a_end)); - __m128d b8 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get3_pd(b_high)); - __m128d c11 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a8,b8))); - _mm_store_sd(out+17,c11); - _mm_store_sd(out+32,c11); - // c_33 - __m128d c12 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b3))); - _mm_store_sd(out+21,c12); - // c_34 - __m128d c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b4))); - _mm_store_sd(out+22,c13); - // c_43 - c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b6))); - _mm_store_sd(out+27,c13); - // c_35 - __m128d c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,_mm_shuffle_pd(b5,b5,0x1)))); - _mm_store_sd(out+23,c14); - // c_53 - c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b6))); - _mm_store_sd(out+33,c14); - // c_44 - __m128d c15 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b7))); - _mm_store_sd(out+28,c15); - // c_45 - __m128d c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b8))); - _mm_store_sd(out+29,c16); - c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b7))); - _mm_store_sd(out+34,c16); - // c_55 - __m128d c17 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,_mm_shuffle_pd(b8,b8,0x1)))); - _mm_store_sd(out+35,c17); -} - -#endif - -} - -#endif // CYCLIC_0_H - diff --git a/noarch/include/Fastor/backend/determinant.h b/noarch/include/Fastor/backend/determinant.h deleted file mode 100644 index e3cd2f2e..00000000 --- a/noarch/include/Fastor/backend/determinant.h +++ /dev/null @@ -1,116 +0,0 @@ -#ifndef DETERMINANT_H -#define DETERMINANT_H - -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/meta/tensor_meta.h" - -namespace Fastor { - - -#ifndef FASTOR_AVX_IMPL -template::type=0> -#else -template::value && - !std::is_same::value && M==2 && N==2, bool>::type=0> -#endif -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { - return a[0] * a[3] - a[1] * a[2]; -} - - -#ifndef FASTOR_AVX_IMPL -template::type=0> -#else -template::value && - !std::is_same::value && M==3 && N==3, bool>::type=0> -#endif -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { - return a[0]*a[4]*a[8] + a[1]*a[5]*a[6] + a[2]*a[3]*a[7] - a[2]*a[4]*a[6] - a[1]*a[3]*a[8] - a[0]*a[5]*a[7]; -} - -template::type=0> -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT m) { - return m[12] * m[9] * m[6] * m[3] - m[8] * m[13] * m[6] * m[3] - - m[12] * m[5] * m[10] * m[3] + m[4] * m[13] * m[10] * m[3] + - m[8] * m[5] * m[14] * m[3] - m[4] * m[9] * m[14] * m[3] - - m[12] * m[9] * m[2] * m[7] + m[8] * m[13] * m[2] * m[7] + - m[12] * m[1] * m[10] * m[7] - m[0] * m[13] * m[10] * m[7] - - m[8] * m[1] * m[14] * m[7] + m[0] * m[9] * m[14] * m[7] + - m[12] * m[5] * m[2] * m[11] - m[4] * m[13] * m[2] * m[11] - - m[12] * m[1] * m[6] * m[11] + m[0] * m[13] * m[6] * m[11] + - m[4] * m[1] * m[14] * m[11] - m[0] * m[5] * m[14] * m[11] - - m[8] * m[5] * m[2] * m[15] + m[4] * m[9] * m[2] * m[15] + - m[8] * m[1] * m[6] * m[15] - m[0] * m[9] * m[6] * m[15] - - m[4] * m[1] * m[10] * m[15] + m[0] * m[5] * m[10] * m[15]; -} - -template::value || is_greater::value, bool>::type=0> -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { - static_assert(M==N, "2D TENSOR MUST BE SQUARE"); - assert(false && "2D TENSOR MUST BE SQUARE"); -} - - -#ifdef FASTOR_AVX_IMPL -template::value && - std::is_same::value && M==2 && N==2, bool>::type=0> -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { - // 10 OPS - __m128 a1 = _mm_load_ps(a); - __m128 a2 = _mm_shuffle_ps(a1,a1,_MM_SHUFFLE(0,1,2,3)); - __m128 a3 = _mm_mul_ps(a1,a2); - return _mm_cvtss_f32(_mm_sub_ss(a3,_mm_shuffle_ps(a3,a3,_MM_SHUFFLE(0,0,0,1)))); -} - -template::value && - std::is_same::value && M==3 && N==3, bool>::type=0> -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { - // ?? OPS - __m128 r0 = {a[2],a[1],a[0],0.}; - __m128 r1 = {a[3],a[5],a[4],0.}; - __m128 r2 = {a[7],a[6],a[8],0.}; - - __m128 r3 = {a[6],a[7],a[8],0.}; - __m128 r4 = {a[4],a[5],a[3],0.}; - __m128 r5 = {a[2],a[0],a[1],0.}; - - __m128 out0 = _mm_mul_ps(r2,_mm_mul_ps(r0,r1)); - __m128 out1 = _mm_mul_ps(r3,_mm_mul_ps(r4,r5)); - - return _mm_cvtss_f32(_mm_sub_ss(_add_ps(out0),_add_ps(out1))); -} - -template::value && - !std::is_same::value && M==2 && N==2, bool>::type=0> -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { - // 10 OPS - __m128d a1 = _mm_load_pd(a); - __m128d a2 = _mm_load_pd(a+2); - __m128d a3 = _mm_mul_pd(a1,_mm_shuffle_pd(a2,a2,1)); - return _mm_cvtsd_f64(_mm_sub_pd(a3,_mm_shuffle_pd(a3,a3,0x1))); -} - -template::value && - !std::is_same::value && M==3 && N==3, bool>::type=0> -FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) { - // ?? OPS - __m256d r0 = {a[2],a[1],a[0],0.}; - __m256d r1 = {a[3],a[5],a[4],0.}; - __m256d r2 = {a[7],a[6],a[8],0.}; - - __m256d r3 = {a[6],a[7],a[8],0.}; - __m256d r4 = {a[4],a[5],a[3],0.}; - __m256d r5 = {a[2],a[0],a[1],0.}; - - __m256d out0 = _mm256_mul_pd(r2,_mm256_mul_pd(r0,r1)); - __m256d out1 = _mm256_mul_pd(r3,_mm256_mul_pd(r4,r5)); - - return _mm_cvtsd_f64(_mm_sub_sd(_add_pd(out0),_add_pd(out1))); -} -#endif - -} - -#endif // DETERMINANT_H - diff --git a/noarch/include/Fastor/backend/doublecontract.h b/noarch/include/Fastor/backend/doublecontract.h deleted file mode 100644 index a2cf1599..00000000 --- a/noarch/include/Fastor/backend/doublecontract.h +++ /dev/null @@ -1,128 +0,0 @@ -#ifndef DOUBLECONTRACT_H -#define DOUBLECONTRACT_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" - -namespace Fastor { - - -template,M*N>::Size, M*N >, bool> = false> -FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { - - constexpr size_t Size = M*N; - using V = choose_best_simd_t,Size>; - - V omm0; - size_t i = 0; - for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) { - const V amm0(&a[i],false); - const V bmm0(&b[i],false); - - omm0 = fmadd(amm0,bmm0,omm0); - } - T scalar = static_cast(0); - for (; i < Size; ++i) { - scalar += a[i]*b[i]; - } - return omm0.sum() + scalar; -} - -template,M*N>::Size, M*N >, bool> = false> -FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { - - constexpr size_t Size = M*N; - using V = choose_best_simd_t,Size>; - - V omm0, omm1, omm2, omm3; - size_t i = 0; - for (; i< ROUND_DOWN(Size,4*V::Size); i+=4*V::Size) { - const V amm0(&a[i],false); - const V amm1(&a[i+V::Size],false); - const V amm2(&a[i+2*V::Size],false); - const V amm3(&a[i+3*V::Size],false); - - const V bmm0(&b[i],false); - const V bmm1(&b[i+V::Size],false); - const V bmm2(&b[i+2*V::Size],false); - const V bmm3(&b[i+3*V::Size],false); - - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm1,bmm1,omm1); - omm2 = fmadd(amm2,bmm2,omm2); - omm3 = fmadd(amm3,bmm3,omm3); - } - for (; i< ROUND_DOWN(Size,2*V::Size); i+=2*V::Size) { - const V amm0(&a[i],false); - const V amm1(&a[i+V::Size],false); - - const V bmm0(&b[i],false); - const V bmm1(&b[i+V::Size],false); - - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm1,bmm1,omm1); - } - for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) { - const V amm0(&a[i],false); - const V bmm0(&b[i],false); - - omm0 = fmadd(amm0,bmm0,omm0); - } - T scalar = static_cast(0); - for (; i < Size; ++i) { - scalar += a[i]*b[i]; - } - return (omm0 + omm1 + omm2 + omm3).sum() + scalar; -} - - -#ifdef FASTOR_AVX_IMPL - -template<> -FASTOR_INLINE float _doublecontract(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) { - return _mm_sum_ps(_mm_mul_ps(_mm_loadu_ps(a),_mm_loadu_ps(b))); -} - -template<> -FASTOR_INLINE float _doublecontract(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) { - float r1 = _mm256_sum_ps(_mm256_mul_ps(_mm256_loadu_ps(a),_mm256_loadu_ps(b))); - float r2 = _mm_sum_ps(_mm_mul_ss(_mm_load_ss(a+8),_mm_load_ss(b+8))); - return r1+r2; -} - -template<> -FASTOR_INLINE double _doublecontract(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) { - return _mm256_sum_pd(_mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b))); -} - -template<> -FASTOR_INLINE double _doublecontract(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) { - __m256d r1 = _mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b)); - __m256d r2 = _mm256_mul_pd(_mm256_loadu_pd(a+4),_mm256_loadu_pd(b+4)); - __m128d r3 = _mm_mul_sd(_mm_load_sd(a+8),_mm_load_sd(b+8)); - __m128d r4 = _add_pd(_mm256_add_pd(r1,r2)); - __m128d summ = _mm_add_pd(_add_pd(r3),r4); - return _mm_cvtsd_f64(summ); -} - - -#endif - -// doublecontract and transpose -template -FASTOR_INLINE T _doublecontract_transpose(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { - T dc = static_cast(0); - for (FASTOR_INDEX i=0; i -FASTOR_INLINE -void _dyadic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - using V = typename internal::choose_best_simd_type,SizeB>::type; - // constexpr size_t unrollOuterloop = 4UL; - // constexpr size_t M0 = SizeA / unrollOuterloop * unrollOuterloop; - // Unrolling the inner loop beyond 4 does not give any benefit neither on AVX - // nor on AVX512 - - size_t i = 0; - for (; i -FASTOR_INLINE -void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - - __m128 vec_a = _mm_load_ps(a); - __m128 vec_b = _mm_load_ps(b); - - __m128 a00 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0)); - __m128 a01 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1)); - __m128 a10 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2)); - __m128 a11 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(3,3,3,3)); - - __m256 a0001 = _mm256_castps128_ps256(a00); - a0001 = _mm256_insertf128_ps(a0001,a01,0x1); - __m256 a1011 = _mm256_castps128_ps256(a10); - a1011 = _mm256_insertf128_ps(a1011,a11,0x1); - - __m256 vec_b2 = _mm256_castps128_ps256(vec_b); - vec_b2 = _mm256_insertf128_ps(vec_b2,vec_b,0x1); - - _mm256_store_ps(out,_mm256_mul_ps(a0001,vec_b2)); - _mm256_store_ps(out+8,_mm256_mul_ps(a1011,vec_b2)); -} - - -// Outer product (2x2) x (2x2) -template<> -FASTOR_INLINE -void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - - __m256d vec_b = _mm256_loadu_pd(b); - - __m256d a00 = _mm256_set1_pd(a[0]); - __m256d a01 = _mm256_set1_pd(a[1]); - __m256d a10 = _mm256_set1_pd(a[2]); - __m256d a11 = _mm256_set1_pd(a[3]); - - _mm256_storeu_pd(out,_mm256_mul_pd(a00,vec_b)); - _mm256_storeu_pd(out+4,_mm256_mul_pd(a01,vec_b)); - _mm256_storeu_pd(out+8,_mm256_mul_pd(a10,vec_b)); - _mm256_storeu_pd(out+12,_mm256_mul_pd(a11,vec_b)); -} - - - -// Outer product (1x2) x (1x2) [for vectors] -template<> -FASTOR_INLINE -void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - // 7 OPS - __m128 vec_a = _mm_loadu_ps(a); - __m128 vec_b = _mm_loadu_ps(b); - - vec_a = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,0,0)); - vec_b = _mm_shuffle_ps(vec_b,vec_b,_MM_SHUFFLE(1,0,1,0)); - - _mm_storeu_ps(out,_mm_mul_ps(vec_a,vec_b)); -} - - -// Outer product (1x2) x (1x2) [for vectors] -template<> -FASTOR_INLINE -void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - // IVY 9 OPS / HW 13 OPS - __m128d vec_a = _mm_loadu_pd(a); - __m128d vec_b = _mm_loadu_pd(b); - - __m128d a0 = _mm_shuffle_pd(vec_a,vec_a,0x0); - __m128d a1 = _mm_shuffle_pd(vec_a,vec_a,0x3); - __m256d as = _mm256_castpd128_pd256(a0); - as = _mm256_insertf128_pd(as,a1,0x1); - - __m256d bs = _mm256_castpd128_pd256(vec_b); - bs = _mm256_insertf128_pd(bs,vec_b,0x1); - - _mm256_storeu_pd(out,_mm256_mul_pd(as,bs)); -} - - -// Outer product (1x3) x (1x3) [for vectors] -template<> -FASTOR_INLINE -void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - // 18 OPS - __m128 vec_a = _mm_loadu_ps(a); - __m128 vec_b = _mm_loadu_ps(b); - - __m128 a0 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0)); - __m128 a1 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1)); - __m128 a2 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2)); - - _mm_storeu_ps(out,_mm_mul_ps(a0,vec_b)); - _mm_storeu_ps(out+3,_mm_mul_ps(a1,vec_b)); - _mm_storeu_ps(out+6,_mm_mul_ps(a2,vec_b)); -} - - -// Outer product (1x3) x (1x3) [for vectors] -template<> -FASTOR_INLINE -void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - // 15 OPS + set OPS - __m256d vec_b = _mm256_loadu_pd(b); - __m256d a0 = _mm256_set1_pd(a[0]); - __m256d a1 = _mm256_set1_pd(a[1]); - __m256d a2 = _mm256_set1_pd(a[2]); - - _mm256_storeu_pd(out,_mm256_mul_pd(a0,vec_b)); - _mm256_storeu_pd(out+3,_mm256_mul_pd(a1,vec_b)); - _mm256_storeu_pd(out+6,_mm256_mul_pd(a2,vec_b)); -} - -#endif -//--------------------------------------------------------------------------------------------------- - - -//--------------------------------------------------------------------------------------------------- -// Outer product of scalars -template<> -FASTOR_INLINE -void _dyadic(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - out[0] = a[0]*b[0]; -} -template<> -FASTOR_INLINE -void _dyadic(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - out[0] = a[0]*b[0]; -} -//--------------------------------------------------------------------------------------------------- - -} - -#endif // DYADIC_H diff --git a/noarch/include/Fastor/backend/inner.h b/noarch/include/Fastor/backend/inner.h deleted file mode 100644 index cb0619b3..00000000 --- a/noarch/include/Fastor/backend/inner.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef INNER_H_ -#define INNER_H_ - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/doublecontract.h" - -namespace Fastor { - -/* The dependency on doublecontract here is on purpose - as it creates a necessary layer of indirection to avoid - the case where M == 0 -*/ -template, bool> = false> -FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { - return _doublecontract(a,b); -} - -template = false> -FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) { - return (*a)*(*b); -} - -} // end of namespace Fastor - -#endif // INNER_H_ diff --git a/noarch/include/Fastor/backend/inverse.h b/noarch/include/Fastor/backend/inverse.h deleted file mode 100644 index 48274664..00000000 --- a/noarch/include/Fastor/backend/inverse.h +++ /dev/null @@ -1,450 +0,0 @@ -#ifndef INVERSE_H -#define INVERSE_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/extintrin.h" - -namespace Fastor { - -template, bool> = false> -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); - -template, bool> = false> -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { - *dst = T(1) / (*src); -} - -#ifdef FASTOR_SSE2_IMPL -template && !is_same_v_ && !is_same_v_, bool> = false> -#else -template, bool> = false> -#endif -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T det; - - T src0 = src[0]; - T src1 = src[1]; - T src2 = src[2]; - T src3 = src[3]; - - /* Compute adjoint: */ - dst[0] = + src3; - dst[1] = - src1; - dst[2] = - src2; - dst[3] = + src0; - - /* Compute determinant: */ - det = src0 * dst[0] + src1 * dst[2]; - - /* Multiply adjoint with reciprocal of determinant: */ - det = T(1.0) / det; - dst[0] *= det; - dst[1] *= det; - dst[2] *= det; - dst[3] *= det; -} - -#ifdef FASTOR_SSE2_IMPL -template && is_same_v_, bool> = false> -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - // This is much superior to the scalar code as - // gcc/clang can't auto-vectorise the scalar code - - // 6 shuffles + 1 add + 1 mul + 1 div - // Sky 6 + 4 + 4 + 11 = 25 - - __m128 mat = _mm_loadu_ps(src); - // xor to swap off-diagonals sings - __m128 nmat = _mm_neg_ps(mat); - // two shuffles to get adjoint - __m128 adj = _mm_shuffle_ps(mat, nmat, 0x009C ); - adj = _mm_shuffle_ps(adj, adj , 0x39 ); - - // compute determinat - __m128 tmp0 = _mm_shuffle_ps(mat , mat , 0x00D8); - tmp0 = _mm_mul_ps (adj , tmp0 ); - __m128 tmp1 = _mm_shuffle_ps(tmp0, tmp0, 0x1 ); - __m128 det = _mm_div_ss (ONEPS, _mm_add_ss(tmp0,tmp1)); - // broadcast det to all elements of __m128 - det = _mm_shuffle_ps(det, det, 0x0 ); - // divide adjoint by determinant - __m128 inv = _mm_mul_ps (adj, det); - - _mm_storeu_ps(dst, inv); -} - -template && is_same_v_, bool> = false> -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - // This is much superior to the scalar code as - // gcc/clang can't auto-vectorise the scalar code - - // 8 shuffles + 1 add + 3 mul + 1 div - // Sky 8 + 4 + 12 + 14 = 38 - - __m128d row0 = _mm_loadu_pd(src); - __m128d row1 = _mm_loadu_pd(src+2); - - __m128d tmp = row0; - row0 = _mm_shuffle_pd(row0,_mm_neg_pd(row0),0x2); - row1 = _mm_shuffle_pd(_mm_neg_pd(row1),row1,0x2); - // these two registers hold the adjoint - __m128d irow0 = _mm_shuffle_pd(row1,row0,0x3); - __m128d irow1 = _mm_shuffle_pd(row1,row0,0x0); - // dot product to compute determinant - __m128d det = _mm_mul_pd(tmp,_mm_reverse_pd(row1)); - det = _mm_add_pd(det,_mm_reverse_pd(det)); - // one by determinant - __m128d invdet = _mm_div_pd(_mm_set1_pd(1.0),det); - // scale - irow0 = _mm_mul_pd(irow0,invdet); - irow1 = _mm_mul_pd(irow1,invdet); - - _mm_storeu_pd(dst ,irow0); - _mm_storeu_pd(dst+2,irow1); -} -#endif - -template, bool> = false> -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T det; - - T src0 = src[0]; - T src1 = src[1]; - T src2 = src[2]; - T src3 = src[3]; - T src4 = src[4]; - T src5 = src[5]; - T src6 = src[6]; - T src7 = src[7]; - T src8 = src[8]; - - /* Compute adjoint: */ - dst[0] = + src4 * src8 - src5 * src7; - dst[1] = - src1 * src8 + src2 * src7; - dst[2] = + src1 * src5 - src2 * src4; - dst[3] = - src3 * src8 + src5 * src6; - dst[4] = + src0 * src8 - src2 * src6; - dst[5] = - src0 * src5 + src2 * src3; - dst[6] = + src3 * src7 - src4 * src6; - dst[7] = - src0 * src7 + src1 * src6; - dst[8] = + src0 * src4 - src1 * src3; - - /* Compute determinant: */ - det = src0 * dst[0] + src1 * dst[3] + src2 * dst[6]; - - /* Multiply adjoint with reciprocal of determinant: */ - det = T(1.0) / det; - - dst[0] *= det; - dst[1] *= det; - dst[2] *= det; - dst[3] *= det; - dst[4] *= det; - dst[5] *= det; - dst[6] *= det; - dst[7] *= det; - dst[8] *= det; -} - - -#ifdef FASTOR_SSE2_IMPL -template && !is_same_v_ && !is_same_v_, bool> = false> -#else -template, bool> = false> -#endif -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2]; - T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1]; - T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1]; - - dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3; - dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3; - - T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0]; - T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0]; - - dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5; - dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5; - - t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0]; - - dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1; - dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1; - dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1; - dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1; - - t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2]; - t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1]; - t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1]; - - dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3; - dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3; - - t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0]; - t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0]; - - dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5; - dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5; - - t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0]; - - dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1; - dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1; - dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1; - dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1; - - const T __det = src[0]*dst[0] + src[1]*dst[4] + src[2]*dst[8] + src[3]*dst[12]; - const T __invdet = T(1)/__det; - for (int i=0; i<16; ++i) - dst[i] *= __invdet; -} - - -#ifdef FASTOR_SSE2_IMPL -template && is_same_v_, bool> = false> -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - // From Intel's SSE matrix library - // The inverse is calculated using "Divide and Conquer" technique. The - // original matrix is divide into four 2x2 sub-matrices. Since each - // register of the matrix holds two elements, the smaller matrices are - // consisted of two registers. Hence we get a better locality of the - // calculations. - - const __m128 p4f_sign_PNNP = _mm_castsi128_ps(_mm_set_epi32(0x00000000, 0x80000000, 0x80000000, 0x00000000)); - - // Load the full matrix into registers - __m128 _L1 = _mm_loadu_ps(src + 0); - __m128 _L2 = _mm_loadu_ps(src + 4); - __m128 _L3 = _mm_loadu_ps(src + 8); - __m128 _L4 = _mm_loadu_ps(src + 12); - - __m128 A, B, C, D; // the four sub-matrices - - A = _mm_movelh_ps(_L1, _L2); - B = _mm_movehl_ps(_L2, _L1); - C = _mm_movelh_ps(_L3, _L4); - D = _mm_movehl_ps(_L4, _L3); - - // partial inverse of the sub-matrices - __m128 iA, iB, iC, iD, DC, AB; - __m128 dA, dB, dC, dD; // determinant of the sub-matrices - __m128 det, d, d1, d2; - __m128 rd; // reciprocal of the determinant - - // AB = A# * B - AB = _mm_mul_ps(_mm_shuffle_ps(A,A,0x0F), B); - AB = _mm_sub_ps(AB,_mm_mul_ps(_mm_shuffle_ps(A,A,0xA5), _mm_shuffle_ps(B,B,0x4E))); - // DC = D# * C - DC = _mm_mul_ps(_mm_shuffle_ps(D,D,0x0F), C); - DC = _mm_sub_ps(DC,_mm_mul_ps(_mm_shuffle_ps(D,D,0xA5), _mm_shuffle_ps(C,C,0x4E))); - - // dA = |A| - dA = _mm_mul_ps(_mm_shuffle_ps(A, A, 0x5F),A); - dA = _mm_sub_ss(dA, _mm_movehl_ps(dA,dA)); - // dB = |B| - dB = _mm_mul_ps(_mm_shuffle_ps(B, B, 0x5F),B); - dB = _mm_sub_ss(dB, _mm_movehl_ps(dB,dB)); - - // dC = |C| - dC = _mm_mul_ps(_mm_shuffle_ps(C, C, 0x5F),C); - dC = _mm_sub_ss(dC, _mm_movehl_ps(dC,dC)); - // dD = |D| - dD = _mm_mul_ps(_mm_shuffle_ps(D, D, 0x5F),D); - dD = _mm_sub_ss(dD, _mm_movehl_ps(dD,dD)); - - // d = trace(AB*DC) = trace(A#*B*D#*C) - d = _mm_mul_ps(_mm_shuffle_ps(DC,DC,0xD8),AB); - - // iD = C*A#*B - iD = _mm_mul_ps(_mm_shuffle_ps(C,C,0xA0), _mm_movelh_ps(AB,AB)); - iD = _mm_add_ps(iD,_mm_mul_ps(_mm_shuffle_ps(C,C,0xF5), _mm_movehl_ps(AB,AB))); - // iA = B*D#*C - iA = _mm_mul_ps(_mm_shuffle_ps(B,B,0xA0), _mm_movelh_ps(DC,DC)); - iA = _mm_add_ps(iA,_mm_mul_ps(_mm_shuffle_ps(B,B,0xF5), _mm_movehl_ps(DC,DC))); - - // d = trace(AB*DC) = trace(A#*B*D#*C) [continue] - d = _mm_add_ps(d, _mm_movehl_ps(d, d)); - d = _mm_add_ss(d, _mm_shuffle_ps(d, d, 1)); - d1 = _mm_mul_ss(dA,dD); - d2 = _mm_mul_ss(dB,dC); - - // iD = D*|A| - C*A#*B - iD = _mm_sub_ps(_mm_mul_ps(D,_mm_shuffle_ps(dA,dA,0)), iD); - - // iA = A*|D| - B*D#*C; - iA = _mm_sub_ps(_mm_mul_ps(A,_mm_shuffle_ps(dD,dD,0)), iA); - - // det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C) - det = _mm_sub_ss(_mm_add_ss(d1,d2),d); - rd = _mm_div_ss(_mm_set_ss(1.0f), det); - - // iB = D * (A#B)# = D*B#*A - iB = _mm_mul_ps(D, _mm_shuffle_ps(AB,AB,0x33)); - iB = _mm_sub_ps(iB, _mm_mul_ps(_mm_shuffle_ps(D,D,0xB1), _mm_shuffle_ps(AB,AB,0x66))); - // iC = A * (D#C)# = A*C#*D - iC = _mm_mul_ps(A, _mm_shuffle_ps(DC,DC,0x33)); - iC = _mm_sub_ps(iC, _mm_mul_ps(_mm_shuffle_ps(A,A,0xB1), _mm_shuffle_ps(DC,DC,0x66))); - - rd = _mm_shuffle_ps(rd,rd,0); - rd = _mm_xor_ps(rd, p4f_sign_PNNP); - - // iB = C*|B| - D*B#*A - iB = _mm_sub_ps(_mm_mul_ps(C,_mm_shuffle_ps(dB,dB,0)), iB); - - // iC = B*|C| - A*C#*D; - iC = _mm_sub_ps(_mm_mul_ps(B,_mm_shuffle_ps(dC,dC,0)), iC); - - // iX = iX / det - iA = _mm_mul_ps(rd,iA); - iB = _mm_mul_ps(rd,iB); - iC = _mm_mul_ps(rd,iC); - iD = _mm_mul_ps(rd,iD); - - _mm_storeu_ps(dst+0, _mm_shuffle_ps(iA,iB,0x77)); - _mm_storeu_ps(dst+4, _mm_shuffle_ps(iA,iB,0x22)); - _mm_storeu_ps(dst+8, _mm_shuffle_ps(iC,iD,0x77)); - _mm_storeu_ps(dst+12, _mm_shuffle_ps(iC,iD,0x22)); -} - - -template && is_same_v_, bool> = false> -FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - // From Intel's SSE matrix library - // The inverse is calculated using "Divide and Conquer" technique. The - // original matrix is divide into four 2x2 sub-matrices. Since each - // register of the matrix holds two elements, the smaller matrices are - // consisted of two registers. Hence we get a better locality of the - // calculations. - - const __m128d _Sign_NP = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0)); - const __m128d _Sign_PN = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0)); - - // the four sub-matrices - __m128d A1, A2, B1, B2, C1, C2, D1, D2; - - A1 = _mm_loadu_pd(src + 0); B1 = _mm_loadu_pd(src + 2); - A2 = _mm_loadu_pd(src + 4); B2 = _mm_loadu_pd(src + 6); - C1 = _mm_loadu_pd(src + 8); D1 = _mm_loadu_pd(src +10); - C2 = _mm_loadu_pd(src +12); D2 = _mm_loadu_pd(src +14); - - // partial inverse of the sub-matrices - __m128d iA1, iA2, iB1, iB2, iC1, iC2, iD1, iD2, DC1, DC2, AB1, AB2; - __m128d dA, dB, dC, dD; // determinant of the sub-matrices - __m128d det, d1, d2, rd; - - // dA = |A| - dA = _mm_shuffle_pd(A2, A2, 1); - dA = _mm_mul_pd(A1, dA); - dA = _mm_sub_sd(dA, _mm_shuffle_pd(dA,dA,3)); - // dB = |B| - dB = _mm_shuffle_pd(B2, B2, 1); - dB = _mm_mul_pd(B1, dB); - dB = _mm_sub_sd(dB, _mm_shuffle_pd(dB,dB,3)); - - // AB = A# * B - AB1 = _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,3)); - AB2 = _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,0)); - AB1 = _mm_sub_pd(AB1, _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,3))); - AB2 = _mm_sub_pd(AB2, _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,0))); - - // dC = |C| - dC = _mm_shuffle_pd(C2, C2, 1); - dC = _mm_mul_pd(C1, dC); - dC = _mm_sub_sd(dC, _mm_shuffle_pd(dC,dC,3)); - // dD = |D| - dD = _mm_shuffle_pd(D2, D2, 1); - dD = _mm_mul_pd(D1, dD); - dD = _mm_sub_sd(dD, _mm_shuffle_pd(dD,dD,3)); - - // DC = D# * C - DC1 = _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,3)); - DC2 = _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,0)); - DC1 = _mm_sub_pd(DC1, _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,3))); - DC2 = _mm_sub_pd(DC2, _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,0))); - - // rd = trace(AB*DC) = trace(A#*B*D#*C) - d1 = _mm_mul_pd(AB1, _mm_shuffle_pd(DC1, DC2, 0)); - d2 = _mm_mul_pd(AB2, _mm_shuffle_pd(DC1, DC2, 3)); - rd = _mm_add_pd(d1, d2); - rd = _mm_add_sd(rd, _mm_shuffle_pd(rd, rd,3)); - - // iD = C*A#*B - iD1 = _mm_mul_pd(AB1, _mm_shuffle_pd(C1,C1,0)); - iD2 = _mm_mul_pd(AB1, _mm_shuffle_pd(C2,C2,0)); - iD1 = _mm_add_pd(iD1, _mm_mul_pd(AB2, _mm_shuffle_pd(C1,C1,3))); - iD2 = _mm_add_pd(iD2, _mm_mul_pd(AB2, _mm_shuffle_pd(C2,C2,3))); - - // iA = B*D#*C - iA1 = _mm_mul_pd(DC1, _mm_shuffle_pd(B1,B1,0)); - iA2 = _mm_mul_pd(DC1, _mm_shuffle_pd(B2,B2,0)); - iA1 = _mm_add_pd(iA1, _mm_mul_pd(DC2, _mm_shuffle_pd(B1,B1,3))); - iA2 = _mm_add_pd(iA2, _mm_mul_pd(DC2, _mm_shuffle_pd(B2,B2,3))); - - // iD = D*|A| - C*A#*B - dA = _mm_shuffle_pd(dA,dA,0); - iD1 = _mm_sub_pd(_mm_mul_pd(D1, dA), iD1); - iD2 = _mm_sub_pd(_mm_mul_pd(D2, dA), iD2); - - // iA = A*|D| - B*D#*C; - dD = _mm_shuffle_pd(dD,dD,0); - iA1 = _mm_sub_pd(_mm_mul_pd(A1, dD), iA1); - iA2 = _mm_sub_pd(_mm_mul_pd(A2, dD), iA2); - - d1 = _mm_mul_sd(dA, dD); - d2 = _mm_mul_sd(dB, dC); - - // iB = D * (A#B)# = D*B#*A - iB1 = _mm_mul_pd(D1, _mm_shuffle_pd(AB2,AB1,1)); - iB2 = _mm_mul_pd(D2, _mm_shuffle_pd(AB2,AB1,1)); - iB1 = _mm_sub_pd(iB1, _mm_mul_pd(_mm_shuffle_pd(D1,D1,1), _mm_shuffle_pd(AB2,AB1,2))); - iB2 = _mm_sub_pd(iB2, _mm_mul_pd(_mm_shuffle_pd(D2,D2,1), _mm_shuffle_pd(AB2,AB1,2))); - - // det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C) - det = _mm_add_sd(d1, d2); - det = _mm_sub_sd(det, rd); - - // iC = A * (D#C)# = A*C#*D - iC1 = _mm_mul_pd(A1, _mm_shuffle_pd(DC2,DC1,1)); - iC2 = _mm_mul_pd(A2, _mm_shuffle_pd(DC2,DC1,1)); - iC1 = _mm_sub_pd(iC1, _mm_mul_pd(_mm_shuffle_pd(A1,A1,1), _mm_shuffle_pd(DC2,DC1,2))); - iC2 = _mm_sub_pd(iC2, _mm_mul_pd(_mm_shuffle_pd(A2,A2,1), _mm_shuffle_pd(DC2,DC1,2))); - - rd = _mm_div_sd(_mm_set_sd(1.0), det); - rd = _mm_shuffle_pd(rd,rd,0); - - // iB = C*|B| - D*B#*A - dB = _mm_shuffle_pd(dB,dB,0); - iB1 = _mm_sub_pd(_mm_mul_pd(C1, dB), iB1); - iB2 = _mm_sub_pd(_mm_mul_pd(C2, dB), iB2); - - d1 = _mm_xor_pd(rd, _Sign_PN); - d2 = _mm_xor_pd(rd, _Sign_NP); - - // iC = B*|C| - A*C#*D; - dC = _mm_shuffle_pd(dC,dC,0); - iC1 = _mm_sub_pd(_mm_mul_pd(B1, dC), iC1); - iC2 = _mm_sub_pd(_mm_mul_pd(B2, dC), iC2); - - _mm_storeu_pd(dst+0, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 3), d1)); - _mm_storeu_pd(dst+4, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 0), d2)); - _mm_storeu_pd(dst+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 3), d1)); - _mm_storeu_pd(dst+4+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 0), d2)); - _mm_storeu_pd(dst+2*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 3), d1)); - _mm_storeu_pd(dst+3*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 0), d2)); - _mm_storeu_pd(dst+2*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 3), d1)); - _mm_storeu_pd(dst+3*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 0), d2)); -} -#endif - - -} // end of namespace Fastor - -#endif // INVERSE_H diff --git a/noarch/include/Fastor/backend/lufact.h b/noarch/include/Fastor/backend/lufact.h deleted file mode 100644 index 1d8422c4..00000000 --- a/noarch/include/Fastor/backend/lufact.h +++ /dev/null @@ -1,753 +0,0 @@ -#ifndef LUFACT_H -#define LUFACT_H - -#include "Fastor/meta/meta.h" -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" - -namespace Fastor { - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u); - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u) { - *l = 1; - *u = *a; -} - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { - - // [a11 a12] [1 0] [u11 u12] - // [a21 a22] [l21 1] [0 u22] - - const T L21 = A[2]/A[0]; - - L[0] = 1; - L[1] = 0; - L[2] = L21; - L[3] = 1; - - - U[0] = A[0]; - U[1] = A[1]; - U[2] = 0; - U[3] = A[3] - L21 * A[1]; -} - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { - - const T A00 = A[0]; - const T L21 = A[3]/A00; - const T L31 = A[6]/A00; - - const T U22 = A[4] - L21 * A[1]; - const T U23 = A[5] - L21 * A[2]; - - const T L32 = (A[7] - L31 * A[1]) / U22; - - const T U33 = A[8] - L31 * A[2] - L32 * U23; - - L[0] = 1; - L[1] = 0; - L[2] = 0; - L[3] = L21; - L[4] = 1; - L[5] = 0; - L[6] = L31; - L[7] = L32; - L[8] = 1; - - U[0] = A00; - U[1] = A[1]; - U[2] = A[2]; - U[3] = 0; - U[4] = U22; - U[5] = U23; - U[6] = 0; - U[7] = 0; - U[8] = U33; -} - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { - - const T A00 = A[0]; - const T L21 = A[4] /A00; - const T L31 = A[8] /A00; - const T L41 = A[12]/A00; - - const T U22 = A[5] - L21 * A[1]; - const T U23 = A[6] - L21 * A[2]; - const T U24 = A[7] - L21 * A[3]; - - const T L32 = (A[9 ] - L31 * A[1]) / U22; - const T L42 = (A[13] - L41 * A[1]) / U22; - - const T U33 = A[10] - L31 * A[2] - L32 * U23; - const T U34 = A[11] - L31 * A[3] - L32 * U24; - - const T L43 = (A[14] - L41 * A[2] - L42 * U23) / U33; - - const T U44 = A[15] - L41 * A[3] - L42 * U24 - L43 * U34; - - L[0] = 1; - L[1] = 0; - L[2] = 0; - L[3] = 0; - L[4] = L21; - L[5] = 1; - L[6] = 0; - L[7] = 0; - L[8] = L31; - L[9] = L32; - L[10] = 1; - L[11] = 0; - L[12] = L41; - L[13] = L42; - L[14] = L43; - L[15] = 1; - - U[0] = A00; - U[1] = A[1]; - U[2] = A[2]; - U[3] = A[3]; - U[4] = 0; - U[5] = U22; - U[6] = U23; - U[7] = U24; - U[8] = 0; - U[9] = 0; - U[10] = U33; - U[11] = U34; - U[12] = 0; - U[13] = 0; - U[14] = 0; - U[15] = U44; -} - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { - - const T A00 = A[0*N]; - const T L21 = A[1*N] /A00; - const T L31 = A[2*N] /A00; - const T L41 = A[3*N] /A00; - const T L51 = A[4*N] /A00; - - const T U22 = A[1*N+1] - L21 * A[1]; - const T U23 = A[1*N+2] - L21 * A[2]; - const T U24 = A[1*N+3] - L21 * A[3]; - const T U25 = A[1*N+4] - L21 * A[4]; - - const T L32 = (A[2*N+1] - L31 * A[1]) / U22; - const T L42 = (A[3*N+1] - L41 * A[1]) / U22; - const T L52 = (A[4*N+1] - L51 * A[1]) / U22; - - const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; - const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; - const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; - - const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; - const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; - - const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; - const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; - - const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; - - const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; - - // L - L[0*N+0] = 1; - L[0*N+1] = 0; - L[0*N+2] = 0; - L[0*N+3] = 0; - L[0*N+4] = 0; - - L[1*N+0] = L21; - L[1*N+1] = 1; - L[1*N+2] = 0; - L[1*N+3] = 0; - L[1*N+4] = 0; - - L[2*N+0] = L31; - L[2*N+1] = L32; - L[2*N+2] = 1; - L[2*N+3] = 0; - L[2*N+4] = 0; - - L[3*N+0] = L41; - L[3*N+1] = L42; - L[3*N+2] = L43; - L[3*N+3] = 1; - L[3*N+4] = 0; - - L[4*N+0] = L51; - L[4*N+1] = L52; - L[4*N+2] = L53; - L[4*N+3] = L54; - L[4*N+4] = 1; - - // U - U[0*N+0] = A00; - U[0*N+1] = A[1]; - U[0*N+2] = A[2]; - U[0*N+3] = A[3]; - U[0*N+4] = A[4]; - - U[1*N+0] = 0; - U[1*N+1] = U22; - U[1*N+2] = U23; - U[1*N+3] = U24; - U[1*N+4] = U25; - - U[2*N+0] = 0; - U[2*N+1] = 0; - U[2*N+2] = U33; - U[2*N+3] = U34; - U[2*N+4] = U35; - - U[3*N+0] = 0; - U[3*N+1] = 0; - U[3*N+2] = 0; - U[3*N+3] = U44; - U[3*N+4] = U45; - - U[4*N+0] = 0; - U[4*N+1] = 0; - U[4*N+2] = 0; - U[4*N+3] = 0; - U[4*N+4] = U55; -} - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { - - const T A00 = A[0*N]; - const T L21 = A[1*N] /A00; - const T L31 = A[2*N] /A00; - const T L41 = A[3*N] /A00; - const T L51 = A[4*N] /A00; - const T L61 = A[5*N] /A00; - - const T U22 = A[1*N+1] - L21 * A[1]; - const T U23 = A[1*N+2] - L21 * A[2]; - const T U24 = A[1*N+3] - L21 * A[3]; - const T U25 = A[1*N+4] - L21 * A[4]; - const T U26 = A[1*N+5] - L21 * A[5]; - - const T L32 = (A[2*N+1] - L31 * A[1]) / U22; - const T L42 = (A[3*N+1] - L41 * A[1]) / U22; - const T L52 = (A[4*N+1] - L51 * A[1]) / U22; - const T L62 = (A[5*N+1] - L61 * A[1]) / U22; - - const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; - const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; - const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; - const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26; - - const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; - const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; - const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33; - - const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; - const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; - const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36; - - const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; - const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44; - - const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; - const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46; - - const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55; - - const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56; - - // L - L[0*N+0] = 1; - L[0*N+1] = 0; - L[0*N+2] = 0; - L[0*N+3] = 0; - L[0*N+4] = 0; - L[0*N+5] = 0; - - L[1*N+0] = L21; - L[1*N+1] = 1; - L[1*N+2] = 0; - L[1*N+3] = 0; - L[1*N+4] = 0; - L[1*N+5] = 0; - - L[2*N+0] = L31; - L[2*N+1] = L32; - L[2*N+2] = 1; - L[2*N+3] = 0; - L[2*N+4] = 0; - L[2*N+5] = 0; - - L[3*N+0] = L41; - L[3*N+1] = L42; - L[3*N+2] = L43; - L[3*N+3] = 1; - L[3*N+4] = 0; - L[3*N+5] = 0; - - L[4*N+0] = L51; - L[4*N+1] = L52; - L[4*N+2] = L53; - L[4*N+3] = L54; - L[4*N+4] = 1; - L[4*N+5] = 0; - - L[5*N+0] = L61; - L[5*N+1] = L62; - L[5*N+2] = L63; - L[5*N+3] = L64; - L[5*N+4] = L65; - L[5*N+5] = 1; - - // U - U[0*N+0] = A00; - U[0*N+1] = A[1]; - U[0*N+2] = A[2]; - U[0*N+3] = A[3]; - U[0*N+4] = A[4]; - U[0*N+5] = A[5]; - - U[1*N+0] = 0; - U[1*N+1] = U22; - U[1*N+2] = U23; - U[1*N+3] = U24; - U[1*N+4] = U25; - U[1*N+5] = U26; - - U[2*N+0] = 0; - U[2*N+1] = 0; - U[2*N+2] = U33; - U[2*N+3] = U34; - U[2*N+4] = U35; - U[2*N+5] = U36; - - U[3*N+0] = 0; - U[3*N+1] = 0; - U[3*N+2] = 0; - U[3*N+3] = U44; - U[3*N+4] = U45; - U[3*N+5] = U46; - - U[4*N+0] = 0; - U[4*N+1] = 0; - U[4*N+2] = 0; - U[4*N+3] = 0; - U[4*N+4] = U55; - U[4*N+5] = U56; - - U[5*N+0] = 0; - U[5*N+1] = 0; - U[5*N+2] = 0; - U[5*N+3] = 0; - U[5*N+4] = 0; - U[5*N+5] = U66; -} - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { - - const T A00 = A[0*N]; - const T L21 = A[1*N] /A00; - const T L31 = A[2*N] /A00; - const T L41 = A[3*N] /A00; - const T L51 = A[4*N] /A00; - const T L61 = A[5*N] /A00; - const T L71 = A[6*N] /A00; - - const T U22 = A[1*N+1] - L21 * A[1]; - const T U23 = A[1*N+2] - L21 * A[2]; - const T U24 = A[1*N+3] - L21 * A[3]; - const T U25 = A[1*N+4] - L21 * A[4]; - const T U26 = A[1*N+5] - L21 * A[5]; - const T U27 = A[1*N+6] - L21 * A[6]; - - const T L32 = (A[2*N+1] - L31 * A[1]) / U22; - const T L42 = (A[3*N+1] - L41 * A[1]) / U22; - const T L52 = (A[4*N+1] - L51 * A[1]) / U22; - const T L62 = (A[5*N+1] - L61 * A[1]) / U22; - const T L72 = (A[6*N+1] - L71 * A[1]) / U22; - - const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; - const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; - const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; - const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26; - const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27; - - const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; - const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; - const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33; - const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33; - - const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; - const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; - const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36; - const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37; - - const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; - const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44; - const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44; - - const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; - const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46; - const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47; - - const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55; - const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55; - - const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56; - const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57; - - const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66; - - const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67; - - // L - L[0*N+0] = 1; - L[0*N+1] = 0; - L[0*N+2] = 0; - L[0*N+3] = 0; - L[0*N+4] = 0; - L[0*N+5] = 0; - L[0*N+6] = 0; - - L[1*N+0] = L21; - L[1*N+1] = 1; - L[1*N+2] = 0; - L[1*N+3] = 0; - L[1*N+4] = 0; - L[1*N+5] = 0; - L[1*N+6] = 0; - - L[2*N+0] = L31; - L[2*N+1] = L32; - L[2*N+2] = 1; - L[2*N+3] = 0; - L[2*N+4] = 0; - L[2*N+5] = 0; - L[2*N+6] = 0; - - L[3*N+0] = L41; - L[3*N+1] = L42; - L[3*N+2] = L43; - L[3*N+3] = 1; - L[3*N+4] = 0; - L[3*N+5] = 0; - L[3*N+6] = 0; - - L[4*N+0] = L51; - L[4*N+1] = L52; - L[4*N+2] = L53; - L[4*N+3] = L54; - L[4*N+4] = 1; - L[4*N+5] = 0; - L[4*N+6] = 0; - - L[5*N+0] = L61; - L[5*N+1] = L62; - L[5*N+2] = L63; - L[5*N+3] = L64; - L[5*N+4] = L65; - L[5*N+5] = 1; - L[5*N+6] = 0; - - L[6*N+0] = L71; - L[6*N+1] = L72; - L[6*N+2] = L73; - L[6*N+3] = L74; - L[6*N+4] = L75; - L[6*N+5] = L76; - L[6*N+6] = 1; - - // U - U[0*N+0] = A00; - U[0*N+1] = A[1]; - U[0*N+2] = A[2]; - U[0*N+3] = A[3]; - U[0*N+4] = A[4]; - U[0*N+5] = A[5]; - U[0*N+6] = A[6]; - - U[1*N+0] = 0; - U[1*N+1] = U22; - U[1*N+2] = U23; - U[1*N+3] = U24; - U[1*N+4] = U25; - U[1*N+5] = U26; - U[1*N+6] = U27; - - U[2*N+0] = 0; - U[2*N+1] = 0; - U[2*N+2] = U33; - U[2*N+3] = U34; - U[2*N+4] = U35; - U[2*N+5] = U36; - U[2*N+6] = U37; - - U[3*N+0] = 0; - U[3*N+1] = 0; - U[3*N+2] = 0; - U[3*N+3] = U44; - U[3*N+4] = U45; - U[3*N+5] = U46; - U[3*N+6] = U47; - - U[4*N+0] = 0; - U[4*N+1] = 0; - U[4*N+2] = 0; - U[4*N+3] = 0; - U[4*N+4] = U55; - U[4*N+5] = U56; - U[4*N+6] = U57; - - U[5*N+0] = 0; - U[5*N+1] = 0; - U[5*N+2] = 0; - U[5*N+3] = 0; - U[5*N+4] = 0; - U[5*N+5] = U66; - U[5*N+6] = U67; - - U[6*N+0] = 0; - U[6*N+1] = 0; - U[6*N+2] = 0; - U[6*N+3] = 0; - U[6*N+4] = 0; - U[6*N+5] = 0; - U[6*N+6] = U77; -} - -template, bool> = false> -FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) { - - const T A00 = A[0*N]; - const T L21 = A[1*N] /A00; - const T L31 = A[2*N] /A00; - const T L41 = A[3*N] /A00; - const T L51 = A[4*N] /A00; - const T L61 = A[5*N] /A00; - const T L71 = A[6*N] /A00; - const T L81 = A[7*N] /A00; - - const T U22 = A[1*N+1] - L21 * A[1]; - const T U23 = A[1*N+2] - L21 * A[2]; - const T U24 = A[1*N+3] - L21 * A[3]; - const T U25 = A[1*N+4] - L21 * A[4]; - const T U26 = A[1*N+5] - L21 * A[5]; - const T U27 = A[1*N+6] - L21 * A[6]; - const T U28 = A[1*N+7] - L21 * A[7]; - - const T L32 = (A[2*N+1] - L31 * A[1]) / U22; - const T L42 = (A[3*N+1] - L41 * A[1]) / U22; - const T L52 = (A[4*N+1] - L51 * A[1]) / U22; - const T L62 = (A[5*N+1] - L61 * A[1]) / U22; - const T L72 = (A[6*N+1] - L71 * A[1]) / U22; - const T L82 = (A[7*N+1] - L81 * A[1]) / U22; - - const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23; - const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24; - const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25; - const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26; - const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27; - const T U38 = A[2*N+7] - L31 * A[7] - L32 * U28; - - const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33; - const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33; - const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33; - const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33; - const T L83 = (A[7*N+2] - L81 * A[2] - L82 * U23) / U33; - - const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34; - const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35; - const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36; - const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37; - const T U48 = A[3*N+7] - L41 * A[7] - L42 * U28 - L43 * U38; - - const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44; - const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44; - const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44; - const T L84 = (A[7*N+3] - L81 * A[3] - L82 * U24 - L83 * U34) / U44; - - const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45; - const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46; - const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47; - const T U58 = A[4*N+7] - L51 * A[7] - L52 * U28 - L53 * U38 - L54 * U48; - - const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55; - const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55; - const T L85 = (A[7*N+4] - L81 * A[4] - L82 * U25 - L83 * U35 - L84 * U45) / U55; - - const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56; - const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57; - const T U68 = A[5*N+7] - L61 * A[7] - L62 * U28 - L63 * U38 - L64 * U48 - L65 * U58; - - const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66; - const T L86 = (A[7*N+5] - L81 * A[5] - L82 * U26 - L83 * U36 - L84 * U46 - L85 * U56) / U66; - - const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67; - const T U78 = A[6*N+7] - L71 * A[7] - L72 * U28 - L73 * U38 - L74 * U48 - L75 * U58 - L76 * U68; - - const T L87 = (A[7*N+6] - L81 * A[6] - L82 * U27 - L83 * U37 - L84 * U47 - L85 * U57 - L86 * U67) / U77; - - const T U88 = A[7*N+7] - L81 * A[7] - L82 * U28 - L83 * U38 - L84 * U48 - L85 * U58 - L86 * U68 - L87 * U78; - - // L - L[0*N+0] = 1; - L[0*N+1] = 0; - L[0*N+2] = 0; - L[0*N+3] = 0; - L[0*N+4] = 0; - L[0*N+5] = 0; - L[0*N+6] = 0; - L[0*N+7] = 0; - - L[1*N+0] = L21; - L[1*N+1] = 1; - L[1*N+2] = 0; - L[1*N+3] = 0; - L[1*N+4] = 0; - L[1*N+5] = 0; - L[1*N+6] = 0; - L[1*N+7] = 0; - - L[2*N+0] = L31; - L[2*N+1] = L32; - L[2*N+2] = 1; - L[2*N+3] = 0; - L[2*N+4] = 0; - L[2*N+5] = 0; - L[2*N+6] = 0; - L[2*N+7] = 0; - - L[3*N+0] = L41; - L[3*N+1] = L42; - L[3*N+2] = L43; - L[3*N+3] = 1; - L[3*N+4] = 0; - L[3*N+5] = 0; - L[3*N+6] = 0; - L[3*N+7] = 0; - - L[4*N+0] = L51; - L[4*N+1] = L52; - L[4*N+2] = L53; - L[4*N+3] = L54; - L[4*N+4] = 1; - L[4*N+5] = 0; - L[4*N+6] = 0; - L[4*N+7] = 0; - - L[5*N+0] = L61; - L[5*N+1] = L62; - L[5*N+2] = L63; - L[5*N+3] = L64; - L[5*N+4] = L65; - L[5*N+5] = 1; - L[5*N+6] = 0; - L[5*N+7] = 0; - - L[6*N+0] = L71; - L[6*N+1] = L72; - L[6*N+2] = L73; - L[6*N+3] = L74; - L[6*N+4] = L75; - L[6*N+5] = L76; - L[6*N+6] = 1; - L[6*N+7] = 0; - - L[7*N+0] = L81; - L[7*N+1] = L82; - L[7*N+2] = L83; - L[7*N+3] = L84; - L[7*N+4] = L85; - L[7*N+5] = L86; - L[7*N+6] = L87; - L[7*N+7] = 1; - - // U - U[0*N+0] = A00; - U[0*N+1] = A[1]; - U[0*N+2] = A[2]; - U[0*N+3] = A[3]; - U[0*N+4] = A[4]; - U[0*N+5] = A[5]; - U[0*N+6] = A[6]; - U[0*N+7] = A[7]; - - U[1*N+0] = 0; - U[1*N+1] = U22; - U[1*N+2] = U23; - U[1*N+3] = U24; - U[1*N+4] = U25; - U[1*N+5] = U26; - U[1*N+6] = U27; - U[1*N+7] = U28; - - U[2*N+0] = 0; - U[2*N+1] = 0; - U[2*N+2] = U33; - U[2*N+3] = U34; - U[2*N+4] = U35; - U[2*N+5] = U36; - U[2*N+6] = U37; - U[2*N+7] = U38; - - U[3*N+0] = 0; - U[3*N+1] = 0; - U[3*N+2] = 0; - U[3*N+3] = U44; - U[3*N+4] = U45; - U[3*N+5] = U46; - U[3*N+6] = U47; - U[3*N+7] = U48; - - U[4*N+0] = 0; - U[4*N+1] = 0; - U[4*N+2] = 0; - U[4*N+3] = 0; - U[4*N+4] = U55; - U[4*N+5] = U56; - U[4*N+6] = U57; - U[4*N+7] = U58; - - U[5*N+0] = 0; - U[5*N+1] = 0; - U[5*N+2] = 0; - U[5*N+3] = 0; - U[5*N+4] = 0; - U[5*N+5] = U66; - U[5*N+6] = U67; - U[5*N+7] = U68; - - U[6*N+0] = 0; - U[6*N+1] = 0; - U[6*N+2] = 0; - U[6*N+3] = 0; - U[6*N+4] = 0; - U[6*N+5] = 0; - U[6*N+6] = U77; - U[6*N+7] = U78; - - U[7*N+0] = 0; - U[7*N+1] = 0; - U[7*N+2] = 0; - U[7*N+3] = 0; - U[7*N+4] = 0; - U[7*N+5] = 0; - U[7*N+6] = 0; - U[7*N+7] = U88; -} - -} // end of namespace Fastor - -#endif // LUFACT_H diff --git a/noarch/include/Fastor/backend/lut_inverse.h b/noarch/include/Fastor/backend/lut_inverse.h deleted file mode 100644 index 5debb56d..00000000 --- a/noarch/include/Fastor/backend/lut_inverse.h +++ /dev/null @@ -1,80 +0,0 @@ -#ifndef LOWUNITRI_INVERSE_H -#define LOWUNITRI_INVERSE_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" - -namespace Fastor { - -template, bool> = false> -FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst); - -template, bool> = false> -FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) { - *dst = T(1); -} - -template, bool> = false> -FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - /* Compute adjoint: */ - dst[0] = 1; - dst[1] = 0; - dst[2] = - src[2]; - dst[3] = 1; -} - -template, bool> = false> -FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - T src3 = src[3]; - T src6 = src[6]; - T src7 = src[7]; - T src8 = src[8]; - - /* Compute adjoint: */ - dst[0] = 1; - dst[1] = 0; - dst[2] = 0; - dst[3] = - src3 * src8; - dst[4] = 1; - dst[5] = 0; - dst[6] = + src3 * src7 - src6; - dst[7] = - src7; - dst[8] = 1; -} - - -template, bool> = false> -FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) -{ - dst[0] = 1; - dst[1] = 0; - dst[2] = 0; - dst[3] = 0; - - dst[4] = - src[1*4+0]; - dst[5] = 1; - dst[6] = 0; - dst[7] = 0; - - T t2 = src[2*4+1]; - T t3 = src[2*4+1]*src[3*4+2] - src[3*4+1]; - - T t4 = src[2*4+0]; - T t5 = src[2*4+0]*src[3*4+2] - src[3*4+0]; - - dst[8] = src[1*4+0]*t2 - t4; - dst[9] = - t2; - dst[10] = 1; - dst[11] = 0; - - dst[12] = t5 - src[1*4+0]*t3; - dst[13] = t3; - dst[14] = - src[3*4+2]; - dst[15] = 1; -} - -} // end of namespace Fastor - -#endif // LOWUNITRI_INVERSE_H diff --git a/noarch/include/Fastor/backend/matmul/libxsmm_backend.h b/noarch/include/Fastor/backend/matmul/libxsmm_backend.h deleted file mode 100644 index fe79c560..00000000 --- a/noarch/include/Fastor/backend/matmul/libxsmm_backend.h +++ /dev/null @@ -1,209 +0,0 @@ -#ifndef LIBXSMM_BACKEND_H -#define LIBXSMM_BACKEND_H - -#include - -#ifdef FASTOR_USE_LIBXSMM - -#include - -namespace Fastor { -namespace blas { - -// single -template -FASTOR_INLINE -void matmulNN_libxsmm( - const float * FASTOR_RESTRICT a_data, - const float * FASTOR_RESTRICT b_data, - float * FASTOR_RESTRICT out_data) { - - constexpr int MM= M; - constexpr int KK= K; - constexpr int NN= N; - constexpr float alpha = 1.0; - constexpr float beta = 0.0; - - constexpr char transa = 'N'; - constexpr char transb = 'N'; - - libxsmm_sgemm( - &transa /*transa*/, - &transb /*transb*/, - &NN /*required*/, - &MM /*required*/, - &KK /*required*/, - &alpha /*alpha*/, - b_data /*required*/, - &NN /*lda*/, - a_data /*required*/, - &KK /*ldb*/, - &beta /*beta*/, - out_data /*required*/, - &NN /*ldc*/ - ); -} - -// double -template -FASTOR_INLINE -void matmulNN_libxsmm( - const double * FASTOR_RESTRICT a_data, - const double * FASTOR_RESTRICT b_data, - double * FASTOR_RESTRICT out_data) { - - constexpr int MM= M; - constexpr int KK= K; - constexpr int NN= N; - constexpr double alpha = 1.0; - constexpr double beta = 0.0; - - constexpr char transa = 'N'; - constexpr char transb = 'N'; - - libxsmm_dgemm( - &transa /*transa*/, - &transb /*transb*/, - &NN /*required*/, - &MM /*required*/, - &KK /*required*/, - &alpha /*alpha*/, - b_data /*required*/, - &NN /*lda*/, - a_data /*required*/, - &KK /*ldb*/, - &beta /*beta*/, - out_data /*required*/, - &NN /*ldc*/ - ); -} - - -template -FASTOR_INLINE -void matmulTN_libxsmm( - const double * FASTOR_RESTRICT a_data, - const double * FASTOR_RESTRICT b_data, - double * FASTOR_RESTRICT out_data) { - - constexpr int MM= M; - constexpr int KK= K; - constexpr int NN= N; - constexpr double alpha = 1.0; - constexpr double beta = 0.0; - - constexpr char transa = 'N'; - constexpr char transb = 'Y'; - - libxsmm_dgemm( - &transa /*transa*/, - &transb /*transb*/, - &NN /*required*/, - &MM /*required*/, - &KK /*required*/, - &alpha /*alpha*/, - b_data /*required*/, - &NN /*lda*/, - a_data /*required*/, - &MM /*ldb*/, - &beta /*beta*/, - out_data /*required*/, - &NN /*ldc*/ - ); -} - - -template -FASTOR_INLINE -void matmulNT_libxsmm( - const double * FASTOR_RESTRICT a_data, - const double * FASTOR_RESTRICT b_data, - double * FASTOR_RESTRICT out_data) { - - constexpr int MM= M; - constexpr int KK= K; - constexpr int NN= N; - constexpr double alpha = 1.0; - constexpr double beta = 0.0; - - constexpr char transa = 'Y'; - constexpr char transb = 'N'; - - libxsmm_dgemm( - &transa /*transa*/, - &transb /*transb*/, - &NN /*required*/, - &MM /*required*/, - &KK /*required*/, - &alpha /*alpha*/, - b_data /*required*/, - &KK /*lda*/, - a_data /*required*/, - &KK /*ldb*/, - &beta /*beta*/, - out_data /*required*/, - &NN /*ldc*/ - ); -} - - -template -FASTOR_INLINE -void matmulTT_libxsmm( - const double * FASTOR_RESTRICT a_data, - const double * FASTOR_RESTRICT b_data, - double * FASTOR_RESTRICT out_data) { - - constexpr int MM= M; - constexpr int KK= K; - constexpr int NN= N; - constexpr double alpha = 1.0; - constexpr double beta = 0.0; - - constexpr char transa = 'Y'; - constexpr char transb = 'Y'; - - libxsmm_dgemm( - &transa /*transa*/, - &transb /*transb*/, - &NN /*required*/, - &MM /*required*/, - &KK /*required*/, - &alpha /*alpha*/, - b_data /*required*/, - &KK /*lda*/, - a_data /*required*/, - &MM /*ldb*/, - &beta /*beta*/, - out_data /*required*/, - &NN /*ldc*/ - ); -} - - -template::value,bool>::type=0> -FASTOR_INLINE -void matmul_libxsmm( - const T * FASTOR_RESTRICT a_data, - const T * FASTOR_RESTRICT b_data, - T * FASTOR_RESTRICT out_data) { - matmulNN_libxsmm(a_data,b_data,out_data); -} - -template::value,bool>::type=0> -FASTOR_INLINE -void matmul_libxsmm( - const T * FASTOR_RESTRICT a_data, - const T * FASTOR_RESTRICT b_data, - T * FASTOR_RESTRICT out_data) { - matmulNN_libxsmm(a_data,b_data,out_data); -} - -} // end of namespace blas -} // end of namespace Fastor - -#endif // FASTOR_USE_LIBXSMM - -#endif // LIBXSMM_BACKEND_H diff --git a/noarch/include/Fastor/backend/matmul/matmul.h b/noarch/include/Fastor/backend/matmul/matmul.h deleted file mode 100644 index 9526b171..00000000 --- a/noarch/include/Fastor/backend/matmul/matmul.h +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef MATMUL_H -#define MATMUL_H - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/matmul/matmul_kernels.h" - -#ifdef FASTOR_USE_LIBXSMM -#include "Fastor/backend/matmul/libxsmm_backend.h" -#endif -#ifdef FASTOR_USE_MKL -#include "Fastor/backend/matmul/mkl_backend.h" -#endif - -namespace Fastor { - - - -// Forward declare -//----------------------------------------------------------------------------------------------------------- -namespace internal { -template -FASTOR_INLINE -void _matvecmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out); -} // internal -//----------------------------------------------------------------------------------------------------------- - - -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -#if !defined(FASTOR_USE_LIBXSMM) && !defined(FASTOR_USE_MKL) -template || is_same_v_) ),bool> = 0> -#else -template || is_same_v_) ) - && is_less_equal::value,1>::value, - bool> = 0> -#endif -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - // Non-primitive types - FASTOR_IF_CONSTEXPR (!is_primitive_v_) { - internal::_matmul_base_non_primitive(a,b,out); - return; - } - - // Matrix-vector specialisation - FASTOR_IF_CONSTEXPR (N==1UL) { - internal::_matvecmul(a,b,out); - return; - } - - using nativeV = SIMDVector; - using V = choose_best_simd_t; - - // Use specialised kernels - FASTOR_IF_CONSTEXPR((N==V::Size || N==2*V::Size || N==3*V::Size || N==4*V::Size || N==5*V::Size) && V::Size!=1UL) { - internal::_matmul_mk_smalln(a,b,out); - return; - } - -#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) - FASTOR_IF_CONSTEXPR((N<5*V::Size && N!=1UL)) { - internal::_matmul_mk_smalln(a,b,out); - return; - } -#endif - -#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) - FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size <= 1UL) { - internal::_matmul_base(a,b,out); - return; - } - else FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size > 1UL) { - internal::_matmul_base_masked(a,b,out); - return; - } -#else - FASTOR_IF_CONSTEXPR( M*N*K > 27UL ) { - internal::_matmul_base(a,b,out); - return; - } -#endif - else - { - // For all other cases where M,N,K is too small - // this simple version is sufficient - constexpr int ROUND_ = ROUND_DOWN(N,V::Size); - for (size_t j=0; j || is_same_v_) ) - && is_greater::value,1>::value, - bool> = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - blas::matmul_libxsmm(a,b,c); -} -#endif - -#if !defined(FASTOR_USE_LIBXSMM) && defined(FASTOR_USE_MKL) -template || is_same_v_) ) - && is_greater::value,1>::value, - bool> = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - blas::matmul_mkl(a,b,c); -} -#endif -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- - - -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -#include "Fastor/backend/matmul/matmul_specialisations_kernels.h" -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- - - -} // end of namespace - -#endif // MATMUL_H diff --git a/noarch/include/Fastor/backend/matmul/matmul_kernels.h b/noarch/include/Fastor/backend/matmul/matmul_kernels.h deleted file mode 100644 index da10cefd..00000000 --- a/noarch/include/Fastor/backend/matmul/matmul_kernels.h +++ /dev/null @@ -1,823 +0,0 @@ -#ifndef MATMUL_KERNELS_H -#define MATMUL_KERNELS_H - - -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/meta/tensor_meta.h" - - -namespace Fastor { - -namespace internal { - - -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- - -// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel) -// unroll the inner-most loop (on unrollOuterloop) -//----------------------------------------------------------------------------------------------------------- -template::type = false> -FASTOR_INLINE -void interior_block_matmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - - const V bmm0(&b[k*N+j],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - } - } -} - -template::type = false> -FASTOR_INLINE -void interior_block_matmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_matmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - const V bmm2(&b[k*N+j+2*V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_matmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - const V bmm2(&b[k*N+j+2*V::Size],false); - const V bmm3(&b[k*N+j+3*V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); - c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); - c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_matmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - const V bmm2(&b[k*N+j+2*V::Size],false); - const V bmm3(&b[k*N+j+3*V::Size],false); - const V bmm4(&b[k*N+j+4*V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); - c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); - c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); - c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); - c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_matmul_scalar_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - T c_ij[unrollOuterloop*numSIMDCols] = {}; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - const T bmm0(b[k*N+j]); - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] += amm0*bmm0; - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n]; - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_matmul_mask_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j, const int (&maska)[V::Size]) { - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - - const V bmm0(maskload(&b[k*N+j],maska)); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_matmul_mask_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j, const MaskT mask) { - - V bmm0; - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = 0; k < K; ++k) { - - bmm0.mask_load(&b[k*N+j],mask,false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false); - } - } -} -//----------------------------------------------------------------------------------------------------------- - - - -//----------------------------------------------------------------------------------------------------------- -// This is the base implementation of matrix-matrix multiplication for all 2D tensors and -// higher order tensor products that can be expressed as gemm -// The function uses two level unrolling one based on block sizes and one based on register widths -// with any remainder left treated in a scalar fashion -template -FASTOR_INLINE -void _matmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - - using V = typename internal::choose_best_simd_type,N>::type; - - // This parameter can be adjusted and does not need to be 4UL/8UL etc - // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; - constexpr size_t unrollOuterloop = 4UL; - -#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE - // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] - constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); -#else - constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; -#endif -#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE - // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] - constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; -#else - constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; -#endif - - // The goal is to get 10 parallel independent chains of accumulators - // to saturate the pipeline by having a completely unrolled block of - // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of - // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains - // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. - // However, most recent X86/64 architectures can do 2 FMAs per load so - // so unrolling with numSIMDCols > 2 is not beneficial - - constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; - // Number of rows of c (M) that can be safely unrolled with this block size. - constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; - - constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; - // Number of columns of c (N) that can be safely unrolled with this block size - constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - size_t i = 0; - for (; i < M0; i += unrollOuterBlock) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N1 columns - for (; j < N; ++j) { - interior_block_matmul_scalar_impl(a,b,c,i,j); - } - - } - - // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. - // This is necessary as for small sizes the earlier block loop may not be - // triggered if the size of the block is bigger than the number of rows of - // (a and c) i.e. M - constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); - for (; i < M1; i += unrollOuterloop) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - V c_ij[unrollOuterloop]; - for (size_t k = 0; k < K; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i + n)*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; ++j) { - T c_ij[unrollOuterloop] = {}; - for (size_t k = 0; k < K; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] += a[(i + n)*K+k] * b[k*N+j]; - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c[(i + n)*N+j] = c_ij[n]; - } - } - } - - // Now treat the remaining M-M1 rows - FASTOR_IF_CONSTEXPR (M-M1 > 0) { - // Hack to get around zero length array issue - constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - // If MM1==0 the function never gets invoked anyway - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - V c_ij[MM1]; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); - c_ij[n-M1].store(&c[n*N+j],false); - } - } - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1].store(&c[n*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; ++j) { - T c_ij[MM1] = {}; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1] += a[n*K+k] * b[k*N+j]; - c[n*N+j] = c_ij[n-M1]; - } - } - for (size_t n = M1; n < M; ++n) { - c[n*N+j] = c_ij[n-M1]; - } - } - } -} -//----------------------------------------------------------------------------------------------------------- - - - -//----------------------------------------------------------------------------------------------------------- -// This is the base implementation of matrix-matrix multiplication for all 2D tensors and -// higher order tensor products that can be expressed as gemm -// The function uses two level unrolling one based on block sizes and one based on register widths -// with any remainder left treated in vector mode with masked and conditional load/stores. -// Note that conditional load/store requires at least AVX intrinsics -template -FASTOR_INLINE -void _matmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - - using V = typename internal::choose_best_simd_type,N>::type; - - // This parameter can be adjusted and does not need to be 4UL/8UL etc - // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; - constexpr size_t unrollOuterloop = 4UL; - -#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE - // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] - constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); -#else - constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; -#endif -#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE - // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] - constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; -#else - constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; -#endif - - // The goal is to get 10 parallel independent chains of accumulators - // to saturate the pipeline by having a completely unrolled block of - // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of - // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains - // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. - // However, most recent X86/64 architectures can do 2 FMAs per load so - // so unrolling with numSIMDCols > 2 is not beneficial - - constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; - // Number of rows of c (M) that can be safely unrolled with this block size. - constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; - - constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; - // Number of columns of c (N) that can be safely unrolled with this block size - constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - int maska[V::Size]; - std::fill(maska,&maska[V::Size], -1); - for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; -#ifdef FASTOR_HAS_AVX512_MASKS - const auto mask = array_to_mask(maska); -#endif - - size_t i = 0; - for (; i < M0; i += unrollOuterBlock) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N1 columns - for (; j < N; j+= N-N1) { -#ifdef FASTOR_HAS_AVX512_MASKS - interior_block_matmul_mask_impl(a,b,c,i,j,mask); -#else - interior_block_matmul_mask_impl(a,b,c,i,j,maska); -#endif - } - } - - // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. - // This is necessary as for small sizes the earlier block loop may not be - // triggered if the size of the block is bigger than the number of rows of - // (a and c) i.e. M - constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); - for (; i < M1; i += unrollOuterloop) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - V c_ij[unrollOuterloop]; - for (size_t k = 0; k < K; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i + n)*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; j+=N-N1) { - V c_ij[unrollOuterloop]; - for (size_t k = 0; k < K; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; bmm0.mask_load(&b[k*N+j],mask); -#else - const V bmm0(maskload(&b[k*N+j],maska)); -#endif - const V amm0 = a[(i + n)*K+k]; - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - c_ij[n].mask_store(&c[(i+n)*N+j],mask,false); -#else - maskstore(&c[(i+n)*N+j],maska,c_ij[n]); -#endif - } - } - } - - // Now treat the remaining M-M1 rows - FASTOR_IF_CONSTEXPR (M-M1 > 0) { - // Hack to get around zero length array issue - constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - // If MM1==0 the function never gets invoked anyway - interior_block_matmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - V c_ij[MM1]; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); - c_ij[n-M1].store(&c[n*N+j],false); - } - } - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1].store(&c[n*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; j+=N-N1) { - V c_ij[MM1] = {}; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; bmm0.mask_load(&b[k*N+j],mask); -#else - const V bmm0(maskload(&b[k*N+j],maska)); -#endif - const V amm0 = a[n*K+k]; - c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]); - } - } - for (size_t n = M1; n < M; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - c_ij[n-M1].mask_store(&c[n*N+j],mask,false); -#else - maskstore(&c[n*N+j],maska,c_ij[n-M1]); -#endif - } - } - } -} -//----------------------------------------------------------------------------------------------------------- - - - - -//----------------------------------------------------------------------------------------------------------- -// matmul kernel for non-fundamental types -// The assumption here is that non-fundamental types are not SIMD vectorisable for instance -// Tensor,3,3> or Tensor,...> plus they cannot fuse [do fused-add-multiply] -// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in -// the inner most loops of matmul -template -FASTOR_INLINE -void _matmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - // There is no SIMD here as V::Size == 1 anyway - // No outer loop unrolling otherwise the innermost loop - // will create unnecessary temporaries - for (size_t i=0; i -FASTOR_INLINE -void _matmul_mkn_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - - using V = typename internal::choose_best_simd_type,N>::type; - - // Get 10 parallel independent chains of accumulators for bigger matrices - constexpr size_t unrollOuterloop = M >= 64 ? 10UL : (M % 8 == 0 ? 8UL : V::Size); - - // The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore - // the last rows may need special treatment if M is not a multiple of unrollOuterloop. - // M0 is the number of rows that can safely be iterated with a stride of - // unrollOuterloop. - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - for (size_t i = 0; i < M0; i += unrollOuterloop) { - // The iteration over the column index of b and c uses a stride of V::Size. This - // enables row-vector loads (from b) and stores (to c). The matrix storage is - // padded accordingly, ensuring correct bounds and alignment. - for (size_t j = 0; j < N; j += V::Size) { - // This temporary variables are used to accumulate the results of the products - // producing the new values for the c matrix. This variable is necessary - // because we need a V object for data-parallel accumulation. Storing to c - // directly stores to scalar objects and thus would drop the ability for - // data-parallel (SIMD) addition. - V c_ij[unrollOuterloop]; - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] = a[(i + n)*K]*V(&b[j]); - } - for (size_t k = 1; k < K - 1; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j]); - c_ij[n].store(&c[(i + n)*N+j]); - } - } - } -} - -// This is the same implementation as the above case but does not unroll on block sizes and does not require -// the registers to be zeroed out but K must be !=1 -template -FASTOR_INLINE -void _matmul_mkn_non_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - - // This variant strictly cannot deal outer-product i.e. with K==1 - - using V = typename internal::choose_best_simd_type,N>::type; - - // Get 10 parallel independent chains of accumulators for bigger matrices - constexpr size_t unrollOuterloop = M < V::Size ? 1UL : - (( M >= 64 && K > 10 && N > V::Size ) ? 10UL : (M % 8 == 0 && N > V::Size ? 8UL : V::Size)); - constexpr bool isPadded = N % V::Size == 0; - - // The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore - // the last rows may need special treatment if M is not a multiple of unrollOuterloop. - // M0 is the number of rows that can safely be iterated with a stride of - // unrollOuterloop. - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - constexpr size_t N0 = N / V::Size * V::Size; - - for (size_t i = 0; i < M0; i += unrollOuterloop) { - // The iteration over the column index of b and c uses a stride of V::size(). This - // enables row-vector loads (from b) and stores (to c). The matrix storage is - // padded accordingly, ensuring correct bounds and alignment. - size_t j = 0; - for (; j < N0; j += V::Size) { - // This temporary variables are used to accumulate the results of the products - // producing the new values for the c matrix. This variable is necessary - // because we need a V object for data-parallel accumulation. Storing to c - // directly stores to scalar objects and thus would drop the ability for - // data-parallel (SIMD) addition. - V c_ij[unrollOuterloop]; - for (size_t n = 0; n < unrollOuterloop; ++n) { // correct - c_ij[n] = a[(i + n)*K]*V(&b[j], isPadded); - } - for (size_t k = 1; k < K - 1; ++k) { // correct - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j], false); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { // correct - c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j], false); - c_ij[n].store(&c[(i + n)*N+j], isPadded); - } - } - - // Remainder N - N0 columns - for (; j < N; ++j) { - T c_ij[unrollOuterloop]; - for (size_t n = 0; n < unrollOuterloop; ++n) { // correct - c_ij[n] = a[(i + n)*K]*b[j]; - } - for (size_t k = 1; k < K - 1; ++k) { // correct - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] += a[(i + n)*K+k] * b[k*N+j]; - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { // correct - c_ij[n] += a[(i + n)*K+(K - 1)] * b[(K - 1)*N+j]; - c[(i + n)*N+j] = c_ij[n]; - } - } - } - - // This final loop treats the remaining M - M0 rows. - size_t j = 0; - for (; j < N0; j += V::Size) { - V c_ij[M-M0]; - for (size_t n = M0; n < M; ++n) { // correct - c_ij[n - M0] = a[n*K] * V(&b[j], isPadded); - } - for (size_t k = 1; k < K - 1; ++k) { // correct - for (size_t n = M0; n < M; ++n) { // correct - c_ij[n - M0] += a[n*K+k] * V(&b[k*N+j], false); - } - } - for (size_t n = M0; n < M; ++n) { // correct - c_ij[n - M0] += a[n*K+(K - 1)] * V(&b[(K - 1)*N+j], false); - c_ij[n - M0].store(&c[n*N+j], isPadded); - } - } - - for (; j < N; ++j) { - T c_ij[M-M0]; - for (size_t n = M0; n < M; ++n) { // correct - c_ij[n - M0] = a[n*K] * b[j]; - } - for (size_t k = 1; k < K - 1; ++k) { // correct - for (size_t n = M0; n < M; ++n) { // correct - c_ij[n - M0] += a[n*K+k] * b[k*N+j]; - } - } - for (size_t n = M0; n < M; ++n) { // correct - c_ij[n - M0] += a[n*K+(K - 1)] * b[(K - 1)*N+j]; - c[n*N+j] = c_ij[n - M0]; - } - } - -} -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- - -} // end of namespace internal - -} // end of namespace Fastor - -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -#include "Fastor/backend/matmul/matmul_mk_smalln.h" -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- - - -#endif // MATMUL_KERNELS_H diff --git a/noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h b/noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h deleted file mode 100644 index c6810c98..00000000 --- a/noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h +++ /dev/null @@ -1,3271 +0,0 @@ -#ifndef MATMUL_MK_SMALLODDN_H -#define MATMUL_MK_SMALLODDN_H - - -#include "Fastor/simd_vector/simd_vector_abi.h" -#include "Fastor/simd_vector/SIMDVector.h" - -namespace Fastor { - -namespace internal { - - - -// This implementation covers all matrix-matrix multiplications with any M and K and -// and N<=5*SIMDVector::Size. Given that it uses choose_best_simd_type it can switch -// between SSE, AVX and AVX512 to cover all ranges of N - - -//----------------------------------------------------------------------------------------------------------- -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - const V amm6(a[(j+6)*K]); - const V amm7(a[(j+7)*K]); - const V amm8(a[(j+8)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - // row 6 - V omm6(amm6*bmm0); - // row 7 - V omm7(amm7*bmm0); - // row 8 - V omm8(amm8*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - const V amm4(a[(j+4)*K+i]); - const V amm5(a[(j+5)*K+i]); - const V amm6(a[(j+6)*K+i]); - const V amm7(a[(j+7)*K+i]); - - const V amm8(a[(j+8)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - // row 3 - omm3 = fmadd(amm3,bmm0,omm3); - // row 4 - omm4 = fmadd(amm4,bmm0,omm4); - // row 5 - omm5 = fmadd(amm5,bmm0,omm5); - // row 6 - omm6 = fmadd(amm6,bmm0,omm6); - // row 7 - omm7 = fmadd(amm7,bmm0,omm7); - // row 8 - omm8 = fmadd(amm8,bmm0,omm8); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); - omm3.mask_store(&out[(j+3)*N],mask,false); - omm4.mask_store(&out[(j+4)*N],mask,false); - omm5.mask_store(&out[(j+5)*N],mask,false); - omm6.mask_store(&out[(j+6)*N],mask,false); - omm7.mask_store(&out[(j+7)*N],mask,false); - omm8.mask_store(&out[(j+8)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); - maskstore(&out[(j+3)*N],maska,omm3); - maskstore(&out[(j+4)*N],maska,omm4); - maskstore(&out[(j+5)*N],maska,omm5); - maskstore(&out[(j+6)*N],maska,omm6); - maskstore(&out[(j+7)*N],maska,omm7); - maskstore(&out[(j+8)*N],maska,omm8); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - const V amm6(a[(j+6)*K]); - const V amm7(a[(j+7)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - // row 6 - V omm6(amm6*bmm0); - // row 7 - V omm7(amm7*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - const V amm4(a[(j+4)*K+i]); - const V amm5(a[(j+5)*K+i]); - const V amm6(a[(j+6)*K+i]); - const V amm7(a[(j+7)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - // row 3 - omm3 = fmadd(amm3,bmm0,omm3); - // row 4 - omm4 = fmadd(amm4,bmm0,omm4); - // row 5 - omm5 = fmadd(amm5,bmm0,omm5); - // row 6 - omm6 = fmadd(amm6,bmm0,omm6); - // row 7 - omm7 = fmadd(amm7,bmm0,omm7); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); - omm3.mask_store(&out[(j+3)*N],mask,false); - omm4.mask_store(&out[(j+4)*N],mask,false); - omm5.mask_store(&out[(j+5)*N],mask,false); - omm6.mask_store(&out[(j+6)*N],mask,false); - omm7.mask_store(&out[(j+7)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); - maskstore(&out[(j+3)*N],maska,omm3); - maskstore(&out[(j+4)*N],maska,omm4); - maskstore(&out[(j+5)*N],maska,omm5); - maskstore(&out[(j+6)*N],maska,omm6); - maskstore(&out[(j+7)*N],maska,omm7); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - const V amm6(a[(j+6)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - // row 6 - V omm6(amm6*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - const V amm4(a[(j+4)*K+i]); - const V amm5(a[(j+5)*K+i]); - const V amm6(a[(j+6)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - // row 3 - omm3 = fmadd(amm3,bmm0,omm3); - // row 4 - omm4 = fmadd(amm4,bmm0,omm4); - // row 5 - omm5 = fmadd(amm5,bmm0,omm5); - // row 6 - omm6 = fmadd(amm6,bmm0,omm6); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); - omm3.mask_store(&out[(j+3)*N],mask,false); - omm4.mask_store(&out[(j+4)*N],mask,false); - omm5.mask_store(&out[(j+5)*N],mask,false); - omm6.mask_store(&out[(j+6)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); - maskstore(&out[(j+3)*N],maska,omm3); - maskstore(&out[(j+4)*N],maska,omm4); - maskstore(&out[(j+5)*N],maska,omm5); - maskstore(&out[(j+6)*N],maska,omm6); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - const V amm4(a[(j+4)*K+i]); - const V amm5(a[(j+5)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - // row 3 - omm3 = fmadd(amm3,bmm0,omm3); - // row 4 - omm4 = fmadd(amm4,bmm0,omm4); - // row 5 - omm5 = fmadd(amm5,bmm0,omm5); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); - omm3.mask_store(&out[(j+3)*N],mask,false); - omm4.mask_store(&out[(j+4)*N],mask,false); - omm5.mask_store(&out[(j+5)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); - maskstore(&out[(j+3)*N],maska,omm3); - maskstore(&out[(j+4)*N],maska,omm4); - maskstore(&out[(j+5)*N],maska,omm5); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - const V amm4(a[(j+4)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - // row 3 - omm3 = fmadd(amm3,bmm0,omm3); - // row 4 - omm4 = fmadd(amm4,bmm0,omm4); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); - omm3.mask_store(&out[(j+3)*N],mask,false); - omm4.mask_store(&out[(j+4)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); - maskstore(&out[(j+3)*N],maska,omm3); - maskstore(&out[(j+4)*N],maska,omm4); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - // row 3 - omm3 = fmadd(amm3,bmm0,omm3); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); - omm3.mask_store(&out[(j+3)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); - maskstore(&out[(j+3)*N],maska,omm3); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); -#endif - return; -} - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; - bmm0.mask_load(&b[0],mask,false); -#else - const V bmm0(maskload(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - - // row 0 - V omm0(amm0*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - } - -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); -#endif - return; -} - - - -#ifdef FASTOR_HAS_AVX512_MASKS -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const MaskType mask, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#else -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, const int (&maska)[V::Size], - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { -#endif - return; -} - - -template,N>::type::Size>::value,bool> = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - using V = typename choose_best_simd_type,N>::type; - // using V = SIMDVector; - // Unroll a by 10 - constexpr size_t unrollOuterloop = 10UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - int maska[V::Size]; - std::fill(maska,&maska[V::Size], -1); - for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; -#ifdef FASTOR_HAS_AVX512_MASKS - const auto mask = array_to_mask(maska); - V bmm0; -#endif - - size_t j=0; - for (; j(&b[0],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - const V amm6(a[(j+6)*K]); - const V amm7(a[(j+7)*K]); - const V amm8(a[(j+8)*K]); - const V amm9(a[(j+9)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - // row 6 - V omm6(amm6*bmm0); - // row 7 - V omm7(amm7*bmm0); - // row 8 - V omm8(amm8*bmm0); - // row 9 - V omm9(amm9*bmm0); - - for (size_t i=1; i(&b[i*N],maska)); -#endif - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - const V amm4(a[(j+4)*K+i]); - const V amm5(a[(j+5)*K+i]); - const V amm6(a[(j+6)*K+i]); - const V amm7(a[(j+7)*K+i]); - const V amm8(a[(j+8)*K+i]); - const V amm9(a[(j+9)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - // row 1 - omm1 = fmadd(amm1,bmm0,omm1); - // row 2 - omm2 = fmadd(amm2,bmm0,omm2); - // row 3 - omm3 = fmadd(amm3,bmm0,omm3); - // row 4 - omm4 = fmadd(amm4,bmm0,omm4); - // row 5 - omm5 = fmadd(amm5,bmm0,omm5); - // row 6 - omm6 = fmadd(amm6,bmm0,omm6); - // row 7 - omm7 = fmadd(amm7,bmm0,omm7); - // row 8 - omm8 = fmadd(amm8,bmm0,omm8); - // row 9 - omm9 = fmadd(amm9,bmm0,omm9); - } - - // Not all aside from the last vector - // needs to be mask stored, however clang - // just does not like -#ifdef FASTOR_HAS_AVX512_MASKS - omm0.mask_store(&out[(j )*N],mask,false); - omm1.mask_store(&out[(j+1)*N],mask,false); - omm2.mask_store(&out[(j+2)*N],mask,false); - omm3.mask_store(&out[(j+3)*N],mask,false); - omm4.mask_store(&out[(j+4)*N],mask,false); - omm5.mask_store(&out[(j+5)*N],mask,false); - omm6.mask_store(&out[(j+6)*N],mask,false); - omm7.mask_store(&out[(j+7)*N],mask,false); - omm8.mask_store(&out[(j+8)*N],mask,false); - omm9.mask_store(&out[(j+9)*N],mask,false); -#else - maskstore(&out[(j )*N],maska,omm0); - maskstore(&out[(j+1)*N],maska,omm1); - maskstore(&out[(j+2)*N],maska,omm2); - maskstore(&out[(j+3)*N],maska,omm3); - maskstore(&out[(j+4)*N],maska,omm4); - maskstore(&out[(j+5)*N],maska,omm5); - maskstore(&out[(j+6)*N],maska,omm6); - maskstore(&out[(j+7)*N],maska,omm7); - maskstore(&out[(j+8)*N],maska,omm8); - maskstore(&out[(j+9)*N],maska,omm9); -#endif - } - -#ifdef FASTOR_HAS_AVX512_MASKS - matmul_mk_uptosimd_remainder_kernel(j,mask,a,b,out); -#else - matmul_mk_uptosimd_remainder_kernel(j,maska,a,b,out); -#endif -} -//----------------------------------------------------------------------------------------------------------- - - - - - - - - - - - - - - -// Take care of N==V::Size -//----------------------------------------------------------------------------------------------------------- -template = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - const V amm6(a[(j+6)*K]); - const V amm7(a[(j+7)*K]); - const V amm8(a[(j+8)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - // row 6 - V omm6(amm6*bmm0); - // row 7 - V omm7(amm7*bmm0); - // row 8 - V omm8(amm8*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - const V amm6(a[(j+6)*K]); - const V amm7(a[(j+7)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - // row 6 - V omm6(amm6*bmm0); - // row 7 - V omm7(amm7*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - const V amm6(a[(j+6)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - // row 6 - V omm6(amm6*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - const V amm5(a[(j+5)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - // row 5 - V omm5(amm5*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - // row 4 - V omm4(amm4*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - // row 3 - V omm3(amm3*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - // row 2 - V omm2(amm2*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - - // row 0 - V omm0(amm0*bmm0); - // row 1 - V omm1(amm1*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - - const V bmm0(&b[0],false); - - const V amm0(a[(j )*K]); - - // row 0 - V omm0(amm0*bmm0); - - for (size_t i=1; i = false> -FASTOR_INLINE void matmul_mk_uptosimd_remainder_kernel(const size_t j, - const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b, T* FASTOR_RESTRICT out) { - return; -} - - -template,N>::type::Size,bool> = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - using V = typename choose_best_simd_type,N>::type; - - // Unroll a by 10 - constexpr size_t unrollOuterloop = 10UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - - size_t j=0; - for (; j(j,a,b,out); -} -//----------------------------------------------------------------------------------------------------------- - - - - -// Take care of [V::Size < N < 2*V::Size] -// The function implements standard loop unrolling over M. It uses conditional -// loads and store using masks and requires at least AVX. The efficiency of the method comes from -// the fact that it attempts to achieve exact two FMA per load. Both GCC and Clang emit excellent -// code for this at O3 -// A recursive implementation of this using compile time unrolling is available at: -// https://gist.github.com/romeric/a176e28127a8348c3c37c5a369051451 -//----------------------------------------------------------------------------------------------------------- -template,N>::type::Size>::value && - is_greater,N>::type::Size>::value),bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - using V = typename internal::choose_best_simd_type,N>::type; - // We unroll a by 5 and load 2 simd wide columns of b to get two FMA per load - // Unrolling by 5 does not hurt as the fall back cases 4,3,2,1 are also explicitly - // unrolled - constexpr size_t unrollOuterloop = 5UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - int maska[V::Size]; - std::fill(maska,&maska[V::Size], -1); - for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; -#ifdef FASTOR_HAS_AVX512_MASKS - const auto mask = array_to_mask(maska); - V bmm1; -#endif - - size_t j=0; - for (; j(&b[V::Size],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - const V amm4(a[(j+4)*K]); - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - // row 1 - V omm2(amm1*bmm0); - V omm3(amm1*bmm1); - // row 2 - V omm4(amm2*bmm0); - V omm5(amm2*bmm1); - // row 3 - V omm6(amm3*bmm0); - V omm7(amm3*bmm1); - // row 4 - V omm8(amm4*bmm0); - V omm9(amm4*bmm1); - - for (size_t i=1; i(&b[i*N+V::Size],maska)); -#endif - - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - const V amm4(a[(j+4)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - // row 1 - omm2 = fmadd(amm1,bmm0,omm2); - omm3 = fmadd(amm1,bmm1,omm3); - // row 2 - omm4 = fmadd(amm2,bmm0,omm4); - omm5 = fmadd(amm2,bmm1,omm5); - // row 3 - omm6 = fmadd(amm3,bmm0,omm6); - omm7 = fmadd(amm3,bmm1,omm7); - // row 4 - omm8 = fmadd(amm4,bmm0,omm8); - omm9 = fmadd(amm4,bmm1,omm9); - } - - omm0.store(&out[(j )*N],false); - omm1.store(&out[(j )*N+V::Size],false); - omm2.store(&out[(j+1)*N],false); - omm3.store(&out[(j+1)*N+V::Size],false); - omm4.store(&out[(j+2)*N],false); - omm5.store(&out[(j+2)*N+V::Size],false); - omm6.store(&out[(j+3)*N],false); - omm7.store(&out[(j+3)*N+V::Size],false); - omm8.store(&out[(j+4)*N],false); -#ifdef FASTOR_HAS_AVX512_MASKS - omm9.mask_store(&out[(j+4)*N+V::Size],mask,false); -#else - maskstore(&out[(j+4)*N+V::Size],maska,omm9); -#endif - } - - // Remainder M-M0 rows - // Explicitly unroll remaining loops, there is going to be atmost 4 - FASTOR_IF_CONSTEXPR (M-M0==4) { - const V bmm0(&b[0], false); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm1.mask_load(&b[V::Size],mask,false); -#else - const V bmm1(maskload(&b[V::Size],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - const V amm3(a[(j+3)*K]); - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - // row 1 - V omm2(amm1*bmm0); - V omm3(amm1*bmm1); - // row 2 - V omm4(amm2*bmm0); - V omm5(amm2*bmm1); - // row 3 - V omm6(amm3*bmm0); - V omm7(amm3*bmm1); - - for (size_t i=1; i(&b[i*N+V::Size],maska)); -#endif - - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - // row 1 - omm2 = fmadd(amm1,bmm0,omm2); - omm3 = fmadd(amm1,bmm1,omm3); - // row 2 - omm4 = fmadd(amm2,bmm0,omm4); - omm5 = fmadd(amm2,bmm1,omm5); - // row 3 - omm6 = fmadd(amm3,bmm0,omm6); - omm7 = fmadd(amm3,bmm1,omm7); - } - - omm0.store(&out[(j )*N],false); - omm1.store(&out[(j )*N+V::Size],false); - omm2.store(&out[(j+1)*N],false); - omm3.store(&out[(j+1)*N+V::Size],false); - omm4.store(&out[(j+2)*N],false); - omm5.store(&out[(j+2)*N+V::Size],false); - omm6.store(&out[(j+3)*N],false); -#ifdef FASTOR_HAS_AVX512_MASKS - omm7.mask_store(&out[(j+3)*N+V::Size],mask,false); -#else - maskstore(&out[(j+3)*N+V::Size],maska,omm7); -#endif - } - - else FASTOR_IF_CONSTEXPR (M-M0==3) { - const V bmm0(&b[0], false); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm1.mask_load(&b[V::Size],mask,false); -#else - const V bmm1(maskload(&b[V::Size],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - // row 1 - V omm2(amm1*bmm0); - V omm3(amm1*bmm1); - // row 2 - V omm4(amm2*bmm0); - V omm5(amm2*bmm1); - - for (size_t i=1; i(&b[i*N+V::Size],maska)); -#endif - - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - // row 1 - omm2 = fmadd(amm1,bmm0,omm2); - omm3 = fmadd(amm1,bmm1,omm3); - // row 2 - omm4 = fmadd(amm2,bmm0,omm4); - omm5 = fmadd(amm2,bmm1,omm5); - } - - omm0.store(&out[(j )*N],false); - omm1.store(&out[(j )*N+V::Size],false); - omm2.store(&out[(j+1)*N],false); - omm3.store(&out[(j+1)*N+V::Size],false); - omm4.store(&out[(j+2)*N],false); -#ifdef FASTOR_HAS_AVX512_MASKS - omm5.mask_store(&out[(j+2)*N+V::Size],mask,false); -#else - maskstore(&out[(j+2)*N+V::Size],maska,omm5); -#endif - } - - else FASTOR_IF_CONSTEXPR (M-M0==2) { - const V bmm0(&b[0], false); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm1.mask_load(&b[V::Size],mask,false); -#else - const V bmm1(maskload(&b[V::Size],maska)); -#endif - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - // row 1 - V omm2(amm1*bmm0); - V omm3(amm1*bmm1); - - for (size_t i=1; i(&b[i*N+V::Size],maska)); -#endif - - const V amm0(a[(j )*K+i]); - const V amm1(a[(j+1)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - // row 1 - omm2 = fmadd(amm1,bmm0,omm2); - omm3 = fmadd(amm1,bmm1,omm3); - } - - omm0.store(&out[(j )*N],false); - omm1.store(&out[(j )*N+V::Size],false); - omm2.store(&out[(j+1)*N],false); -#ifdef FASTOR_HAS_AVX512_MASKS - omm3.mask_store(&out[(j+1)*N+V::Size],mask,false); -#else - maskstore(&out[(j+1)*N+V::Size],maska,omm3); -#endif - } - - else FASTOR_IF_CONSTEXPR (M-M0==1) { - const V bmm0(&b[0], false); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm1.mask_load(&b[V::Size],mask,false); -#else - const V bmm1(maskload(&b[V::Size],maska)); -#endif - - const V amm0(a[(j )*K]); - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - - for (size_t i=1; i(&b[i*N+V::Size],maska)); -#endif - - const V amm0(a[(j )*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - } - - omm0.store(&out[(j )*N],false); -#ifdef FASTOR_HAS_AVX512_MASKS - omm1.mask_store(&out[(j)*N+V::Size],mask,false); -#else - maskstore(&out[(j)*N+V::Size],maska,omm1); -#endif - } -} -//----------------------------------------------------------------------------------------------------------- - - - -// Take care of 2*V::Size cases -//----------------------------------------------------------------------------------------------------------- -template,N>::type::Size,bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - using V = typename internal::choose_best_simd_type,N>::type; - // We unroll a by 5 and load 2 simd wide columns of b to get two FMA per load - // Unrolling by 5 does not hurt as the fall back cases 4,3,2,1 are also explicitly - // unrolled - constexpr size_t unrollOuterloop = 5UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - - size_t j=0; - for (; j,N>::type::Size>::value && - is_less,N>::type::Size>::value),bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - // Unrolling by 4 to get 12 independent fma - using V = typename internal::choose_best_simd_type,N>::type; - constexpr size_t unrollOuterloop = 4UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - // constexpr size_t remainder = M < unrollOuterloop ? 0 : M0-unrollOuterloop; - constexpr bool isBAligned = false; - constexpr bool isCAligned = false; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - int maska[V::Size]; - std::fill(maska,&maska[V::Size], -1); - for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; -#ifdef FASTOR_HAS_AVX512_MASKS - const auto mask = array_to_mask(maska); - V bmm2; -#endif - - size_t j=0; - for (; j(&b[2*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - // row 1 - V omm3(amm1*bmm0); - V omm4(amm1*bmm1); - V omm5(amm1*bmm2); - // row 2 - V omm6(amm2*bmm0); - V omm7(amm2*bmm1); - V omm8(amm2*bmm2); - // row 3 - V omm9 (amm3*bmm0); - V omm10(amm3*bmm1); - V omm11(amm3*bmm2); - - for (size_t i=1; i(&b[i*N+2*V::Size],maska)); -#endif - - const V amm0(a[j*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - const V amm3(a[(j+3)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - // row 1 - omm3 = fmadd(amm1,bmm0,omm3); - omm4 = fmadd(amm1,bmm1,omm4); - omm5 = fmadd(amm1,bmm2,omm5); - // row 2 - omm6 = fmadd(amm2,bmm0,omm6); - omm7 = fmadd(amm2,bmm1,omm7); - omm8 = fmadd(amm2,bmm2,omm8); - // row 3 - omm9 = fmadd(amm3,bmm0,omm9); - omm10 = fmadd(amm3,bmm1,omm10); - omm11 = fmadd(amm3,bmm2,omm11); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); - - omm3.store(&out[(j+1)*N],isCAligned); - omm4.store(&out[(j+1)*N+V::Size],isCAligned); - omm5.store(&out[(j+1)*N+2*V::Size],isCAligned); - - omm6.store(&out[(j+2)*N],isCAligned); - omm7.store(&out[(j+2)*N+V::Size],isCAligned); - omm8.store(&out[(j+2)*N+2*V::Size],isCAligned); - - omm9.store(&out[(j+3)*N],isCAligned); - omm10.store(&out[(j+3)*N+V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm11.mask_store(&out[(j+3)*N+2*V::Size],mask,false); -#else - maskstore(&out[(j+3)*N+2*V::Size],maska,omm11); -#endif - } - - FASTOR_IF_CONSTEXPR (M-M0==3) { - const V amm0(a[j*K]); - const V amm1(a[(j+1)*K]); - const V amm2(a[(j+2)*K]); - - const V bmm0(&b[0], isBAligned); - const V bmm1((&b[V::Size]),isBAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm2.mask_load(&b[2*V::Size],mask,false); -#else - const V bmm2(maskload(&b[2*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - // row 1 - V omm3(amm1*bmm0); - V omm4(amm1*bmm1); - V omm5(amm1*bmm2); - // row 2 - V omm6(amm2*bmm0); - V omm7(amm2*bmm1); - V omm8(amm2*bmm2); - - for (size_t i=1; i(&b[i*N+2*V::Size],maska)); -#endif - - const V amm0(a[j*K+i]); - const V amm1(a[(j+1)*K+i]); - const V amm2(a[(j+2)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - // row 1 - omm3 = fmadd(amm1,bmm0,omm3); - omm4 = fmadd(amm1,bmm1,omm4); - omm5 = fmadd(amm1,bmm2,omm5); - // row 2 - omm6 = fmadd(amm2,bmm0,omm6); - omm7 = fmadd(amm2,bmm1,omm7); - omm8 = fmadd(amm2,bmm2,omm8); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); - - omm3.store(&out[(j+1)*N],isCAligned); - omm4.store(&out[(j+1)*N+V::Size],isCAligned); - omm5.store(&out[(j+1)*N+2*V::Size],isCAligned); - - omm6.store(&out[(j+2)*N],isCAligned); - omm7.store(&out[(j+2)*N+V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm8.mask_store(&out[(j+2)*N+2*V::Size],mask,false); -#else - maskstore(&out[(j+2)*N+2*V::Size],maska,omm8); -#endif - } - - else FASTOR_IF_CONSTEXPR (M-M0==2) { - - const V amm0(a[j*K]); - const V amm1(a[(j+1)*K]); - - const V bmm0(&b[0], isBAligned); - const V bmm1((&b[V::Size]),isBAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm2.mask_load(&b[2*V::Size],mask,false); -#else - const V bmm2(maskload(&b[2*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - // row 1 - V omm3(amm1*bmm0); - V omm4(amm1*bmm1); - V omm5(amm1*bmm2); - - for (size_t i=1; i(&b[i*N+2*V::Size],maska)); -#endif - - const V amm0(a[j*K+i]); - const V amm1(a[(j+1)*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - // row 1 - omm3 = fmadd(amm1,bmm0,omm3); - omm4 = fmadd(amm1,bmm1,omm4); - omm5 = fmadd(amm1,bmm2,omm5); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); - - omm3.store(&out[(j+1)*N],isCAligned); - omm4.store(&out[(j+1)*N+V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm5.mask_store(&out[(j+1)*N+2*V::Size],mask,false); -#else - maskstore(&out[(j+1)*N+2*V::Size],maska,omm5); -#endif - } - - FASTOR_IF_CONSTEXPR (M-M0==1) { - - const V amm0(a[j*K]); - - const V bmm0(&b[0], isBAligned); - const V bmm1((&b[V::Size]),isBAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm2.mask_load(&b[2*V::Size],mask,false); -#else - const V bmm2(maskload(&b[2*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - - for (size_t i=1; i(&b[i*N+2*V::Size],maska)); -#endif - - const V amm0(a[j*K+i]); - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - -#ifdef FASTOR_HAS_AVX512_MASKS - omm2.mask_store(&out[j*N+2*V::Size],mask,false); -#else - maskstore(&out[j*N+2*V::Size],maska,omm2); -#endif - } -} -//----------------------------------------------------------------------------------------------------------- - - - -// Take care of 3*V::Size cases -// Note that you get the exact same performance by tuning _matmul_base parameters as -// [unrollOuterloop = 2, nSIMDRows=1 and nSIMDCols=3] however these parameters affect -// the universal behaviour of that method specially for big matrices -// This method unrolls M by 2 only as when N==3*V::Size unrolling the entire column of c -// and doing 2 rows at a time is the most benificial. Unrolling M any further hurts the -// performance really bad -//----------------------------------------------------------------------------------------------------------- -template,N>::type::Size,bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - // Unrolling by 4 to get 12 independent fma - using V = typename internal::choose_best_simd_type,N>::type; - constexpr size_t unrollOuterloop = 4UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - constexpr bool isBAligned = false; - constexpr bool isCAligned = false; - - size_t j=0; - for (; j,N>::type::Size>::value && - is_less,N>::type::Size>::value),bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - using V = typename internal::choose_best_simd_type,N>::type; - constexpr size_t unrollOuterloop = 3UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - constexpr bool isBAligned = false; - constexpr bool isCAligned = false; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - int maska[V::Size]; - std::fill(maska,&maska[V::Size], -1); - for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; -#ifdef FASTOR_HAS_AVX512_MASKS - const auto mask = array_to_mask(maska); - V bmm3; -#endif - - size_t j=0; - for (; j(&b[3*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - V omm3(amm0*bmm3); - // row 1 - V omm4(amm1*bmm0); - V omm5(amm1*bmm1); - V omm6(amm1*bmm2); - V omm7(amm1*bmm3); - // row 2 - V omm8 (amm2*bmm0); - V omm9 (amm2*bmm1); - V omm10(amm2*bmm2); - V omm11(amm2*bmm3); - - for (size_t i=1; i(&b[i*N+3*V::Size],maska)); -#endif - - const V amm0 = a[(j )*K+i]; - const V amm1 = a[(j+1)*K+i]; - const V amm2 = a[(j+2)*K+i]; - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - omm3 = fmadd(amm0,bmm3,omm3); - // row 1 - omm4 = fmadd(amm1,bmm0,omm4); - omm5 = fmadd(amm1,bmm1,omm5); - omm6 = fmadd(amm1,bmm2,omm6); - omm7 = fmadd(amm1,bmm3,omm7); - // row 2 - omm8 = fmadd(amm2,bmm0,omm8); - omm9 = fmadd(amm2,bmm1,omm9); - omm10 = fmadd(amm2,bmm2,omm10); - omm11 = fmadd(amm2,bmm3,omm11); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); - omm3.store(&out[j*N+3*V::Size],isCAligned); - - omm4.store(&out[(j+1)*N],isCAligned); - omm5.store(&out[(j+1)*N+V::Size],isCAligned); - omm6.store(&out[(j+1)*N+2*V::Size],isCAligned); - omm7.store(&out[(j+1)*N+3*V::Size],isCAligned); - - omm8.store (&out[(j+2)*N],isCAligned); - omm9.store(&out[(j+2)*N+V::Size],isCAligned); - omm10.store(&out[(j+2)*N+2*V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm11.mask_store(&out[(j+2)*N+3*V::Size],mask,false); -#else - maskstore(&out[(j+2)*N+3*V::Size],maska,omm11); -#endif - } - - FASTOR_IF_CONSTEXPR (M-M0==2) { - - const V amm0(a[(j )*K]); - const V amm1(a[(j+1)*K]); - - const V bmm0(&b[0], isBAligned); - const V bmm1(&b[V::Size],isBAligned); - const V bmm2(&b[2*V::Size],isBAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm3.mask_load(&b[3*V::Size],mask,false); -#else - const V bmm3(maskload(&b[3*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - V omm3(amm0*bmm3); - // row 1 - V omm4(amm1*bmm0); - V omm5(amm1*bmm1); - V omm6(amm1*bmm2); - V omm7(amm1*bmm3); - - for (size_t i=1; i(&b[i*N+3*V::Size],maska)); -#endif - - const V amm0 = a[(j )*K+i]; - const V amm1 = a[(j+1)*K+i]; - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - omm3 = fmadd(amm0,bmm3,omm3); - // row 1 - omm4 = fmadd(amm1,bmm0,omm4); - omm5 = fmadd(amm1,bmm1,omm5); - omm6 = fmadd(amm1,bmm2,omm6); - omm7 = fmadd(amm1,bmm3,omm7); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); - omm3.store(&out[j*N+3*V::Size],isCAligned); - - omm4.store(&out[(j+1)*N],isCAligned); - omm5.store(&out[(j+1)*N+V::Size],isCAligned); - omm6.store(&out[(j+1)*N+2*V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm7.mask_store(&out[(j+1)*N+3*V::Size],mask,false); -#else - maskstore(&out[(j+1)*N+3*V::Size],maska,omm7); -#endif - } - - else FASTOR_IF_CONSTEXPR (M-M0==1) { - - const V amm0(a[(j )*K]); - - const V bmm0(&b[0], isBAligned); - const V bmm1(&b[V::Size],isBAligned); - const V bmm2(&b[2*V::Size],isBAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm3.mask_load(&b[3*V::Size],mask,false); -#else - const V bmm3(maskload(&b[3*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - V omm3(amm0*bmm3); - - for (size_t i=1; i(&b[i*N+3*V::Size],maska)); -#endif - - const V amm0 = a[(j )*K+i]; - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - omm3 = fmadd(amm0,bmm3,omm3); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm3.mask_store(&out[(j)*N+3*V::Size],mask,false); -#else - maskstore(&out[(j)*N+3*V::Size],maska,omm3); -#endif - } -} -//----------------------------------------------------------------------------------------------------------- - - - - -// Take care of 4*V::Size cases -// Note that you get the exact same performance by tuning _matmul_base parameters as -// [unrollOuterloop = 2, nSIMDRows=1 and nSIMDCols=4] however these parameters affect -// the universal behaviour of that method specially for big matrices -// This method unrolls M by 2 only as when N==4*V::Size unrolling the entire column of c -// and doing 2 rows at a time is the most benificial. Unrolling M any further hurts the -// performance really bad -//----------------------------------------------------------------------------------------------------------- -template,N>::type::Size,bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - - using V = typename internal::choose_best_simd_type,N>::type; - constexpr size_t unrollOuterloop = 3UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - constexpr bool isBAligned = false; - constexpr bool isCAligned = false; - - size_t j=0; - for (; j,N>::type::Size>::value && - is_less,N>::type::Size>::value),bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - - using V = typename internal::choose_best_simd_type,N>::type; - constexpr size_t unrollOuterloop = 2UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - // constexpr size_t remainder = M < unrollOuterloop ? 0 : M0-unrollOuterloop; - constexpr bool isBAligned = false; - constexpr bool isCAligned = false; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - int maska[V::Size]; - std::fill(maska,&maska[V::Size], -1); - for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; -#ifdef FASTOR_HAS_AVX512_MASKS - const auto mask = array_to_mask(maska); - V bmm4; -#endif - - size_t j=0; - for (; j(&b[4*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - V omm3(amm0*bmm3); - V omm4(amm0*bmm4); - // row 1 - V omm5(amm1*bmm0); - V omm6(amm1*bmm1); - V omm7(amm1*bmm2); - V omm8(amm1*bmm3); - V omm9(amm1*bmm4); - - - for (size_t i=1; i(&b[i*N+4*V::Size],maska)); -#endif - - const V amm0 = a[j*K+i]; - const V amm1 = a[(j+1)*K+i]; - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - omm3 = fmadd(amm0,bmm3,omm3); - omm4 = fmadd(amm0,bmm4,omm4); - // row 1 - omm5 = fmadd(amm1,bmm0,omm5); - omm6 = fmadd(amm1,bmm1,omm6); - omm7 = fmadd(amm1,bmm2,omm7); - omm8 = fmadd(amm1,bmm3,omm8); - omm9 = fmadd(amm1,bmm4,omm9); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); - omm3.store(&out[j*N+3*V::Size],isCAligned); - omm4.store(&out[j*N+4*V::Size],isCAligned); - - omm5.store(&out[(j+1)*N],isCAligned); - omm6.store(&out[(j+1)*N+V::Size],isCAligned); - omm7.store(&out[(j+1)*N+2*V::Size],isCAligned); - omm8.store(&out[(j+1)*N+3*V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm9.mask_store(&out[(j+1)*N+4*V::Size],mask,false); -#else - maskstore(&out[(j+1)*N+4*V::Size],maska,omm9); -#endif - } - - FASTOR_IF_CONSTEXPR (M-M0==1) { - V amm0(a[j*K]); - - const V bmm0(&b[0], isBAligned); - const V bmm1((&b[V::Size]),isBAligned); - const V bmm2((&b[2*V::Size]),isBAligned); - const V bmm3((&b[3*V::Size]),isBAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - bmm4.mask_load(&b[4*V::Size],mask,false); -#else - const V bmm4(maskload(&b[4*V::Size],maska)); -#endif - - // row 0 - V omm0(amm0*bmm0); - V omm1(amm0*bmm1); - V omm2(amm0*bmm2); - V omm3(amm0*bmm3); - V omm4(amm0*bmm4); - - for (size_t i=1; i(&b[i*N+4*V::Size],maska)); -#endif - - const V amm0 = a[j*K+i]; - - // row 0 - omm0 = fmadd(amm0,bmm0,omm0); - omm1 = fmadd(amm0,bmm1,omm1); - omm2 = fmadd(amm0,bmm2,omm2); - omm3 = fmadd(amm0,bmm3,omm3); - omm4 = fmadd(amm0,bmm4,omm4); - } - - omm0.store(&out[j*N],isCAligned); - omm1.store(&out[j*N+V::Size],isCAligned); - omm2.store(&out[j*N+2*V::Size],isCAligned); - omm3.store(&out[j*N+3*V::Size],isCAligned); -#ifdef FASTOR_HAS_AVX512_MASKS - omm4.mask_store(&out[j*N+4*V::Size],mask,false); -#else - maskstore(&out[j*N+4*V::Size],maska,omm4); -#endif - } -} -//----------------------------------------------------------------------------------------------------------- - - - -// N==5*V::Size case -//----------------------------------------------------------------------------------------------------------- -template,N>::type::Size,bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - - using V = typename internal::choose_best_simd_type,N>::type; - constexpr size_t unrollOuterloop = 2UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - // constexpr size_t remainder = M < unrollOuterloop ? 0 : M0-unrollOuterloop; - constexpr bool isBAligned = false; - constexpr bool isCAligned = false; - - size_t j=0; - for (; j,N>::type::Size>::value,bool>::type = 0> -FASTOR_INLINE -void _matmul_mk_smalln(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - _matmul_base_masked(a,b,out); -} - - - -} // internal - -} // Fastor - - -#endif // MATMUL_MK_SMALLODDN_H diff --git a/noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h b/noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h deleted file mode 100644 index fb916f41..00000000 --- a/noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h +++ /dev/null @@ -1,1187 +0,0 @@ -#ifndef MATMUL_SPECIALISATIONS_KERNELS_H -#define MATMUL_SPECIALISATIONS_KERNELS_H - - -// Forward declare -//----------------------------------------------------------------------------------------------------------- -namespace internal { -template::value,bool>::type = 0 > -FASTOR_INLINE -void _matmul8k8_float(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out); - -template::value,bool>::type = 0 > -FASTOR_INLINE -void _matmul8k8_double(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out); - -} // internal -//----------------------------------------------------------------------------------------------------------- - - - -#ifdef FASTOR_SSE2_IMPL -// (2xk) x (kx2) matrices -template::value),bool>::type = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m128d out_row0 = ZEROPD; - __m128d out_row1 = ZEROPD; - - for (size_t i=0; i::value),bool>::type = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m128 out_row0 = ZEROPS; - __m128 out_row1 = ZEROPS; - - __m128 brow = ZEROPS; - for (size_t i=0; i::type = 0> -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - internal::_matmul_base(a,b,out); -} -#endif - -#ifdef FASTOR_AVX_IMPL -// (3xk) x (kx3) matrices -template::value),bool>::type = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m256d out_row0 = VZEROPD; - __m256d out_row1 = VZEROPD; - __m256d out_row2 = VZEROPD; - - for (size_t i=0; i::value,bool>::type = 0> -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - internal::_matmul_base(a,b,out); -} -#endif - -#ifdef FASTOR_SSE2_IMPL -// (3xk) x (kx3) matrices -template::value),bool>::type = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m128 out_row0 = ZEROPS; - __m128 out_row1 = ZEROPS; - __m128 out_row2 = ZEROPS; - - for (size_t i=0; i::value),bool>::type = 0> -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - internal::_matmul_base(a,b,out); -} -#endif - -#ifdef FASTOR_AVX_IMPL -// (4xk) x (kx4) matrices -template::value),bool>::type = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m256d out_row0 = VZEROPD; - __m256d out_row1 = VZEROPD; - __m256d out_row2 = VZEROPD; - __m256d out_row3 = VZEROPD; - - for (size_t i=0; i::value,bool>::type = 0> -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - internal::_matmul_base(a,b,out); -} -#endif - -#ifdef FASTOR_SSE2_IMPL -// (4xk) x (kx4) matrices -template::value),bool>::type = 0> -FASTOR_INLINE -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m128 out_row0 = ZEROPS; - __m128 out_row1 = ZEROPS; - __m128 out_row2 = ZEROPS; - __m128 out_row3 = ZEROPS; - - for (size_t i=0; i::value,bool>::type = 0> -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - internal::_matmul_base(a,b,out); -} -#endif - - -// (8xk) x (kx8) matrices -template::value,bool>::type = 0> -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { -#ifdef FASTOR_AVX512_IMPL - FASTOR_IF_CONSTEXPR(K<=64) - internal::_matmul8k8_double(a,b,out); - else - internal::_matmul_mk_smalln(a,b,out); -#else - internal::_matmul_mk_smalln(a,b,out); -#endif -} - -template::value,bool>::type = 0> -void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { -#ifdef FASTOR_AVX_IMPL - internal::_matmul8k8_float(a,b,out); -#else - internal::_matmul_base(a,b,out); -#endif -} - - - - - -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE -void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - - // Ivy 17 - Sky 16 [12 if fused] OPS - __m128 ar = _mm_loadu_ps(a); - __m128 br = _mm_loadu_ps(b); - __m128 ar0 = _mm_shuffle_ps(ar,ar,_MM_SHUFFLE(2,2,0,0)); - __m128 ar1 = _mm_shuffle_ps(ar,ar,_MM_SHUFFLE(3,3,1,1)); - __m128 br0 = _mm_shuffle_ps(br,br,_MM_SHUFFLE(1,0,1,0)); - __m128 br1 = _mm_shuffle_ps(br,br,_MM_SHUFFLE(3,2,3,2)); - __m128 res = _mm_add_ps(_mm_mul_ps(ar0,br0),_mm_mul_ps(ar1,br1)); - _mm_storeu_ps(out,res); -} - -template<> -FASTOR_INLINE -void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - - // Ivy 63 - HW 45 - Sky 36 OPS - // Fully vectorised approach that reduces - // (27 scalar mul + 18 scalar add) to - // (9 SSE mul + 6 SEE add) [non-FMA] - // (3 SSE mul + 6 SEE FMA) [ FMA] - - __m128 bmm0 = _mm_loadu_ps (b ); - __m128 bmm1 = _mm_loadu_ps (b+3); - __m128 bmm2 = _mm_loadul3_ps(b+6); - - __m128 omm0(_mm_mul_ps(_mm_set1_ps(a[0]),bmm0)); - __m128 omm1(_mm_mul_ps(_mm_set1_ps(a[3]),bmm0)); - __m128 omm2(_mm_mul_ps(_mm_set1_ps(a[6]),bmm0)); -#ifndef FASTOR_FMA_IMPL - omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[1]),bmm1)); - omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[4]),bmm1)); - omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[7]),bmm1)); - - omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[2]),bmm2)); - omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[5]),bmm2)); - omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[8]),bmm2)); -#else - omm0 = _mm_fmadd_ps(_mm_set1_ps(a[1]),bmm1,omm0); - omm1 = _mm_fmadd_ps(_mm_set1_ps(a[4]),bmm1,omm1); - omm2 = _mm_fmadd_ps(_mm_set1_ps(a[7]),bmm1,omm2); - - omm0 = _mm_fmadd_ps(_mm_set1_ps(a[2]),bmm2,omm0); - omm1 = _mm_fmadd_ps(_mm_set1_ps(a[5]),bmm2,omm1); - omm2 = _mm_fmadd_ps(_mm_set1_ps(a[8]),bmm2,omm2); -#endif - - _mm_storeu_ps (out , omm0); - _mm_storeu_ps (out+3, omm1); - _mm_storeul3_ps(out+6, omm2); -} - -template<> -FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - - // Ivy 116 - HW 80 - Sky 64 OPS - // Fully vectorised approach that reduces - // (64 scalar mul + 48 scalar add) to - // (16 SSE mul + 12 SEE add) [non-FMA] - // (4 SSE mul + 12 SEE FMA) [ FMA] - - __m128 bmm0 = _mm_loadu_ps (b ); - __m128 bmm1 = _mm_loadu_ps (b+4 ); - __m128 bmm2 = _mm_loadu_ps (b+8 ); - __m128 bmm3 = _mm_loadu_ps (b+12); - - __m128 omm0(_mm_mul_ps(_mm_set1_ps(a[0 ]),bmm0)); - __m128 omm1(_mm_mul_ps(_mm_set1_ps(a[4 ]),bmm0)); - __m128 omm2(_mm_mul_ps(_mm_set1_ps(a[8 ]),bmm0)); - __m128 omm3(_mm_mul_ps(_mm_set1_ps(a[12]),bmm0)); -#ifndef FASTOR_FMA_IMPL - omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[1 ]),bmm1)); - omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[5 ]),bmm1)); - omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[9 ]),bmm1)); - omm3 = _mm_add_ps(omm3,_mm_mul_ps(_mm_set1_ps(a[13]),bmm1)); - - omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[2 ]),bmm2)); - omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[6 ]),bmm2)); - omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[10]),bmm2)); - omm3 = _mm_add_ps(omm3,_mm_mul_ps(_mm_set1_ps(a[14]),bmm2)); - - omm0 = _mm_add_ps(omm0,_mm_mul_ps(_mm_set1_ps(a[3 ]),bmm3)); - omm1 = _mm_add_ps(omm1,_mm_mul_ps(_mm_set1_ps(a[7 ]),bmm3)); - omm2 = _mm_add_ps(omm2,_mm_mul_ps(_mm_set1_ps(a[11]),bmm3)); - omm3 = _mm_add_ps(omm3,_mm_mul_ps(_mm_set1_ps(a[15]),bmm3)); -#else - omm0 = _mm_fmadd_ps(_mm_set1_ps(a[1 ]),bmm1,omm0); - omm1 = _mm_fmadd_ps(_mm_set1_ps(a[5 ]),bmm1,omm1); - omm2 = _mm_fmadd_ps(_mm_set1_ps(a[9 ]),bmm1,omm2); - omm3 = _mm_fmadd_ps(_mm_set1_ps(a[13]),bmm1,omm3); - - omm0 = _mm_fmadd_ps(_mm_set1_ps(a[2 ]),bmm2,omm0); - omm1 = _mm_fmadd_ps(_mm_set1_ps(a[6 ]),bmm2,omm1); - omm2 = _mm_fmadd_ps(_mm_set1_ps(a[10]),bmm2,omm2); - omm3 = _mm_fmadd_ps(_mm_set1_ps(a[14]),bmm2,omm3); - - omm0 = _mm_fmadd_ps(_mm_set1_ps(a[3 ]),bmm3,omm0); - omm1 = _mm_fmadd_ps(_mm_set1_ps(a[7 ]),bmm3,omm1); - omm2 = _mm_fmadd_ps(_mm_set1_ps(a[11]),bmm3,omm2); - omm3 = _mm_fmadd_ps(_mm_set1_ps(a[15]),bmm3,omm3); -#endif - - _mm_storeu_ps (out , omm0); - _mm_storeu_ps (out+4 , omm1); - _mm_storeu_ps (out+8 , omm2); - _mm_storeu_ps (out+12, omm3); -} -#endif - -#ifdef FASTOR_AVX_IMPL -namespace internal { -// This is the common interface for 8k8 matmul and not only for 888 so do not -// make it specific to 888 floats -template::value,bool>::type> -FASTOR_INLINE -void _matmul8k8_float(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m256 out_row0 = VZEROPS; - __m256 out_row1 = VZEROPS; - __m256 out_row2 = VZEROPS; - __m256 out_row3 = VZEROPS; - __m256 out_row4 = VZEROPS; - __m256 out_row5 = VZEROPS; - __m256 out_row6 = VZEROPS; - __m256 out_row7 = VZEROPS; - - for (size_t i=0; i -FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - internal::_matmul8k8_float(a,b,out); - return; -} -#endif - - -#ifdef FASTOR_AVX512_IMPL -namespace internal { -template::value,bool>::type> -FASTOR_INLINE -void _matmul8k8_double(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - __m512d out_row0 = _mm512_setzero_pd(); - __m512d out_row1 = _mm512_setzero_pd(); - __m512d out_row2 = _mm512_setzero_pd(); - __m512d out_row3 = _mm512_setzero_pd(); - __m512d out_row4 = _mm512_setzero_pd(); - __m512d out_row5 = _mm512_setzero_pd(); - __m512d out_row6 = _mm512_setzero_pd(); - __m512d out_row7 = _mm512_setzero_pd(); - - for (size_t i=0; i -FASTOR_INLINE void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - internal::_matmul8k8_double(a,b,out); - return; -} -#endif - - -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE -void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - - // Ivy 26 - HW 20 - Sky 16 OPS - - __m128d bmm0 = _mm_loadu_pd (b ); - __m128d bmm1 = _mm_loadu_pd (b+2); - - __m128d omm0(_mm_mul_pd(_mm_set1_pd(a[0]),bmm0)); - __m128d omm1(_mm_mul_pd(_mm_set1_pd(a[2]),bmm0)); -#ifndef FASTOR_FMA_IMPL - omm0 = _mm_add_pd(omm0,_mm_mul_pd(_mm_set1_pd(a[1]),bmm1)); - omm1 = _mm_add_pd(omm1,_mm_mul_pd(_mm_set1_pd(a[3]),bmm1)); -#else - omm0 = _mm_fmadd_pd(_mm_set1_pd(a[1]),bmm1,omm0); - omm1 = _mm_fmadd_pd(_mm_set1_pd(a[3]),bmm1,omm1); -#endif - _mm_storeu_pd (out , omm0); - _mm_storeu_pd (out+2, omm1); -} -#endif - -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE -void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - - // Ivy 63 - HW 45 - Sky 36 OPS - // Fully vectorised approach that reduces - // (27 scalar mul + 18 scalar add) to - // (9 AVX mul + 6 AVX add) [non-FMA] - // (3 AVX mul + 6 AVX FMA) [ FMA] - - __m256d bmm0 = _mm256_loadu_pd (b ); - __m256d bmm1 = _mm256_loadu_pd (b+3); - __m256d bmm2 = _mm256_loadul3_pd(b+6); - - __m256d omm0(_mm256_mul_pd(_mm256_set1_pd(a[0]),bmm0)); - __m256d omm1(_mm256_mul_pd(_mm256_set1_pd(a[3]),bmm0)); - __m256d omm2(_mm256_mul_pd(_mm256_set1_pd(a[6]),bmm0)); -#ifndef FASTOR_FMA_IMPL - omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[1]),bmm1)); - omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[4]),bmm1)); - omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[7]),bmm1)); - - omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[2]),bmm2)); - omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[5]),bmm2)); - omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[8]),bmm2)); -#else - omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[1]),bmm1,omm0); - omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[4]),bmm1,omm1); - omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[7]),bmm1,omm2); - - omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[2]),bmm2,omm0); - omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[5]),bmm2,omm1); - omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[8]),bmm2,omm2); -#endif - - _mm256_storeu_pd (out , omm0); - _mm256_storeu_pd (out+3, omm1); - _mm256_storeul3_pd(out+6, omm2); -} - -template<> -FASTOR_INLINE -void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - - // Ivy 116 - HW 80 - Sky 64 OPS - // Fully vectorised approach that reduces - // (64 scalar mul + 48 scalar add) to - // (16 SSE mul + 12 SEE add) [non-FMA] - // (4 SSE mul + 12 SEE FMA) [ FMA] - - __m256d bmm0 = _mm256_loadu_pd (b ); - __m256d bmm1 = _mm256_loadu_pd (b+4 ); - __m256d bmm2 = _mm256_loadu_pd (b+8 ); - __m256d bmm3 = _mm256_loadu_pd (b+12); - - __m256d omm0(_mm256_mul_pd(_mm256_set1_pd(a[0 ]),bmm0)); - __m256d omm1(_mm256_mul_pd(_mm256_set1_pd(a[4 ]),bmm0)); - __m256d omm2(_mm256_mul_pd(_mm256_set1_pd(a[8 ]),bmm0)); - __m256d omm3(_mm256_mul_pd(_mm256_set1_pd(a[12]),bmm0)); -#ifndef FASTOR_FMA_IMPL - omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[1 ]),bmm1)); - omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[5 ]),bmm1)); - omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[9 ]),bmm1)); - omm3 = _mm256_add_pd(omm3,_mm256_mul_pd(_mm256_set1_pd(a[13]),bmm1)); - - omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[2 ]),bmm2)); - omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[6 ]),bmm2)); - omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[10]),bmm2)); - omm3 = _mm256_add_pd(omm3,_mm256_mul_pd(_mm256_set1_pd(a[14]),bmm2)); - - omm0 = _mm256_add_pd(omm0,_mm256_mul_pd(_mm256_set1_pd(a[3 ]),bmm3)); - omm1 = _mm256_add_pd(omm1,_mm256_mul_pd(_mm256_set1_pd(a[7 ]),bmm3)); - omm2 = _mm256_add_pd(omm2,_mm256_mul_pd(_mm256_set1_pd(a[11]),bmm3)); - omm3 = _mm256_add_pd(omm3,_mm256_mul_pd(_mm256_set1_pd(a[15]),bmm3)); -#else - omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[1 ]),bmm1,omm0); - omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[5 ]),bmm1,omm1); - omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[9 ]),bmm1,omm2); - omm3 = _mm256_fmadd_pd(_mm256_set1_pd(a[13]),bmm1,omm3); - - omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[2 ]),bmm2,omm0); - omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[6 ]),bmm2,omm1); - omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[10]),bmm2,omm2); - omm3 = _mm256_fmadd_pd(_mm256_set1_pd(a[14]),bmm2,omm3); - - omm0 = _mm256_fmadd_pd(_mm256_set1_pd(a[3 ]),bmm3,omm0); - omm1 = _mm256_fmadd_pd(_mm256_set1_pd(a[7 ]),bmm3,omm1); - omm2 = _mm256_fmadd_pd(_mm256_set1_pd(a[11]),bmm3,omm2); - omm3 = _mm256_fmadd_pd(_mm256_set1_pd(a[15]),bmm3,omm3); -#endif - - _mm256_storeu_pd (out , omm0); - _mm256_storeu_pd (out+4 , omm1); - _mm256_storeu_pd (out+8 , omm2); - _mm256_storeu_pd (out+12, omm3); -} -#endif - - - - - -//! Matrix-vector multiplication -//----------------------------------------------------------------------------------------------------------- -// Don't call this function directly as it's name is unconventional -// It gets called from within matmul anyway so always call matmul -namespace internal { - -template -FASTOR_INLINE -void _matvecmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - using V = typename choose_best_simd_type,N>::type; - constexpr size_t unrollOuterloop = 8UL; - constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - constexpr bool isAAligned = false; - constexpr bool isBAligned = false; - - size_t i=0; - FASTOR_IF_CONSTEXPR(N < V::Size) { - - for (; i -FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - // 11 OPS - __m128 amm = _mm_loadu_ps(a); - __m128 bmm = ZEROPS; - bmm = _mm_loadl_pi(bmm, (__m64*)b); - - __m128 res = _mm_mul_ps(amm,_mm_movelh_ps(bmm,bmm)); - __m128 res2 = _mm_shuffle_ps(res ,res ,_MM_SHUFFLE(2,3,0,1)); - res2 = _mm_add_ps(res,res2); - __m128 res3 = _mm_shuffle_ps(res2,res2,_MM_SHUFFLE(3,1,2,0)); - _mm_storel_pi((__m64*) out,res3); -} - -template<> -FASTOR_INLINE void _matmul(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - // IVY/HW 47 OPS - // Mask loads on AVX and AVX512 otherwise 3 single loads each - __m128 amm0 = _mm_loadul3_ps(a); - __m128 amm1 = _mm_loadul3_ps(&a[3]); - __m128 amm2 = _mm_loadul3_ps(&a[6]); - __m128 bmm = _mm_loadul3_ps(b); - - out[0] = _mm_sum_ps(_mm_mul_ps(amm0,bmm)); - out[1] = _mm_sum_ps(_mm_mul_ps(amm1,bmm)); - out[2] = _mm_sum_ps(_mm_mul_ps(amm2,bmm)); -} - -template<> -FASTOR_INLINE void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - // IVY/HW/SKY 19 OPS - __m128d amm0 = _mm_loadu_pd(a ); - __m128d amm1 = _mm_loadu_pd(a+2); - __m128d bmm0 = _mm_loadu_pd(b ); - - __m128d omm0 = _mm_mul_pd(amm0,bmm0); - __m128d omm1 = _mm_mul_pd(amm1,bmm0); - omm0 = _mm_add_pd(omm0, _mm_reverse_pd(omm0)); - omm1 = _mm_add_pd(omm1, _mm_reverse_pd(omm1)); - - _mm_storeu_pd(out,_mm_shuffle_pd(omm0,omm1,0x1)); -} -#endif - - -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE void _matmul(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - // SKY 49 - ICE 87 OPS - // Mask loads on AVX and AVX512 - __m256d amm0 = _mm256_loadul3_pd(a); - __m256d amm1 = _mm256_loadul3_pd(&a[3]); - __m256d amm2 = _mm256_loadul3_pd(&a[6]); - __m256d bmm = _mm256_loadul3_pd(b); - - out[0] = _mm256_sum_pd(_mm256_mul_pd(amm0,bmm)); - out[1] = _mm256_sum_pd(_mm256_mul_pd(amm1,bmm)); - out[2] = _mm256_sum_pd(_mm256_mul_pd(amm2,bmm)); -} -#endif - -#endif // MATMUL_SPECIALISATIONS_KERNELS_H diff --git a/noarch/include/Fastor/backend/matmul/mkl_backend.h b/noarch/include/Fastor/backend/matmul/mkl_backend.h deleted file mode 100644 index e2eb88fe..00000000 --- a/noarch/include/Fastor/backend/matmul/mkl_backend.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef MKL_BACKEND_H -#define MKL_BACKEND_H - -#include - -#ifdef FASTOR_USE_MKL - -// Explicitly activate the jit -#ifndef MKL_DIRECT_CALL_SEQ_JIT -#define MKL_DIRECT_CALL_SEQ_JIT -#endif - -#include - -namespace Fastor { -namespace blas { - -// single -template,bool> = false> -void matmul_mkl( - const float * FASTOR_RESTRICT a_data, - const float * FASTOR_RESTRICT b_data, - float * FASTOR_RESTRICT out_data) { - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N); - -} - -// double -template,bool> = false> -void matmul_mkl( - const double * FASTOR_RESTRICT a_data, - const double * FASTOR_RESTRICT b_data, - double * FASTOR_RESTRICT out_data) { - - cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N); - -} - - -#if 0 -// dedicated jit api, but the jit kernel has to be created before hand -template -Tensor matmul_mkl_jit_api(const Tensor &a, const Tensor &b) { - - Tensor out; - void* jitter; - mkl_jit_status_t status = mkl_jit_create_dgemm(&jitter, MKL_ROW_MAJOR, MKL_NOTRANS, MKL_NOTRANS, M, N, K, 1.0, K, N, 0.0, N); - - dgemm_jit_kernel_t _dgemm_kernel = mkl_jit_get_dgemm_ptr(jitter); - - _dgemm_kernel(jitter, a.data(), b.data(), out.data()); - - mkl_jit_destroy(jitter); - - return out; -} -#endif - -} // end of namespace blas -} // end of namespace Fastor - -#endif // FASTOR_USE_MKL - -#endif // MKL_BACKEND_H diff --git a/noarch/include/Fastor/backend/matmul/tmatmul.h b/noarch/include/Fastor/backend/matmul/tmatmul.h deleted file mode 100644 index 96dd6072..00000000 --- a/noarch/include/Fastor/backend/matmul/tmatmul.h +++ /dev/null @@ -1,803 +0,0 @@ -#ifndef MATMUL_KERNELS2_H -#define MATMUL_KERNELS2_H - - -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/meta/meta.h" -#include "Fastor/meta/tensor_meta.h" - - -namespace Fastor { - -namespace internal { - -// TRMM implementation of Fastor - matrix-matrix multiplication when either or both operands are -// lower or upper triangular. The matrices do not need to be square and trapezoidal cases are also -// covered. For big matrices the speed-up is 2X or even better over matmul for when one operand is -// is triangular and nearly 4X for when both operands are triangular. -// For small matrices due to aggressive unrolling for SIMD the matrices cannot be exactly traversed -// within their triangular part(s) and a bit of the non-triangular part(s) need(s) to be loaded as well -// hence, the performance may not be exactly 2X/4X over the general matmul case - - -// The functions here are exact replica of those in matmul_kernels.h and will be eventually -// merged together as these variants have no associated overhead for the general case of matmul -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -/* - -For triangular matmul only the iteration span of K is modified -using the following logic [lt = Lower, ut=Upper] - -// if lhs == lt -const size_t kfirst = 0; -const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K); - -// if lhs == ut -const size_t kfirst = i; -const size_t klast = K; - -// if rhs == lt -const size_t kfirst = j; -const size_t klast = K; - -// if rhs == ut -const size_t kfirst = 0; -const size_t klast = min(j+1,K); // or min(j+unrollOuterloop,K); - - -// both lower -const size_t kfirst = j; -const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K); - -// if lhs == lt && rhs == ut -const size_t kfirst = 0; -const size_t klast = min(min(i+1,j+1),K); // or min(min(i+unrollOuterloop,j+unrollInnerloop),K); - -// if lhs == ut && rhs == lt -const size_t kfirst = max(i,j); -const size_t klast = K; - -// if both upper -const size_t kfirst = i; -const size_t klast = min(j+1,K); // or min(j+unrollInnerloop,K); - -*/ - -template -constexpr FASTOR_INLINE T find_kfirst(const T i, const T j) { - return is_same_v_ || is_same_v_ ? - ( is_same_v_ ? j : 0UL ) : - (is_same_v_ ? ( is_same_v_ ? internal::max_(i,j) : i ) : 0UL ); -} -template -constexpr FASTOR_INLINE T find_klast(const T i, const T j) { - return is_same_v_ ? - ( is_same_v_ ? internal::min_(internal::min_(i+unrollOuterloop,j+unrollInnerloop),K) : internal::min_(i+unrollOuterloop,K) ) : - (is_same_v_ || (is_same_v_) ? - ( is_same_v_ ? internal::min_(j+unrollInnerloop,K) : K ) : K ); -} - - -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- - -// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel) -// unroll the inner-most loop (on unrollOuterloop) -//----------------------------------------------------------------------------------------------------------- -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - - const V bmm0(&b[k*N+j],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - } - } -} - -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - const V bmm2(&b[k*N+j+2*V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - const V bmm2(&b[k*N+j+2*V::Size],false); - const V bmm3(&b[k*N+j+3*V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); - c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); - c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - - const V bmm0(&b[k*N+j],false); - const V bmm1(&b[k*N+j+V::Size],false); - const V bmm2(&b[k*N+j+2*V::Size],false); - const V bmm3(&b[k*N+j+3*V::Size],false); - const V bmm4(&b[k*N+j+4*V::Size],false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]); - c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]); - c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]); - c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false); - c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false); - c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false); - c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false); - c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_scalar_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - T c_ij[unrollOuterloop*numSIMDCols] = {}; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - const T bmm0(b[k*N+j]); - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] += amm0*bmm0; - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n]; - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_mask_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j, const int (&maska)[V::Size]) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - - const V bmm0(maskload(&b[k*N+j],maska)); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]); - } - } -} - - -template::type = false> -FASTOR_INLINE -void interior_block_tmatmul_mask_impl( - const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c, - const size_t i, const size_t j, const MaskT mask) { - - V bmm0; - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - for (size_t ii = 0; ii < numSIMDRows; ++ii) { - - V c_ij[unrollOuterloop*numSIMDCols]; - // Loop over columns of a (rows of b) - for (size_t k = kfirst; k < klast; ++k) { - - bmm0.mask_load(&b[k*N+j],mask,false); - - for (size_t n = 0; n < unrollOuterloop; ++n) { - - const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k]; - - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false); - } - } -} -//----------------------------------------------------------------------------------------------------------- - - - -//----------------------------------------------------------------------------------------------------------- -// This is the base implementation of triangular matrix-matrix multiplication for all 2D tensors and -// higher order tensor products that can be expressed as trmm -// The function uses two level unrolling one based on block sizes and one based on register widths -// with any remainder left treated in a scalar fashion -template -FASTOR_INLINE -void _tmatmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - - using V = choose_best_simd_t,N>; - - // This parameter can be adjusted and does not need to be 4UL/8UL etc - // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; - constexpr size_t unrollOuterloop = 4UL; - -#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE - // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] - constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); -#else - constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; -#endif -#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE - // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] - constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; -#else - constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; -#endif - - // The goal is to get 10 parallel independent chains of accumulators - // to saturate the pipeline by having a completely unrolled block of - // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of - // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains - // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. - // However, most recent X86/64 architectures can do 2 FMAs per load so - // so unrolling with numSIMDCols > 2 is not beneficial - - constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; - // Number of rows of c (M) that can be safely unrolled with this block size. - constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; - - constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; - // Number of columns of c (N) that can be safely unrolled with this block size - constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - size_t i = 0; - for (; i < M0; i += unrollOuterBlock) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N1 columns - for (; j < N; ++j) { - interior_block_tmatmul_scalar_impl(a,b,c,i,j); - } - - } - - // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. - // This is necessary as for small sizes the earlier block loop may not be - // triggered if the size of the block is bigger than the number of rows of - // (a and c) i.e. M - constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); - for (; i < M1; i += unrollOuterloop) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - V c_ij[unrollOuterloop]; - for (size_t k = kfirst; k < klast; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i + n)*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; ++j) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - T c_ij[unrollOuterloop] = {}; - for (size_t k = kfirst; k < klast; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] += a[(i + n)*K+k] * b[k*N+j]; - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c[(i + n)*N+j] = c_ij[n]; - } - } - } - - // Now treat the remaining M-M1 rows - here the klast - kfirst range is not used - // so the implementation is exactly the same as matmul_base - FASTOR_IF_CONSTEXPR (M-M1 > 0) { - // Hack to get around zero length array issue - constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - // If MM1==0 the function never gets invoked anyway - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - - V c_ij[MM1]; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); - c_ij[n-M1].store(&c[n*N+j],false); - } - } - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1].store(&c[n*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; ++j) { - - T c_ij[MM1] = {}; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1] += a[n*K+k] * b[k*N+j]; - c[n*N+j] = c_ij[n-M1]; - } - } - for (size_t n = M1; n < M; ++n) { - c[n*N+j] = c_ij[n-M1]; - } - } - } -} -//----------------------------------------------------------------------------------------------------------- - - - - -//----------------------------------------------------------------------------------------------------------- -// This the base implementation of triangular matrix-matrix multiplication for all 2D tensors and -// higher order tensor products that can be expressed as trmm -// The function uses two level unrolling one based on block sizes and one based on register widths -// with any remainder left treated in vector mode with masked and conditional load/stores. -// Note that conditional load/store requires at least AVX intrinsics -template -FASTOR_INLINE -void _tmatmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - - using V = typename internal::choose_best_simd_type,N>::type; - - // This parameter can be adjusted and does not need to be 4UL/8UL etc - // constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL; - constexpr size_t unrollOuterloop = 4UL; - -#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE - // Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size] - constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL); -#else - constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE; -#endif -#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE - // Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size] - constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL; -#else - constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE; -#endif - - // The goal is to get 10 parallel independent chains of accumulators - // to saturate the pipeline by having a completely unrolled block of - // [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of - // unrollOuterloop=4 ensures a minimum of 8 independent parallel chains - // while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively. - // However, most recent X86/64 architectures can do 2 FMAs per load so - // so unrolling with numSIMDCols > 2 is not beneficial - - constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop; - // Number of rows of c (M) that can be safely unrolled with this block size. - constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock; - - constexpr size_t unrollInnerBlock = numSIMDCols*V::Size; - // Number of columns of c (N) that can be safely unrolled with this block size - constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock; - - // Number of columns of c (N) that can be safely unrolled with V::Size - constexpr size_t N1 = N / V::Size * V::Size; - - int maska[V::Size]; - std::fill(maska,&maska[V::Size], -1); - for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0; -#ifdef FASTOR_HAS_AVX512_MASKS - const auto mask = array_to_mask(maska); -#endif - - size_t i = 0; - for (; i < M0; i += unrollOuterBlock) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N1 columns - for (; j < N; j+= N-N1) { -#ifdef FASTOR_HAS_AVX512_MASKS - interior_block_matmul_mask_impl(a,b,c,i,j,mask); -#else - interior_block_matmul_mask_impl(a,b,c,i,j,maska); -#endif - } - } - - // The remaining M-M0 rows are now unrolled yet again by unrollOuterloop. - // This is necessary as for small sizes the earlier block loop may not be - // triggered if the size of the block is bigger than the number of rows of - // (a and c) i.e. M - constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop); - for (; i < M1; i += unrollOuterloop) { - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - V c_ij[unrollOuterloop]; - for (size_t k = kfirst; k < klast; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { - c_ij[n].store(&c[(i + n)*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; j+=N-N1) { - - const size_t kfirst = find_kfirst(i,j); - const size_t klast = find_klast (i,j); - - V c_ij[unrollOuterloop]; - for (size_t k = kfirst; k < klast; ++k) { - for (size_t n = 0; n < unrollOuterloop; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; bmm0.mask_load(&b[k*N+j],mask); -#else - const V bmm0(maskload(&b[k*N+j],maska)); -#endif - const V amm0 = a[(i + n)*K+k]; - c_ij[n] = fmadd(amm0,bmm0,c_ij[n]); - } - } - for (size_t n = 0; n < unrollOuterloop; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - c_ij[n].mask_store(&c[(i+n)*N+j],mask,false); -#else - maskstore(&c[(i+n)*N+j],maska,c_ij[n]); -#endif - } - } - } - - // Now treat the remaining M-M1 rows - here the klast - kfirst range is not used - // so the implementation is exactly the same as matmul_base - FASTOR_IF_CONSTEXPR (M-M1 > 0) { - // Hack to get around zero length array issue - constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1; - size_t j = 0; - for (; j < N0; j += unrollInnerBlock) { - // If MM1==0 the function never gets invoked anyway - interior_block_tmatmul_impl(a,b,c,i,j); - } - - // Remaining N - N0 columns - for (; j < N1; j += V::Size) { - V c_ij[MM1]; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]); - c_ij[n-M1].store(&c[n*N+j],false); - } - } - for (size_t n = M1; n < M; ++n) { - c_ij[n-M1].store(&c[n*N+j],false); - } - } - - // Remaining N - N1 columns - for (; j < N; j+=N-N1) { - V c_ij[MM1] = {}; - for (size_t k = 0; k < K; ++k) { - for (size_t n = M1; n < M; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - V bmm0; bmm0.mask_load(&b[k*N+j],mask); -#else - const V bmm0(maskload(&b[k*N+j],maska)); -#endif - const V amm0 = a[n*K+k]; - c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]); - } - } - for (size_t n = M1; n < M; ++n) { -#ifdef FASTOR_HAS_AVX512_MASKS - c_ij[n-M1].mask_store(&c[n*N+j],mask,false); -#else - maskstore(&c[n*N+j],maska,c_ij[n-M1]); -#endif - } - } - } -} -//----------------------------------------------------------------------------------------------------------- - - - - -//----------------------------------------------------------------------------------------------------------- -// tmatmul kernel for non-fundamental types -// The assumption here is that non-fundamental types are not SIMD vectorisable for instance -// Tensor,3,3> or Tensor,...> plus they cannot fuse [do fused-add-multiply] -// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in -// the inner most loops of matmul -template -FASTOR_INLINE -void _tmatmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - // There is no SIMD here as V::Size == 1 anyway - // No outer loop unrolling otherwise the innermost loop - // will create unnecessary temporaries - for (size_t i=0; i(i,j); - const size_t klast = find_klast (i,j); - // This could potentially cost as opposed to directly writing in to c - T tmp {}; - for (size_t k=kfirst; k -FASTOR_INLINE -void _tmatmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - - // Non-primitive types - FASTOR_IF_CONSTEXPR (!is_primitive_v_) { - internal::_tmatmul_base_non_primitive(a,b,out); - return; - } - - // Use specialised kernels -#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) - - using nativeV = SIMDVector; - using V = choose_best_simd_t; - - FASTOR_IF_CONSTEXPR(N % V::Size <= 1UL) { - internal::_tmatmul_base(a,b,out); - return; - } - else { - internal::_tmatmul_base_masked(a,b,out); - return; - } -#else - internal::_tmatmul_base(a,b,out); - return; -#endif -} -//----------------------------------------------------------------------------------------------------------- -//----------------------------------------------------------------------------------------------------------- -} // end of namespace Fastor - - -#endif // MATMUL_KERNELS_H diff --git a/noarch/include/Fastor/backend/norm.h b/noarch/include/Fastor/backend/norm.h deleted file mode 100644 index 3381177e..00000000 --- a/noarch/include/Fastor/backend/norm.h +++ /dev/null @@ -1,149 +0,0 @@ -#ifndef NORM_H -#define NORM_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/SIMDVector.h" - -namespace Fastor { - -template,N>::type::Size, N >, bool> = false> -#else - enable_if_t_,N>::type::Size, N >, bool> = false> -#endif -FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) { - - using V = typename internal::choose_best_simd_type,N>::type; - T _scal = 0; -#ifdef FASTOR_AVX512_IMPL - V omm0, omm1, omm2, omm3, omm4, omm5, omm6, omm7; -#else - V omm0, omm1, omm2, omm3; -#endif - FASTOR_INDEX i = 0; - - // With AVX utilises all the 16 registers but hurts the performance - // due to spill if eval has created temporary registers so only - // activated for AVX512 -#ifdef FASTOR_AVX512_IMPL - for (; i < ROUND_DOWN(N,8*V::Size); i+=8*V::Size) { - const V smm0(&a[i] , false); - const V smm1(&a[i+V::Size] , false); - const V smm2(&a[i+2*V::Size] , false); - const V smm3(&a[i+3*V::Size] , false); - const V smm4(&a[i+4*V::Size] , false); - const V smm5(&a[i+5*V::Size] , false); - const V smm6(&a[i+6*V::Size] , false); - const V smm7(&a[i+7*V::Size] , false); - omm0 = fmadd(smm0,smm0,omm0); - omm1 = fmadd(smm1,smm1,omm1); - omm2 = fmadd(smm2,smm2,omm2); - omm3 = fmadd(smm3,smm3,omm3); - omm4 = fmadd(smm4,smm4,omm4); - omm5 = fmadd(smm5,smm5,omm5); - omm6 = fmadd(smm6,smm6,omm6); - omm7 = fmadd(smm7,smm7,omm7); - } -#endif - for (; i < ROUND_DOWN(N,4*V::Size); i+=4*V::Size) { - const V smm0(&a[i] , false); - const V smm1(&a[i+V::Size] , false); - const V smm2(&a[i+2*V::Size] , false); - const V smm3(&a[i+3*V::Size] , false); - omm0 = fmadd(smm0,smm0,omm0); - omm1 = fmadd(smm1,smm1,omm1); - omm2 = fmadd(smm2,smm2,omm2); - omm3 = fmadd(smm3,smm3,omm3); - } - for (; i < ROUND_DOWN(N,2*V::Size); i+=2*V::Size) { - const V smm0(&a[i] , false); - const V smm1(&a[i+V::Size] , false); - omm0 = fmadd(smm0,smm0,omm0); - omm1 = fmadd(smm1,smm1,omm1); - } - for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) { - const V smm0(&a[i] , false); - omm0 = fmadd(smm0,smm0,omm0); - } - for (; i < N; ++i) { - const auto smm0(a[i]); - _scal += smm0*smm0; - } -#ifdef FASTOR_AVX512_IMPL - return sqrts( (omm0 + omm1 + omm2 + omm3 + omm4 + omm5 + omm6 + omm7).sum() + _scal); -#else - return sqrts( (omm0 + omm1 + omm2 + omm3).sum() + _scal); -#endif -} - -template,N>::type::Size, N >, bool> = false> -#else - enable_if_t_,N>::type::Size, N >, bool> = false> -#endif -FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) { - - using V = typename internal::choose_best_simd_type,N>::type; - T _scal = 0; - V omm0; - FASTOR_INDEX i = 0; - for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) { - const V smm0(&a[i] , false); - omm0 = fmadd(smm0,smm0,omm0); - } - for (; i < N; ++i) { - const auto smm0(a[i]); - _scal += smm0*smm0; - } - return sqrts( omm0.sum() + _scal); -} - -#ifdef FASTOR_SSE4_2_IMPL -template<> -FASTOR_INLINE float _norm(const float * FASTOR_RESTRICT a) { - // IVY 33 OPS / HW 31 OPS - __m128 a_reg = _mm_load_ps(a); - return _mm_cvtss_f32(_mm_sqrt_ps(_add_ps(_mm_mul_ps(a_reg,a_reg)))); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE float _norm(const float * FASTOR_RESTRICT a) { - // IVY & HW 61 OPS - __m256 a_reg = _mm256_loadu_ps(a); - __m128 a_end = _mm_load_ss(a+8); - __m128 a0 = _add_ps(_mm256_mul_ps(a_reg,a_reg)); - __m128 a1 = _add_ps(_mm_mul_ps(a_end,a_end)); - return _mm_cvtss_f32(_mm_sqrt_ps(_mm_add_ss(a0,a1))); -} - - -template<> -FASTOR_INLINE double _norm(const double * FASTOR_RESTRICT a) { - // IVY 34 OPS / HW 36 OPS - __m256d a_reg = _mm256_loadu_pd(a); - return _mm_cvtsd_f64(_mm_sqrt_pd(_add_pd(_mm256_mul_pd(a_reg,a_reg)))); -} - -template<> -FASTOR_INLINE double _norm(const double * FASTOR_RESTRICT a) { - // IVY 63 OPS / HW 67 OPS - __m256d a_low = _mm256_loadu_pd(a); - __m256d a_high = _mm256_loadu_pd(a+4); - __m128d a_end = _mm_load_sd(a+8); - __m128d a0 = _add_pd(_mm256_mul_pd(a_low,a_low)); - __m128d a1 = _add_pd(_mm256_mul_pd(a_high,a_high)); - __m128d a2 = _add_pd(_mm_mul_pd(a_end,a_end)); - return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_add_sd(a2,(_mm_add_sd(a0,a1))))); -} -#endif - - -} - -#endif // NORM_H - diff --git a/noarch/include/Fastor/backend/outer.h b/noarch/include/Fastor/backend/outer.h deleted file mode 100644 index 5952059d..00000000 --- a/noarch/include/Fastor/backend/outer.h +++ /dev/null @@ -1,292 +0,0 @@ -#ifndef OUTER_H -#define OUTER_H - -#include "Fastor/config/config.h" - -namespace Fastor { - -template -FASTOR_HINT_INLINE void _outer(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) { - for (size_t i=0; i -FASTOR_HINT_INLINE void _outer(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - // 31 OPS - // Fetch a to L1-cache -// _mm_prefetch(a,_MM_HINT_T0); - __m128 a0 = _mm_set1_ps(a[0]); - __m128 a1 = _mm_set1_ps(a[1]); -// __m128 a2 = _mm_set1_ps(a[2]); - __m128 a3 = _mm_set1_ps(a[3]); - __m128 bs = _mm_load_ps(b); - - __m128 r0 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(1,2,3,0)); - __m128 r1 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(3,1,2,0)); - __m128 r2 = _mm_mul_ps(HALFPS,_mm_add_ps(r1,bs)); - r2 = _mm_shuffle_ps(r2,r2,_MM_SHUFFLE(3,0,2,1)); -// __m128 r3 = _mm_shift1_ps(_mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0))); - __m128 r3 = _mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0)); - - // row0 - __m128 row0 = _mm_mul_ps(a0,r3); - _mm_store_ps(out, row0); - // row1 - __m128 row1 = _mm_mul_ps(a3,r3); - row1 = _mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,3,2,1)); - _mm_store_ss(out+3,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,1))); -// _mm_storeu_ps(out+4,row1); - _mm_store_ps(out+4,row1); - // row2 - __m128 row2 = _mm_mul_ps(a1,r3); - row2 = _mm_shuffle_ps(row2,row2,_MM_SHUFFLE(1,0,3,2)); - _mm_store_ss(out+6,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,2))); - _mm_store_ss(out+7,_mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,0,0,1))); - _mm_store_ss(out+8, row2); -} - -template<> -FASTOR_HINT_INLINE void _outer(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) { - // 81 OPS - // Fetch a to L1-cache -// _mm_prefetch(a,_MM_HINT_T0); - - __m128 a0 = _mm_set1_ps(a[0]); - __m128 a1 = _mm_set1_ps(a[1]); - __m128 a2 = _mm_set1_ps(a[2]); -// __m128 a3 = _mm_set1_ps(a[3]); - __m128 a4 = _mm_set1_ps(a[4]); - __m128 a5 = _mm_set1_ps(a[5]); -// __m128 a6 = _mm_set1_ps(a[6]); -// __m128 a7 = _mm_set1_ps(a[7]); - __m128 a8 = _mm_set1_ps(a[8]); - - __m128 b_low = _mm_load_ps(b); - __m128 b_high =_mm_load_ps(b+4); - __m128 b_end = _mm_load_ss(b+8); - - __m128 b_diag = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(1,0,1,0)); - b_diag = _mm_shuffle_ps(b_diag,b_end,_MM_SHUFFLE(1,0,2,0)); - __m128 ofdiag_str = _mm_shift1_ps(_mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,1,2,1))); - __m128 ofdiag_rev = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,2,2,3)); - ofdiag_rev = _mm_shift1_ps(_mm_shuffle_ps(ofdiag_rev,ofdiag_rev,_MM_SHUFFLE(3,3,2,0))); - - // Compute this only once - __m128 half_add_diag = _mm_mul_ps(HALFPS,_mm_add_ps(ofdiag_str,ofdiag_rev)); - - // row0 - __m128 c0_d = _mm_mul_ps(b_diag,a0); - __m128 c0_off = _mm_mul_ps(a0,half_add_diag); - c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1)); - _mm_store_ps(out,c0_d); - _mm_storeu_ps(out+3,c0_off); - // row1 - __m128 c1_d = _mm_mul_ps(b_diag,a4); - __m128 c1_off = _mm_mul_ps(a4,half_add_diag); - c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1)); - c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1)); - _mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1))); - _mm_storeu_ps(out+7,c1_d); - _mm_storeu_ps(out+9,c1_off); - // row2 - __m128 c2_d = _mm_mul_ps(b_diag,a8); - __m128 c2_off = _mm_mul_ps(a8,half_add_diag); - c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1)); - _mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2))); - _mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1))); - _mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2))); - _mm_storeu_ps(out+15,c2_off); - // row3 - __m128 c3_off = _mm_mul_ps(a1,half_add_diag); - c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1)); - _mm_store_ss(out+18,c0_off); - _mm_store_ss(out+19,c1_off); - _mm_store_ss(out+20,c2_off); - _mm_storeu_ps(out+21,c3_off); - // row4 - __m128 c4_off = _mm_mul_ps(a2,half_add_diag); - c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1)); - _mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1))); - _mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1))); - _mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1))); - _mm_storeu_ps(out+27,c4_off); - // row5 - __m128 c5_off = _mm_mul_ps(a5,half_add_diag); - c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1)); - _mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2))); - _mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2))); - _mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2))); - _mm_storeu_ps(out+33,c5_off); - - // row0 -// __m128 c0_d = _mm_mul_ps(b_diag,a0); -// __m128 c_os = _mm_mul_ps(ofdiag_str,a0); -// __m128 c_or = _mm_mul_ps(ofdiag_rev,a0); -// __m128 c0_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); -// c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1)); -// _mm_store_ps(out,c0_d); -// _mm_storeu_ps(out+3,c0_off); - -// // row1 -// __m128 c1_d = _mm_mul_ps(b_diag,a4); -// c_os = _mm_mul_ps(ofdiag_str,a4); -// c_or = _mm_mul_ps(ofdiag_rev,a4); -// __m128 c1_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); -// c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1)); -// c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1)); -// _mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1))); -// _mm_storeu_ps(out+7,c1_d); -// _mm_storeu_ps(out+9,c1_off); - -// // row2 -// __m128 c2_d = _mm_mul_ps(b_diag,a8); -// c_os = _mm_mul_ps(ofdiag_str,a8); -// c_or = _mm_mul_ps(ofdiag_rev,a8); -// __m128 c2_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); -// c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1)); -// _mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2))); -// _mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1))); -// _mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2))); -// _mm_storeu_ps(out+15,c2_off); - -// // row3 -// c_os = _mm_mul_ps(ofdiag_str,a1); -// c_or = _mm_mul_ps(ofdiag_rev,a1); -// __m128 c3_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); -// c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1)); -// _mm_store_ss(out+18,c0_off); -// _mm_store_ss(out+19,c1_off); -// _mm_store_ss(out+20,c2_off); -// _mm_storeu_ps(out+21,c3_off); - -// // row4 -// c_os = _mm_mul_ps(ofdiag_str,a2); -// c_or = _mm_mul_ps(ofdiag_rev,a2); -// __m128 c4_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); -// c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1)); -// _mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1))); -// _mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1))); -// _mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1))); -// _mm_storeu_ps(out+27,c4_off); - -// // row5 -// c_os = _mm_mul_ps(ofdiag_str,a5); -// c_or = _mm_mul_ps(ofdiag_rev,a5); -// __m128 c5_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or)); -// c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1)); -// _mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2))); -// _mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2))); -// _mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2))); -// _mm_storeu_ps(out+33,c5_off); - -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_HINT_INLINE void _outer(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - // Fetch a to L1-cache -// _mm_prefetch(a,_MM_HINT_T0); - __m256d a0 = _mm256_set1_pd(a[0]); - __m256d a1 = _mm256_set1_pd(a[1]); - __m256d a3 = _mm256_set1_pd(a[3]); - __m256d bs = _mm256_load_pd(b); - - __m128d r0 = _mm_setr_pd(_mm256_get2_pd(bs),_mm256_get1_pd(bs)); - __m128d r1 = _mm_setr_pd(_mm256_get1_pd(bs),_mm256_get2_pd(bs)); - __m128d r2 = _mm_mul_pd(HALFPD,_mm_add_pd(r0,r1)); - __m256d r3 = _mm256_setr_pd(_mm256_get0_pd(bs),_mm256_get3_pd(bs),_mm_get0_pd(r2),0.0); - - // row0 - __m256d row0 = _mm256_mul_pd(a0,r3); - _mm256_store_pd(out, row0); - // row1 - __m256d row1 = _mm256_mul_pd(a3,r3); - _mm_store_sd(out+3,_mm_set_sd(_mm256_get1_pd(row0))); - _mm_store_pd(out+4,_mm_setr_pd(_mm256_get1_pd(row1),_mm256_get2_pd(row1))); - // row2 - __m256d row2 = _mm256_mul_pd(a1,r3); - _mm_store_sd(out+6,_mm256_extractf128_pd(row0,0x1)); - _mm_store_sd(out+7,_mm256_extractf128_pd(row1,0x1)); - _mm_store_sd(out+8,_mm256_extractf128_pd(row2,0x1)); -} - -template<> -FASTOR_HINT_INLINE void _outer(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) { - // OPS -// _mm_prefetch(a,_MM_HINT_T0); - - __m256d a0 = _mm256_set1_pd(a[0]); - __m256d a1 = _mm256_set1_pd(a[1]); - __m256d a2 = _mm256_set1_pd(a[2]); - __m256d a4 = _mm256_set1_pd(a[4]); - __m256d a5 = _mm256_set1_pd(a[5]); - __m256d a8 = _mm256_set1_pd(a[8]); - - __m256d b_low = _mm256_load_pd(b); - __m256d b_high = _mm256_load_pd(b+4); - __m128d b_end = _mm_load_sd(b+8); - - __m256d b_diag = _mm256_setr_pd(_mm_cvtsd_f64(_mm256_castpd256_pd128(b_low)), - _mm_cvtsd_f64(_mm256_castpd256_pd128(b_high)), - _mm_cvtsd_f64(b_end),0.0); - - __m256d ofdiag_str = _mm256_setr_pd(b[1],b[2],b[5],0.0); - __m256d ofdiag_rev = _mm256_setr_pd(b[3],b[6],b[7],0.0); - // Compute this only once - __m256d half_add_diag = _mm256_mul_pd(VHALFPD,_mm256_add_pd(ofdiag_str,ofdiag_rev)); - - // row0 - __m256d c0_d = _mm256_mul_pd(b_diag,a0); - __m256d c0_off = _mm256_mul_pd(a0,half_add_diag); - _mm256_store_pd(out,c0_d); - _mm256_storeu_pd(out+3,c0_off); - // row1 - __m256d c1_d = _mm256_mul_pd(b_diag,a4); - __m256d c1_off = _mm256_mul_pd(a4,half_add_diag); - _mm256_storeu_pd(out+6,c1_d); - _mm_store_sd(out+6,_mm256_castpd256_pd128(_mm256_shuffle_pd(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1)))); - _mm256_storeu_pd(out+9,c1_off); - // row2 - __m256d c2_d = _mm256_mul_pd(b_diag,a8); - __m256d c2_off = _mm256_mul_pd(a8,half_add_diag); - _mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0_d))); - _mm_store_sd(out+13,_mm_set_sd(_mm256_get2_pd(c1_d))); - _mm_store_sd(out+14,_mm_set_sd(_mm256_get2_pd(c2_d))); - _mm256_storeu_pd(out+15,c2_off); - // row3 - __m256d c3_off = _mm256_mul_pd(a1,half_add_diag); - _mm_store_sd(out+18,_mm256_castpd256_pd128(c0_off)); - _mm_store_sd(out+19,_mm256_castpd256_pd128(c1_off)); - _mm_store_sd(out+20,_mm256_castpd256_pd128(c2_off)); - _mm256_storeu_pd(out+21,c3_off); - // row4 - __m256d c4_off = _mm256_mul_pd(a2,half_add_diag); - _mm_store_sd(out+24,_mm_set_sd(_mm256_get1_pd(c0_off))); - _mm_store_sd(out+25,_mm_set_sd(_mm256_get1_pd(c1_off))); - _mm_store_sd(out+26,_mm_set_sd(_mm256_get1_pd(c2_off))); - _mm256_storeu_pd(out+27,c4_off); - // row4 - __m256d c5_off = _mm256_mul_pd(a5,half_add_diag); - _mm_store_sd(out+30,_mm_set_sd(_mm256_get2_pd(c0_off))); - _mm_store_sd(out+31,_mm_set_sd(_mm256_get2_pd(c1_off))); - _mm_store_sd(out+32,_mm_set_sd(_mm256_get2_pd(c2_off))); - _mm256_storeu_pd(out+33,c5_off); -} -#endif - -} - -#endif // OUTER_H - diff --git a/noarch/include/Fastor/backend/tensor_cross.h b/noarch/include/Fastor/backend/tensor_cross.h deleted file mode 100644 index d14a20e3..00000000 --- a/noarch/include/Fastor/backend/tensor_cross.h +++ /dev/null @@ -1,1843 +0,0 @@ -#ifndef TENSOR_CROSS_H -#define TENSOR_CROSS_H - -#include "Fastor/meta/meta.h" -#include "Fastor/config/config.h" - -namespace Fastor { - -// classic vector cross product -//--------------------------------------------------------------------------------------------------------// -template = false> -FASTOR_INLINE void _vector_crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - c[0] = a[1] * b[2] - a[2] * b[1]; - c[1] = a[2] * b[0] - a[0] * b[2]; - c[2] = a[0] * b[1] - a[1] * b[0]; -} - -template = false> -FASTOR_INLINE void _vector_crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - c[0] = 0; - c[1] = 0; - c[2] = a[0] * b[1] - a[1] * b[0]; -} -//--------------------------------------------------------------------------------------------------------// - - -// tensor cross products -//--------------------------------------------------------------------------------------------------------// -template && is_equal_v_ && is_equal_v_,bool> = false> -FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - T ___A00 = a[0]; - T ___A01 = a[1]; - T ___A02 = a[2]; - T ___A10 = a[3]; - T ___A11 = a[4]; - T ___A12 = a[5]; - T ___A20 = a[6]; - T ___A21 = a[7]; - T ___A22 = a[8]; - - T ___B00 = b[0]; - T ___B01 = b[1]; - T ___B02 = b[2]; - T ___B10 = b[3]; - T ___B11 = b[4]; - T ___B12 = b[5]; - T ___B20 = b[6]; - T ___B21 = b[7]; - T ___B22 = b[8]; - - c[0] = ___A11*___B22 - ___A12*___B21 - ___A21*___B12 + ___A22*___B11; - c[1] = ___A12*___B20 - ___A10*___B22 + ___A20*___B12 - ___A22*___B10; - c[2] = ___A10*___B21 - ___A11*___B20 - ___A20*___B11 + ___A21*___B10; - c[3] = ___A02*___B21 - ___A01*___B22 + ___A21*___B02 - ___A22*___B01; - c[4] = ___A00*___B22 - ___A02*___B20 - ___A20*___B02 + ___A22*___B00; - c[5] = ___A01*___B20 - ___A00*___B21 + ___A20*___B01 - ___A21*___B00; - c[6] = ___A01*___B12 - ___A02*___B11 - ___A11*___B02 + ___A12*___B01; - c[7] = ___A02*___B10 - ___A00*___B12 + ___A10*___B02 - ___A12*___B00; - c[8] = ___A00*___B11 - ___A01*___B10 - ___A10*___B01 + ___A11*___B00; -} - -template && is_equal_v_ && is_equal_v_,bool> = false> -FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - T ___A00 = a[0]; - T ___A01 = a[1]; - T ___A10 = a[3]; - T ___A11 = a[4]; - - T ___B00 = b[0]; - T ___B01 = b[1]; - T ___B10 = b[3]; - T ___B11 = b[4]; - - c[0] = 0; - c[1] = 0; - c[2] = 0; - c[3] = 0; - c[4] = 0; - c[5] = 0; - c[6] = 0; - c[7] = 0; - c[8] = ___A00*___B11 - ___A01*___B10 - ___A10*___B01 + ___A11*___B00; -} - -// vectors and 2nd order tensors -//--------------------------------------------------------------------------------------------------------// -template && is_equal_v_ && is_equal_v_,bool> = false> -FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - // vector-tensor cross product - regitster based - T ___A1 = a[0]; - T ___A2 = a[1]; - T ___A3 = a[2]; - - T B1_1 = b[0]; - T B1_2 = b[1]; - T B1_3 = b[2]; - T B2_1 = b[3]; - T B2_2 = b[4]; - T B2_3 = b[5]; - T B3_1 = b[6]; - T B3_2 = b[7]; - T B3_3 = b[8]; - - c[0] = ___A2*B3_1 - ___A3*B2_1; - c[1] = ___A2*B3_2 - ___A3*B2_2; - c[2] = ___A2*B3_3 - ___A3*B2_3; - c[3] = ___A3*B1_1 - ___A1*B3_1; - c[4] = ___A3*B1_2 - ___A1*B3_2; - c[5] = ___A3*B1_3 - ___A1*B3_3; - c[6] = ___A1*B2_1 - ___A2*B1_1; - c[7] = ___A1*B2_2 - ___A2*B1_2; - c[8] = ___A1*B2_3 - ___A2*B1_3; - -// [ A2*B3_1 - A3*B2_1, A2*B3_2 - A3*B2_2, A2*B3_3 - A3*B2_3] -// [ A3*B1_1 - A1*B3_1, A3*B1_2 - A1*B3_2, A3*B1_3 - A1*B3_3] -// [ A1*B2_1 - A2*B1_1, A1*B2_2 - A2*B1_2, A1*B2_3 - A2*B1_3] -} - -template && is_equal_v_ && is_equal_v_,bool> = false> -FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - // vector-tensor cross product - regitster based - T ___A1 = a[0]; - T ___A2 = a[1]; - - T B1_1 = b[0]; - T B1_2 = b[1]; - T B2_1 = b[3]; - T B2_2 = b[4]; - - c[0] = 0; - c[1] = 0; - c[2] = 0; - c[3] = 0; - c[4] = 0; - c[5] = 0; - c[6] = ___A1*B2_1 - ___A2*B1_1; - c[7] = ___A1*B2_2 - ___A2*B1_2; - c[8] = 0; -} - -template && is_equal_v_ && is_equal_v_,bool> = false> -FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - // tensor-vector cross product - regitster based - T ___B1 = b[0]; - T ___B2 = b[1]; - T ___B3 = b[2]; - - T A1_1 = a[0]; - T A1_2 = a[1]; - T A1_3 = a[2]; - T A2_1 = a[3]; - T A2_2 = a[4]; - T A2_3 = a[5]; - T A3_1 = a[6]; - T A3_2 = a[7]; - T A3_3 = a[8]; - - c[0] = A1_2*___B3 - A1_3*___B2; - c[1] = A1_3*___B1 - A1_1*___B3; - c[2] = A1_1*___B2 - A1_2*___B1; - c[3] = A2_2*___B3 - A2_3*___B2; - c[4] = A2_3*___B1 - A2_1*___B3; - c[5] = A2_1*___B2 - A2_2*___B1; - c[6] = A3_2*___B3 - A3_3*___B2; - c[7] = A3_3*___B1 - A3_1*___B3; - c[8] = A3_1*___B2 - A3_2*___B1; - -// [ A1_2*B3 - A1_3*B2, A1_3*B1 - A1_1*B3, A1_1*B2 - A1_2*B1] -// [ A2_2*B3 - A2_3*B2, A2_3*B1 - A2_1*B3, A2_1*B2 - A2_2*B1] -// [ A3_2*B3 - A3_3*B2, A3_3*B1 - A3_1*B3, A3_1*B2 - A3_2*B1] -} - -template && is_equal_v_ && is_equal_v_,bool> = false> -FASTOR_INLINE void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - // tensor-vector cross product - regitster based - T ___B1 = b[0]; - T ___B2 = b[1]; - - T A1_1 = a[0]; - T A1_2 = a[1]; - T A2_1 = a[3]; - T A2_2 = a[4]; - - c[0] = 0; - c[1] = 0; - c[2] = A1_1*___B2 - A1_2*___B1; - c[3] = 0; - c[4] = 0; - c[5] = A2_1*___B2 - A2_2*___B1; - c[6] = 0; - c[7] = 0; - c[8] = 0; -} -//--------------------------------------------------------------------------------------------------------// - - - -//--------------------------------------------------------------------------------------------------------// -// for plane strain problems -template -FASTOR_INLINE -void _crossproduct(const T *FASTOR_RESTRICT a, const T *FASTOR_RESTRICT b, T *FASTOR_RESTRICT c) { - // For plane strain problems a and b need to be 3D with last element a[8]=b[8]=1 - // Note that in ultimate case you might not need the last row/column of the output matrix hence computing c22 - // and storing it is un-necessary, hence the (c) array should really be 2x2=4 in length - // Load a data - T ___A00 = a[0]; - T ___A11 = a[4]; - T ___A22 = a[8]; - T ___A01 = a[1]; - T ___A02 = a[2]; - T ___A12 = a[5]; - T ___A10 = a[3]; - T ___A20 = a[6]; - T ___A21 = a[7]; - - T ___B00 = b[0]; - T ___B11 = b[4]; - T ___B22 = b[8]; - T ___B01 = b[1]; - T ___B02 = b[2]; - T ___B12 = b[5]; - T ___B10 = b[3]; - T ___B20 = b[6]; - T ___B21 = b[7]; - - c[0] = ___A11*___B22 - ___A12*___B21 - ___A21*___B12 + ___A22*___B11; - c[1] = ___A12*___B20 - ___A10*___B22 + ___A20*___B12 - ___A22*___B10; - c[0] = 0; - c[3] = ___A02*___B21 - ___A01*___B22 + ___A21*___B02 - ___A22*___B01; - c[4] = ___A00*___B22 - ___A02*___B20 - ___A20*___B02 + ___A22*___B00; - c[5] = 0; - c[6] = 0; - c[7] = 0; - // c[8] = A00*B11 - A01*B10 - A10*B01 + A11*B00; - c[8] = 0; // 1 ? -} -//--------------------------------------------------------------------------------------------------------// - - - -//--------------------------------------------------------------------------------------------------------// -//(AxB)_{PiIQ} = E_{ijk}E_{IJK}A_{PjJ}B_{kKQ} -// tensor cross product of 3rd order tensors -template -FASTOR_HINT_INLINE void _crossproduct(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { - -T _A000 = A[0]; - T _A001 = A[1]; - T _A002 = A[2]; - T _A010 = A[3]; - T _A011 = A[4]; - T _A012 = A[5]; - T _A020 = A[6]; - T _A021 = A[7]; - T _A022 = A[8]; - T _A100 = A[9]; - T _A101 = A[10]; - T _A102 = A[11]; - T _A110 = A[12]; - T _A111 = A[13]; - T _A112 = A[14]; - T _A120 = A[15]; - T _A121 = A[16]; - T _A122 = A[17]; - T _A200 = A[18]; - T _A201 = A[19]; - T _A202 = A[20]; - T _A210 = A[21]; - T _A211 = A[22]; - T _A212 = A[23]; - T _A220 = A[24]; - T _A221 = A[25]; - T _A222 = A[26]; - - - T _B000 = B[0]; - T _B001 = B[1]; - T _B002 = B[2]; - T _B010 = B[3]; - T _B011 = B[4]; - T _B012 = B[5]; - T _B020 = B[6]; - T _B021 = B[7]; - T _B022 = B[8]; - T _B100 = B[9]; - T _B101 = B[10]; - T _B102 = B[11]; - T _B110 = B[12]; - T _B111 = B[13]; - T _B112 = B[14]; - T _B120 = B[15]; - T _B121 = B[16]; - T _B122 = B[17]; - T _B200 = B[18]; - T _B201 = B[19]; - T _B202 = B[20]; - T _B210 = B[21]; - T _B211 = B[22]; - T _B212 = B[23]; - T _B220 = B[24]; - T _B221 = B[25]; - T _B222 = B[26]; - - - C[0] = _A022*_B110 - _A021*_B120 - _A012*_B210 + _A011*_B220; - C[1] = _A022*_B111 - _A021*_B121 - _A012*_B211 + _A011*_B221; - C[2] = _A022*_B112 - _A021*_B122 - _A012*_B212 + _A011*_B222; - C[3] = _A020*_B120 - _A022*_B100 + _A012*_B200 - _A010*_B220; - C[4] = _A020*_B121 - _A022*_B101 + _A012*_B201 - _A010*_B221; - C[5] = _A020*_B122 - _A022*_B102 + _A012*_B202 - _A010*_B222; - C[6] = _A021*_B100 - _A020*_B110 - _A011*_B200 + _A010*_B210; - C[7] = _A021*_B101 - _A020*_B111 - _A011*_B201 + _A010*_B211; - C[8] = _A021*_B102 - _A020*_B112 - _A011*_B202 + _A010*_B212; - C[9] = _A021*_B020 - _A022*_B010 + _A002*_B210 - _A001*_B220; - C[10] = _A021*_B021 - _A022*_B011 + _A002*_B211 - _A001*_B221; - C[11] = _A021*_B022 - _A022*_B012 + _A002*_B212 - _A001*_B222; - C[12] = _A022*_B000 - _A020*_B020 - _A002*_B200 + _A000*_B220; - C[13] = _A022*_B001 - _A020*_B021 - _A002*_B201 + _A000*_B221; - C[14] = _A022*_B002 - _A020*_B022 - _A002*_B202 + _A000*_B222; - C[15] = _A020*_B010 - _A021*_B000 + _A001*_B200 - _A000*_B210; - C[16] = _A020*_B011 - _A021*_B001 + _A001*_B201 - _A000*_B211; - C[17] = _A020*_B012 - _A021*_B002 + _A001*_B202 - _A000*_B212; - C[18] = _A012*_B010 - _A011*_B020 - _A002*_B110 + _A001*_B120; - C[19] = _A012*_B011 - _A011*_B021 - _A002*_B111 + _A001*_B121; - C[20] = _A012*_B012 - _A011*_B022 - _A002*_B112 + _A001*_B122; - C[21] = _A010*_B020 - _A012*_B000 + _A002*_B100 - _A000*_B120; - C[22] = _A010*_B021 - _A012*_B001 + _A002*_B101 - _A000*_B121; - C[23] = _A010*_B022 - _A012*_B002 + _A002*_B102 - _A000*_B122; - C[24] = _A011*_B000 - _A010*_B010 - _A001*_B100 + _A000*_B110; - C[25] = _A011*_B001 - _A010*_B011 - _A001*_B101 + _A000*_B111; - C[26] = _A011*_B002 - _A010*_B012 - _A001*_B102 + _A000*_B112; - C[27] = _A122*_B110 - _A121*_B120 - _A112*_B210 + _A111*_B220; - C[28] = _A122*_B111 - _A121*_B121 - _A112*_B211 + _A111*_B221; - C[29] = _A122*_B112 - _A121*_B122 - _A112*_B212 + _A111*_B222; - C[30] = _A120*_B120 - _A122*_B100 + _A112*_B200 - _A110*_B220; - C[31] = _A120*_B121 - _A122*_B101 + _A112*_B201 - _A110*_B221; - C[32] = _A120*_B122 - _A122*_B102 + _A112*_B202 - _A110*_B222; - C[33] = _A121*_B100 - _A120*_B110 - _A111*_B200 + _A110*_B210; - C[34] = _A121*_B101 - _A120*_B111 - _A111*_B201 + _A110*_B211; - C[35] = _A121*_B102 - _A120*_B112 - _A111*_B202 + _A110*_B212; - C[36] = _A121*_B020 - _A122*_B010 + _A102*_B210 - _A101*_B220; - C[37] = _A121*_B021 - _A122*_B011 + _A102*_B211 - _A101*_B221; - C[38] = _A121*_B022 - _A122*_B012 + _A102*_B212 - _A101*_B222; - C[39] = _A122*_B000 - _A120*_B020 - _A102*_B200 + _A100*_B220; - C[40] = _A122*_B001 - _A120*_B021 - _A102*_B201 + _A100*_B221; - C[41] = _A122*_B002 - _A120*_B022 - _A102*_B202 + _A100*_B222; - C[42] = _A120*_B010 - _A121*_B000 + _A101*_B200 - _A100*_B210; - C[43] = _A120*_B011 - _A121*_B001 + _A101*_B201 - _A100*_B211; - C[44] = _A120*_B012 - _A121*_B002 + _A101*_B202 - _A100*_B212; - C[45] = _A112*_B010 - _A111*_B020 - _A102*_B110 + _A101*_B120; - C[46] = _A112*_B011 - _A111*_B021 - _A102*_B111 + _A101*_B121; - C[47] = _A112*_B012 - _A111*_B022 - _A102*_B112 + _A101*_B122; - C[48] = _A110*_B020 - _A112*_B000 + _A102*_B100 - _A100*_B120; - C[49] = _A110*_B021 - _A112*_B001 + _A102*_B101 - _A100*_B121; - C[50] = _A110*_B022 - _A112*_B002 + _A102*_B102 - _A100*_B122; - C[51] = _A111*_B000 - _A110*_B010 - _A101*_B100 + _A100*_B110; - C[52] = _A111*_B001 - _A110*_B011 - _A101*_B101 + _A100*_B111; - C[53] = _A111*_B002 - _A110*_B012 - _A101*_B102 + _A100*_B112; - C[54] = _A222*_B110 - _A221*_B120 - _A212*_B210 + _A211*_B220; - C[55] = _A222*_B111 - _A221*_B121 - _A212*_B211 + _A211*_B221; - C[56] = _A222*_B112 - _A221*_B122 - _A212*_B212 + _A211*_B222; - C[57] = _A220*_B120 - _A222*_B100 + _A212*_B200 - _A210*_B220; - C[58] = _A220*_B121 - _A222*_B101 + _A212*_B201 - _A210*_B221; - C[59] = _A220*_B122 - _A222*_B102 + _A212*_B202 - _A210*_B222; - C[60] = _A221*_B100 - _A220*_B110 - _A211*_B200 + _A210*_B210; - C[61] = _A221*_B101 - _A220*_B111 - _A211*_B201 + _A210*_B211; - C[62] = _A221*_B102 - _A220*_B112 - _A211*_B202 + _A210*_B212; - C[63] = _A221*_B020 - _A222*_B010 + _A202*_B210 - _A201*_B220; - C[64] = _A221*_B021 - _A222*_B011 + _A202*_B211 - _A201*_B221; - C[65] = _A221*_B022 - _A222*_B012 + _A202*_B212 - _A201*_B222; - C[66] = _A222*_B000 - _A220*_B020 - _A202*_B200 + _A200*_B220; - C[67] = _A222*_B001 - _A220*_B021 - _A202*_B201 + _A200*_B221; - C[68] = _A222*_B002 - _A220*_B022 - _A202*_B202 + _A200*_B222; - C[69] = _A220*_B010 - _A221*_B000 + _A201*_B200 - _A200*_B210; - C[70] = _A220*_B011 - _A221*_B001 + _A201*_B201 - _A200*_B211; - C[71] = _A220*_B012 - _A221*_B002 + _A201*_B202 - _A200*_B212; - C[72] = _A212*_B010 - _A211*_B020 - _A202*_B110 + _A201*_B120; - C[73] = _A212*_B011 - _A211*_B021 - _A202*_B111 + _A201*_B121; - C[74] = _A212*_B012 - _A211*_B022 - _A202*_B112 + _A201*_B122; - C[75] = _A210*_B020 - _A212*_B000 + _A202*_B100 - _A200*_B120; - C[76] = _A210*_B021 - _A212*_B001 + _A202*_B101 - _A200*_B121; - C[77] = _A210*_B022 - _A212*_B002 + _A202*_B102 - _A200*_B122; - C[78] = _A211*_B000 - _A210*_B010 - _A201*_B100 + _A200*_B110; - C[79] = _A211*_B001 - _A210*_B011 - _A201*_B101 + _A200*_B111; - C[80] = _A211*_B002 - _A210*_B012 - _A201*_B102 + _A200*_B112; -} -//--------------------------------------------------------------------------------------------------------// - - - -//--------------------------------------------------------------------------------------------------------// -//(AxB)_{pPiIqQ} = E_{ijk}E_{IJK}A_{pPjJ}B_{kKqQ} -// tensor cross product of 4th order tensors -template -FASTOR_HINT_INLINE void _crossproduct(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { - - T _A0000 = A[0]; - T _A0001 = A[1]; - T _A0002 = A[2]; - T _A0010 = A[3]; - T _A0011 = A[4]; - T _A0012 = A[5]; - T _A0020 = A[6]; - T _A0021 = A[7]; - T _A0022 = A[8]; - T _A0100 = A[9]; - T _A0101 = A[10]; - T _A0102 = A[11]; - T _A0110 = A[12]; - T _A0111 = A[13]; - T _A0112 = A[14]; - T _A0120 = A[15]; - T _A0121 = A[16]; - T _A0122 = A[17]; - T _A0200 = A[18]; - T _A0201 = A[19]; - T _A0202 = A[20]; - T _A0210 = A[21]; - T _A0211 = A[22]; - T _A0212 = A[23]; - T _A0220 = A[24]; - T _A0221 = A[25]; - T _A0222 = A[26]; - T _A1000 = A[27]; - T _A1001 = A[28]; - T _A1002 = A[29]; - T _A1010 = A[30]; - T _A1011 = A[31]; - T _A1012 = A[32]; - T _A1020 = A[33]; - T _A1021 = A[34]; - T _A1022 = A[35]; - T _A1100 = A[36]; - T _A1101 = A[37]; - T _A1102 = A[38]; - T _A1110 = A[39]; - T _A1111 = A[40]; - T _A1112 = A[41]; - T _A1120 = A[42]; - T _A1121 = A[43]; - T _A1122 = A[44]; - T _A1200 = A[45]; - T _A1201 = A[46]; - T _A1202 = A[47]; - T _A1210 = A[48]; - T _A1211 = A[49]; - T _A1212 = A[50]; - T _A1220 = A[51]; - T _A1221 = A[52]; - T _A1222 = A[53]; - T _A2000 = A[54]; - T _A2001 = A[55]; - T _A2002 = A[56]; - T _A2010 = A[57]; - T _A2011 = A[58]; - T _A2012 = A[59]; - T _A2020 = A[60]; - T _A2021 = A[61]; - T _A2022 = A[62]; - T _A2100 = A[63]; - T _A2101 = A[64]; - T _A2102 = A[65]; - T _A2110 = A[66]; - T _A2111 = A[67]; - T _A2112 = A[68]; - T _A2120 = A[69]; - T _A2121 = A[70]; - T _A2122 = A[71]; - T _A2200 = A[72]; - T _A2201 = A[73]; - T _A2202 = A[74]; - T _A2210 = A[75]; - T _A2211 = A[76]; - T _A2212 = A[77]; - T _A2220 = A[78]; - T _A2221 = A[79]; - T _A2222 = A[80]; - - - T _B0000 = B[0]; - T _B0001 = B[1]; - T _B0002 = B[2]; - T _B0010 = B[3]; - T _B0011 = B[4]; - T _B0012 = B[5]; - T _B0020 = B[6]; - T _B0021 = B[7]; - T _B0022 = B[8]; - T _B0100 = B[9]; - T _B0101 = B[10]; - T _B0102 = B[11]; - T _B0110 = B[12]; - T _B0111 = B[13]; - T _B0112 = B[14]; - T _B0120 = B[15]; - T _B0121 = B[16]; - T _B0122 = B[17]; - T _B0200 = B[18]; - T _B0201 = B[19]; - T _B0202 = B[20]; - T _B0210 = B[21]; - T _B0211 = B[22]; - T _B0212 = B[23]; - T _B0220 = B[24]; - T _B0221 = B[25]; - T _B0222 = B[26]; - T _B1000 = B[27]; - T _B1001 = B[28]; - T _B1002 = B[29]; - T _B1010 = B[30]; - T _B1011 = B[31]; - T _B1012 = B[32]; - T _B1020 = B[33]; - T _B1021 = B[34]; - T _B1022 = B[35]; - T _B1100 = B[36]; - T _B1101 = B[37]; - T _B1102 = B[38]; - T _B1110 = B[39]; - T _B1111 = B[40]; - T _B1112 = B[41]; - T _B1120 = B[42]; - T _B1121 = B[43]; - T _B1122 = B[44]; - T _B1200 = B[45]; - T _B1201 = B[46]; - T _B1202 = B[47]; - T _B1210 = B[48]; - T _B1211 = B[49]; - T _B1212 = B[50]; - T _B1220 = B[51]; - T _B1221 = B[52]; - T _B1222 = B[53]; - T _B2000 = B[54]; - T _B2001 = B[55]; - T _B2002 = B[56]; - T _B2010 = B[57]; - T _B2011 = B[58]; - T _B2012 = B[59]; - T _B2020 = B[60]; - T _B2021 = B[61]; - T _B2022 = B[62]; - T _B2100 = B[63]; - T _B2101 = B[64]; - T _B2102 = B[65]; - T _B2110 = B[66]; - T _B2111 = B[67]; - T _B2112 = B[68]; - T _B2120 = B[69]; - T _B2121 = B[70]; - T _B2122 = B[71]; - T _B2200 = B[72]; - T _B2201 = B[73]; - T _B2202 = B[74]; - T _B2210 = B[75]; - T _B2211 = B[76]; - T _B2212 = B[77]; - T _B2220 = B[78]; - T _B2221 = B[79]; - T _B2222 = B[80]; - - - C[0] = _A0022*_B1100 - _A0021*_B1200 - _A0012*_B2100 + _A0011*_B2200; - C[1] = _A0022*_B1101 - _A0021*_B1201 - _A0012*_B2101 + _A0011*_B2201; - C[2] = _A0022*_B1102 - _A0021*_B1202 - _A0012*_B2102 + _A0011*_B2202; - C[3] = _A0022*_B1110 - _A0021*_B1210 - _A0012*_B2110 + _A0011*_B2210; - C[4] = _A0022*_B1111 - _A0021*_B1211 - _A0012*_B2111 + _A0011*_B2211; - C[5] = _A0022*_B1112 - _A0021*_B1212 - _A0012*_B2112 + _A0011*_B2212; - C[6] = _A0022*_B1120 - _A0021*_B1220 - _A0012*_B2120 + _A0011*_B2220; - C[7] = _A0022*_B1121 - _A0021*_B1221 - _A0012*_B2121 + _A0011*_B2221; - C[8] = _A0022*_B1122 - _A0021*_B1222 - _A0012*_B2122 + _A0011*_B2222; - C[9] = _A0020*_B1200 - _A0022*_B1000 + _A0012*_B2000 - _A0010*_B2200; - C[10] = _A0020*_B1201 - _A0022*_B1001 + _A0012*_B2001 - _A0010*_B2201; - C[11] = _A0020*_B1202 - _A0022*_B1002 + _A0012*_B2002 - _A0010*_B2202; - C[12] = _A0020*_B1210 - _A0022*_B1010 + _A0012*_B2010 - _A0010*_B2210; - C[13] = _A0020*_B1211 - _A0022*_B1011 + _A0012*_B2011 - _A0010*_B2211; - C[14] = _A0020*_B1212 - _A0022*_B1012 + _A0012*_B2012 - _A0010*_B2212; - C[15] = _A0020*_B1220 - _A0022*_B1020 + _A0012*_B2020 - _A0010*_B2220; - C[16] = _A0020*_B1221 - _A0022*_B1021 + _A0012*_B2021 - _A0010*_B2221; - C[17] = _A0020*_B1222 - _A0022*_B1022 + _A0012*_B2022 - _A0010*_B2222; - C[18] = _A0021*_B1000 - _A0020*_B1100 - _A0011*_B2000 + _A0010*_B2100; - C[19] = _A0021*_B1001 - _A0020*_B1101 - _A0011*_B2001 + _A0010*_B2101; - C[20] = _A0021*_B1002 - _A0020*_B1102 - _A0011*_B2002 + _A0010*_B2102; - C[21] = _A0021*_B1010 - _A0020*_B1110 - _A0011*_B2010 + _A0010*_B2110; - C[22] = _A0021*_B1011 - _A0020*_B1111 - _A0011*_B2011 + _A0010*_B2111; - C[23] = _A0021*_B1012 - _A0020*_B1112 - _A0011*_B2012 + _A0010*_B2112; - C[24] = _A0021*_B1020 - _A0020*_B1120 - _A0011*_B2020 + _A0010*_B2120; - C[25] = _A0021*_B1021 - _A0020*_B1121 - _A0011*_B2021 + _A0010*_B2121; - C[26] = _A0021*_B1022 - _A0020*_B1122 - _A0011*_B2022 + _A0010*_B2122; - C[27] = _A0021*_B0200 - _A0022*_B0100 + _A0002*_B2100 - _A0001*_B2200; - C[28] = _A0021*_B0201 - _A0022*_B0101 + _A0002*_B2101 - _A0001*_B2201; - C[29] = _A0021*_B0202 - _A0022*_B0102 + _A0002*_B2102 - _A0001*_B2202; - C[30] = _A0021*_B0210 - _A0022*_B0110 + _A0002*_B2110 - _A0001*_B2210; - C[31] = _A0021*_B0211 - _A0022*_B0111 + _A0002*_B2111 - _A0001*_B2211; - C[32] = _A0021*_B0212 - _A0022*_B0112 + _A0002*_B2112 - _A0001*_B2212; - C[33] = _A0021*_B0220 - _A0022*_B0120 + _A0002*_B2120 - _A0001*_B2220; - C[34] = _A0021*_B0221 - _A0022*_B0121 + _A0002*_B2121 - _A0001*_B2221; - C[35] = _A0021*_B0222 - _A0022*_B0122 + _A0002*_B2122 - _A0001*_B2222; - C[36] = _A0022*_B0000 - _A0020*_B0200 - _A0002*_B2000 + _A0000*_B2200; - C[37] = _A0022*_B0001 - _A0020*_B0201 - _A0002*_B2001 + _A0000*_B2201; - C[38] = _A0022*_B0002 - _A0020*_B0202 - _A0002*_B2002 + _A0000*_B2202; - C[39] = _A0022*_B0010 - _A0020*_B0210 - _A0002*_B2010 + _A0000*_B2210; - C[40] = _A0022*_B0011 - _A0020*_B0211 - _A0002*_B2011 + _A0000*_B2211; - C[41] = _A0022*_B0012 - _A0020*_B0212 - _A0002*_B2012 + _A0000*_B2212; - C[42] = _A0022*_B0020 - _A0020*_B0220 - _A0002*_B2020 + _A0000*_B2220; - C[43] = _A0022*_B0021 - _A0020*_B0221 - _A0002*_B2021 + _A0000*_B2221; - C[44] = _A0022*_B0022 - _A0020*_B0222 - _A0002*_B2022 + _A0000*_B2222; - C[45] = _A0020*_B0100 - _A0021*_B0000 + _A0001*_B2000 - _A0000*_B2100; - C[46] = _A0020*_B0101 - _A0021*_B0001 + _A0001*_B2001 - _A0000*_B2101; - C[47] = _A0020*_B0102 - _A0021*_B0002 + _A0001*_B2002 - _A0000*_B2102; - C[48] = _A0020*_B0110 - _A0021*_B0010 + _A0001*_B2010 - _A0000*_B2110; - C[49] = _A0020*_B0111 - _A0021*_B0011 + _A0001*_B2011 - _A0000*_B2111; - C[50] = _A0020*_B0112 - _A0021*_B0012 + _A0001*_B2012 - _A0000*_B2112; - C[51] = _A0020*_B0120 - _A0021*_B0020 + _A0001*_B2020 - _A0000*_B2120; - C[52] = _A0020*_B0121 - _A0021*_B0021 + _A0001*_B2021 - _A0000*_B2121; - C[53] = _A0020*_B0122 - _A0021*_B0022 + _A0001*_B2022 - _A0000*_B2122; - C[54] = _A0012*_B0100 - _A0011*_B0200 - _A0002*_B1100 + _A0001*_B1200; - C[55] = _A0012*_B0101 - _A0011*_B0201 - _A0002*_B1101 + _A0001*_B1201; - C[56] = _A0012*_B0102 - _A0011*_B0202 - _A0002*_B1102 + _A0001*_B1202; - C[57] = _A0012*_B0110 - _A0011*_B0210 - _A0002*_B1110 + _A0001*_B1210; - C[58] = _A0012*_B0111 - _A0011*_B0211 - _A0002*_B1111 + _A0001*_B1211; - C[59] = _A0012*_B0112 - _A0011*_B0212 - _A0002*_B1112 + _A0001*_B1212; - C[60] = _A0012*_B0120 - _A0011*_B0220 - _A0002*_B1120 + _A0001*_B1220; - C[61] = _A0012*_B0121 - _A0011*_B0221 - _A0002*_B1121 + _A0001*_B1221; - C[62] = _A0012*_B0122 - _A0011*_B0222 - _A0002*_B1122 + _A0001*_B1222; - C[63] = _A0010*_B0200 - _A0012*_B0000 + _A0002*_B1000 - _A0000*_B1200; - C[64] = _A0010*_B0201 - _A0012*_B0001 + _A0002*_B1001 - _A0000*_B1201; - C[65] = _A0010*_B0202 - _A0012*_B0002 + _A0002*_B1002 - _A0000*_B1202; - C[66] = _A0010*_B0210 - _A0012*_B0010 + _A0002*_B1010 - _A0000*_B1210; - C[67] = _A0010*_B0211 - _A0012*_B0011 + _A0002*_B1011 - _A0000*_B1211; - C[68] = _A0010*_B0212 - _A0012*_B0012 + _A0002*_B1012 - _A0000*_B1212; - C[69] = _A0010*_B0220 - _A0012*_B0020 + _A0002*_B1020 - _A0000*_B1220; - C[70] = _A0010*_B0221 - _A0012*_B0021 + _A0002*_B1021 - _A0000*_B1221; - C[71] = _A0010*_B0222 - _A0012*_B0022 + _A0002*_B1022 - _A0000*_B1222; - C[72] = _A0011*_B0000 - _A0010*_B0100 - _A0001*_B1000 + _A0000*_B1100; - C[73] = _A0011*_B0001 - _A0010*_B0101 - _A0001*_B1001 + _A0000*_B1101; - C[74] = _A0011*_B0002 - _A0010*_B0102 - _A0001*_B1002 + _A0000*_B1102; - C[75] = _A0011*_B0010 - _A0010*_B0110 - _A0001*_B1010 + _A0000*_B1110; - C[76] = _A0011*_B0011 - _A0010*_B0111 - _A0001*_B1011 + _A0000*_B1111; - C[77] = _A0011*_B0012 - _A0010*_B0112 - _A0001*_B1012 + _A0000*_B1112; - C[78] = _A0011*_B0020 - _A0010*_B0120 - _A0001*_B1020 + _A0000*_B1120; - C[79] = _A0011*_B0021 - _A0010*_B0121 - _A0001*_B1021 + _A0000*_B1121; - C[80] = _A0011*_B0022 - _A0010*_B0122 - _A0001*_B1022 + _A0000*_B1122; - C[81] = _A0122*_B1100 - _A0121*_B1200 - _A0112*_B2100 + _A0111*_B2200; - C[82] = _A0122*_B1101 - _A0121*_B1201 - _A0112*_B2101 + _A0111*_B2201; - C[83] = _A0122*_B1102 - _A0121*_B1202 - _A0112*_B2102 + _A0111*_B2202; - C[84] = _A0122*_B1110 - _A0121*_B1210 - _A0112*_B2110 + _A0111*_B2210; - C[85] = _A0122*_B1111 - _A0121*_B1211 - _A0112*_B2111 + _A0111*_B2211; - C[86] = _A0122*_B1112 - _A0121*_B1212 - _A0112*_B2112 + _A0111*_B2212; - C[87] = _A0122*_B1120 - _A0121*_B1220 - _A0112*_B2120 + _A0111*_B2220; - C[88] = _A0122*_B1121 - _A0121*_B1221 - _A0112*_B2121 + _A0111*_B2221; - C[89] = _A0122*_B1122 - _A0121*_B1222 - _A0112*_B2122 + _A0111*_B2222; - C[90] = _A0120*_B1200 - _A0122*_B1000 + _A0112*_B2000 - _A0110*_B2200; - C[91] = _A0120*_B1201 - _A0122*_B1001 + _A0112*_B2001 - _A0110*_B2201; - C[92] = _A0120*_B1202 - _A0122*_B1002 + _A0112*_B2002 - _A0110*_B2202; - C[93] = _A0120*_B1210 - _A0122*_B1010 + _A0112*_B2010 - _A0110*_B2210; - C[94] = _A0120*_B1211 - _A0122*_B1011 + _A0112*_B2011 - _A0110*_B2211; - C[95] = _A0120*_B1212 - _A0122*_B1012 + _A0112*_B2012 - _A0110*_B2212; - C[96] = _A0120*_B1220 - _A0122*_B1020 + _A0112*_B2020 - _A0110*_B2220; - C[97] = _A0120*_B1221 - _A0122*_B1021 + _A0112*_B2021 - _A0110*_B2221; - C[98] = _A0120*_B1222 - _A0122*_B1022 + _A0112*_B2022 - _A0110*_B2222; - C[99] = _A0121*_B1000 - _A0120*_B1100 - _A0111*_B2000 + _A0110*_B2100; - C[100] = _A0121*_B1001 - _A0120*_B1101 - _A0111*_B2001 + _A0110*_B2101; - C[101] = _A0121*_B1002 - _A0120*_B1102 - _A0111*_B2002 + _A0110*_B2102; - C[102] = _A0121*_B1010 - _A0120*_B1110 - _A0111*_B2010 + _A0110*_B2110; - C[103] = _A0121*_B1011 - _A0120*_B1111 - _A0111*_B2011 + _A0110*_B2111; - C[104] = _A0121*_B1012 - _A0120*_B1112 - _A0111*_B2012 + _A0110*_B2112; - C[105] = _A0121*_B1020 - _A0120*_B1120 - _A0111*_B2020 + _A0110*_B2120; - C[106] = _A0121*_B1021 - _A0120*_B1121 - _A0111*_B2021 + _A0110*_B2121; - C[107] = _A0121*_B1022 - _A0120*_B1122 - _A0111*_B2022 + _A0110*_B2122; - C[108] = _A0121*_B0200 - _A0122*_B0100 + _A0102*_B2100 - _A0101*_B2200; - C[109] = _A0121*_B0201 - _A0122*_B0101 + _A0102*_B2101 - _A0101*_B2201; - C[110] = _A0121*_B0202 - _A0122*_B0102 + _A0102*_B2102 - _A0101*_B2202; - C[111] = _A0121*_B0210 - _A0122*_B0110 + _A0102*_B2110 - _A0101*_B2210; - C[112] = _A0121*_B0211 - _A0122*_B0111 + _A0102*_B2111 - _A0101*_B2211; - C[113] = _A0121*_B0212 - _A0122*_B0112 + _A0102*_B2112 - _A0101*_B2212; - C[114] = _A0121*_B0220 - _A0122*_B0120 + _A0102*_B2120 - _A0101*_B2220; - C[115] = _A0121*_B0221 - _A0122*_B0121 + _A0102*_B2121 - _A0101*_B2221; - C[116] = _A0121*_B0222 - _A0122*_B0122 + _A0102*_B2122 - _A0101*_B2222; - C[117] = _A0122*_B0000 - _A0120*_B0200 - _A0102*_B2000 + _A0100*_B2200; - C[118] = _A0122*_B0001 - _A0120*_B0201 - _A0102*_B2001 + _A0100*_B2201; - C[119] = _A0122*_B0002 - _A0120*_B0202 - _A0102*_B2002 + _A0100*_B2202; - C[120] = _A0122*_B0010 - _A0120*_B0210 - _A0102*_B2010 + _A0100*_B2210; - C[121] = _A0122*_B0011 - _A0120*_B0211 - _A0102*_B2011 + _A0100*_B2211; - C[122] = _A0122*_B0012 - _A0120*_B0212 - _A0102*_B2012 + _A0100*_B2212; - C[123] = _A0122*_B0020 - _A0120*_B0220 - _A0102*_B2020 + _A0100*_B2220; - C[124] = _A0122*_B0021 - _A0120*_B0221 - _A0102*_B2021 + _A0100*_B2221; - C[125] = _A0122*_B0022 - _A0120*_B0222 - _A0102*_B2022 + _A0100*_B2222; - C[126] = _A0120*_B0100 - _A0121*_B0000 + _A0101*_B2000 - _A0100*_B2100; - C[127] = _A0120*_B0101 - _A0121*_B0001 + _A0101*_B2001 - _A0100*_B2101; - C[128] = _A0120*_B0102 - _A0121*_B0002 + _A0101*_B2002 - _A0100*_B2102; - C[129] = _A0120*_B0110 - _A0121*_B0010 + _A0101*_B2010 - _A0100*_B2110; - C[130] = _A0120*_B0111 - _A0121*_B0011 + _A0101*_B2011 - _A0100*_B2111; - C[131] = _A0120*_B0112 - _A0121*_B0012 + _A0101*_B2012 - _A0100*_B2112; - C[132] = _A0120*_B0120 - _A0121*_B0020 + _A0101*_B2020 - _A0100*_B2120; - C[133] = _A0120*_B0121 - _A0121*_B0021 + _A0101*_B2021 - _A0100*_B2121; - C[134] = _A0120*_B0122 - _A0121*_B0022 + _A0101*_B2022 - _A0100*_B2122; - C[135] = _A0112*_B0100 - _A0111*_B0200 - _A0102*_B1100 + _A0101*_B1200; - C[136] = _A0112*_B0101 - _A0111*_B0201 - _A0102*_B1101 + _A0101*_B1201; - C[137] = _A0112*_B0102 - _A0111*_B0202 - _A0102*_B1102 + _A0101*_B1202; - C[138] = _A0112*_B0110 - _A0111*_B0210 - _A0102*_B1110 + _A0101*_B1210; - C[139] = _A0112*_B0111 - _A0111*_B0211 - _A0102*_B1111 + _A0101*_B1211; - C[140] = _A0112*_B0112 - _A0111*_B0212 - _A0102*_B1112 + _A0101*_B1212; - C[141] = _A0112*_B0120 - _A0111*_B0220 - _A0102*_B1120 + _A0101*_B1220; - C[142] = _A0112*_B0121 - _A0111*_B0221 - _A0102*_B1121 + _A0101*_B1221; - C[143] = _A0112*_B0122 - _A0111*_B0222 - _A0102*_B1122 + _A0101*_B1222; - C[144] = _A0110*_B0200 - _A0112*_B0000 + _A0102*_B1000 - _A0100*_B1200; - C[145] = _A0110*_B0201 - _A0112*_B0001 + _A0102*_B1001 - _A0100*_B1201; - C[146] = _A0110*_B0202 - _A0112*_B0002 + _A0102*_B1002 - _A0100*_B1202; - C[147] = _A0110*_B0210 - _A0112*_B0010 + _A0102*_B1010 - _A0100*_B1210; - C[148] = _A0110*_B0211 - _A0112*_B0011 + _A0102*_B1011 - _A0100*_B1211; - C[149] = _A0110*_B0212 - _A0112*_B0012 + _A0102*_B1012 - _A0100*_B1212; - C[150] = _A0110*_B0220 - _A0112*_B0020 + _A0102*_B1020 - _A0100*_B1220; - C[151] = _A0110*_B0221 - _A0112*_B0021 + _A0102*_B1021 - _A0100*_B1221; - C[152] = _A0110*_B0222 - _A0112*_B0022 + _A0102*_B1022 - _A0100*_B1222; - C[153] = _A0111*_B0000 - _A0110*_B0100 - _A0101*_B1000 + _A0100*_B1100; - C[154] = _A0111*_B0001 - _A0110*_B0101 - _A0101*_B1001 + _A0100*_B1101; - C[155] = _A0111*_B0002 - _A0110*_B0102 - _A0101*_B1002 + _A0100*_B1102; - C[156] = _A0111*_B0010 - _A0110*_B0110 - _A0101*_B1010 + _A0100*_B1110; - C[157] = _A0111*_B0011 - _A0110*_B0111 - _A0101*_B1011 + _A0100*_B1111; - C[158] = _A0111*_B0012 - _A0110*_B0112 - _A0101*_B1012 + _A0100*_B1112; - C[159] = _A0111*_B0020 - _A0110*_B0120 - _A0101*_B1020 + _A0100*_B1120; - C[160] = _A0111*_B0021 - _A0110*_B0121 - _A0101*_B1021 + _A0100*_B1121; - C[161] = _A0111*_B0022 - _A0110*_B0122 - _A0101*_B1022 + _A0100*_B1122; - C[162] = _A0222*_B1100 - _A0221*_B1200 - _A0212*_B2100 + _A0211*_B2200; - C[163] = _A0222*_B1101 - _A0221*_B1201 - _A0212*_B2101 + _A0211*_B2201; - C[164] = _A0222*_B1102 - _A0221*_B1202 - _A0212*_B2102 + _A0211*_B2202; - C[165] = _A0222*_B1110 - _A0221*_B1210 - _A0212*_B2110 + _A0211*_B2210; - C[166] = _A0222*_B1111 - _A0221*_B1211 - _A0212*_B2111 + _A0211*_B2211; - C[167] = _A0222*_B1112 - _A0221*_B1212 - _A0212*_B2112 + _A0211*_B2212; - C[168] = _A0222*_B1120 - _A0221*_B1220 - _A0212*_B2120 + _A0211*_B2220; - C[169] = _A0222*_B1121 - _A0221*_B1221 - _A0212*_B2121 + _A0211*_B2221; - C[170] = _A0222*_B1122 - _A0221*_B1222 - _A0212*_B2122 + _A0211*_B2222; - C[171] = _A0220*_B1200 - _A0222*_B1000 + _A0212*_B2000 - _A0210*_B2200; - C[172] = _A0220*_B1201 - _A0222*_B1001 + _A0212*_B2001 - _A0210*_B2201; - C[173] = _A0220*_B1202 - _A0222*_B1002 + _A0212*_B2002 - _A0210*_B2202; - C[174] = _A0220*_B1210 - _A0222*_B1010 + _A0212*_B2010 - _A0210*_B2210; - C[175] = _A0220*_B1211 - _A0222*_B1011 + _A0212*_B2011 - _A0210*_B2211; - C[176] = _A0220*_B1212 - _A0222*_B1012 + _A0212*_B2012 - _A0210*_B2212; - C[177] = _A0220*_B1220 - _A0222*_B1020 + _A0212*_B2020 - _A0210*_B2220; - C[178] = _A0220*_B1221 - _A0222*_B1021 + _A0212*_B2021 - _A0210*_B2221; - C[179] = _A0220*_B1222 - _A0222*_B1022 + _A0212*_B2022 - _A0210*_B2222; - C[180] = _A0221*_B1000 - _A0220*_B1100 - _A0211*_B2000 + _A0210*_B2100; - C[181] = _A0221*_B1001 - _A0220*_B1101 - _A0211*_B2001 + _A0210*_B2101; - C[182] = _A0221*_B1002 - _A0220*_B1102 - _A0211*_B2002 + _A0210*_B2102; - C[183] = _A0221*_B1010 - _A0220*_B1110 - _A0211*_B2010 + _A0210*_B2110; - C[184] = _A0221*_B1011 - _A0220*_B1111 - _A0211*_B2011 + _A0210*_B2111; - C[185] = _A0221*_B1012 - _A0220*_B1112 - _A0211*_B2012 + _A0210*_B2112; - C[186] = _A0221*_B1020 - _A0220*_B1120 - _A0211*_B2020 + _A0210*_B2120; - C[187] = _A0221*_B1021 - _A0220*_B1121 - _A0211*_B2021 + _A0210*_B2121; - C[188] = _A0221*_B1022 - _A0220*_B1122 - _A0211*_B2022 + _A0210*_B2122; - C[189] = _A0221*_B0200 - _A0222*_B0100 + _A0202*_B2100 - _A0201*_B2200; - C[190] = _A0221*_B0201 - _A0222*_B0101 + _A0202*_B2101 - _A0201*_B2201; - C[191] = _A0221*_B0202 - _A0222*_B0102 + _A0202*_B2102 - _A0201*_B2202; - C[192] = _A0221*_B0210 - _A0222*_B0110 + _A0202*_B2110 - _A0201*_B2210; - C[193] = _A0221*_B0211 - _A0222*_B0111 + _A0202*_B2111 - _A0201*_B2211; - C[194] = _A0221*_B0212 - _A0222*_B0112 + _A0202*_B2112 - _A0201*_B2212; - C[195] = _A0221*_B0220 - _A0222*_B0120 + _A0202*_B2120 - _A0201*_B2220; - C[196] = _A0221*_B0221 - _A0222*_B0121 + _A0202*_B2121 - _A0201*_B2221; - C[197] = _A0221*_B0222 - _A0222*_B0122 + _A0202*_B2122 - _A0201*_B2222; - C[198] = _A0222*_B0000 - _A0220*_B0200 - _A0202*_B2000 + _A0200*_B2200; - C[199] = _A0222*_B0001 - _A0220*_B0201 - _A0202*_B2001 + _A0200*_B2201; - C[200] = _A0222*_B0002 - _A0220*_B0202 - _A0202*_B2002 + _A0200*_B2202; - C[201] = _A0222*_B0010 - _A0220*_B0210 - _A0202*_B2010 + _A0200*_B2210; - C[202] = _A0222*_B0011 - _A0220*_B0211 - _A0202*_B2011 + _A0200*_B2211; - C[203] = _A0222*_B0012 - _A0220*_B0212 - _A0202*_B2012 + _A0200*_B2212; - C[204] = _A0222*_B0020 - _A0220*_B0220 - _A0202*_B2020 + _A0200*_B2220; - C[205] = _A0222*_B0021 - _A0220*_B0221 - _A0202*_B2021 + _A0200*_B2221; - C[206] = _A0222*_B0022 - _A0220*_B0222 - _A0202*_B2022 + _A0200*_B2222; - C[207] = _A0220*_B0100 - _A0221*_B0000 + _A0201*_B2000 - _A0200*_B2100; - C[208] = _A0220*_B0101 - _A0221*_B0001 + _A0201*_B2001 - _A0200*_B2101; - C[209] = _A0220*_B0102 - _A0221*_B0002 + _A0201*_B2002 - _A0200*_B2102; - C[210] = _A0220*_B0110 - _A0221*_B0010 + _A0201*_B2010 - _A0200*_B2110; - C[211] = _A0220*_B0111 - _A0221*_B0011 + _A0201*_B2011 - _A0200*_B2111; - C[212] = _A0220*_B0112 - _A0221*_B0012 + _A0201*_B2012 - _A0200*_B2112; - C[213] = _A0220*_B0120 - _A0221*_B0020 + _A0201*_B2020 - _A0200*_B2120; - C[214] = _A0220*_B0121 - _A0221*_B0021 + _A0201*_B2021 - _A0200*_B2121; - C[215] = _A0220*_B0122 - _A0221*_B0022 + _A0201*_B2022 - _A0200*_B2122; - C[216] = _A0212*_B0100 - _A0211*_B0200 - _A0202*_B1100 + _A0201*_B1200; - C[217] = _A0212*_B0101 - _A0211*_B0201 - _A0202*_B1101 + _A0201*_B1201; - C[218] = _A0212*_B0102 - _A0211*_B0202 - _A0202*_B1102 + _A0201*_B1202; - C[219] = _A0212*_B0110 - _A0211*_B0210 - _A0202*_B1110 + _A0201*_B1210; - C[220] = _A0212*_B0111 - _A0211*_B0211 - _A0202*_B1111 + _A0201*_B1211; - C[221] = _A0212*_B0112 - _A0211*_B0212 - _A0202*_B1112 + _A0201*_B1212; - C[222] = _A0212*_B0120 - _A0211*_B0220 - _A0202*_B1120 + _A0201*_B1220; - C[223] = _A0212*_B0121 - _A0211*_B0221 - _A0202*_B1121 + _A0201*_B1221; - C[224] = _A0212*_B0122 - _A0211*_B0222 - _A0202*_B1122 + _A0201*_B1222; - C[225] = _A0210*_B0200 - _A0212*_B0000 + _A0202*_B1000 - _A0200*_B1200; - C[226] = _A0210*_B0201 - _A0212*_B0001 + _A0202*_B1001 - _A0200*_B1201; - C[227] = _A0210*_B0202 - _A0212*_B0002 + _A0202*_B1002 - _A0200*_B1202; - C[228] = _A0210*_B0210 - _A0212*_B0010 + _A0202*_B1010 - _A0200*_B1210; - C[229] = _A0210*_B0211 - _A0212*_B0011 + _A0202*_B1011 - _A0200*_B1211; - C[230] = _A0210*_B0212 - _A0212*_B0012 + _A0202*_B1012 - _A0200*_B1212; - C[231] = _A0210*_B0220 - _A0212*_B0020 + _A0202*_B1020 - _A0200*_B1220; - C[232] = _A0210*_B0221 - _A0212*_B0021 + _A0202*_B1021 - _A0200*_B1221; - C[233] = _A0210*_B0222 - _A0212*_B0022 + _A0202*_B1022 - _A0200*_B1222; - C[234] = _A0211*_B0000 - _A0210*_B0100 - _A0201*_B1000 + _A0200*_B1100; - C[235] = _A0211*_B0001 - _A0210*_B0101 - _A0201*_B1001 + _A0200*_B1101; - C[236] = _A0211*_B0002 - _A0210*_B0102 - _A0201*_B1002 + _A0200*_B1102; - C[237] = _A0211*_B0010 - _A0210*_B0110 - _A0201*_B1010 + _A0200*_B1110; - C[238] = _A0211*_B0011 - _A0210*_B0111 - _A0201*_B1011 + _A0200*_B1111; - C[239] = _A0211*_B0012 - _A0210*_B0112 - _A0201*_B1012 + _A0200*_B1112; - C[240] = _A0211*_B0020 - _A0210*_B0120 - _A0201*_B1020 + _A0200*_B1120; - C[241] = _A0211*_B0021 - _A0210*_B0121 - _A0201*_B1021 + _A0200*_B1121; - C[242] = _A0211*_B0022 - _A0210*_B0122 - _A0201*_B1022 + _A0200*_B1122; - C[243] = _A1022*_B1100 - _A1021*_B1200 - _A1012*_B2100 + _A1011*_B2200; - C[244] = _A1022*_B1101 - _A1021*_B1201 - _A1012*_B2101 + _A1011*_B2201; - C[245] = _A1022*_B1102 - _A1021*_B1202 - _A1012*_B2102 + _A1011*_B2202; - C[246] = _A1022*_B1110 - _A1021*_B1210 - _A1012*_B2110 + _A1011*_B2210; - C[247] = _A1022*_B1111 - _A1021*_B1211 - _A1012*_B2111 + _A1011*_B2211; - C[248] = _A1022*_B1112 - _A1021*_B1212 - _A1012*_B2112 + _A1011*_B2212; - C[249] = _A1022*_B1120 - _A1021*_B1220 - _A1012*_B2120 + _A1011*_B2220; - C[250] = _A1022*_B1121 - _A1021*_B1221 - _A1012*_B2121 + _A1011*_B2221; - C[251] = _A1022*_B1122 - _A1021*_B1222 - _A1012*_B2122 + _A1011*_B2222; - C[252] = _A1020*_B1200 - _A1022*_B1000 + _A1012*_B2000 - _A1010*_B2200; - C[253] = _A1020*_B1201 - _A1022*_B1001 + _A1012*_B2001 - _A1010*_B2201; - C[254] = _A1020*_B1202 - _A1022*_B1002 + _A1012*_B2002 - _A1010*_B2202; - C[255] = _A1020*_B1210 - _A1022*_B1010 + _A1012*_B2010 - _A1010*_B2210; - C[256] = _A1020*_B1211 - _A1022*_B1011 + _A1012*_B2011 - _A1010*_B2211; - C[257] = _A1020*_B1212 - _A1022*_B1012 + _A1012*_B2012 - _A1010*_B2212; - C[258] = _A1020*_B1220 - _A1022*_B1020 + _A1012*_B2020 - _A1010*_B2220; - C[259] = _A1020*_B1221 - _A1022*_B1021 + _A1012*_B2021 - _A1010*_B2221; - C[260] = _A1020*_B1222 - _A1022*_B1022 + _A1012*_B2022 - _A1010*_B2222; - C[261] = _A1021*_B1000 - _A1020*_B1100 - _A1011*_B2000 + _A1010*_B2100; - C[262] = _A1021*_B1001 - _A1020*_B1101 - _A1011*_B2001 + _A1010*_B2101; - C[263] = _A1021*_B1002 - _A1020*_B1102 - _A1011*_B2002 + _A1010*_B2102; - C[264] = _A1021*_B1010 - _A1020*_B1110 - _A1011*_B2010 + _A1010*_B2110; - C[265] = _A1021*_B1011 - _A1020*_B1111 - _A1011*_B2011 + _A1010*_B2111; - C[266] = _A1021*_B1012 - _A1020*_B1112 - _A1011*_B2012 + _A1010*_B2112; - C[267] = _A1021*_B1020 - _A1020*_B1120 - _A1011*_B2020 + _A1010*_B2120; - C[268] = _A1021*_B1021 - _A1020*_B1121 - _A1011*_B2021 + _A1010*_B2121; - C[269] = _A1021*_B1022 - _A1020*_B1122 - _A1011*_B2022 + _A1010*_B2122; - C[270] = _A1021*_B0200 - _A1022*_B0100 + _A1002*_B2100 - _A1001*_B2200; - C[271] = _A1021*_B0201 - _A1022*_B0101 + _A1002*_B2101 - _A1001*_B2201; - C[272] = _A1021*_B0202 - _A1022*_B0102 + _A1002*_B2102 - _A1001*_B2202; - C[273] = _A1021*_B0210 - _A1022*_B0110 + _A1002*_B2110 - _A1001*_B2210; - C[274] = _A1021*_B0211 - _A1022*_B0111 + _A1002*_B2111 - _A1001*_B2211; - C[275] = _A1021*_B0212 - _A1022*_B0112 + _A1002*_B2112 - _A1001*_B2212; - C[276] = _A1021*_B0220 - _A1022*_B0120 + _A1002*_B2120 - _A1001*_B2220; - C[277] = _A1021*_B0221 - _A1022*_B0121 + _A1002*_B2121 - _A1001*_B2221; - C[278] = _A1021*_B0222 - _A1022*_B0122 + _A1002*_B2122 - _A1001*_B2222; - C[279] = _A1022*_B0000 - _A1020*_B0200 - _A1002*_B2000 + _A1000*_B2200; - C[280] = _A1022*_B0001 - _A1020*_B0201 - _A1002*_B2001 + _A1000*_B2201; - C[281] = _A1022*_B0002 - _A1020*_B0202 - _A1002*_B2002 + _A1000*_B2202; - C[282] = _A1022*_B0010 - _A1020*_B0210 - _A1002*_B2010 + _A1000*_B2210; - C[283] = _A1022*_B0011 - _A1020*_B0211 - _A1002*_B2011 + _A1000*_B2211; - C[284] = _A1022*_B0012 - _A1020*_B0212 - _A1002*_B2012 + _A1000*_B2212; - C[285] = _A1022*_B0020 - _A1020*_B0220 - _A1002*_B2020 + _A1000*_B2220; - C[286] = _A1022*_B0021 - _A1020*_B0221 - _A1002*_B2021 + _A1000*_B2221; - C[287] = _A1022*_B0022 - _A1020*_B0222 - _A1002*_B2022 + _A1000*_B2222; - C[288] = _A1020*_B0100 - _A1021*_B0000 + _A1001*_B2000 - _A1000*_B2100; - C[289] = _A1020*_B0101 - _A1021*_B0001 + _A1001*_B2001 - _A1000*_B2101; - C[290] = _A1020*_B0102 - _A1021*_B0002 + _A1001*_B2002 - _A1000*_B2102; - C[291] = _A1020*_B0110 - _A1021*_B0010 + _A1001*_B2010 - _A1000*_B2110; - C[292] = _A1020*_B0111 - _A1021*_B0011 + _A1001*_B2011 - _A1000*_B2111; - C[293] = _A1020*_B0112 - _A1021*_B0012 + _A1001*_B2012 - _A1000*_B2112; - C[294] = _A1020*_B0120 - _A1021*_B0020 + _A1001*_B2020 - _A1000*_B2120; - C[295] = _A1020*_B0121 - _A1021*_B0021 + _A1001*_B2021 - _A1000*_B2121; - C[296] = _A1020*_B0122 - _A1021*_B0022 + _A1001*_B2022 - _A1000*_B2122; - C[297] = _A1012*_B0100 - _A1011*_B0200 - _A1002*_B1100 + _A1001*_B1200; - C[298] = _A1012*_B0101 - _A1011*_B0201 - _A1002*_B1101 + _A1001*_B1201; - C[299] = _A1012*_B0102 - _A1011*_B0202 - _A1002*_B1102 + _A1001*_B1202; - C[300] = _A1012*_B0110 - _A1011*_B0210 - _A1002*_B1110 + _A1001*_B1210; - C[301] = _A1012*_B0111 - _A1011*_B0211 - _A1002*_B1111 + _A1001*_B1211; - C[302] = _A1012*_B0112 - _A1011*_B0212 - _A1002*_B1112 + _A1001*_B1212; - C[303] = _A1012*_B0120 - _A1011*_B0220 - _A1002*_B1120 + _A1001*_B1220; - C[304] = _A1012*_B0121 - _A1011*_B0221 - _A1002*_B1121 + _A1001*_B1221; - C[305] = _A1012*_B0122 - _A1011*_B0222 - _A1002*_B1122 + _A1001*_B1222; - C[306] = _A1010*_B0200 - _A1012*_B0000 + _A1002*_B1000 - _A1000*_B1200; - C[307] = _A1010*_B0201 - _A1012*_B0001 + _A1002*_B1001 - _A1000*_B1201; - C[308] = _A1010*_B0202 - _A1012*_B0002 + _A1002*_B1002 - _A1000*_B1202; - C[309] = _A1010*_B0210 - _A1012*_B0010 + _A1002*_B1010 - _A1000*_B1210; - C[310] = _A1010*_B0211 - _A1012*_B0011 + _A1002*_B1011 - _A1000*_B1211; - C[311] = _A1010*_B0212 - _A1012*_B0012 + _A1002*_B1012 - _A1000*_B1212; - C[312] = _A1010*_B0220 - _A1012*_B0020 + _A1002*_B1020 - _A1000*_B1220; - C[313] = _A1010*_B0221 - _A1012*_B0021 + _A1002*_B1021 - _A1000*_B1221; - C[314] = _A1010*_B0222 - _A1012*_B0022 + _A1002*_B1022 - _A1000*_B1222; - C[315] = _A1011*_B0000 - _A1010*_B0100 - _A1001*_B1000 + _A1000*_B1100; - C[316] = _A1011*_B0001 - _A1010*_B0101 - _A1001*_B1001 + _A1000*_B1101; - C[317] = _A1011*_B0002 - _A1010*_B0102 - _A1001*_B1002 + _A1000*_B1102; - C[318] = _A1011*_B0010 - _A1010*_B0110 - _A1001*_B1010 + _A1000*_B1110; - C[319] = _A1011*_B0011 - _A1010*_B0111 - _A1001*_B1011 + _A1000*_B1111; - C[320] = _A1011*_B0012 - _A1010*_B0112 - _A1001*_B1012 + _A1000*_B1112; - C[321] = _A1011*_B0020 - _A1010*_B0120 - _A1001*_B1020 + _A1000*_B1120; - C[322] = _A1011*_B0021 - _A1010*_B0121 - _A1001*_B1021 + _A1000*_B1121; - C[323] = _A1011*_B0022 - _A1010*_B0122 - _A1001*_B1022 + _A1000*_B1122; - C[324] = _A1122*_B1100 - _A1121*_B1200 - _A1112*_B2100 + _A1111*_B2200; - C[325] = _A1122*_B1101 - _A1121*_B1201 - _A1112*_B2101 + _A1111*_B2201; - C[326] = _A1122*_B1102 - _A1121*_B1202 - _A1112*_B2102 + _A1111*_B2202; - C[327] = _A1122*_B1110 - _A1121*_B1210 - _A1112*_B2110 + _A1111*_B2210; - C[328] = _A1122*_B1111 - _A1121*_B1211 - _A1112*_B2111 + _A1111*_B2211; - C[329] = _A1122*_B1112 - _A1121*_B1212 - _A1112*_B2112 + _A1111*_B2212; - C[330] = _A1122*_B1120 - _A1121*_B1220 - _A1112*_B2120 + _A1111*_B2220; - C[331] = _A1122*_B1121 - _A1121*_B1221 - _A1112*_B2121 + _A1111*_B2221; - C[332] = _A1122*_B1122 - _A1121*_B1222 - _A1112*_B2122 + _A1111*_B2222; - C[333] = _A1120*_B1200 - _A1122*_B1000 + _A1112*_B2000 - _A1110*_B2200; - C[334] = _A1120*_B1201 - _A1122*_B1001 + _A1112*_B2001 - _A1110*_B2201; - C[335] = _A1120*_B1202 - _A1122*_B1002 + _A1112*_B2002 - _A1110*_B2202; - C[336] = _A1120*_B1210 - _A1122*_B1010 + _A1112*_B2010 - _A1110*_B2210; - C[337] = _A1120*_B1211 - _A1122*_B1011 + _A1112*_B2011 - _A1110*_B2211; - C[338] = _A1120*_B1212 - _A1122*_B1012 + _A1112*_B2012 - _A1110*_B2212; - C[339] = _A1120*_B1220 - _A1122*_B1020 + _A1112*_B2020 - _A1110*_B2220; - C[340] = _A1120*_B1221 - _A1122*_B1021 + _A1112*_B2021 - _A1110*_B2221; - C[341] = _A1120*_B1222 - _A1122*_B1022 + _A1112*_B2022 - _A1110*_B2222; - C[342] = _A1121*_B1000 - _A1120*_B1100 - _A1111*_B2000 + _A1110*_B2100; - C[343] = _A1121*_B1001 - _A1120*_B1101 - _A1111*_B2001 + _A1110*_B2101; - C[344] = _A1121*_B1002 - _A1120*_B1102 - _A1111*_B2002 + _A1110*_B2102; - C[345] = _A1121*_B1010 - _A1120*_B1110 - _A1111*_B2010 + _A1110*_B2110; - C[346] = _A1121*_B1011 - _A1120*_B1111 - _A1111*_B2011 + _A1110*_B2111; - C[347] = _A1121*_B1012 - _A1120*_B1112 - _A1111*_B2012 + _A1110*_B2112; - C[348] = _A1121*_B1020 - _A1120*_B1120 - _A1111*_B2020 + _A1110*_B2120; - C[349] = _A1121*_B1021 - _A1120*_B1121 - _A1111*_B2021 + _A1110*_B2121; - C[350] = _A1121*_B1022 - _A1120*_B1122 - _A1111*_B2022 + _A1110*_B2122; - C[351] = _A1121*_B0200 - _A1122*_B0100 + _A1102*_B2100 - _A1101*_B2200; - C[352] = _A1121*_B0201 - _A1122*_B0101 + _A1102*_B2101 - _A1101*_B2201; - C[353] = _A1121*_B0202 - _A1122*_B0102 + _A1102*_B2102 - _A1101*_B2202; - C[354] = _A1121*_B0210 - _A1122*_B0110 + _A1102*_B2110 - _A1101*_B2210; - C[355] = _A1121*_B0211 - _A1122*_B0111 + _A1102*_B2111 - _A1101*_B2211; - C[356] = _A1121*_B0212 - _A1122*_B0112 + _A1102*_B2112 - _A1101*_B2212; - C[357] = _A1121*_B0220 - _A1122*_B0120 + _A1102*_B2120 - _A1101*_B2220; - C[358] = _A1121*_B0221 - _A1122*_B0121 + _A1102*_B2121 - _A1101*_B2221; - C[359] = _A1121*_B0222 - _A1122*_B0122 + _A1102*_B2122 - _A1101*_B2222; - C[360] = _A1122*_B0000 - _A1120*_B0200 - _A1102*_B2000 + _A1100*_B2200; - C[361] = _A1122*_B0001 - _A1120*_B0201 - _A1102*_B2001 + _A1100*_B2201; - C[362] = _A1122*_B0002 - _A1120*_B0202 - _A1102*_B2002 + _A1100*_B2202; - C[363] = _A1122*_B0010 - _A1120*_B0210 - _A1102*_B2010 + _A1100*_B2210; - C[364] = _A1122*_B0011 - _A1120*_B0211 - _A1102*_B2011 + _A1100*_B2211; - C[365] = _A1122*_B0012 - _A1120*_B0212 - _A1102*_B2012 + _A1100*_B2212; - C[366] = _A1122*_B0020 - _A1120*_B0220 - _A1102*_B2020 + _A1100*_B2220; - C[367] = _A1122*_B0021 - _A1120*_B0221 - _A1102*_B2021 + _A1100*_B2221; - C[368] = _A1122*_B0022 - _A1120*_B0222 - _A1102*_B2022 + _A1100*_B2222; - C[369] = _A1120*_B0100 - _A1121*_B0000 + _A1101*_B2000 - _A1100*_B2100; - C[370] = _A1120*_B0101 - _A1121*_B0001 + _A1101*_B2001 - _A1100*_B2101; - C[371] = _A1120*_B0102 - _A1121*_B0002 + _A1101*_B2002 - _A1100*_B2102; - C[372] = _A1120*_B0110 - _A1121*_B0010 + _A1101*_B2010 - _A1100*_B2110; - C[373] = _A1120*_B0111 - _A1121*_B0011 + _A1101*_B2011 - _A1100*_B2111; - C[374] = _A1120*_B0112 - _A1121*_B0012 + _A1101*_B2012 - _A1100*_B2112; - C[375] = _A1120*_B0120 - _A1121*_B0020 + _A1101*_B2020 - _A1100*_B2120; - C[376] = _A1120*_B0121 - _A1121*_B0021 + _A1101*_B2021 - _A1100*_B2121; - C[377] = _A1120*_B0122 - _A1121*_B0022 + _A1101*_B2022 - _A1100*_B2122; - C[378] = _A1112*_B0100 - _A1111*_B0200 - _A1102*_B1100 + _A1101*_B1200; - C[379] = _A1112*_B0101 - _A1111*_B0201 - _A1102*_B1101 + _A1101*_B1201; - C[380] = _A1112*_B0102 - _A1111*_B0202 - _A1102*_B1102 + _A1101*_B1202; - C[381] = _A1112*_B0110 - _A1111*_B0210 - _A1102*_B1110 + _A1101*_B1210; - C[382] = _A1112*_B0111 - _A1111*_B0211 - _A1102*_B1111 + _A1101*_B1211; - C[383] = _A1112*_B0112 - _A1111*_B0212 - _A1102*_B1112 + _A1101*_B1212; - C[384] = _A1112*_B0120 - _A1111*_B0220 - _A1102*_B1120 + _A1101*_B1220; - C[385] = _A1112*_B0121 - _A1111*_B0221 - _A1102*_B1121 + _A1101*_B1221; - C[386] = _A1112*_B0122 - _A1111*_B0222 - _A1102*_B1122 + _A1101*_B1222; - C[387] = _A1110*_B0200 - _A1112*_B0000 + _A1102*_B1000 - _A1100*_B1200; - C[388] = _A1110*_B0201 - _A1112*_B0001 + _A1102*_B1001 - _A1100*_B1201; - C[389] = _A1110*_B0202 - _A1112*_B0002 + _A1102*_B1002 - _A1100*_B1202; - C[390] = _A1110*_B0210 - _A1112*_B0010 + _A1102*_B1010 - _A1100*_B1210; - C[391] = _A1110*_B0211 - _A1112*_B0011 + _A1102*_B1011 - _A1100*_B1211; - C[392] = _A1110*_B0212 - _A1112*_B0012 + _A1102*_B1012 - _A1100*_B1212; - C[393] = _A1110*_B0220 - _A1112*_B0020 + _A1102*_B1020 - _A1100*_B1220; - C[394] = _A1110*_B0221 - _A1112*_B0021 + _A1102*_B1021 - _A1100*_B1221; - C[395] = _A1110*_B0222 - _A1112*_B0022 + _A1102*_B1022 - _A1100*_B1222; - C[396] = _A1111*_B0000 - _A1110*_B0100 - _A1101*_B1000 + _A1100*_B1100; - C[397] = _A1111*_B0001 - _A1110*_B0101 - _A1101*_B1001 + _A1100*_B1101; - C[398] = _A1111*_B0002 - _A1110*_B0102 - _A1101*_B1002 + _A1100*_B1102; - C[399] = _A1111*_B0010 - _A1110*_B0110 - _A1101*_B1010 + _A1100*_B1110; - C[400] = _A1111*_B0011 - _A1110*_B0111 - _A1101*_B1011 + _A1100*_B1111; - C[401] = _A1111*_B0012 - _A1110*_B0112 - _A1101*_B1012 + _A1100*_B1112; - C[402] = _A1111*_B0020 - _A1110*_B0120 - _A1101*_B1020 + _A1100*_B1120; - C[403] = _A1111*_B0021 - _A1110*_B0121 - _A1101*_B1021 + _A1100*_B1121; - C[404] = _A1111*_B0022 - _A1110*_B0122 - _A1101*_B1022 + _A1100*_B1122; - C[405] = _A1222*_B1100 - _A1221*_B1200 - _A1212*_B2100 + _A1211*_B2200; - C[406] = _A1222*_B1101 - _A1221*_B1201 - _A1212*_B2101 + _A1211*_B2201; - C[407] = _A1222*_B1102 - _A1221*_B1202 - _A1212*_B2102 + _A1211*_B2202; - C[408] = _A1222*_B1110 - _A1221*_B1210 - _A1212*_B2110 + _A1211*_B2210; - C[409] = _A1222*_B1111 - _A1221*_B1211 - _A1212*_B2111 + _A1211*_B2211; - C[410] = _A1222*_B1112 - _A1221*_B1212 - _A1212*_B2112 + _A1211*_B2212; - C[411] = _A1222*_B1120 - _A1221*_B1220 - _A1212*_B2120 + _A1211*_B2220; - C[412] = _A1222*_B1121 - _A1221*_B1221 - _A1212*_B2121 + _A1211*_B2221; - C[413] = _A1222*_B1122 - _A1221*_B1222 - _A1212*_B2122 + _A1211*_B2222; - C[414] = _A1220*_B1200 - _A1222*_B1000 + _A1212*_B2000 - _A1210*_B2200; - C[415] = _A1220*_B1201 - _A1222*_B1001 + _A1212*_B2001 - _A1210*_B2201; - C[416] = _A1220*_B1202 - _A1222*_B1002 + _A1212*_B2002 - _A1210*_B2202; - C[417] = _A1220*_B1210 - _A1222*_B1010 + _A1212*_B2010 - _A1210*_B2210; - C[418] = _A1220*_B1211 - _A1222*_B1011 + _A1212*_B2011 - _A1210*_B2211; - C[419] = _A1220*_B1212 - _A1222*_B1012 + _A1212*_B2012 - _A1210*_B2212; - C[420] = _A1220*_B1220 - _A1222*_B1020 + _A1212*_B2020 - _A1210*_B2220; - C[421] = _A1220*_B1221 - _A1222*_B1021 + _A1212*_B2021 - _A1210*_B2221; - C[422] = _A1220*_B1222 - _A1222*_B1022 + _A1212*_B2022 - _A1210*_B2222; - C[423] = _A1221*_B1000 - _A1220*_B1100 - _A1211*_B2000 + _A1210*_B2100; - C[424] = _A1221*_B1001 - _A1220*_B1101 - _A1211*_B2001 + _A1210*_B2101; - C[425] = _A1221*_B1002 - _A1220*_B1102 - _A1211*_B2002 + _A1210*_B2102; - C[426] = _A1221*_B1010 - _A1220*_B1110 - _A1211*_B2010 + _A1210*_B2110; - C[427] = _A1221*_B1011 - _A1220*_B1111 - _A1211*_B2011 + _A1210*_B2111; - C[428] = _A1221*_B1012 - _A1220*_B1112 - _A1211*_B2012 + _A1210*_B2112; - C[429] = _A1221*_B1020 - _A1220*_B1120 - _A1211*_B2020 + _A1210*_B2120; - C[430] = _A1221*_B1021 - _A1220*_B1121 - _A1211*_B2021 + _A1210*_B2121; - C[431] = _A1221*_B1022 - _A1220*_B1122 - _A1211*_B2022 + _A1210*_B2122; - C[432] = _A1221*_B0200 - _A1222*_B0100 + _A1202*_B2100 - _A1201*_B2200; - C[433] = _A1221*_B0201 - _A1222*_B0101 + _A1202*_B2101 - _A1201*_B2201; - C[434] = _A1221*_B0202 - _A1222*_B0102 + _A1202*_B2102 - _A1201*_B2202; - C[435] = _A1221*_B0210 - _A1222*_B0110 + _A1202*_B2110 - _A1201*_B2210; - C[436] = _A1221*_B0211 - _A1222*_B0111 + _A1202*_B2111 - _A1201*_B2211; - C[437] = _A1221*_B0212 - _A1222*_B0112 + _A1202*_B2112 - _A1201*_B2212; - C[438] = _A1221*_B0220 - _A1222*_B0120 + _A1202*_B2120 - _A1201*_B2220; - C[439] = _A1221*_B0221 - _A1222*_B0121 + _A1202*_B2121 - _A1201*_B2221; - C[440] = _A1221*_B0222 - _A1222*_B0122 + _A1202*_B2122 - _A1201*_B2222; - C[441] = _A1222*_B0000 - _A1220*_B0200 - _A1202*_B2000 + _A1200*_B2200; - C[442] = _A1222*_B0001 - _A1220*_B0201 - _A1202*_B2001 + _A1200*_B2201; - C[443] = _A1222*_B0002 - _A1220*_B0202 - _A1202*_B2002 + _A1200*_B2202; - C[444] = _A1222*_B0010 - _A1220*_B0210 - _A1202*_B2010 + _A1200*_B2210; - C[445] = _A1222*_B0011 - _A1220*_B0211 - _A1202*_B2011 + _A1200*_B2211; - C[446] = _A1222*_B0012 - _A1220*_B0212 - _A1202*_B2012 + _A1200*_B2212; - C[447] = _A1222*_B0020 - _A1220*_B0220 - _A1202*_B2020 + _A1200*_B2220; - C[448] = _A1222*_B0021 - _A1220*_B0221 - _A1202*_B2021 + _A1200*_B2221; - C[449] = _A1222*_B0022 - _A1220*_B0222 - _A1202*_B2022 + _A1200*_B2222; - C[450] = _A1220*_B0100 - _A1221*_B0000 + _A1201*_B2000 - _A1200*_B2100; - C[451] = _A1220*_B0101 - _A1221*_B0001 + _A1201*_B2001 - _A1200*_B2101; - C[452] = _A1220*_B0102 - _A1221*_B0002 + _A1201*_B2002 - _A1200*_B2102; - C[453] = _A1220*_B0110 - _A1221*_B0010 + _A1201*_B2010 - _A1200*_B2110; - C[454] = _A1220*_B0111 - _A1221*_B0011 + _A1201*_B2011 - _A1200*_B2111; - C[455] = _A1220*_B0112 - _A1221*_B0012 + _A1201*_B2012 - _A1200*_B2112; - C[456] = _A1220*_B0120 - _A1221*_B0020 + _A1201*_B2020 - _A1200*_B2120; - C[457] = _A1220*_B0121 - _A1221*_B0021 + _A1201*_B2021 - _A1200*_B2121; - C[458] = _A1220*_B0122 - _A1221*_B0022 + _A1201*_B2022 - _A1200*_B2122; - C[459] = _A1212*_B0100 - _A1211*_B0200 - _A1202*_B1100 + _A1201*_B1200; - C[460] = _A1212*_B0101 - _A1211*_B0201 - _A1202*_B1101 + _A1201*_B1201; - C[461] = _A1212*_B0102 - _A1211*_B0202 - _A1202*_B1102 + _A1201*_B1202; - C[462] = _A1212*_B0110 - _A1211*_B0210 - _A1202*_B1110 + _A1201*_B1210; - C[463] = _A1212*_B0111 - _A1211*_B0211 - _A1202*_B1111 + _A1201*_B1211; - C[464] = _A1212*_B0112 - _A1211*_B0212 - _A1202*_B1112 + _A1201*_B1212; - C[465] = _A1212*_B0120 - _A1211*_B0220 - _A1202*_B1120 + _A1201*_B1220; - C[466] = _A1212*_B0121 - _A1211*_B0221 - _A1202*_B1121 + _A1201*_B1221; - C[467] = _A1212*_B0122 - _A1211*_B0222 - _A1202*_B1122 + _A1201*_B1222; - C[468] = _A1210*_B0200 - _A1212*_B0000 + _A1202*_B1000 - _A1200*_B1200; - C[469] = _A1210*_B0201 - _A1212*_B0001 + _A1202*_B1001 - _A1200*_B1201; - C[470] = _A1210*_B0202 - _A1212*_B0002 + _A1202*_B1002 - _A1200*_B1202; - C[471] = _A1210*_B0210 - _A1212*_B0010 + _A1202*_B1010 - _A1200*_B1210; - C[472] = _A1210*_B0211 - _A1212*_B0011 + _A1202*_B1011 - _A1200*_B1211; - C[473] = _A1210*_B0212 - _A1212*_B0012 + _A1202*_B1012 - _A1200*_B1212; - C[474] = _A1210*_B0220 - _A1212*_B0020 + _A1202*_B1020 - _A1200*_B1220; - C[475] = _A1210*_B0221 - _A1212*_B0021 + _A1202*_B1021 - _A1200*_B1221; - C[476] = _A1210*_B0222 - _A1212*_B0022 + _A1202*_B1022 - _A1200*_B1222; - C[477] = _A1211*_B0000 - _A1210*_B0100 - _A1201*_B1000 + _A1200*_B1100; - C[478] = _A1211*_B0001 - _A1210*_B0101 - _A1201*_B1001 + _A1200*_B1101; - C[479] = _A1211*_B0002 - _A1210*_B0102 - _A1201*_B1002 + _A1200*_B1102; - C[480] = _A1211*_B0010 - _A1210*_B0110 - _A1201*_B1010 + _A1200*_B1110; - C[481] = _A1211*_B0011 - _A1210*_B0111 - _A1201*_B1011 + _A1200*_B1111; - C[482] = _A1211*_B0012 - _A1210*_B0112 - _A1201*_B1012 + _A1200*_B1112; - C[483] = _A1211*_B0020 - _A1210*_B0120 - _A1201*_B1020 + _A1200*_B1120; - C[484] = _A1211*_B0021 - _A1210*_B0121 - _A1201*_B1021 + _A1200*_B1121; - C[485] = _A1211*_B0022 - _A1210*_B0122 - _A1201*_B1022 + _A1200*_B1122; - C[486] = _A2022*_B1100 - _A2021*_B1200 - _A2012*_B2100 + _A2011*_B2200; - C[487] = _A2022*_B1101 - _A2021*_B1201 - _A2012*_B2101 + _A2011*_B2201; - C[488] = _A2022*_B1102 - _A2021*_B1202 - _A2012*_B2102 + _A2011*_B2202; - C[489] = _A2022*_B1110 - _A2021*_B1210 - _A2012*_B2110 + _A2011*_B2210; - C[490] = _A2022*_B1111 - _A2021*_B1211 - _A2012*_B2111 + _A2011*_B2211; - C[491] = _A2022*_B1112 - _A2021*_B1212 - _A2012*_B2112 + _A2011*_B2212; - C[492] = _A2022*_B1120 - _A2021*_B1220 - _A2012*_B2120 + _A2011*_B2220; - C[493] = _A2022*_B1121 - _A2021*_B1221 - _A2012*_B2121 + _A2011*_B2221; - C[494] = _A2022*_B1122 - _A2021*_B1222 - _A2012*_B2122 + _A2011*_B2222; - C[495] = _A2020*_B1200 - _A2022*_B1000 + _A2012*_B2000 - _A2010*_B2200; - C[496] = _A2020*_B1201 - _A2022*_B1001 + _A2012*_B2001 - _A2010*_B2201; - C[497] = _A2020*_B1202 - _A2022*_B1002 + _A2012*_B2002 - _A2010*_B2202; - C[498] = _A2020*_B1210 - _A2022*_B1010 + _A2012*_B2010 - _A2010*_B2210; - C[499] = _A2020*_B1211 - _A2022*_B1011 + _A2012*_B2011 - _A2010*_B2211; - C[500] = _A2020*_B1212 - _A2022*_B1012 + _A2012*_B2012 - _A2010*_B2212; - C[501] = _A2020*_B1220 - _A2022*_B1020 + _A2012*_B2020 - _A2010*_B2220; - C[502] = _A2020*_B1221 - _A2022*_B1021 + _A2012*_B2021 - _A2010*_B2221; - C[503] = _A2020*_B1222 - _A2022*_B1022 + _A2012*_B2022 - _A2010*_B2222; - C[504] = _A2021*_B1000 - _A2020*_B1100 - _A2011*_B2000 + _A2010*_B2100; - C[505] = _A2021*_B1001 - _A2020*_B1101 - _A2011*_B2001 + _A2010*_B2101; - C[506] = _A2021*_B1002 - _A2020*_B1102 - _A2011*_B2002 + _A2010*_B2102; - C[507] = _A2021*_B1010 - _A2020*_B1110 - _A2011*_B2010 + _A2010*_B2110; - C[508] = _A2021*_B1011 - _A2020*_B1111 - _A2011*_B2011 + _A2010*_B2111; - C[509] = _A2021*_B1012 - _A2020*_B1112 - _A2011*_B2012 + _A2010*_B2112; - C[510] = _A2021*_B1020 - _A2020*_B1120 - _A2011*_B2020 + _A2010*_B2120; - C[511] = _A2021*_B1021 - _A2020*_B1121 - _A2011*_B2021 + _A2010*_B2121; - C[512] = _A2021*_B1022 - _A2020*_B1122 - _A2011*_B2022 + _A2010*_B2122; - C[513] = _A2021*_B0200 - _A2022*_B0100 + _A2002*_B2100 - _A2001*_B2200; - C[514] = _A2021*_B0201 - _A2022*_B0101 + _A2002*_B2101 - _A2001*_B2201; - C[515] = _A2021*_B0202 - _A2022*_B0102 + _A2002*_B2102 - _A2001*_B2202; - C[516] = _A2021*_B0210 - _A2022*_B0110 + _A2002*_B2110 - _A2001*_B2210; - C[517] = _A2021*_B0211 - _A2022*_B0111 + _A2002*_B2111 - _A2001*_B2211; - C[518] = _A2021*_B0212 - _A2022*_B0112 + _A2002*_B2112 - _A2001*_B2212; - C[519] = _A2021*_B0220 - _A2022*_B0120 + _A2002*_B2120 - _A2001*_B2220; - C[520] = _A2021*_B0221 - _A2022*_B0121 + _A2002*_B2121 - _A2001*_B2221; - C[521] = _A2021*_B0222 - _A2022*_B0122 + _A2002*_B2122 - _A2001*_B2222; - C[522] = _A2022*_B0000 - _A2020*_B0200 - _A2002*_B2000 + _A2000*_B2200; - C[523] = _A2022*_B0001 - _A2020*_B0201 - _A2002*_B2001 + _A2000*_B2201; - C[524] = _A2022*_B0002 - _A2020*_B0202 - _A2002*_B2002 + _A2000*_B2202; - C[525] = _A2022*_B0010 - _A2020*_B0210 - _A2002*_B2010 + _A2000*_B2210; - C[526] = _A2022*_B0011 - _A2020*_B0211 - _A2002*_B2011 + _A2000*_B2211; - C[527] = _A2022*_B0012 - _A2020*_B0212 - _A2002*_B2012 + _A2000*_B2212; - C[528] = _A2022*_B0020 - _A2020*_B0220 - _A2002*_B2020 + _A2000*_B2220; - C[529] = _A2022*_B0021 - _A2020*_B0221 - _A2002*_B2021 + _A2000*_B2221; - C[530] = _A2022*_B0022 - _A2020*_B0222 - _A2002*_B2022 + _A2000*_B2222; - C[531] = _A2020*_B0100 - _A2021*_B0000 + _A2001*_B2000 - _A2000*_B2100; - C[532] = _A2020*_B0101 - _A2021*_B0001 + _A2001*_B2001 - _A2000*_B2101; - C[533] = _A2020*_B0102 - _A2021*_B0002 + _A2001*_B2002 - _A2000*_B2102; - C[534] = _A2020*_B0110 - _A2021*_B0010 + _A2001*_B2010 - _A2000*_B2110; - C[535] = _A2020*_B0111 - _A2021*_B0011 + _A2001*_B2011 - _A2000*_B2111; - C[536] = _A2020*_B0112 - _A2021*_B0012 + _A2001*_B2012 - _A2000*_B2112; - C[537] = _A2020*_B0120 - _A2021*_B0020 + _A2001*_B2020 - _A2000*_B2120; - C[538] = _A2020*_B0121 - _A2021*_B0021 + _A2001*_B2021 - _A2000*_B2121; - C[539] = _A2020*_B0122 - _A2021*_B0022 + _A2001*_B2022 - _A2000*_B2122; - C[540] = _A2012*_B0100 - _A2011*_B0200 - _A2002*_B1100 + _A2001*_B1200; - C[541] = _A2012*_B0101 - _A2011*_B0201 - _A2002*_B1101 + _A2001*_B1201; - C[542] = _A2012*_B0102 - _A2011*_B0202 - _A2002*_B1102 + _A2001*_B1202; - C[543] = _A2012*_B0110 - _A2011*_B0210 - _A2002*_B1110 + _A2001*_B1210; - C[544] = _A2012*_B0111 - _A2011*_B0211 - _A2002*_B1111 + _A2001*_B1211; - C[545] = _A2012*_B0112 - _A2011*_B0212 - _A2002*_B1112 + _A2001*_B1212; - C[546] = _A2012*_B0120 - _A2011*_B0220 - _A2002*_B1120 + _A2001*_B1220; - C[547] = _A2012*_B0121 - _A2011*_B0221 - _A2002*_B1121 + _A2001*_B1221; - C[548] = _A2012*_B0122 - _A2011*_B0222 - _A2002*_B1122 + _A2001*_B1222; - C[549] = _A2010*_B0200 - _A2012*_B0000 + _A2002*_B1000 - _A2000*_B1200; - C[550] = _A2010*_B0201 - _A2012*_B0001 + _A2002*_B1001 - _A2000*_B1201; - C[551] = _A2010*_B0202 - _A2012*_B0002 + _A2002*_B1002 - _A2000*_B1202; - C[552] = _A2010*_B0210 - _A2012*_B0010 + _A2002*_B1010 - _A2000*_B1210; - C[553] = _A2010*_B0211 - _A2012*_B0011 + _A2002*_B1011 - _A2000*_B1211; - C[554] = _A2010*_B0212 - _A2012*_B0012 + _A2002*_B1012 - _A2000*_B1212; - C[555] = _A2010*_B0220 - _A2012*_B0020 + _A2002*_B1020 - _A2000*_B1220; - C[556] = _A2010*_B0221 - _A2012*_B0021 + _A2002*_B1021 - _A2000*_B1221; - C[557] = _A2010*_B0222 - _A2012*_B0022 + _A2002*_B1022 - _A2000*_B1222; - C[558] = _A2011*_B0000 - _A2010*_B0100 - _A2001*_B1000 + _A2000*_B1100; - C[559] = _A2011*_B0001 - _A2010*_B0101 - _A2001*_B1001 + _A2000*_B1101; - C[560] = _A2011*_B0002 - _A2010*_B0102 - _A2001*_B1002 + _A2000*_B1102; - C[561] = _A2011*_B0010 - _A2010*_B0110 - _A2001*_B1010 + _A2000*_B1110; - C[562] = _A2011*_B0011 - _A2010*_B0111 - _A2001*_B1011 + _A2000*_B1111; - C[563] = _A2011*_B0012 - _A2010*_B0112 - _A2001*_B1012 + _A2000*_B1112; - C[564] = _A2011*_B0020 - _A2010*_B0120 - _A2001*_B1020 + _A2000*_B1120; - C[565] = _A2011*_B0021 - _A2010*_B0121 - _A2001*_B1021 + _A2000*_B1121; - C[566] = _A2011*_B0022 - _A2010*_B0122 - _A2001*_B1022 + _A2000*_B1122; - C[567] = _A2122*_B1100 - _A2121*_B1200 - _A2112*_B2100 + _A2111*_B2200; - C[568] = _A2122*_B1101 - _A2121*_B1201 - _A2112*_B2101 + _A2111*_B2201; - C[569] = _A2122*_B1102 - _A2121*_B1202 - _A2112*_B2102 + _A2111*_B2202; - C[570] = _A2122*_B1110 - _A2121*_B1210 - _A2112*_B2110 + _A2111*_B2210; - C[571] = _A2122*_B1111 - _A2121*_B1211 - _A2112*_B2111 + _A2111*_B2211; - C[572] = _A2122*_B1112 - _A2121*_B1212 - _A2112*_B2112 + _A2111*_B2212; - C[573] = _A2122*_B1120 - _A2121*_B1220 - _A2112*_B2120 + _A2111*_B2220; - C[574] = _A2122*_B1121 - _A2121*_B1221 - _A2112*_B2121 + _A2111*_B2221; - C[575] = _A2122*_B1122 - _A2121*_B1222 - _A2112*_B2122 + _A2111*_B2222; - C[576] = _A2120*_B1200 - _A2122*_B1000 + _A2112*_B2000 - _A2110*_B2200; - C[577] = _A2120*_B1201 - _A2122*_B1001 + _A2112*_B2001 - _A2110*_B2201; - C[578] = _A2120*_B1202 - _A2122*_B1002 + _A2112*_B2002 - _A2110*_B2202; - C[579] = _A2120*_B1210 - _A2122*_B1010 + _A2112*_B2010 - _A2110*_B2210; - C[580] = _A2120*_B1211 - _A2122*_B1011 + _A2112*_B2011 - _A2110*_B2211; - C[581] = _A2120*_B1212 - _A2122*_B1012 + _A2112*_B2012 - _A2110*_B2212; - C[582] = _A2120*_B1220 - _A2122*_B1020 + _A2112*_B2020 - _A2110*_B2220; - C[583] = _A2120*_B1221 - _A2122*_B1021 + _A2112*_B2021 - _A2110*_B2221; - C[584] = _A2120*_B1222 - _A2122*_B1022 + _A2112*_B2022 - _A2110*_B2222; - C[585] = _A2121*_B1000 - _A2120*_B1100 - _A2111*_B2000 + _A2110*_B2100; - C[586] = _A2121*_B1001 - _A2120*_B1101 - _A2111*_B2001 + _A2110*_B2101; - C[587] = _A2121*_B1002 - _A2120*_B1102 - _A2111*_B2002 + _A2110*_B2102; - C[588] = _A2121*_B1010 - _A2120*_B1110 - _A2111*_B2010 + _A2110*_B2110; - C[589] = _A2121*_B1011 - _A2120*_B1111 - _A2111*_B2011 + _A2110*_B2111; - C[590] = _A2121*_B1012 - _A2120*_B1112 - _A2111*_B2012 + _A2110*_B2112; - C[591] = _A2121*_B1020 - _A2120*_B1120 - _A2111*_B2020 + _A2110*_B2120; - C[592] = _A2121*_B1021 - _A2120*_B1121 - _A2111*_B2021 + _A2110*_B2121; - C[593] = _A2121*_B1022 - _A2120*_B1122 - _A2111*_B2022 + _A2110*_B2122; - C[594] = _A2121*_B0200 - _A2122*_B0100 + _A2102*_B2100 - _A2101*_B2200; - C[595] = _A2121*_B0201 - _A2122*_B0101 + _A2102*_B2101 - _A2101*_B2201; - C[596] = _A2121*_B0202 - _A2122*_B0102 + _A2102*_B2102 - _A2101*_B2202; - C[597] = _A2121*_B0210 - _A2122*_B0110 + _A2102*_B2110 - _A2101*_B2210; - C[598] = _A2121*_B0211 - _A2122*_B0111 + _A2102*_B2111 - _A2101*_B2211; - C[599] = _A2121*_B0212 - _A2122*_B0112 + _A2102*_B2112 - _A2101*_B2212; - C[600] = _A2121*_B0220 - _A2122*_B0120 + _A2102*_B2120 - _A2101*_B2220; - C[601] = _A2121*_B0221 - _A2122*_B0121 + _A2102*_B2121 - _A2101*_B2221; - C[602] = _A2121*_B0222 - _A2122*_B0122 + _A2102*_B2122 - _A2101*_B2222; - C[603] = _A2122*_B0000 - _A2120*_B0200 - _A2102*_B2000 + _A2100*_B2200; - C[604] = _A2122*_B0001 - _A2120*_B0201 - _A2102*_B2001 + _A2100*_B2201; - C[605] = _A2122*_B0002 - _A2120*_B0202 - _A2102*_B2002 + _A2100*_B2202; - C[606] = _A2122*_B0010 - _A2120*_B0210 - _A2102*_B2010 + _A2100*_B2210; - C[607] = _A2122*_B0011 - _A2120*_B0211 - _A2102*_B2011 + _A2100*_B2211; - C[608] = _A2122*_B0012 - _A2120*_B0212 - _A2102*_B2012 + _A2100*_B2212; - C[609] = _A2122*_B0020 - _A2120*_B0220 - _A2102*_B2020 + _A2100*_B2220; - C[610] = _A2122*_B0021 - _A2120*_B0221 - _A2102*_B2021 + _A2100*_B2221; - C[611] = _A2122*_B0022 - _A2120*_B0222 - _A2102*_B2022 + _A2100*_B2222; - C[612] = _A2120*_B0100 - _A2121*_B0000 + _A2101*_B2000 - _A2100*_B2100; - C[613] = _A2120*_B0101 - _A2121*_B0001 + _A2101*_B2001 - _A2100*_B2101; - C[614] = _A2120*_B0102 - _A2121*_B0002 + _A2101*_B2002 - _A2100*_B2102; - C[615] = _A2120*_B0110 - _A2121*_B0010 + _A2101*_B2010 - _A2100*_B2110; - C[616] = _A2120*_B0111 - _A2121*_B0011 + _A2101*_B2011 - _A2100*_B2111; - C[617] = _A2120*_B0112 - _A2121*_B0012 + _A2101*_B2012 - _A2100*_B2112; - C[618] = _A2120*_B0120 - _A2121*_B0020 + _A2101*_B2020 - _A2100*_B2120; - C[619] = _A2120*_B0121 - _A2121*_B0021 + _A2101*_B2021 - _A2100*_B2121; - C[620] = _A2120*_B0122 - _A2121*_B0022 + _A2101*_B2022 - _A2100*_B2122; - C[621] = _A2112*_B0100 - _A2111*_B0200 - _A2102*_B1100 + _A2101*_B1200; - C[622] = _A2112*_B0101 - _A2111*_B0201 - _A2102*_B1101 + _A2101*_B1201; - C[623] = _A2112*_B0102 - _A2111*_B0202 - _A2102*_B1102 + _A2101*_B1202; - C[624] = _A2112*_B0110 - _A2111*_B0210 - _A2102*_B1110 + _A2101*_B1210; - C[625] = _A2112*_B0111 - _A2111*_B0211 - _A2102*_B1111 + _A2101*_B1211; - C[626] = _A2112*_B0112 - _A2111*_B0212 - _A2102*_B1112 + _A2101*_B1212; - C[627] = _A2112*_B0120 - _A2111*_B0220 - _A2102*_B1120 + _A2101*_B1220; - C[628] = _A2112*_B0121 - _A2111*_B0221 - _A2102*_B1121 + _A2101*_B1221; - C[629] = _A2112*_B0122 - _A2111*_B0222 - _A2102*_B1122 + _A2101*_B1222; - C[630] = _A2110*_B0200 - _A2112*_B0000 + _A2102*_B1000 - _A2100*_B1200; - C[631] = _A2110*_B0201 - _A2112*_B0001 + _A2102*_B1001 - _A2100*_B1201; - C[632] = _A2110*_B0202 - _A2112*_B0002 + _A2102*_B1002 - _A2100*_B1202; - C[633] = _A2110*_B0210 - _A2112*_B0010 + _A2102*_B1010 - _A2100*_B1210; - C[634] = _A2110*_B0211 - _A2112*_B0011 + _A2102*_B1011 - _A2100*_B1211; - C[635] = _A2110*_B0212 - _A2112*_B0012 + _A2102*_B1012 - _A2100*_B1212; - C[636] = _A2110*_B0220 - _A2112*_B0020 + _A2102*_B1020 - _A2100*_B1220; - C[637] = _A2110*_B0221 - _A2112*_B0021 + _A2102*_B1021 - _A2100*_B1221; - C[638] = _A2110*_B0222 - _A2112*_B0022 + _A2102*_B1022 - _A2100*_B1222; - C[639] = _A2111*_B0000 - _A2110*_B0100 - _A2101*_B1000 + _A2100*_B1100; - C[640] = _A2111*_B0001 - _A2110*_B0101 - _A2101*_B1001 + _A2100*_B1101; - C[641] = _A2111*_B0002 - _A2110*_B0102 - _A2101*_B1002 + _A2100*_B1102; - C[642] = _A2111*_B0010 - _A2110*_B0110 - _A2101*_B1010 + _A2100*_B1110; - C[643] = _A2111*_B0011 - _A2110*_B0111 - _A2101*_B1011 + _A2100*_B1111; - C[644] = _A2111*_B0012 - _A2110*_B0112 - _A2101*_B1012 + _A2100*_B1112; - C[645] = _A2111*_B0020 - _A2110*_B0120 - _A2101*_B1020 + _A2100*_B1120; - C[646] = _A2111*_B0021 - _A2110*_B0121 - _A2101*_B1021 + _A2100*_B1121; - C[647] = _A2111*_B0022 - _A2110*_B0122 - _A2101*_B1022 + _A2100*_B1122; - C[648] = _A2222*_B1100 - _A2221*_B1200 - _A2212*_B2100 + _A2211*_B2200; - C[649] = _A2222*_B1101 - _A2221*_B1201 - _A2212*_B2101 + _A2211*_B2201; - C[650] = _A2222*_B1102 - _A2221*_B1202 - _A2212*_B2102 + _A2211*_B2202; - C[651] = _A2222*_B1110 - _A2221*_B1210 - _A2212*_B2110 + _A2211*_B2210; - C[652] = _A2222*_B1111 - _A2221*_B1211 - _A2212*_B2111 + _A2211*_B2211; - C[653] = _A2222*_B1112 - _A2221*_B1212 - _A2212*_B2112 + _A2211*_B2212; - C[654] = _A2222*_B1120 - _A2221*_B1220 - _A2212*_B2120 + _A2211*_B2220; - C[655] = _A2222*_B1121 - _A2221*_B1221 - _A2212*_B2121 + _A2211*_B2221; - C[656] = _A2222*_B1122 - _A2221*_B1222 - _A2212*_B2122 + _A2211*_B2222; - C[657] = _A2220*_B1200 - _A2222*_B1000 + _A2212*_B2000 - _A2210*_B2200; - C[658] = _A2220*_B1201 - _A2222*_B1001 + _A2212*_B2001 - _A2210*_B2201; - C[659] = _A2220*_B1202 - _A2222*_B1002 + _A2212*_B2002 - _A2210*_B2202; - C[660] = _A2220*_B1210 - _A2222*_B1010 + _A2212*_B2010 - _A2210*_B2210; - C[661] = _A2220*_B1211 - _A2222*_B1011 + _A2212*_B2011 - _A2210*_B2211; - C[662] = _A2220*_B1212 - _A2222*_B1012 + _A2212*_B2012 - _A2210*_B2212; - C[663] = _A2220*_B1220 - _A2222*_B1020 + _A2212*_B2020 - _A2210*_B2220; - C[664] = _A2220*_B1221 - _A2222*_B1021 + _A2212*_B2021 - _A2210*_B2221; - C[665] = _A2220*_B1222 - _A2222*_B1022 + _A2212*_B2022 - _A2210*_B2222; - C[666] = _A2221*_B1000 - _A2220*_B1100 - _A2211*_B2000 + _A2210*_B2100; - C[667] = _A2221*_B1001 - _A2220*_B1101 - _A2211*_B2001 + _A2210*_B2101; - C[668] = _A2221*_B1002 - _A2220*_B1102 - _A2211*_B2002 + _A2210*_B2102; - C[669] = _A2221*_B1010 - _A2220*_B1110 - _A2211*_B2010 + _A2210*_B2110; - C[670] = _A2221*_B1011 - _A2220*_B1111 - _A2211*_B2011 + _A2210*_B2111; - C[671] = _A2221*_B1012 - _A2220*_B1112 - _A2211*_B2012 + _A2210*_B2112; - C[672] = _A2221*_B1020 - _A2220*_B1120 - _A2211*_B2020 + _A2210*_B2120; - C[673] = _A2221*_B1021 - _A2220*_B1121 - _A2211*_B2021 + _A2210*_B2121; - C[674] = _A2221*_B1022 - _A2220*_B1122 - _A2211*_B2022 + _A2210*_B2122; - C[675] = _A2221*_B0200 - _A2222*_B0100 + _A2202*_B2100 - _A2201*_B2200; - C[676] = _A2221*_B0201 - _A2222*_B0101 + _A2202*_B2101 - _A2201*_B2201; - C[677] = _A2221*_B0202 - _A2222*_B0102 + _A2202*_B2102 - _A2201*_B2202; - C[678] = _A2221*_B0210 - _A2222*_B0110 + _A2202*_B2110 - _A2201*_B2210; - C[679] = _A2221*_B0211 - _A2222*_B0111 + _A2202*_B2111 - _A2201*_B2211; - C[680] = _A2221*_B0212 - _A2222*_B0112 + _A2202*_B2112 - _A2201*_B2212; - C[681] = _A2221*_B0220 - _A2222*_B0120 + _A2202*_B2120 - _A2201*_B2220; - C[682] = _A2221*_B0221 - _A2222*_B0121 + _A2202*_B2121 - _A2201*_B2221; - C[683] = _A2221*_B0222 - _A2222*_B0122 + _A2202*_B2122 - _A2201*_B2222; - C[684] = _A2222*_B0000 - _A2220*_B0200 - _A2202*_B2000 + _A2200*_B2200; - C[685] = _A2222*_B0001 - _A2220*_B0201 - _A2202*_B2001 + _A2200*_B2201; - C[686] = _A2222*_B0002 - _A2220*_B0202 - _A2202*_B2002 + _A2200*_B2202; - C[687] = _A2222*_B0010 - _A2220*_B0210 - _A2202*_B2010 + _A2200*_B2210; - C[688] = _A2222*_B0011 - _A2220*_B0211 - _A2202*_B2011 + _A2200*_B2211; - C[689] = _A2222*_B0012 - _A2220*_B0212 - _A2202*_B2012 + _A2200*_B2212; - C[690] = _A2222*_B0020 - _A2220*_B0220 - _A2202*_B2020 + _A2200*_B2220; - C[691] = _A2222*_B0021 - _A2220*_B0221 - _A2202*_B2021 + _A2200*_B2221; - C[692] = _A2222*_B0022 - _A2220*_B0222 - _A2202*_B2022 + _A2200*_B2222; - C[693] = _A2220*_B0100 - _A2221*_B0000 + _A2201*_B2000 - _A2200*_B2100; - C[694] = _A2220*_B0101 - _A2221*_B0001 + _A2201*_B2001 - _A2200*_B2101; - C[695] = _A2220*_B0102 - _A2221*_B0002 + _A2201*_B2002 - _A2200*_B2102; - C[696] = _A2220*_B0110 - _A2221*_B0010 + _A2201*_B2010 - _A2200*_B2110; - C[697] = _A2220*_B0111 - _A2221*_B0011 + _A2201*_B2011 - _A2200*_B2111; - C[698] = _A2220*_B0112 - _A2221*_B0012 + _A2201*_B2012 - _A2200*_B2112; - C[699] = _A2220*_B0120 - _A2221*_B0020 + _A2201*_B2020 - _A2200*_B2120; - C[700] = _A2220*_B0121 - _A2221*_B0021 + _A2201*_B2021 - _A2200*_B2121; - C[701] = _A2220*_B0122 - _A2221*_B0022 + _A2201*_B2022 - _A2200*_B2122; - C[702] = _A2212*_B0100 - _A2211*_B0200 - _A2202*_B1100 + _A2201*_B1200; - C[703] = _A2212*_B0101 - _A2211*_B0201 - _A2202*_B1101 + _A2201*_B1201; - C[704] = _A2212*_B0102 - _A2211*_B0202 - _A2202*_B1102 + _A2201*_B1202; - C[705] = _A2212*_B0110 - _A2211*_B0210 - _A2202*_B1110 + _A2201*_B1210; - C[706] = _A2212*_B0111 - _A2211*_B0211 - _A2202*_B1111 + _A2201*_B1211; - C[707] = _A2212*_B0112 - _A2211*_B0212 - _A2202*_B1112 + _A2201*_B1212; - C[708] = _A2212*_B0120 - _A2211*_B0220 - _A2202*_B1120 + _A2201*_B1220; - C[709] = _A2212*_B0121 - _A2211*_B0221 - _A2202*_B1121 + _A2201*_B1221; - C[710] = _A2212*_B0122 - _A2211*_B0222 - _A2202*_B1122 + _A2201*_B1222; - C[711] = _A2210*_B0200 - _A2212*_B0000 + _A2202*_B1000 - _A2200*_B1200; - C[712] = _A2210*_B0201 - _A2212*_B0001 + _A2202*_B1001 - _A2200*_B1201; - C[713] = _A2210*_B0202 - _A2212*_B0002 + _A2202*_B1002 - _A2200*_B1202; - C[714] = _A2210*_B0210 - _A2212*_B0010 + _A2202*_B1010 - _A2200*_B1210; - C[715] = _A2210*_B0211 - _A2212*_B0011 + _A2202*_B1011 - _A2200*_B1211; - C[716] = _A2210*_B0212 - _A2212*_B0012 + _A2202*_B1012 - _A2200*_B1212; - C[717] = _A2210*_B0220 - _A2212*_B0020 + _A2202*_B1020 - _A2200*_B1220; - C[718] = _A2210*_B0221 - _A2212*_B0021 + _A2202*_B1021 - _A2200*_B1221; - C[719] = _A2210*_B0222 - _A2212*_B0022 + _A2202*_B1022 - _A2200*_B1222; - C[720] = _A2211*_B0000 - _A2210*_B0100 - _A2201*_B1000 + _A2200*_B1100; - C[721] = _A2211*_B0001 - _A2210*_B0101 - _A2201*_B1001 + _A2200*_B1101; - C[722] = _A2211*_B0002 - _A2210*_B0102 - _A2201*_B1002 + _A2200*_B1102; - C[723] = _A2211*_B0010 - _A2210*_B0110 - _A2201*_B1010 + _A2200*_B1110; - C[724] = _A2211*_B0011 - _A2210*_B0111 - _A2201*_B1011 + _A2200*_B1111; - C[725] = _A2211*_B0012 - _A2210*_B0112 - _A2201*_B1012 + _A2200*_B1112; - C[726] = _A2211*_B0020 - _A2210*_B0120 - _A2201*_B1020 + _A2200*_B1120; - C[727] = _A2211*_B0021 - _A2210*_B0121 - _A2201*_B1021 + _A2200*_B1121; - C[728] = _A2211*_B0022 - _A2210*_B0122 - _A2201*_B1022 + _A2200*_B1122; - -} -//--------------------------------------------------------------------------------------------------------// - - - - - - - - - - -//--------------------------------------------------------------------------------------------------------// -//(AxB)_{IjJ} = E_{jpq}E_{JPQ}A_{IpP}B_{qQ} -// tensor cross product of 3rd and 2nd order tensors -template -FASTOR_HINT_INLINE void _crossproduct32(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { - - T _A000 = A[0]; - T _A001 = A[1]; - T _A002 = A[2]; - T _A010 = A[3]; - T _A011 = A[4]; - T _A012 = A[5]; - T _A020 = A[6]; - T _A021 = A[7]; - T _A022 = A[8]; - T _A100 = A[9]; - T _A101 = A[10]; - T _A102 = A[11]; - T _A110 = A[12]; - T _A111 = A[13]; - T _A112 = A[14]; - T _A120 = A[15]; - T _A121 = A[16]; - T _A122 = A[17]; - T _A200 = A[18]; - T _A201 = A[19]; - T _A202 = A[20]; - T _A210 = A[21]; - T _A211 = A[22]; - T _A212 = A[23]; - T _A220 = A[24]; - T _A221 = A[25]; - T _A222 = A[26]; - - - T _B00 = B[0]; - T _B01 = B[1]; - T _B02 = B[2]; - T _B10 = B[3]; - T _B11 = B[4]; - T _B12 = B[5]; - T _B20 = B[6]; - T _B21 = B[7]; - T _B22 = B[8]; - - - C[0] = _A011*_B22 - _A012*_B21 - _A021*_B12 + _A022*_B11; - C[1] = _A012*_B20 - _A010*_B22 + _A020*_B12 - _A022*_B10; - C[2] = _A010*_B21 - _A011*_B20 - _A020*_B11 + _A021*_B10; - C[3] = _A002*_B21 - _A001*_B22 + _A021*_B02 - _A022*_B01; - C[4] = _A000*_B22 - _A002*_B20 - _A020*_B02 + _A022*_B00; - C[5] = _A001*_B20 - _A000*_B21 + _A020*_B01 - _A021*_B00; - C[6] = _A001*_B12 - _A002*_B11 - _A011*_B02 + _A012*_B01; - C[7] = _A002*_B10 - _A000*_B12 + _A010*_B02 - _A012*_B00; - C[8] = _A000*_B11 - _A001*_B10 - _A010*_B01 + _A011*_B00; - C[9] = _A111*_B22 - _A112*_B21 - _A121*_B12 + _A122*_B11; - C[10] = _A112*_B20 - _A110*_B22 + _A120*_B12 - _A122*_B10; - C[11] = _A110*_B21 - _A111*_B20 - _A120*_B11 + _A121*_B10; - C[12] = _A102*_B21 - _A101*_B22 + _A121*_B02 - _A122*_B01; - C[13] = _A100*_B22 - _A102*_B20 - _A120*_B02 + _A122*_B00; - C[14] = _A101*_B20 - _A100*_B21 + _A120*_B01 - _A121*_B00; - C[15] = _A101*_B12 - _A102*_B11 - _A111*_B02 + _A112*_B01; - C[16] = _A102*_B10 - _A100*_B12 + _A110*_B02 - _A112*_B00; - C[17] = _A100*_B11 - _A101*_B10 - _A110*_B01 + _A111*_B00; - C[18] = _A211*_B22 - _A212*_B21 - _A221*_B12 + _A222*_B11; - C[19] = _A212*_B20 - _A210*_B22 + _A220*_B12 - _A222*_B10; - C[20] = _A210*_B21 - _A211*_B20 - _A220*_B11 + _A221*_B10; - C[21] = _A202*_B21 - _A201*_B22 + _A221*_B02 - _A222*_B01; - C[22] = _A200*_B22 - _A202*_B20 - _A220*_B02 + _A222*_B00; - C[23] = _A201*_B20 - _A200*_B21 + _A220*_B01 - _A221*_B00; - C[24] = _A201*_B12 - _A202*_B11 - _A211*_B02 + _A212*_B01; - C[25] = _A202*_B10 - _A200*_B12 + _A210*_B02 - _A212*_B00; - C[26] = _A200*_B11 - _A201*_B10 - _A210*_B01 + _A211*_B00; -} -//--------------------------------------------------------------------------------------------------------// - - - -//--------------------------------------------------------------------------------------------------------// -//(AxB)_{iIJ} = E_{ipq}E_{IPQ}A_{qQJ}B_{pP} -// tensor cross product of 2nd and 3rd order tensors -template -FASTOR_HINT_INLINE void _crossproduct23(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { - - T _A000 = A[0]; - T _A001 = A[1]; - T _A002 = A[2]; - T _A010 = A[3]; - T _A011 = A[4]; - T _A012 = A[5]; - T _A020 = A[6]; - T _A021 = A[7]; - T _A022 = A[8]; - T _A100 = A[9]; - T _A101 = A[10]; - T _A102 = A[11]; - T _A110 = A[12]; - T _A111 = A[13]; - T _A112 = A[14]; - T _A120 = A[15]; - T _A121 = A[16]; - T _A122 = A[17]; - T _A200 = A[18]; - T _A201 = A[19]; - T _A202 = A[20]; - T _A210 = A[21]; - T _A211 = A[22]; - T _A212 = A[23]; - T _A220 = A[24]; - T _A221 = A[25]; - T _A222 = A[26]; - - - T _B00 = B[0]; - T _B01 = B[1]; - T _B02 = B[2]; - T _B10 = B[3]; - T _B11 = B[4]; - T _B12 = B[5]; - T _B20 = B[6]; - T _B21 = B[7]; - T _B22 = B[8]; - - - C[0] = _A110*_B22 - _A120*_B21 - _A210*_B12 + _A220*_B11; - C[1] = _A111*_B22 - _A121*_B21 - _A211*_B12 + _A221*_B11; - C[2] = _A112*_B22 - _A122*_B21 - _A212*_B12 + _A222*_B11; - C[3] = _A120*_B20 - _A100*_B22 + _A200*_B12 - _A220*_B10; - C[4] = _A121*_B20 - _A101*_B22 + _A201*_B12 - _A221*_B10; - C[5] = _A122*_B20 - _A102*_B22 + _A202*_B12 - _A222*_B10; - C[6] = _A100*_B21 - _A110*_B20 - _A200*_B11 + _A210*_B10; - C[7] = _A101*_B21 - _A111*_B20 - _A201*_B11 + _A211*_B10; - C[8] = _A102*_B21 - _A112*_B20 - _A202*_B11 + _A212*_B10; - C[9] = _A020*_B21 - _A010*_B22 + _A210*_B02 - _A220*_B01; - C[10] = _A021*_B21 - _A011*_B22 + _A211*_B02 - _A221*_B01; - C[11] = _A022*_B21 - _A012*_B22 + _A212*_B02 - _A222*_B01; - C[12] = _A000*_B22 - _A020*_B20 - _A200*_B02 + _A220*_B00; - C[13] = _A001*_B22 - _A021*_B20 - _A201*_B02 + _A221*_B00; - C[14] = _A002*_B22 - _A022*_B20 - _A202*_B02 + _A222*_B00; - C[15] = _A010*_B20 - _A000*_B21 + _A200*_B01 - _A210*_B00; - C[16] = _A011*_B20 - _A001*_B21 + _A201*_B01 - _A211*_B00; - C[17] = _A012*_B20 - _A002*_B21 + _A202*_B01 - _A212*_B00; - C[18] = _A010*_B12 - _A020*_B11 - _A110*_B02 + _A120*_B01; - C[19] = _A011*_B12 - _A021*_B11 - _A111*_B02 + _A121*_B01; - C[20] = _A012*_B12 - _A022*_B11 - _A112*_B02 + _A122*_B01; - C[21] = _A020*_B10 - _A000*_B12 + _A100*_B02 - _A120*_B00; - C[22] = _A021*_B10 - _A001*_B12 + _A101*_B02 - _A121*_B00; - C[23] = _A022*_B10 - _A002*_B12 + _A102*_B02 - _A122*_B00; - C[24] = _A000*_B11 - _A010*_B10 - _A100*_B01 + _A110*_B00; - C[25] = _A001*_B11 - _A011*_B10 - _A101*_B01 + _A111*_B00; - C[26] = _A002*_B11 - _A012*_B10 - _A102*_B01 + _A112*_B00; -} -//--------------------------------------------------------------------------------------------------------// - - - - -//--------------------------------------------------------------------------------------------------------// -//(AxB)_{iIjJ} = E_{jpq}E_{JPQ}A_{iIpP}B_{qQ} -// tensor cross product of 4th and 2nd order tensors -template -FASTOR_HINT_INLINE void _crossproduct42(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { - - T _A0000 = A[0]; - T _A0001 = A[1]; - T _A0002 = A[2]; - T _A0010 = A[3]; - T _A0011 = A[4]; - T _A0012 = A[5]; - T _A0020 = A[6]; - T _A0021 = A[7]; - T _A0022 = A[8]; - T _A0100 = A[9]; - T _A0101 = A[10]; - T _A0102 = A[11]; - T _A0110 = A[12]; - T _A0111 = A[13]; - T _A0112 = A[14]; - T _A0120 = A[15]; - T _A0121 = A[16]; - T _A0122 = A[17]; - T _A0200 = A[18]; - T _A0201 = A[19]; - T _A0202 = A[20]; - T _A0210 = A[21]; - T _A0211 = A[22]; - T _A0212 = A[23]; - T _A0220 = A[24]; - T _A0221 = A[25]; - T _A0222 = A[26]; - T _A1000 = A[27]; - T _A1001 = A[28]; - T _A1002 = A[29]; - T _A1010 = A[30]; - T _A1011 = A[31]; - T _A1012 = A[32]; - T _A1020 = A[33]; - T _A1021 = A[34]; - T _A1022 = A[35]; - T _A1100 = A[36]; - T _A1101 = A[37]; - T _A1102 = A[38]; - T _A1110 = A[39]; - T _A1111 = A[40]; - T _A1112 = A[41]; - T _A1120 = A[42]; - T _A1121 = A[43]; - T _A1122 = A[44]; - T _A1200 = A[45]; - T _A1201 = A[46]; - T _A1202 = A[47]; - T _A1210 = A[48]; - T _A1211 = A[49]; - T _A1212 = A[50]; - T _A1220 = A[51]; - T _A1221 = A[52]; - T _A1222 = A[53]; - T _A2000 = A[54]; - T _A2001 = A[55]; - T _A2002 = A[56]; - T _A2010 = A[57]; - T _A2011 = A[58]; - T _A2012 = A[59]; - T _A2020 = A[60]; - T _A2021 = A[61]; - T _A2022 = A[62]; - T _A2100 = A[63]; - T _A2101 = A[64]; - T _A2102 = A[65]; - T _A2110 = A[66]; - T _A2111 = A[67]; - T _A2112 = A[68]; - T _A2120 = A[69]; - T _A2121 = A[70]; - T _A2122 = A[71]; - T _A2200 = A[72]; - T _A2201 = A[73]; - T _A2202 = A[74]; - T _A2210 = A[75]; - T _A2211 = A[76]; - T _A2212 = A[77]; - T _A2220 = A[78]; - T _A2221 = A[79]; - T _A2222 = A[80]; - - - T _B00 = B[0]; - T _B01 = B[1]; - T _B02 = B[2]; - T _B10 = B[3]; - T _B11 = B[4]; - T _B12 = B[5]; - T _B20 = B[6]; - T _B21 = B[7]; - T _B22 = B[8]; - - - C[0] = _A0011*_B22 - _A0012*_B21 - _A0021*_B12 + _A0022*_B11; - C[1] = _A0012*_B20 - _A0010*_B22 + _A0020*_B12 - _A0022*_B10; - C[2] = _A0010*_B21 - _A0011*_B20 - _A0020*_B11 + _A0021*_B10; - C[3] = _A0002*_B21 - _A0001*_B22 + _A0021*_B02 - _A0022*_B01; - C[4] = _A0000*_B22 - _A0002*_B20 - _A0020*_B02 + _A0022*_B00; - C[5] = _A0001*_B20 - _A0000*_B21 + _A0020*_B01 - _A0021*_B00; - C[6] = _A0001*_B12 - _A0002*_B11 - _A0011*_B02 + _A0012*_B01; - C[7] = _A0002*_B10 - _A0000*_B12 + _A0010*_B02 - _A0012*_B00; - C[8] = _A0000*_B11 - _A0001*_B10 - _A0010*_B01 + _A0011*_B00; - C[9] = _A0111*_B22 - _A0112*_B21 - _A0121*_B12 + _A0122*_B11; - C[10] = _A0112*_B20 - _A0110*_B22 + _A0120*_B12 - _A0122*_B10; - C[11] = _A0110*_B21 - _A0111*_B20 - _A0120*_B11 + _A0121*_B10; - C[12] = _A0102*_B21 - _A0101*_B22 + _A0121*_B02 - _A0122*_B01; - C[13] = _A0100*_B22 - _A0102*_B20 - _A0120*_B02 + _A0122*_B00; - C[14] = _A0101*_B20 - _A0100*_B21 + _A0120*_B01 - _A0121*_B00; - C[15] = _A0101*_B12 - _A0102*_B11 - _A0111*_B02 + _A0112*_B01; - C[16] = _A0102*_B10 - _A0100*_B12 + _A0110*_B02 - _A0112*_B00; - C[17] = _A0100*_B11 - _A0101*_B10 - _A0110*_B01 + _A0111*_B00; - C[18] = _A0211*_B22 - _A0212*_B21 - _A0221*_B12 + _A0222*_B11; - C[19] = _A0212*_B20 - _A0210*_B22 + _A0220*_B12 - _A0222*_B10; - C[20] = _A0210*_B21 - _A0211*_B20 - _A0220*_B11 + _A0221*_B10; - C[21] = _A0202*_B21 - _A0201*_B22 + _A0221*_B02 - _A0222*_B01; - C[22] = _A0200*_B22 - _A0202*_B20 - _A0220*_B02 + _A0222*_B00; - C[23] = _A0201*_B20 - _A0200*_B21 + _A0220*_B01 - _A0221*_B00; - C[24] = _A0201*_B12 - _A0202*_B11 - _A0211*_B02 + _A0212*_B01; - C[25] = _A0202*_B10 - _A0200*_B12 + _A0210*_B02 - _A0212*_B00; - C[26] = _A0200*_B11 - _A0201*_B10 - _A0210*_B01 + _A0211*_B00; - C[27] = _A1011*_B22 - _A1012*_B21 - _A1021*_B12 + _A1022*_B11; - C[28] = _A1012*_B20 - _A1010*_B22 + _A1020*_B12 - _A1022*_B10; - C[29] = _A1010*_B21 - _A1011*_B20 - _A1020*_B11 + _A1021*_B10; - C[30] = _A1002*_B21 - _A1001*_B22 + _A1021*_B02 - _A1022*_B01; - C[31] = _A1000*_B22 - _A1002*_B20 - _A1020*_B02 + _A1022*_B00; - C[32] = _A1001*_B20 - _A1000*_B21 + _A1020*_B01 - _A1021*_B00; - C[33] = _A1001*_B12 - _A1002*_B11 - _A1011*_B02 + _A1012*_B01; - C[34] = _A1002*_B10 - _A1000*_B12 + _A1010*_B02 - _A1012*_B00; - C[35] = _A1000*_B11 - _A1001*_B10 - _A1010*_B01 + _A1011*_B00; - C[36] = _A1111*_B22 - _A1112*_B21 - _A1121*_B12 + _A1122*_B11; - C[37] = _A1112*_B20 - _A1110*_B22 + _A1120*_B12 - _A1122*_B10; - C[38] = _A1110*_B21 - _A1111*_B20 - _A1120*_B11 + _A1121*_B10; - C[39] = _A1102*_B21 - _A1101*_B22 + _A1121*_B02 - _A1122*_B01; - C[40] = _A1100*_B22 - _A1102*_B20 - _A1120*_B02 + _A1122*_B00; - C[41] = _A1101*_B20 - _A1100*_B21 + _A1120*_B01 - _A1121*_B00; - C[42] = _A1101*_B12 - _A1102*_B11 - _A1111*_B02 + _A1112*_B01; - C[43] = _A1102*_B10 - _A1100*_B12 + _A1110*_B02 - _A1112*_B00; - C[44] = _A1100*_B11 - _A1101*_B10 - _A1110*_B01 + _A1111*_B00; - C[45] = _A1211*_B22 - _A1212*_B21 - _A1221*_B12 + _A1222*_B11; - C[46] = _A1212*_B20 - _A1210*_B22 + _A1220*_B12 - _A1222*_B10; - C[47] = _A1210*_B21 - _A1211*_B20 - _A1220*_B11 + _A1221*_B10; - C[48] = _A1202*_B21 - _A1201*_B22 + _A1221*_B02 - _A1222*_B01; - C[49] = _A1200*_B22 - _A1202*_B20 - _A1220*_B02 + _A1222*_B00; - C[50] = _A1201*_B20 - _A1200*_B21 + _A1220*_B01 - _A1221*_B00; - C[51] = _A1201*_B12 - _A1202*_B11 - _A1211*_B02 + _A1212*_B01; - C[52] = _A1202*_B10 - _A1200*_B12 + _A1210*_B02 - _A1212*_B00; - C[53] = _A1200*_B11 - _A1201*_B10 - _A1210*_B01 + _A1211*_B00; - C[54] = _A2011*_B22 - _A2012*_B21 - _A2021*_B12 + _A2022*_B11; - C[55] = _A2012*_B20 - _A2010*_B22 + _A2020*_B12 - _A2022*_B10; - C[56] = _A2010*_B21 - _A2011*_B20 - _A2020*_B11 + _A2021*_B10; - C[57] = _A2002*_B21 - _A2001*_B22 + _A2021*_B02 - _A2022*_B01; - C[58] = _A2000*_B22 - _A2002*_B20 - _A2020*_B02 + _A2022*_B00; - C[59] = _A2001*_B20 - _A2000*_B21 + _A2020*_B01 - _A2021*_B00; - C[60] = _A2001*_B12 - _A2002*_B11 - _A2011*_B02 + _A2012*_B01; - C[61] = _A2002*_B10 - _A2000*_B12 + _A2010*_B02 - _A2012*_B00; - C[62] = _A2000*_B11 - _A2001*_B10 - _A2010*_B01 + _A2011*_B00; - C[63] = _A2111*_B22 - _A2112*_B21 - _A2121*_B12 + _A2122*_B11; - C[64] = _A2112*_B20 - _A2110*_B22 + _A2120*_B12 - _A2122*_B10; - C[65] = _A2110*_B21 - _A2111*_B20 - _A2120*_B11 + _A2121*_B10; - C[66] = _A2102*_B21 - _A2101*_B22 + _A2121*_B02 - _A2122*_B01; - C[67] = _A2100*_B22 - _A2102*_B20 - _A2120*_B02 + _A2122*_B00; - C[68] = _A2101*_B20 - _A2100*_B21 + _A2120*_B01 - _A2121*_B00; - C[69] = _A2101*_B12 - _A2102*_B11 - _A2111*_B02 + _A2112*_B01; - C[70] = _A2102*_B10 - _A2100*_B12 + _A2110*_B02 - _A2112*_B00; - C[71] = _A2100*_B11 - _A2101*_B10 - _A2110*_B01 + _A2111*_B00; - C[72] = _A2211*_B22 - _A2212*_B21 - _A2221*_B12 + _A2222*_B11; - C[73] = _A2212*_B20 - _A2210*_B22 + _A2220*_B12 - _A2222*_B10; - C[74] = _A2210*_B21 - _A2211*_B20 - _A2220*_B11 + _A2221*_B10; - C[75] = _A2202*_B21 - _A2201*_B22 + _A2221*_B02 - _A2222*_B01; - C[76] = _A2200*_B22 - _A2202*_B20 - _A2220*_B02 + _A2222*_B00; - C[77] = _A2201*_B20 - _A2200*_B21 + _A2220*_B01 - _A2221*_B00; - C[78] = _A2201*_B12 - _A2202*_B11 - _A2211*_B02 + _A2212*_B01; - C[79] = _A2202*_B10 - _A2200*_B12 + _A2210*_B02 - _A2212*_B00; - C[80] = _A2200*_B11 - _A2201*_B10 - _A2210*_B01 + _A2211*_B00; -} -//--------------------------------------------------------------------------------------------------------// - - - -//--------------------------------------------------------------------------------------------------------// -//(AxB)_{iIjJ} = E_{ipq}E_{IPQ}A_{qQjJ}B_{pP} -// tensor cross product of 2nd and 4th order tensors -template -FASTOR_HINT_INLINE void _crossproduct24(const T *FASTOR_RESTRICT A, const T *FASTOR_RESTRICT B, T *FASTOR_RESTRICT C) { - - T _A0000 = A[0]; - T _A0001 = A[1]; - T _A0002 = A[2]; - T _A0010 = A[3]; - T _A0011 = A[4]; - T _A0012 = A[5]; - T _A0020 = A[6]; - T _A0021 = A[7]; - T _A0022 = A[8]; - T _A0100 = A[9]; - T _A0101 = A[10]; - T _A0102 = A[11]; - T _A0110 = A[12]; - T _A0111 = A[13]; - T _A0112 = A[14]; - T _A0120 = A[15]; - T _A0121 = A[16]; - T _A0122 = A[17]; - T _A0200 = A[18]; - T _A0201 = A[19]; - T _A0202 = A[20]; - T _A0210 = A[21]; - T _A0211 = A[22]; - T _A0212 = A[23]; - T _A0220 = A[24]; - T _A0221 = A[25]; - T _A0222 = A[26]; - T _A1000 = A[27]; - T _A1001 = A[28]; - T _A1002 = A[29]; - T _A1010 = A[30]; - T _A1011 = A[31]; - T _A1012 = A[32]; - T _A1020 = A[33]; - T _A1021 = A[34]; - T _A1022 = A[35]; - T _A1100 = A[36]; - T _A1101 = A[37]; - T _A1102 = A[38]; - T _A1110 = A[39]; - T _A1111 = A[40]; - T _A1112 = A[41]; - T _A1120 = A[42]; - T _A1121 = A[43]; - T _A1122 = A[44]; - T _A1200 = A[45]; - T _A1201 = A[46]; - T _A1202 = A[47]; - T _A1210 = A[48]; - T _A1211 = A[49]; - T _A1212 = A[50]; - T _A1220 = A[51]; - T _A1221 = A[52]; - T _A1222 = A[53]; - T _A2000 = A[54]; - T _A2001 = A[55]; - T _A2002 = A[56]; - T _A2010 = A[57]; - T _A2011 = A[58]; - T _A2012 = A[59]; - T _A2020 = A[60]; - T _A2021 = A[61]; - T _A2022 = A[62]; - T _A2100 = A[63]; - T _A2101 = A[64]; - T _A2102 = A[65]; - T _A2110 = A[66]; - T _A2111 = A[67]; - T _A2112 = A[68]; - T _A2120 = A[69]; - T _A2121 = A[70]; - T _A2122 = A[71]; - T _A2200 = A[72]; - T _A2201 = A[73]; - T _A2202 = A[74]; - T _A2210 = A[75]; - T _A2211 = A[76]; - T _A2212 = A[77]; - T _A2220 = A[78]; - T _A2221 = A[79]; - T _A2222 = A[80]; - - - T _B00 = B[0]; - T _B01 = B[1]; - T _B02 = B[2]; - T _B10 = B[3]; - T _B11 = B[4]; - T _B12 = B[5]; - T _B20 = B[6]; - T _B21 = B[7]; - T _B22 = B[8]; - - - C[0] = _A1100*_B22 - _A1200*_B21 - _A2100*_B12 + _A2200*_B11; - C[1] = _A1101*_B22 - _A1201*_B21 - _A2101*_B12 + _A2201*_B11; - C[2] = _A1102*_B22 - _A1202*_B21 - _A2102*_B12 + _A2202*_B11; - C[3] = _A1110*_B22 - _A1210*_B21 - _A2110*_B12 + _A2210*_B11; - C[4] = _A1111*_B22 - _A1211*_B21 - _A2111*_B12 + _A2211*_B11; - C[5] = _A1112*_B22 - _A1212*_B21 - _A2112*_B12 + _A2212*_B11; - C[6] = _A1120*_B22 - _A1220*_B21 - _A2120*_B12 + _A2220*_B11; - C[7] = _A1121*_B22 - _A1221*_B21 - _A2121*_B12 + _A2221*_B11; - C[8] = _A1122*_B22 - _A1222*_B21 - _A2122*_B12 + _A2222*_B11; - C[9] = _A1200*_B20 - _A1000*_B22 + _A2000*_B12 - _A2200*_B10; - C[10] = _A1201*_B20 - _A1001*_B22 + _A2001*_B12 - _A2201*_B10; - C[11] = _A1202*_B20 - _A1002*_B22 + _A2002*_B12 - _A2202*_B10; - C[12] = _A1210*_B20 - _A1010*_B22 + _A2010*_B12 - _A2210*_B10; - C[13] = _A1211*_B20 - _A1011*_B22 + _A2011*_B12 - _A2211*_B10; - C[14] = _A1212*_B20 - _A1012*_B22 + _A2012*_B12 - _A2212*_B10; - C[15] = _A1220*_B20 - _A1020*_B22 + _A2020*_B12 - _A2220*_B10; - C[16] = _A1221*_B20 - _A1021*_B22 + _A2021*_B12 - _A2221*_B10; - C[17] = _A1222*_B20 - _A1022*_B22 + _A2022*_B12 - _A2222*_B10; - C[18] = _A1000*_B21 - _A1100*_B20 - _A2000*_B11 + _A2100*_B10; - C[19] = _A1001*_B21 - _A1101*_B20 - _A2001*_B11 + _A2101*_B10; - C[20] = _A1002*_B21 - _A1102*_B20 - _A2002*_B11 + _A2102*_B10; - C[21] = _A1010*_B21 - _A1110*_B20 - _A2010*_B11 + _A2110*_B10; - C[22] = _A1011*_B21 - _A1111*_B20 - _A2011*_B11 + _A2111*_B10; - C[23] = _A1012*_B21 - _A1112*_B20 - _A2012*_B11 + _A2112*_B10; - C[24] = _A1020*_B21 - _A1120*_B20 - _A2020*_B11 + _A2120*_B10; - C[25] = _A1021*_B21 - _A1121*_B20 - _A2021*_B11 + _A2121*_B10; - C[26] = _A1022*_B21 - _A1122*_B20 - _A2022*_B11 + _A2122*_B10; - C[27] = _A0200*_B21 - _A0100*_B22 + _A2100*_B02 - _A2200*_B01; - C[28] = _A0201*_B21 - _A0101*_B22 + _A2101*_B02 - _A2201*_B01; - C[29] = _A0202*_B21 - _A0102*_B22 + _A2102*_B02 - _A2202*_B01; - C[30] = _A0210*_B21 - _A0110*_B22 + _A2110*_B02 - _A2210*_B01; - C[31] = _A0211*_B21 - _A0111*_B22 + _A2111*_B02 - _A2211*_B01; - C[32] = _A0212*_B21 - _A0112*_B22 + _A2112*_B02 - _A2212*_B01; - C[33] = _A0220*_B21 - _A0120*_B22 + _A2120*_B02 - _A2220*_B01; - C[34] = _A0221*_B21 - _A0121*_B22 + _A2121*_B02 - _A2221*_B01; - C[35] = _A0222*_B21 - _A0122*_B22 + _A2122*_B02 - _A2222*_B01; - C[36] = _A0000*_B22 - _A0200*_B20 - _A2000*_B02 + _A2200*_B00; - C[37] = _A0001*_B22 - _A0201*_B20 - _A2001*_B02 + _A2201*_B00; - C[38] = _A0002*_B22 - _A0202*_B20 - _A2002*_B02 + _A2202*_B00; - C[39] = _A0010*_B22 - _A0210*_B20 - _A2010*_B02 + _A2210*_B00; - C[40] = _A0011*_B22 - _A0211*_B20 - _A2011*_B02 + _A2211*_B00; - C[41] = _A0012*_B22 - _A0212*_B20 - _A2012*_B02 + _A2212*_B00; - C[42] = _A0020*_B22 - _A0220*_B20 - _A2020*_B02 + _A2220*_B00; - C[43] = _A0021*_B22 - _A0221*_B20 - _A2021*_B02 + _A2221*_B00; - C[44] = _A0022*_B22 - _A0222*_B20 - _A2022*_B02 + _A2222*_B00; - C[45] = _A0100*_B20 - _A0000*_B21 + _A2000*_B01 - _A2100*_B00; - C[46] = _A0101*_B20 - _A0001*_B21 + _A2001*_B01 - _A2101*_B00; - C[47] = _A0102*_B20 - _A0002*_B21 + _A2002*_B01 - _A2102*_B00; - C[48] = _A0110*_B20 - _A0010*_B21 + _A2010*_B01 - _A2110*_B00; - C[49] = _A0111*_B20 - _A0011*_B21 + _A2011*_B01 - _A2111*_B00; - C[50] = _A0112*_B20 - _A0012*_B21 + _A2012*_B01 - _A2112*_B00; - C[51] = _A0120*_B20 - _A0020*_B21 + _A2020*_B01 - _A2120*_B00; - C[52] = _A0121*_B20 - _A0021*_B21 + _A2021*_B01 - _A2121*_B00; - C[53] = _A0122*_B20 - _A0022*_B21 + _A2022*_B01 - _A2122*_B00; - C[54] = _A0100*_B12 - _A0200*_B11 - _A1100*_B02 + _A1200*_B01; - C[55] = _A0101*_B12 - _A0201*_B11 - _A1101*_B02 + _A1201*_B01; - C[56] = _A0102*_B12 - _A0202*_B11 - _A1102*_B02 + _A1202*_B01; - C[57] = _A0110*_B12 - _A0210*_B11 - _A1110*_B02 + _A1210*_B01; - C[58] = _A0111*_B12 - _A0211*_B11 - _A1111*_B02 + _A1211*_B01; - C[59] = _A0112*_B12 - _A0212*_B11 - _A1112*_B02 + _A1212*_B01; - C[60] = _A0120*_B12 - _A0220*_B11 - _A1120*_B02 + _A1220*_B01; - C[61] = _A0121*_B12 - _A0221*_B11 - _A1121*_B02 + _A1221*_B01; - C[62] = _A0122*_B12 - _A0222*_B11 - _A1122*_B02 + _A1222*_B01; - C[63] = _A0200*_B10 - _A0000*_B12 + _A1000*_B02 - _A1200*_B00; - C[64] = _A0201*_B10 - _A0001*_B12 + _A1001*_B02 - _A1201*_B00; - C[65] = _A0202*_B10 - _A0002*_B12 + _A1002*_B02 - _A1202*_B00; - C[66] = _A0210*_B10 - _A0010*_B12 + _A1010*_B02 - _A1210*_B00; - C[67] = _A0211*_B10 - _A0011*_B12 + _A1011*_B02 - _A1211*_B00; - C[68] = _A0212*_B10 - _A0012*_B12 + _A1012*_B02 - _A1212*_B00; - C[69] = _A0220*_B10 - _A0020*_B12 + _A1020*_B02 - _A1220*_B00; - C[70] = _A0221*_B10 - _A0021*_B12 + _A1021*_B02 - _A1221*_B00; - C[71] = _A0222*_B10 - _A0022*_B12 + _A1022*_B02 - _A1222*_B00; - C[72] = _A0000*_B11 - _A0100*_B10 - _A1000*_B01 + _A1100*_B00; - C[73] = _A0001*_B11 - _A0101*_B10 - _A1001*_B01 + _A1101*_B00; - C[74] = _A0002*_B11 - _A0102*_B10 - _A1002*_B01 + _A1102*_B00; - C[75] = _A0010*_B11 - _A0110*_B10 - _A1010*_B01 + _A1110*_B00; - C[76] = _A0011*_B11 - _A0111*_B10 - _A1011*_B01 + _A1111*_B00; - C[77] = _A0012*_B11 - _A0112*_B10 - _A1012*_B01 + _A1112*_B00; - C[78] = _A0020*_B11 - _A0120*_B10 - _A1020*_B01 + _A1120*_B00; - C[79] = _A0021*_B11 - _A0121*_B10 - _A1021*_B01 + _A1121*_B00; - C[80] = _A0022*_B11 - _A0122*_B10 - _A1022*_B01 + _A1122*_B00; -} -//--------------------------------------------------------------------------------------------------------// - -} // end of namespace Fastor - - -#endif // TENSOR_CROSS_H - diff --git a/noarch/include/Fastor/backend/trace.h b/noarch/include/Fastor/backend/trace.h deleted file mode 100644 index e7c4b46b..00000000 --- a/noarch/include/Fastor/backend/trace.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef TRACE_H -#define TRACE_H - -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" - -namespace Fastor { - - -template::type=0> -FASTOR_INLINE T _trace(const T * FASTOR_RESTRICT a) { - T sum = static_cast(0); - for (FASTOR_INDEX i=0; i -FASTOR_INLINE double _trace(const double * FASTOR_RESTRICT a) { - // AVX VERSION - // IVY 5 OPS / HW 7 OPS -// __m256d a_reg = _mm256_load_pd(a); -// __m128d a_high = _mm256_extractf128_pd(a_reg,0x1); -// return _mm_cvtsd_f64(_mm_add_sd(_mm256_castpd256_pd128(a_reg),_mm_shuffle_pd(a_high,a_high,0x1))); - - // SSE VERSION - // 3 OPS - __m128d a0 = _mm_load_sd(a); - __m128d a1 = _mm_load_sd(a+3); - return _mm_cvtsd_f64(_mm_add_pd(a0,a1)); -} - -template<> -FASTOR_INLINE double _trace(const double * FASTOR_RESTRICT a) { - // No benefit in AVX - return _mm_cvtsd_f64(_mm_add_sd(_mm_load_sd(a),_mm_add_sd(_mm_load_sd(a+4),_mm_load_sd(a+8)))); -} - -template<> -FASTOR_INLINE float _trace(const float * FASTOR_RESTRICT a) { - __m128 a_reg = _mm_load_ps(a); - return _mm_cvtss_f32(_mm_add_ss(a_reg,_mm_reverse_ps(a_reg))); -} -#endif - -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE float _trace(const float * FASTOR_RESTRICT a) { - __m256 a_reg = _mm256_load_ps(a); - __m128 sum_two = _mm_add_ps(_mm256_castps256_ps128(a_reg),_mm256_extractf128_ps(a_reg,0x1)); - return _mm_cvtss_f32(_mm_add_ss(sum_two,_mm_load_ss(a+8))); -} -#endif - -} - -#endif // TRACE_H - diff --git a/noarch/include/Fastor/backend/transpose/transpose.h b/noarch/include/Fastor/backend/transpose/transpose.h deleted file mode 100644 index 9e45cfae..00000000 --- a/noarch/include/Fastor/backend/transpose/transpose.h +++ /dev/null @@ -1,448 +0,0 @@ -#ifndef TRANSPOSE_H -#define TRANSPOSE_H - - -#include "Fastor/config/config.h" -#include "Fastor/backend/transpose/transpose_kernels.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/SIMDVector.h" - -namespace Fastor { - -// Forward declare -namespace internal { -template -FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out); -} // internal - - -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_AVX_IMPL - -template -FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { - - using V = SIMDVector; - - // Block sizes of 8x8 i.e. numSIMDRows=1 - // numSIMDCols=1 and innerBlock=outerBlock=1 - // give a much greater speed up, but causes - // significant slow-down for issue #42 - -#ifndef FASTOR_TRANS_OUTER_BLOCK_SIZE - constexpr size_t numSIMDRows = 1UL; -#else - constexpr size_t numSIMDRows = FASTOR_TRANS_OUTER_BLOCK_SIZE; -#endif -#ifndef FASTOR_TRANS_INNER_BLOCK_SIZE - constexpr size_t numSIMDCols = 1UL; -#else - constexpr size_t numSIMDCols = FASTOR_TRANS_INNER_BLOCK_SIZE; -#endif - - constexpr size_t innerBlock = V::Size * numSIMDCols; - constexpr size_t outerBlock = V::Size * numSIMDRows; - - FASTOR_ARCH_ALIGN T pack_a[outerBlock*innerBlock]; - FASTOR_ARCH_ALIGN T pack_out[outerBlock*innerBlock]; - - constexpr size_t M0 = M / innerBlock * innerBlock; - constexpr size_t N0 = N / outerBlock * outerBlock; - V _vec; - - // For row-major matrices we go over N - // and then M to get contiguous writes - size_t j=0; - for (; j(pack_a,pack_out); - // Unpack pack_out to out - for (size_t jj=0; jj -FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { - for (size_t j=0; j -FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - __m128 a_reg = _mm_loadu_ps(a); - _mm_storeu_ps(out,_mm_shuffle_ps(a_reg,a_reg,_MM_SHUFFLE(3,1,2,0))); -} - -template<> -FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { -#ifndef FASTOR_AVX2_IMPL - // 5 OPS - __m128 row0 = _mm_loadu_ps(a); - __m128 row1 = _mm_loadu_ps(a+3); - __m128 row2 = _mm_loadu_ps(a+6); - - __m128 T0 = _mm_unpacklo_ps(row0,row1); - __m128 T1 = _mm_unpackhi_ps(row0,row1); - - row0 = _mm_movelh_ps ( T0,row2 ); - row1 = _mm_shuffle_ps( T0,row2, _MM_SHUFFLE(3,1,3,2) ); - row2 = _mm_shuffle_ps( T1,row2, _MM_SHUFFLE(3,2,1,0) ); - - _mm_storeu_ps(out,row0); - _mm_storeu_ps(out+3,row1); - _mm_storeu_ps(out+6,row2); // out of range for out[9] -#else - // 3 OPS - // gcc/clang emit vpermsps tht operate on (%rsp) - // less pressure on shuffle port perhaps - __m256 trans07 = _mm256_loadu_ps(a); - const __m256i trans_mask = _mm256_setr_epi32( - 0,3,6, - 1,4,7, - 2,5); - // does not shuffle across 256 lanes, only 128 lanes - // __m256 _res = _mm256_permutevar_ps(trans07, trans_mask); - // this one shuffles correctly - __m256 _res = _mm256_permutevar8x32_ps(trans07, trans_mask); - _mm256_storeu_ps(out,_res); - // out[8] = a[8]; - _mm_store_ss(out+8,_mm_load_ss(a+8)); -#endif -} - -template<> -FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { -#ifdef FASTOR_AVX512F_IMPL - __m512 amm = _mm512_loadu_ps(a); - __m512i idx = _mm512_setr_epi32( 0, 4, 8, 12, - 1, 5, 9, 13, - 2, 6, 10, 14, - 3, 7, 11, 15); - __m512 omm = _mm512_permutexvar_ps(idx, amm); - _mm512_storeu_ps(out, omm); -#else - __m128 row1 = _mm_loadu_ps(a); - __m128 row2 = _mm_loadu_ps(a+4); - __m128 row3 = _mm_loadu_ps(a+8); - __m128 row4 = _mm_loadu_ps(a+12); - _MM_TRANSPOSE4_PS(row1, row2, row3, row4); - _mm_storeu_ps(out , row1); - _mm_storeu_ps(out+4 , row2); - _mm_storeu_ps(out+8 , row3); - _mm_storeu_ps(out+12, row4); -#endif -} -#endif - -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - __m256 row1 = _mm256_loadu_ps(a); - __m256 row2 = _mm256_loadu_ps(a+8); - __m256 row3 = _mm256_loadu_ps(a+16); - __m256 row4 = _mm256_loadu_ps(a+24); - __m256 row5 = _mm256_loadu_ps(a+32); - __m256 row6 = _mm256_loadu_ps(a+40); - __m256 row7 = _mm256_loadu_ps(a+48); - __m256 row8 = _mm256_loadu_ps(a+56); - internal::_MM_TRANSPOSE8_PS(row1, row2, row3, row4, row5, row6, row7, row8); - _mm256_storeu_ps(out, row1); - _mm256_storeu_ps(out+8, row2); - _mm256_storeu_ps(out+16, row3); - _mm256_storeu_ps(out+24, row4); - _mm256_storeu_ps(out+32, row5); - _mm256_storeu_ps(out+40, row6); - _mm256_storeu_ps(out+48, row7); - _mm256_storeu_ps(out+56, row8); -} -#endif - -#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL) -template<> -FASTOR_INLINE void _transpose(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - internal::_MM_TRANSPOSE16_PS(a,out); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - - -// Specialisations - double -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE void _transpose(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) { - /*-------------------------------------------------------*/ - // 2 OPS - __m128d row0 = _mm_loadu_pd(a); - __m128d row1 = _mm_loadu_pd(a+2); - __m128d tmp = row0; - row0 = _mm_shuffle_pd(row0,row1,0x0); - row1 = _mm_shuffle_pd(tmp ,row1,0x3); - _mm_storeu_pd(out ,row0); - _mm_storeu_pd(out+2,row1); - /*-------------------------------------------------------*/ - - /*-------------------------------------------------------*/ -// // AVX VERSION - // // IVY 4 OPS / HW 8 OPS - // __m256d a1 = _mm256_loadu_pd(a); - // __m128d a2 = _mm256_castpd256_pd128(a1); - // __m128d a3 = _mm256_extractf128_pd(a1,0x1); - // __m128d a4 = _mm_shuffle_pd(a2,a3,0x0); - // a3 = _mm_shuffle_pd(a2,a3,0x3); - // a1 = _mm256_castpd128_pd256(a4); - // a1 = _mm256_insertf128_pd(a1,a3,0x1); - // _mm256_storeu_pd(out,a1); - /*-------------------------------------------------------*/ -} -#endif - -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE void _transpose(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) { - // AVX512 is the fastest & AVX version despite more instructions is faster than SSE -#if defined(FASTOR_AVX512F_IMPL) - // AVX512 - /*-------------------------------------------------------*/ - __m512d a07 = _mm512_loadu_pd(a); - const __m512i trans_mask = _mm512_setr_epi64( - 0,3,6, - 1,4,7, - 2,5); - __m512d trans07 = _mm512_permutexvar_pd(trans_mask, a07); - _mm512_storeu_pd(out,trans07); - _mm_store_sd(out+8,_mm_load_sd(a+8)); - /*-------------------------------------------------------*/ -#elif defined(FASTOR_AVX_IMPL) - // AVX - /*-------------------------------------------------------*/ - __m256d row1 = _mm256_loadu_pd(a); - __m256d row2 = _mm256_loadu_pd(a+4); - - __m128d a11 = _mm256_castpd256_pd128(row1); - __m128d a12 = _mm256_extractf128_pd(row1,0x1); - __m128d a21 = _mm256_castpd256_pd128(row2); - __m128d a22 = _mm256_extractf128_pd(row2,0x1); - - row1 = _mm256_castpd128_pd256(_mm_shuffle_pd(a11,a12,0x2)); - row1 = _mm256_insertf128_pd(row1,_mm_shuffle_pd(a22,a11,0x2),0x1); - row2 = _mm256_castpd128_pd256(_mm_shuffle_pd(a21,a22,0x2)); - row2 = _mm256_insertf128_pd(row2,_mm_shuffle_pd(a12,a21,0x2),0x1); - - _mm256_storeu_pd(out,row1); - _mm256_storeu_pd(out+4,row2); - _mm_store_sd(out+8,_mm_load_sd(a+8)); - /*-------------------------------------------------------*/ -#else - // SSE - /*-------------------------------------------------------*/ - __m128d a11 = _mm_loadu_pd(a); - __m128d a12 = _mm_loadu_pd(a+2); - __m128d a21 = _mm_loadu_pd(a+4); - __m128d a22 = _mm_loadu_pd(a+6); - - _mm_storeu_pd(out ,_mm_shuffle_pd(a11,a12,0x2)); - _mm_storeu_pd(out+2,_mm_shuffle_pd(a22,a11,0x2)); - _mm_storeu_pd(out+4,_mm_shuffle_pd(a21,a22,0x2)); - _mm_storeu_pd(out+6,_mm_shuffle_pd(a12,a21,0x2)); - _mm_store_sd (out+8,_mm_load_sd(a+8)); - /*-------------------------------------------------------*/ -#endif -} -#endif - -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE void _transpose(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { -#ifdef FASTOR_AVX512F_IMPL - __m512d amm0 = _mm512_loadu_pd(a); - __m512d amm1 = _mm512_loadu_pd(a+8); - __m512i idx0 = _mm512_setr_epi64(0, 4, 8, 12, 1, 5, 9, 13); - __m512i idx1 = _mm512_setr_epi64(2, 6, 10, 14, 3, 7, 11, 15); - __m512d omm0 = _mm512_permutex2var_pd(amm0, idx0, amm1); - __m512d omm1 = _mm512_permutex2var_pd(amm0, idx1, amm1); - _mm512_storeu_pd(out , omm0); - _mm512_storeu_pd(out+8, omm1); -#else - __m256d row1 = _mm256_loadu_pd(a); - __m256d row2 = _mm256_loadu_pd(a+4); - __m256d row3 = _mm256_loadu_pd(a+8); - __m256d row4 = _mm256_loadu_pd(a+12); - internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); - _mm256_storeu_pd(out, row1); - _mm256_storeu_pd(out+4, row2); - _mm256_storeu_pd(out+8, row3); - _mm256_storeu_pd(out+12, row4); -#endif -} -#endif - -#if defined(FASTOR_AVX512F_IMPL) -template<> -FASTOR_INLINE void _transpose(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { - __m512d row0 = _mm512_loadu_pd(a); - __m512d row1 = _mm512_loadu_pd(a+8); - __m512d row2 = _mm512_loadu_pd(a+16); - __m512d row3 = _mm512_loadu_pd(a+24); - __m512d row4 = _mm512_loadu_pd(a+32); - __m512d row5 = _mm512_loadu_pd(a+40); - __m512d row6 = _mm512_loadu_pd(a+48); - __m512d row7 = _mm512_loadu_pd(a+56); - internal::_MM_TRANSPOSE8_PD(row0,row1,row2,row3,row4,row5,row6,row7); - _mm512_storeu_pd(out , row0); - _mm512_storeu_pd(out+8 , row1); - _mm512_storeu_pd(out+16, row2); - _mm512_storeu_pd(out+24, row3); - _mm512_storeu_pd(out+32, row4); - _mm512_storeu_pd(out+40, row5); - _mm512_storeu_pd(out+48, row6); - _mm512_storeu_pd(out+56, row7); -} -#elif defined(FASTOR_AVX_IMPL) -template<> -FASTOR_INLINE void _transpose(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { - - { - __m256d row1 = _mm256_loadu_pd(&a[0]); - __m256d row2 = _mm256_loadu_pd(&a[8]); - __m256d row3 = _mm256_loadu_pd(&a[16]); - __m256d row4 = _mm256_loadu_pd(&a[24]); - internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); - _mm256_storeu_pd(&out[0], row1); - _mm256_storeu_pd(&out[8], row2); - _mm256_storeu_pd(&out[16], row3); - _mm256_storeu_pd(&out[24], row4); - } - - { - __m256d row1 = _mm256_loadu_pd(&a[32]); - __m256d row2 = _mm256_loadu_pd(&a[40]); - __m256d row3 = _mm256_loadu_pd(&a[48]); - __m256d row4 = _mm256_loadu_pd(&a[56]); - internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); - _mm256_storeu_pd(&out[4], row1); - _mm256_storeu_pd(&out[12], row2); - _mm256_storeu_pd(&out[20], row3); - _mm256_storeu_pd(&out[28], row4); - } - - { - __m256d row1 = _mm256_loadu_pd(&a[4]); - __m256d row2 = _mm256_loadu_pd(&a[12]); - __m256d row3 = _mm256_loadu_pd(&a[20]); - __m256d row4 = _mm256_loadu_pd(&a[28]); - internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); - _mm256_storeu_pd(&out[32], row1); - _mm256_storeu_pd(&out[40], row2); - _mm256_storeu_pd(&out[48], row3); - _mm256_storeu_pd(&out[56], row4); - } - - { - __m256d row1 = _mm256_loadu_pd(&a[36]); - __m256d row2 = _mm256_loadu_pd(&a[44]); - __m256d row3 = _mm256_loadu_pd(&a[52]); - __m256d row4 = _mm256_loadu_pd(&a[60]); - internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4); - _mm256_storeu_pd(&out[36], row1); - _mm256_storeu_pd(&out[44], row2); - _mm256_storeu_pd(&out[52], row3); - _mm256_storeu_pd(&out[60], row4); - } -} -#endif -//----------------------------------------------------------------------------------------------------------// - - - - -//----------------------------------------------------------------------------------------------------------// -namespace internal { -// To get around compilers recusive inlining depth issue -template -FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { - for (size_t j=0; j -FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { - _transpose(a,out); -} -template<> -FASTOR_INLINE void _transpose_dispatch(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) { - _transpose(a,out); -} -} // internal -//----------------------------------------------------------------------------------------------------------// - -} - -#endif // TRANSPOSE_H - diff --git a/noarch/include/Fastor/backend/transpose/transpose_kernels.h b/noarch/include/Fastor/backend/transpose/transpose_kernels.h deleted file mode 100644 index c0ecef99..00000000 --- a/noarch/include/Fastor/backend/transpose/transpose_kernels.h +++ /dev/null @@ -1,233 +0,0 @@ -#ifndef TRANSPOSE_KERNELS_H -#define TRANSPOSE_KERNELS_H - -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" - -namespace Fastor { - -namespace internal { - -#ifdef FASTOR_SSE_IMPL -// 4x4 PS - defined -// _MM_TRANSPOSE4_PS -#endif - -#ifdef FASTOR_AVX_IMPL - -// 4x4 PD -FASTOR_INLINE void _MM_TRANSPOSE4_PD(__m256d &row0, __m256d &row1, __m256d &row2, __m256d &row3) -{ - __m256d tmp3, tmp2, tmp1, tmp0; - tmp0 = _mm256_shuffle_pd((row0),(row1), 0x0); - tmp2 = _mm256_shuffle_pd((row0),(row1), 0xF); - tmp1 = _mm256_shuffle_pd((row2),(row3), 0x0); - tmp3 = _mm256_shuffle_pd((row2),(row3), 0xF); - row0 = _mm256_permute2f128_pd(tmp0, tmp1, 0x20); - row1 = _mm256_permute2f128_pd(tmp2, tmp3, 0x20); - row2 = _mm256_permute2f128_pd(tmp0, tmp1, 0x31); - row3 = _mm256_permute2f128_pd(tmp2, tmp3, 0x31); -} - -// 8x8 PS -FASTOR_INLINE void _MM_TRANSPOSE8_PS(__m256 &row0, __m256 &row1, __m256 &row2, __m256 &row3, - __m256 &row4, __m256 &row5, __m256 &row6, __m256 &row7) -{ - __m256 __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7; - __m256 __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7; - __t0 = _mm256_unpacklo_ps(row0, row1); - __t1 = _mm256_unpackhi_ps(row0, row1); - __t2 = _mm256_unpacklo_ps(row2, row3); - __t3 = _mm256_unpackhi_ps(row2, row3); - __t4 = _mm256_unpacklo_ps(row4, row5); - __t5 = _mm256_unpackhi_ps(row4, row5); - __t6 = _mm256_unpacklo_ps(row6, row7); - __t7 = _mm256_unpackhi_ps(row6, row7); - __tt0 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(1,0,1,0)); - __tt1 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(3,2,3,2)); - __tt2 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(1,0,1,0)); - __tt3 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(3,2,3,2)); - __tt4 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(1,0,1,0)); - __tt5 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(3,2,3,2)); - __tt6 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(1,0,1,0)); - __tt7 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(3,2,3,2)); - row0 = _mm256_permute2f128_ps(__tt0, __tt4, 0x20); - row1 = _mm256_permute2f128_ps(__tt1, __tt5, 0x20); - row2 = _mm256_permute2f128_ps(__tt2, __tt6, 0x20); - row3 = _mm256_permute2f128_ps(__tt3, __tt7, 0x20); - row4 = _mm256_permute2f128_ps(__tt0, __tt4, 0x31); - row5 = _mm256_permute2f128_ps(__tt1, __tt5, 0x31); - row6 = _mm256_permute2f128_ps(__tt2, __tt6, 0x31); - row7 = _mm256_permute2f128_ps(__tt3, __tt7, 0x31); -} -#endif - -#ifdef FASTOR_AVX512F_IMPL -// 8x8 PD -inline void _MM_TRANSPOSE8_PD(__m512d &row0, __m512d &row1, __m512d &row2, __m512d &row3, - __m512d &row4, __m512d &row5, __m512d &row6, __m512d &row7) -{ - __m512d __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7; - __m512d __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7; - - FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {0, 8 , 1 , 9 , 4 , 12, 5 , 13}; - FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {2, 10, 3 , 11, 6 , 14, 7 , 15}; - FASTOR_ARCH_ALIGN constexpr int64_t idx3[8] = {0, 1 , 8 , 9 , 4 , 5 , 12, 13}; - FASTOR_ARCH_ALIGN constexpr int64_t idx4[8] = {2, 3 , 10, 11, 6 , 7 , 14, 15}; - FASTOR_ARCH_ALIGN constexpr int64_t idx5[8] = {4, 5 , 6 , 7 , 12, 13, 14, 15}; - - __m512i vidx1 = _mm512_load_epi64(idx1); - __m512i vidx2 = _mm512_load_epi64(idx2); - __m512i vidx3 = _mm512_load_epi64(idx3); - __m512i vidx4 = _mm512_load_epi64(idx4); - __m512i vidx5 = _mm512_load_epi64(idx5); - - __t0 = _mm512_permutex2var_pd(row0, vidx1, row1); - __t1 = _mm512_permutex2var_pd(row0, vidx2, row1); - __t2 = _mm512_permutex2var_pd(row2, vidx1, row3); - __t3 = _mm512_permutex2var_pd(row2, vidx2, row3); - __t4 = _mm512_permutex2var_pd(row4, vidx1, row5); - __t5 = _mm512_permutex2var_pd(row4, vidx2, row5); - __t6 = _mm512_permutex2var_pd(row6, vidx1, row7); - __t7 = _mm512_permutex2var_pd(row6, vidx2, row7); - - __tt0 = _mm512_permutex2var_pd(__t0, vidx3, __t2); - __tt1 = _mm512_permutex2var_pd(__t0, vidx4, __t2); - __tt2 = _mm512_permutex2var_pd(__t1, vidx3, __t3); - __tt3 = _mm512_permutex2var_pd(__t1, vidx4, __t3); - __tt4 = _mm512_permutex2var_pd(__t4, vidx3, __t6); - __tt5 = _mm512_permutex2var_pd(__t4, vidx4, __t6); - __tt6 = _mm512_permutex2var_pd(__t5, vidx3, __t7); - __tt7 = _mm512_permutex2var_pd(__t5, vidx4, __t7); - - row0 = _mm512_insertf64x4(__tt0,_mm512_castpd512_pd256(__tt4),0x1); - row1 = _mm512_insertf64x4(__tt1,_mm512_castpd512_pd256(__tt5),0x1); - row2 = _mm512_insertf64x4(__tt2,_mm512_castpd512_pd256(__tt6),0x1); - row3 = _mm512_insertf64x4(__tt3,_mm512_castpd512_pd256(__tt7),0x1); - row4 = _mm512_permutex2var_pd(__tt0, vidx5, __tt4); - row5 = _mm512_permutex2var_pd(__tt1, vidx5, __tt5); - row6 = _mm512_permutex2var_pd(__tt2, vidx5, __tt6); - row7 = _mm512_permutex2var_pd(__tt3, vidx5, __tt7); -} -#endif - - -#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL) -// 16x16 -FASTOR_INLINE void _MM_TRANSPOSE16_PS(const float * FASTOR_RESTRICT mat, float * FASTOR_RESTRICT matT) -{ - __m512 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, ta, tb, tc, td, te, tf; - __m512 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, ra, rb, rc, rd, re, rf; - - int mask; - FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {2, 3, 0, 1, 6, 7, 4, 5}; - FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {1, 0, 3, 2, 5, 4, 7, 6}; - FASTOR_ARCH_ALIGN constexpr int32_t idx3[16] = {1, 0, 3, 2, 5 ,4 ,7 ,6 ,9 ,8 , 11, 10, 13, 12 ,15, 14}; - __m512i vidx1 = _mm512_load_epi64(idx1); - __m512i vidx2 = _mm512_load_epi64(idx2); - __m512i vidx3 = _mm512_load_epi32(idx3); - - t0 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+0])), _mm256_loadu_ps(&mat[ 8*16+0]), 1); - t1 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+0])), _mm256_loadu_ps(&mat[ 9*16+0]), 1); - t2 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+0])), _mm256_loadu_ps(&mat[10*16+0]), 1); - t3 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+0])), _mm256_loadu_ps(&mat[11*16+0]), 1); - t4 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+0])), _mm256_loadu_ps(&mat[12*16+0]), 1); - t5 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+0])), _mm256_loadu_ps(&mat[13*16+0]), 1); - t6 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+0])), _mm256_loadu_ps(&mat[14*16+0]), 1); - t7 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+0])), _mm256_loadu_ps(&mat[15*16+0]), 1); - - t8 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+8])), _mm256_loadu_ps(&mat[ 8*16+8]), 1); - t9 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+8])), _mm256_loadu_ps(&mat[ 9*16+8]), 1); - ta = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+8])), _mm256_loadu_ps(&mat[10*16+8]), 1); - tb = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+8])), _mm256_loadu_ps(&mat[11*16+8]), 1); - tc = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+8])), _mm256_loadu_ps(&mat[12*16+8]), 1); - td = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+8])), _mm256_loadu_ps(&mat[13*16+8]), 1); - te = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+8])), _mm256_loadu_ps(&mat[14*16+8]), 1); - tf = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+8])), _mm256_loadu_ps(&mat[15*16+8]), 1); - - mask= 0xcc; - r0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t0), (__mmask8)mask, vidx1, _mm512_castps_pd(t4))); - r1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t1), (__mmask8)mask, vidx1, _mm512_castps_pd(t5))); - r2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t2), (__mmask8)mask, vidx1, _mm512_castps_pd(t6))); - r3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t3), (__mmask8)mask, vidx1, _mm512_castps_pd(t7))); - r8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t8), (__mmask8)mask, vidx1, _mm512_castps_pd(tc))); - r9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t9), (__mmask8)mask, vidx1, _mm512_castps_pd(td))); - ra = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ta), (__mmask8)mask, vidx1, _mm512_castps_pd(te))); - rb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tb), (__mmask8)mask, vidx1, _mm512_castps_pd(tf))); - - mask= 0x33; - r4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t4), (__mmask8)mask, vidx1, _mm512_castps_pd(t0))); - r5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t5), (__mmask8)mask, vidx1, _mm512_castps_pd(t1))); - r6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t6), (__mmask8)mask, vidx1, _mm512_castps_pd(t2))); - r7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t7), (__mmask8)mask, vidx1, _mm512_castps_pd(t3))); - rc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tc), (__mmask8)mask, vidx1, _mm512_castps_pd(t8))); - rd = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(td), (__mmask8)mask, vidx1, _mm512_castps_pd(t9))); - re = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(te), (__mmask8)mask, vidx1, _mm512_castps_pd(ta))); - rf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tf), (__mmask8)mask, vidx1, _mm512_castps_pd(tb))); - - mask = 0xaa; - t0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r0), (__mmask8)mask, vidx2, _mm512_castps_pd(r2))); - t1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r1), (__mmask8)mask, vidx2, _mm512_castps_pd(r3))); - t4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r4), (__mmask8)mask, vidx2, _mm512_castps_pd(r6))); - t5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r5), (__mmask8)mask, vidx2, _mm512_castps_pd(r7))); - t8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r8), (__mmask8)mask, vidx2, _mm512_castps_pd(ra))); - t9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r9), (__mmask8)mask, vidx2, _mm512_castps_pd(rb))); - tc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rc), (__mmask8)mask, vidx2, _mm512_castps_pd(re))); - td = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rd), (__mmask8)mask, vidx2, _mm512_castps_pd(rf))); - - mask = 0x55; - t2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r2), (__mmask8)mask, vidx2, _mm512_castps_pd(r0))); - t3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r3), (__mmask8)mask, vidx2, _mm512_castps_pd(r1))); - t6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r6), (__mmask8)mask, vidx2, _mm512_castps_pd(r4))); - t7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r7), (__mmask8)mask, vidx2, _mm512_castps_pd(r5))); - ta = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ra), (__mmask8)mask, vidx2, _mm512_castps_pd(r8))); - tb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rb), (__mmask8)mask, vidx2, _mm512_castps_pd(r9))); - te = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(re), (__mmask8)mask, vidx2, _mm512_castps_pd(rc))); - tf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rf), (__mmask8)mask, vidx2, _mm512_castps_pd(rd))); - - mask = 0xaaaa; - r0 = _mm512_mask_permutexvar_ps(t0, (__mmask16)mask, vidx3, t1); - r2 = _mm512_mask_permutexvar_ps(t2, (__mmask16)mask, vidx3, t3); - r4 = _mm512_mask_permutexvar_ps(t4, (__mmask16)mask, vidx3, t5); - r6 = _mm512_mask_permutexvar_ps(t6, (__mmask16)mask, vidx3, t7); - r8 = _mm512_mask_permutexvar_ps(t8, (__mmask16)mask, vidx3, t9); - ra = _mm512_mask_permutexvar_ps(ta, (__mmask16)mask, vidx3, tb); - rc = _mm512_mask_permutexvar_ps(tc, (__mmask16)mask, vidx3, td); - re = _mm512_mask_permutexvar_ps(te, (__mmask16)mask, vidx3, tf); - - mask = 0x5555; - r1 = _mm512_mask_permutexvar_ps(t1, (__mmask16)mask, vidx3, t0); - r3 = _mm512_mask_permutexvar_ps(t3, (__mmask16)mask, vidx3, t2); - r5 = _mm512_mask_permutexvar_ps(t5, (__mmask16)mask, vidx3, t4); - r7 = _mm512_mask_permutexvar_ps(t7, (__mmask16)mask, vidx3, t6); - r9 = _mm512_mask_permutexvar_ps(t9, (__mmask16)mask, vidx3, t8); - rb = _mm512_mask_permutexvar_ps(tb, (__mmask16)mask, vidx3, ta); - rd = _mm512_mask_permutexvar_ps(td, (__mmask16)mask, vidx3, tc); - rf = _mm512_mask_permutexvar_ps(tf, (__mmask16)mask, vidx3, te); - - _mm512_storeu_ps(&matT[ 0*16], r0); - _mm512_storeu_ps(&matT[ 1*16], r1); - _mm512_storeu_ps(&matT[ 2*16], r2); - _mm512_storeu_ps(&matT[ 3*16], r3); - _mm512_storeu_ps(&matT[ 4*16], r4); - _mm512_storeu_ps(&matT[ 5*16], r5); - _mm512_storeu_ps(&matT[ 6*16], r6); - _mm512_storeu_ps(&matT[ 7*16], r7); - _mm512_storeu_ps(&matT[ 8*16], r8); - _mm512_storeu_ps(&matT[ 9*16], r9); - _mm512_storeu_ps(&matT[10*16], ra); - _mm512_storeu_ps(&matT[11*16], rb); - _mm512_storeu_ps(&matT[12*16], rc); - _mm512_storeu_ps(&matT[13*16], rd); - _mm512_storeu_ps(&matT[14*16], re); - _mm512_storeu_ps(&matT[15*16], rf); -} - -#endif - -} // end of namespace internal -} // end of namespace Fastor - - - -#endif // TRANSPOSE_KERNELS_H diff --git a/noarch/include/Fastor/backend/voigt.h b/noarch/include/Fastor/backend/voigt.h deleted file mode 100644 index c4e436ee..00000000 --- a/noarch/include/Fastor/backend/voigt.h +++ /dev/null @@ -1,182 +0,0 @@ -#ifndef VOIGT_H -#define VOIGT_H - -#include "Fastor/tensor/Tensor.h" - -// Conversion of tensors to symmetrised Voigt forms - -namespace Fastor { - -template -struct VoigtType; -template -struct VoigtType { - using return_type = Tensor; -}; -template -struct VoigtType { - using return_type = Tensor; -}; -template -struct VoigtType { - using return_type = Tensor; -}; -template -struct VoigtType { - using return_type = Tensor; -}; -template -struct VoigtType { - using return_type = Tensor; -}; -template -struct VoigtType { - using return_type = Tensor; -}; - - -template::value == 16 - ,bool>::type=0> -FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) { - VoigtA[0] = a_data[0]; - VoigtA[1] = a_data[3]; - VoigtA[2] = 0.5*(a_data[1]+a_data[2]); - VoigtA[3] = VoigtA[1]; - VoigtA[4] = a_data[15]; - VoigtA[5] = 0.5*(a_data[13]+a_data[14]); - VoigtA[6] = VoigtA[2]; - VoigtA[7] = VoigtA[5]; - VoigtA[8] = 0.5*(a_data[5]+a_data[6]); -} - - -template::value == 81 - ,bool>::type=0> -FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) { - - VoigtA[0] = a_data[0]; - VoigtA[1] = a_data[4]; - VoigtA[2] = a_data[8]; - VoigtA[3] = 0.5*(a_data[1]+a_data[3]); - VoigtA[4] = 0.5*(a_data[2]+a_data[6]); - VoigtA[5] = 0.5*(a_data[5]+a_data[7]); - VoigtA[6] = VoigtA[1]; - VoigtA[7] = a_data[40]; - VoigtA[8] = a_data[44]; - VoigtA[9] = 0.5*(a_data[37]+a_data[39]); - VoigtA[10] = 0.5*(a_data[38]+a_data[42]); - VoigtA[11] = 0.5*(a_data[41]+a_data[43]); - VoigtA[12] = VoigtA[2]; - VoigtA[13] = VoigtA[8]; - VoigtA[14] = a_data[80]; - VoigtA[15] = 0.5*(a_data[73]+a_data[75]); - VoigtA[16] = 0.5*(a_data[74]+a_data[78]); - VoigtA[17] = 0.5*(a_data[77]+a_data[79]); - VoigtA[18] = VoigtA[3]; - VoigtA[19] = VoigtA[9]; - VoigtA[20] = VoigtA[15]; - VoigtA[21] = 0.5*(a_data[10]+a_data[12]); - VoigtA[22] = 0.5*(a_data[11]+a_data[15]); - VoigtA[23] = 0.5*(a_data[14]+a_data[16]); - VoigtA[24] = VoigtA[4]; - VoigtA[25] = VoigtA[10]; - VoigtA[26] = VoigtA[16]; - VoigtA[27] = VoigtA[22]; - VoigtA[28] = 0.5*(a_data[20]+a_data[24]); - VoigtA[29] = 0.5*(a_data[23]+a_data[25]); - VoigtA[30] = VoigtA[5]; - VoigtA[31] = VoigtA[11]; - VoigtA[32] = VoigtA[17]; - VoigtA[33] = VoigtA[23]; - VoigtA[34] = VoigtA[29]; - VoigtA[35] = 0.5*(a_data[50]+a_data[52]); -} - - -template::value == 8 - ,bool>::type=0> -FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { - - // 3rd order tensor to 2nd order tensor - // 3rd order tensor should be symmetric in the first two indices - VoigtA[0] = e[0]; - VoigtA[1] = e[1]; - VoigtA[2] = e[6]; - VoigtA[3] = e[7]; - VoigtA[4] = 0.5*(e[2]+e[4]); - VoigtA[5] = 0.5*(e[3]+e[5]); -} - - -template::value == 27 - ,bool>::type=0> -FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { - - // 3rd order tensor to 2nd order tensor - // 3rd order tensor should be symmetric in the first two indices - VoigtA[0] = e[0]; - VoigtA[1] = e[1]; - VoigtA[2] = e[2]; - VoigtA[3] = e[12]; - VoigtA[4] = e[13]; - VoigtA[5] = e[14]; - VoigtA[6] = e[24]; - VoigtA[7] = e[25]; - VoigtA[8] = e[26]; - VoigtA[9] = 0.5*(e[3]+e[9]); - VoigtA[10] = 0.5*(e[4]+e[10]); - VoigtA[11] = 0.5*(e[5]+e[11]); - VoigtA[12] = 0.5*(e[6]+e[18]); - VoigtA[13] = 0.5*(e[7]+e[19]); - VoigtA[14] = 0.5*(e[8]+e[20]); - VoigtA[15] = 0.5*(e[15]+e[21]); - VoigtA[16] = 0.5*(e[16]+e[22]); - VoigtA[17] = 0.5*(e[17]+e[23]); -} - - -template::value == 4 - ,bool>::type=0> -FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { - - VoigtA[0] = e[0]; - VoigtA[1] = e[3]; - VoigtA[2] = 0.5*(e[1]+e[2]); -} - - -template::value == 9 - ,bool>::type=0> -FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) { - - VoigtA[0] = e[0]; - VoigtA[1] = e[4]; - VoigtA[2] = e[8]; - VoigtA[3] = 0.5*(e[1]+e[3]); - VoigtA[4] = 0.5*(e[2]+e[6]); - VoigtA[5] = 0.5*(e[5]+e[7]); -} - - -template -FASTOR_INLINE auto voigt(const Tensor &a) --> typename VoigtType::return_type { - T *a_data = a.data(); - using ret_type = typename VoigtType::return_type; - ret_type voigt_a; - T *VoigtA = voigt_a.data(); - _voigt(a_data,VoigtA); - - return voigt_a; -} - -} - -#endif // VOIGT_H - diff --git a/noarch/include/Fastor/config/config.h b/noarch/include/Fastor/config/config.h deleted file mode 100644 index c10b3e67..00000000 --- a/noarch/include/Fastor/config/config.h +++ /dev/null @@ -1,359 +0,0 @@ -/* This file is part of the FASTOR library. {{{ -Copyright (c) 2020 Roman Poya - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -}}}*/ - -#ifndef CONFIG_H -#define CONFIG_H - -#include -#include -#include -#include -#include -#include - -//------------------------------------------------------------------------------------------------// -//------------------------------------------------------------------------------------------------// - -// Error out for pre-C++14 compiler versions -//------------------------------------------------------------------------------------------------// -#if defined(_MSC_VER) - #if _MSC_VER < 1920 - #error FASTOR REQUIRES AN ISO C++14 COMPLIANT COMPILER - #endif -#elif defined(__GNUC__) || defined(__GNUG__) - #if __cplusplus < 201402L - #error FASTOR REQUIRES AN ISO C++14 COMPLIANT COMPILER - #endif -#endif -//------------------------------------------------------------------------------------------------// - - -// Compiler define macros -//------------------------------------------------------------------------------------------------// -#ifdef __INTEL_COMPILER -#define FASTOR_INTEL __INTEL_COMPILER_BUILD_DATE -#elif defined(__clang__) && defined(__apple_build_version__) -#define FASTOR_APPLECLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) -#elif defined(__clang__) -#define FASTOR_CLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) -#elif defined(__GNUC__) -#define FASTOR_GCC (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -#elif defined(_MSC_VER) -#define FASTOR_MSVC _MSC_FULL_VER -#else -#define FASTOR_UNSUPPORTED_COMPILER 1 -#endif -//------------------------------------------------------------------------------------------------// - - -// Operating system define macros -//------------------------------------------------------------------------------------------------// -#if defined(_WIN32) -#define FASTOR_WINDOWS32_OS 1 -#endif -#if defined(_WIN64) -#define FASTOR_WINDOWS64_OS 1 -#endif -#if defined(_WIN32) || defined(_WIN64) -#define FASTOR_WINDOWS_OS 1 -#endif -#if defined(__unix__) || defined(__unix) -#define FASTOR_UNIX_OS 1 -#endif -#if defined(__linux__) -#define FASTOR_LINUX_OS 1 -#endif -#if defined(__APPLE__) || defined(__MACH__) -#define FASTOR_APPLE_OS 1 -#endif -#if defined(__ANDROID__) || defined(ANDROID) -#define FASTOR_ANDROID_OS 1 -#endif -#if defined(__CYGWIN__) -#define FASTOR_CYGWIN_OS 1 -#endif -//------------------------------------------------------------------------------------------------// - - -// Determine CXX version -//------------------------------------------------------------------------------------------------// -#if defined(FASTOR_MSVC) -#if defined(_MSVC_LANG) - #if _MSVC_LANG == 199711L - #define FASTOR_CXX_VERSION 1998 - #elif _MSVC_LANG == 201103L - #define FASTOR_CXX_VERSION 2011 - #elif _MSVC_LANG == 201402L - #define FASTOR_CXX_VERSION 2014 - #elif _MSVC_LANG == 201703L - #define FASTOR_CXX_VERSION 2017 - #elif _MSVC_LANG > 201703L - #define FASTOR_CXX_VERSION 2020 - #endif -#endif -#else -#if defined(__cplusplus) - #if __cplusplus == 199711L - #define FASTOR_CXX_VERSION 1998 - #elif __cplusplus == 201103L - #define FASTOR_CXX_VERSION 2011 - #elif __cplusplus == 201402L - #define FASTOR_CXX_VERSION 2014 - #elif __cplusplus == 201703L - #define FASTOR_CXX_VERSION 2017 - #elif __cplusplus > 201703L - #define FASTOR_CXX_VERSION 2020 - #endif -#endif -#endif -//------------------------------------------------------------------------------------------------// - - -// Force inline and no-inline macros -//------------------------------------------------------------------------------------------------// -#if defined(__GNUC__) || defined(__GNUG__) - #define FASTOR_INLINE inline __attribute__((always_inline)) - #define FASTOR_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) - #define FASTOR_INLINE __forceinline - #define FASTOR_NOINLINE __declspec(noinline) -#endif - -#if defined(__GNUC__) || defined(__GNUG__) -#define FASTOR_RESTRICT __restrict__ -#elif defined(_MSC_VER) -#define FASTOR_RESTRICT __restrict -#endif - -// Traditional inline which works will and helps the compiler -// eliminate a lot of code -#define FASTOR_HINT_INLINE inline -//------------------------------------------------------------------------------------------------// - - -// C++17 [if constexpr] define -//------------------------------------------------------------------------------------------------// -#if FASTOR_CXX_VERSION >= 2017 - #define FASTOR_HAS_IF_CONSTEXPR 1 - #define FASTOR_IF_CONSTEXPR if constexpr -#else - #define FASTOR_HAS_IF_CONSTEXPR 0 - #define FASTOR_IF_CONSTEXPR if -#endif -//------------------------------------------------------------------------------------------------// - - - -// Intrinsics defines -//------------------------------------------------------------------------------------------------// -#ifdef FASTOR_MSVC - #ifdef _M_IX86_FP - #if _M_IX86_FP >= 1 - #ifndef __SSE__ - #define __SSE__ 1 - #endif - #endif - #if _M_IX86_FP >= 2 - #ifndef __SSE2__ - #define __SSE2__ 1 - #endif - #endif - #elif defined(_M_AMD64) - #ifndef __SSE__ - #define __SSE__ 1 - #endif - #ifndef __SSE2__ - #define __SSE2__ 1 - #endif - #endif -#endif - -#if defined(__MIC__) - #define FASTOR_MIC_IMPL 1 -#endif -#if defined(__AVX512F__) - #define FASTOR_AVX512F_IMPL 1 -#endif -#if defined(__AVX512CD__) - #define FASTOR_AVX512CD_IMPL 1 -#endif -#if defined(__AVX512BW__) - #define FASTOR_AVX512BW_IMPL 1 -#endif -#if defined(__AVX512DQ__) - #define FASTOR_AVX512DQ_IMPL 1 -#endif -#if defined(__AVX512VL__) - #define FASTOR_AVX512VL_IMPL 1 -#endif -#if defined(__AVX2__) - #define FASTOR_AVX2_IMPL 1 -#endif -#if defined(__AVX__) - #define FASTOR_AVX_IMPL 1 -#endif -#if defined(__SSE4_2__) - #define FASTOR_SSE4_2_IMPL 1 -#endif -#if defined(__SSE4_1__) - #define FASTOR_SSE4_1_IMPL 1 -#endif -#if defined(__SSE3__) - #define FASTOR_SSE3_IMPL 1 -#endif -#if defined(__SSSE3__) - #define FASTOR_SSSE3_IMPL 1 -#endif -#if defined(__SSE2__) - #define FASTOR_SSE2_IMPL 1 -#endif - -#if defined(__FMA4__) - #define FASTOR_FMA4_IMPL 1 -#endif -#if defined(__FMA__) - #define FASTOR_FMA_IMPL 1 -#endif -// #if !defined(__FMA__) && defined(__AVX2__) -// #define __FMA__ 1 -// #endif - - -// Get around MSVC issue -#if defined(FASTOR_WINDOWS_OS) || defined(FASTOR_MSVC) - #if defined(FASTOR_AVX512F_IMPL) - #if !defined(FASTOR_SSE2_IMPL) - #define FASTOR_SSE2_IMPL 1 - #endif - #if !defined(FASTOR_SSE3_IMPL) - #define FASTOR_SSE3_IMPL 1 - #endif - #if !defined(FASTOR_SSSE3_IMPL) - #define FASTOR_SSSE3_IMPL 1 - #endif - #if !defined(FASTOR_SSE4_1_IMPL) - #define FASTOR_SSE4_1_IMPL 1 - #endif - #if !defined(FASTOR_SSE4_2_IMPL) - #define FASTOR_SSE4_2_IMPL 1 - #endif - #if !defined(FASTOR_AVX_IMPL) - #define FASTOR_AVX_IMPL 1 - #endif - #if !defined(FASTOR_AVX2_IMPL) - #define FASTOR_AVX2_IMPL 1 - #endif - #elif defined(FASTOR_AVX_IMPL) || defined(FASTOR_AVX2_IMPL) - #if !defined(FASTOR_SSE2_IMPL) - #define FASTOR_SSE2_IMPL 1 - #endif - #if !defined(FASTOR_SSE3_IMPL) - #define FASTOR_SSE3_IMPL 1 - #endif - #if !defined(FASTOR_SSSE3_IMPL) - #define FASTOR_SSSE3_IMPL 1 - #endif - #if !defined(FASTOR_SSE4_1_IMPL) - #define FASTOR_SSE4_1_IMPL 1 - #endif - #if !defined(FASTOR_SSE4_2_IMPL) - #define FASTOR_SSE4_2_IMPL 1 - #endif - #endif -#endif - - -#if defined(FASTOR_AVX512F_IMPL) || defined(FASTOR_AVX512CD_IMPL) || defined(FASTOR_AVX512BW_IMPL) || defined(FASTOR_AVX512DQ_IMPL) || defined(FASTOR_AVX512VL_IMPL) - #define FASTOR_AVX512_IMPL 1 -#endif -#if defined(FASTOR_AVX2_IMPL) && !defined(FASTOR_AVX_IMPL) - #define FASTOR_AVX_IMPL 1 -#endif -#if defined(FASTOR_SSE2_IMPL) || defined(FASTOR_SSSE3_IMPL) || defined(FASTOR_SSE3_IMPL) || defined(FASTOR_SSE4_1_IMPL) || defined(FASTOR_SSE4_2_IMPL) - #define FASTOR_SSE_IMPL 1 -#endif - -#if !defined(FASTOR_MIC_IMPL) && !defined(FASTOR_AVX512_IMPL) && !defined(FASTOR_AVX_IMPL) && !defined(FASTOR_SSE_IMPL) -#define FASTOR_SCALAR_IMPL 1 -#endif - - -#ifdef FASTOR_SSE2_IMPL -#include -#endif -#ifdef FASTOR_SSE3_IMPL -#include -#endif -#ifdef FASTOR_SSSE3_IMPL -#include -#endif -#ifdef FASTOR_SSE4_1_IMPL -#include -#endif -#ifdef FASTOR_SSE4_2_IMPL -#include -#endif -#ifdef FASTOR_AVX_IMPL -#include -#endif - - -// Mask loading -//------------------------------------------------------------------------------------------------// -#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) - #define FASTOR_HAS_AVX512_MASKS 1 -#endif -//------------------------------------------------------------------------------------------------// - -// Horizontal add -//------------------------------------------------------------------------------------------------// -#if defined(FASTOR_AVX512F_IMPL) -#if defined(FASTOR_INTEL) - #define FASTOR_HAS_AVX512_REDUCE_ADD 1 -#elif defined (FASTOR_GCC) && __GNUC__ >= 7 - #define FASTOR_HAS_AVX512_REDUCE_ADD 1 -#elif defined (FASTOR_GCC) && __clang_major__ >= 4 - #define FASTOR_HAS_AVX512_REDUCE_ADD 1 -#endif -#endif -//------------------------------------------------------------------------------------------------// - -// AVX512 abs -//------------------------------------------------------------------------------------------------// -#if defined(FASTOR_AVX512F_IMPL) -#if defined(FASTOR_INTEL) - #define FASTOR_HAS_AVX512_ABS 1 -#elif defined (FASTOR_GCC) && __GNUC__ >= 7 && __GNUC_MINOR__ >= 4 - #define FASTOR_HAS_AVX512_ABS 1 -#elif defined (FASTOR_GCC) && __clang_major__ >= 4 - #define FASTOR_HAS_AVX512_ABS 1 -#endif -#endif -//------------------------------------------------------------------------------------------------// -//------------------------------------------------------------------------------------------------// - -#include "Fastor/config/macros.h" - -//------------------------------------------------------------------------------------------------// - -#endif // CONFIG_H diff --git a/noarch/include/Fastor/config/cpuid.h b/noarch/include/Fastor/config/cpuid.h deleted file mode 100644 index 498883fe..00000000 --- a/noarch/include/Fastor/config/cpuid.h +++ /dev/null @@ -1,71 +0,0 @@ -/* This file is part of the FASTOR library. {{{ -Copyright (c) 2020 Roman Poya - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -}}}*/ - -#ifndef CPUID_H -#define CPUID_H - -#ifdef _WIN32 -#include -#include -typedef unsigned __int32 uint32_t; -#else -#include -#endif - - -namespace Fastor { - -class CPUID { - uint32_t regs[4]; - -public: - explicit CPUID(unsigned i) { -#ifdef _WIN32 - __cpuid((int *)regs, (int)i); - -#else - asm volatile - ("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3]) - : "a" (i), "c" (0)); - // ECX is set to zero for CPUID function 4 -#endif - } - - const uint32_t& EAX() const {return regs[0];} // cpu base frequency - const uint32_t& EBX() const {return regs[1];} // cpu max frequency - const uint32_t& ECX() const {return regs[2];} // bus (reference) frequency - const uint32_t& EDX() const {return regs[3];} -}; - -// Usage: -// CPUID cpuID(0); -// std::string vendor; -// vendor += std::string((const char *)&cpuID.EBX(), 4); -// vendor += std::string((const char *)&cpuID.EDX(), 4); -// vendor += std::string((const char *)&cpuID.ECX(), 4); -// std::cout << "CPU vendor = " << vendor << std::endl; - - -} // Fastor - -#endif // CPUID_H diff --git a/noarch/include/Fastor/config/macros.h b/noarch/include/Fastor/config/macros.h deleted file mode 100644 index 8e772443..00000000 --- a/noarch/include/Fastor/config/macros.h +++ /dev/null @@ -1,292 +0,0 @@ -/* This file is part of the FASTOR library. {{{ -Copyright (c) 2020 Roman Poya - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -}}}*/ - -#ifndef FASTOR_MACROS_H -#define FASTOR_MACROS_H - - -/* This file contains only the set of macros that Fastor defines and can be used or altered by the user -*/ -//------------------------------------------------------------------------------------------------// -// Compiler version -//------------------------------------------------------------------------------------------------// -#define FASTOR_MAJOR 0 -#define FASTOR_MINOR 6 -#define FASTOR_PATCHLEVEL 4 -#define FASTOR_VERSION (FASTOR_MAJOR * 10000 + FASTOR_MINOR * 100 + FASTOR_PATCHLEVEL) -//------------------------------------------------------------------------------------------------// - -//------------------------------------------------------------------------------------------------// -//------------------------------------------------------------------------------------------------// -//------------------------------------------------------------------------------------------------// -// ICC's default option is fast anyway (i.e. -fp-model fast=1) -// but it does not define the __FAST_MATH__ macro -#if defined(__FAST_MATH__) -#define FASTOR_FAST_MATH -// Use the following for unsafe math -// Only for GCC & Clang, activates fast div/rcp -//#define FASTOR_UNSAFE_MATH -#endif - -// Runtime checks - off by default under release -//------------------------------------------------------------------------------------------------// -#ifndef FASTOR_ENABLE_RUNTIME_CHECKS -#define FASTOR_ENABLE_RUNTIME_CHECKS 0 -#endif -//------------------------------------------------------------------------------------------------// - -// Bounds and shape checking - ON by default under debug -//------------------------------------------------------------------------------------------------// -#if !defined(NDEBUG) || FASTOR_ENABLE_RUNTIME_CHECKS -#ifndef FASTOR_BOUNDS_CHECK -#define FASTOR_BOUNDS_CHECK 1 -#endif -#ifndef FASTOR_SHAPE_CHECK -#define FASTOR_SHAPE_CHECK 1 -#endif -#else -#ifndef FASTOR_BOUNDS_CHECK -#define FASTOR_BOUNDS_CHECK 0 -#endif -#ifndef FASTOR_SHAPE_CHECK -#define FASTOR_SHAPE_CHECK 0 -#endif -#endif -//------------------------------------------------------------------------------------------------// - - -// Aliasing -//------------------------------------------------------------------------------------------------// -#ifndef FASTOR_NO_ALIAS -#define FASTOR_NO_ALIAS 0 -#endif -//------------------------------------------------------------------------------------------------// - - -// Macros used throughout Fastor -//------------------------------------------------------------------------------------------------// -//#define FASTOR_COPY_EXPR -//#define FASTOR_DONT_VECTORISE -//#define FASTOR_DONT_PERFORM_OP_MIN -//#define FASTOR_USE_OLD_OUTER -//#define FASTOR_USE_OLD_INTRINSICS -//#define FASTOR_USE_HADD -//#define FASTOR_USE_VECTORISED_EXPR_ASSIGN // to use vectorised expression assignment -//#define FASTOR_ZERO_INITIALISE -//#define FASTOR_USE_OLD_NDVIEWS -//#define FASTOR_DISPATCH_DIV_TO_MUL_EXPR // change BINARY_DIV_OP to BINARY_MUL_OP for Expression/Number -//#define FASTOR_DISABLE_SPECIALISED_CTR - -//FASTOR_MATMUL_OUTER_BLOCK_SIZE 2 -//FASTOR_MATMUL_INNER_BLOCK_SIZE 2 - -//FASTOR_TRANS_OUTER_BLOCK_SIZE 2 -//FASTOR_TRANS_INNER_BLOCK_SIZE 2 - -#ifndef FASTOR_BLAS_SWITCH_MATRIX_SIZE -#define FASTOR_BLAS_SWITCH_MATRIX_SIZE 16 -#endif - -// FASTOR_NIL -//------------------------------------------------------------------------------------------------// -#define FASTOR_NIL 0 -//------------------------------------------------------------------------------------------------// - - -// Bit sizes -//------------------------------------------------------------------------------------------------// -#define FASTOR_AVX512_BITSIZE 512 -#define FASTOR_AVX_BITSIZE 256 -#define FASTOR_SSE_BITSIZE 128 -#define FASTOR_DOUBLE_BITSIZE (sizeof(double)*8) -#define FASTOR_SINGLE_BITSIZE (sizeof(float)*8) -#ifndef FASTOR_SCALAR_BITSIZE -#define FASTOR_SCALAR_BITSIZE FASTOR_DOUBLE_BITSIZE -#endif -//------------------------------------------------------------------------------------------------// - - -// Alignment -//------------------------------------------------------------------------------------------------// -#ifndef FASTOR_MEMORY_ALIGNMENT_VALUE -#if defined(FASTOR_AVX512_IMPL) -#define FASTOR_MEMORY_ALIGNMENT_VALUE 64 -#elif defined(FASTOR_AVX_IMPL) -#define FASTOR_MEMORY_ALIGNMENT_VALUE 32 -#elif defined(FASTOR_SSE_IMPL) -#define FASTOR_MEMORY_ALIGNMENT_VALUE 16 -#else -#define FASTOR_MEMORY_ALIGNMENT_VALUE 8 -#endif -#endif - -/* User controllable alignment for Fastor containers */ -#define FASTOR_ALIGN alignas((FASTOR_MEMORY_ALIGNMENT_VALUE)) -/* Strict non-controllable alignment for Fastor's internal use */ -#define FASTOR_ARCH_ALIGN alignas((FASTOR_MEMORY_ALIGNMENT_VALUE)) - -// Conservative alignment for SIMD -#if defined(FASTOR_DONT_ALIGN) || defined(FASTOR_DONT_VECTORISE) - #define FASTOR_ALIGNED false -#else - #define FASTOR_ALIGNED true -#endif - -#define FASTOR_ISALIGNED(POINTER, BYTE_COUNT) \ - (((uintptr_t)(const void *)(POINTER)) % (BYTE_COUNT) == 0) -//------------------------------------------------------------------------------------------------// - - -// Assertions -//------------------------------------------------------------------------------------------------// -namespace Fastor { -// Strong unconditional assert -FASTOR_INLINE void FASTOR_EXIT_ASSERT(bool cond, const std::string &msg="") { - if (cond==false) { - throw std::runtime_error(msg); - } -} - -#if !defined(NDEBUG) || FASTOR_ENABLE_RUNTIME_CHECKS -#define FASTOR_ASSERT(COND, MESSAGE) FASTOR_EXIT_ASSERT(COND, MESSAGE) -#else -#define FASTOR_ASSERT(COND, MESSAGE) -#endif - -// Warn -FASTOR_INLINE void FASTOR_WARN(bool cond, const std::string &x) { - if (cond==false) { - std::cout << x << std::endl; - } -} -} // end of namespace Fastor -//------------------------------------------------------------------------------------------------// - - -// Warnings -//------------------------------------------------------------------------------------------------// -// Static warn -#ifndef FASTOR_NO_STATIC_WARNING -#if defined(__GNUC__) - #define FASTOR_DEPRECATE(foo, msg) foo __attribute__((deprecated(msg))) -#elif defined(_MSC_VER) - #define FASTOR_DEPRECATE(foo, msg) __declspec(deprecated(msg)) foo -#else - #error FASTOR STATIC WARNING DOES NOT SUPPORT THIS COMPILER -#endif - -#define FASTOR_CAT(x,y) FASTOR_CAT1_(x,y) -#define FASTOR_CAT1_(x,y) x##y - - -namespace Fastor { -namespace useless -{ - struct true_type {}; - struct false_type {}; - template struct converter : public true_type {}; - template <> struct converter<0> : public false_type {}; -} - -#define FASTOR_STATIC_WARN(cond, msg) \ -struct FASTOR_CAT(static_warning,__LINE__) { \ - FASTOR_DEPRECATE(void _(::useless::false_type const& ),msg) {}; \ - void _(::useless::true_type const& ) {}; \ - FASTOR_CAT(static_warning,__LINE__)() {_(::useless::converter<(cond)>());} \ -} - -} // end of namespace Fastor -#endif // FASTOR_NO_STATIC_WARNING -//------------------------------------------------------------------------------------------------// - - -// Token to string -//------------------------------------------------------------------------------------------------// -#define FASTOR_TOSTRING_(X) #X -#define FASTOR_TOSTRING(X) FASTOR_TOSTRING_(X) -//------------------------------------------------------------------------------------------------// - -// asm comment -//------------------------------------------------------------------------------------------------// -#define FASTOR_ASM(STR) asm(STR ::) -//------------------------------------------------------------------------------------------------// - -// unused -//------------------------------------------------------------------------------------------------// -namespace Fastor { -// clobber -template void unused(T &&x) { -#ifndef _WIN32 - asm("" ::"m"(x)); -#endif -} -template void unused(T&& x, U&& ...y) { unused(x); unused(y...); } -} // end of namespace Fastor -//------------------------------------------------------------------------------------------------// - - -// SIMD round-down -//------------------------------------------------------------------------------------------------// -#define ROUND_DOWN2(x, s) ((x) & ~((s)-1)) -#define ROUND_DOWN(x, s) ROUND_DOWN2(x,s) -//------------------------------------------------------------------------------------------------// - - -// FASTOR CONSTRUCTS -//------------------------------------------------------------------------------------------------// -#include - -namespace Fastor { - -using FASTOR_INDEX = size_t; -using Int64 = int64_t; -using DEFAULT_FLOAT_TYPE = double; -using DFT = DEFAULT_FLOAT_TYPE; -using FASTOR_VINDEX = volatile size_t; - -constexpr int RowMajor = 0; -constexpr int ColumnMajor = 1; - - -constexpr int FASTOR_Symmetric = -100; -constexpr int FASTOR_NonSymmetric = -101; -constexpr int FASTOR_AntiSymmetric = -102; -constexpr int FASTOR_Identity = -103; -constexpr int FASTOR_One = -104; -constexpr int FASTOR_Zero = -105; - -constexpr int FASTOR_ThreeD = -150; -constexpr int FASTOR_TwoD = -151; -constexpr int FASTOR_PlaneStrain = -152; -constexpr int FASTOR_PlaneStress = -153; -constexpr int FASTOR_Voigt = -106; - -constexpr int DepthFirst = -200; -constexpr int NoDepthFirst = -201; - -constexpr double PRECI_TOL = 1e-14; - -} -//------------------------------------------------------------------------------------------------// - -#endif // FASTOR_MACROS_H diff --git a/noarch/include/Fastor/experimental/SingleValueTensor.h b/noarch/include/Fastor/experimental/SingleValueTensor.h deleted file mode 100644 index f59f5b9a..00000000 --- a/noarch/include/Fastor/experimental/SingleValueTensor.h +++ /dev/null @@ -1,274 +0,0 @@ -#ifndef SINGLEVALUE_TENSOR_H -#define SINGLEVALUE_TENSOR_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/TensorIO.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/meta/tensor_meta.h" -#include - -namespace Fastor { - -template -class SingleValueTensor : public AbstractTensor,sizeof...(Rest)> { -public: - using scalar_type = T; - using simd_vector_type = choose_best_simd_vector_t; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = SingleValueTensor; - using dimension_t = std::integral_constant; - static constexpr FASTOR_INLINE FASTOR_INDEX rank() {return sizeof...(Rest);} - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return pack_prod::value;} - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX dim) const { -#if FASTOR_SHAPE_CHECK - FASTOR_ASSERT(dim>=0 && dim < sizeof...(Rest), "TENSOR SHAPE MISMATCH"); -#endif - const FASTOR_INDEX DimensionHolder[sizeof...(Rest)] = {Rest...}; - return DimensionHolder[dim]; - } - - template - SingleValueTensor(U num) : _data{(T)num} {} - SingleValueTensor(const SingleValueTensor &a) : _data{(T)a.data()[0]} {} - FASTOR_INLINE SingleValueTensor(const AbstractTensor,sizeof...(Rest)>& src_) : _data{T(0)} { - } - - constexpr FASTOR_INLINE T* data() const { return const_cast(this->_data.data());} - - // Index retriever - //----------------------------------------------------------------------------------------------------------// - template - FASTOR_INLINE int get_mem_index(U index) const { -#if FASTOR_BOUNDS_CHECK - FASTOR_ASSERT((index>=0 && index::value,bool>::type =0> - FASTOR_INLINE int get_flat_index(Args ... args) const { -#if FASTOR_BOUNDS_CHECK - int largs[sizeof...(Args)] = {args...}; - constexpr int DimensionHolder[dimension_t::value] = {Rest...}; - for (int i=0; i=0 && largs[i] &as) const { -#if FASTOR_BOUNDS_CHECK - constexpr std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; - int index = 0; - for (int i=0; i=0 && index -#define SCALAR_INDEXING_CONST_H - //----------------------------------------------------------------------------------------------------------// - - // Expression templates evaluators - //----------------------------------------------------------------------------------------------------------// -#undef TENSOR_EVALUATOR_H - #include "Fastor/tensor/TensorEvaluator.h" -#define TENSOR_EVALUATOR_H - //----------------------------------------------------------------------------------------------------------// - - // Tensor methods - //----------------------------------------------------------------------------------------------------------// -#undef TENSOR_METHODS_CONST_H - #include "Fastor/tensor/TensorMethods.h" -#define TENSOR_METHODS_CONST_H - //----------------------------------------------------------------------------------------------------------// - - // Converters - //----------------------------------------------------------------------------------------------------------// -#undef PODCONVERTERS_H - #include "Fastor/tensor/PODConverters.h" -#define PODCONVERTERS_H - //----------------------------------------------------------------------------------------------------------// - - // Cast method - //----------------------------------------------------------------------------------------------------------// - template - FASTOR_INLINE SingleValueTensor cast() const { - SingleValueTensor out(static_cast(_data[0])); - return out; - } - //----------------------------------------------------------------------------------------------------------// - - //----------------------------------------------------------------------------------------------------------// -private: - const FASTOR_ALIGN std::array _data; - //----------------------------------------------------------------------------------------------------------// -}; - -// template -// constexpr const T SingleValueTensor::_data[pack_prod::value]; - -template -struct tensor_type_finder> { - using type = SingleValueTensor; -}; - -template -struct scalar_type_finder> { - using type = T; -}; - - -FASTOR_MAKE_OS_STREAM_TENSOR0(SingleValueTensor) -FASTOR_MAKE_OS_STREAM_TENSOR1(SingleValueTensor) -FASTOR_MAKE_OS_STREAM_TENSOR2(SingleValueTensor) -FASTOR_MAKE_OS_STREAM_TENSORn(SingleValueTensor) - - -template -FASTOR_INLINE SingleValueTensor transpose(const SingleValueTensor &a) { - return SingleValueTensor(a(0,0)); -} - -template -T trace(const SingleValueTensor &a) { - return M*a(0,0); -} - -template -FASTOR_INLINE T determinant(const SingleValueTensor &a) { - // determinant of a single value tensor is 0 - return 0.; -} - -template -FASTOR_INLINE double norm(const SingleValueTensor &a) { - return a(0,0)*std::sqrt(double(M*N)); -} - -template -FASTOR_INLINE Tensor inverse(const SingleValueTensor &a) { - // A single value tensor is not invertible - Tensor out(std::numeric_limits::quiet_NaN()); - return out; -} - -template -FASTOR_INLINE Tensor matmul(const Tensor &a, const SingleValueTensor &b) { - - using V = SIMDVector; - - Tensor out; - T *out_data = out.data(); - const T *a_data = a.data(); - const T b_value = b(0,0); - - for (size_t i=0; i -FASTOR_INLINE Tensor matmul(const SingleValueTensor &a, const Tensor &b) { - return transpose(matmul(transpose(b),transpose(a))); -} - -template -FASTOR_INLINE SingleValueTensor matmul(const SingleValueTensor &a, const SingleValueTensor &b) { - - const T a_value = a(0,0); - const T b_value = b(0,0); - // matmul is just this - SingleValueTensor out(a_value*b_value*K); - - // Not necessary - // using V = SIMDVector; - // V vec_out; - // size_t j=0; - // for (; j out(out_value); - - return out; -} - - - - -// This one is almost like a compile time einsum -template -FASTOR_INLINE -typename contraction_impl::type,SingleValueTensor, - typename std_ext::make_index_sequence::type>::type -einsum(const SingleValueTensor &a, const SingleValueTensor &b) { - - static_assert(einsum_index_checker::type>::value, - "INDICES FOR EINSUM FUNCTION CANNOT APPEAR MORE THAN TWICE. USE CONTRACTION INSTEAD"); - - std::array idx0; std::copy_n(Index_I::_IndexHolder,Index_I::NoIndices,idx0.begin()); - std::array idx1; std::copy_n(Index_J::_IndexHolder,Index_J::NoIndices,idx1.begin()); - std::array dims0 = {Rest0...}; - - // n^2 but it is okay as this is a small loop with compile time spans - size_t total = 1; - for (int i=0; i::type,SingleValueTensor, - typename std_ext::make_index_sequence::type>::type; - OutTensor out(out_value); - return out; -} - - -} // end of namespace Fastor - - -#endif // SINGLEVALUE_TENSOR_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_add_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_add_op.h deleted file mode 100644 index 4f9d6cd8..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_add_op.h +++ /dev/null @@ -1,247 +0,0 @@ -#ifndef BINARY_ADD_OP_H -#define BINARY_ADD_OP_H - -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/expressions/expression_traits.h" - - - -namespace Fastor { - -template -struct BinaryAddOp: public AbstractTensor,DIM0> { -private: - expression_t _lhs; - expression_t _rhs; -public: - - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = binary_op_simd_vector_t >; - using simd_abi_type = typename simd_vector_type::abi_type; - - FASTOR_INLINE BinaryAddOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} - - FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} - template::value && - !std::is_arithmetic::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); -#endif - return _rhs.size(); - } - - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} - template::value && - !std::is_arithmetic::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); -#endif - return _rhs.dimension(i); - } - constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} - constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} - - // Generic version of eval - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - // Delay evaluation using a helper function to fully inform BinaryOp about _lhs and _rhs - return helper(i); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return _lhs.template eval(i) + _rhs.template eval(i); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return (U)_lhs + _rhs.template eval(i); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return _lhs.template eval(i) + (U)_rhs; - } - - - // scalar based - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return helper_s(i); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) + _rhs.template eval_s(i); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return (U)_lhs + _rhs.template eval_s(i); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) + (U)_rhs; - } - - - // for 2D tensors - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper(i,j); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval(i,j) + _rhs.template eval(i,j); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return (U)_lhs + _rhs.template eval(i,j); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval(i,j) + (U)_rhs; - } - - // scalar based (for 2D tensors) - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper_s(i,j); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) + _rhs.template eval_s(i,j); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return (U)_lhs + _rhs.template eval_s(i,j); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) + (U)_rhs; - } - - // for nD tensors - template - FASTOR_INLINE SIMDVector teval(const std::array &as) const { - return thelper(as); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) + _rhs.template teval(as); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return (U)_lhs + _rhs.template teval(as); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) + (U)_rhs; - } - - // scalar based (for nD tensors) - template - FASTOR_INLINE U teval_s(const std::array &as) const { - return thelper_s(as); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) + _rhs.template teval_s(as); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return (U)_lhs + _rhs.template teval_s(as); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) + (U)_rhs; - } -}; - -template::value && - !std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinaryAddOp operator+(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinaryAddOp(_lhs.self(), _rhs.self()); -} -template::value && - std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinaryAddOp operator+(const AbstractTensor &_lhs, TRhs bb) { - return BinaryAddOp(_lhs.self(), bb); -} -template::value && - !std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinaryAddOp operator+(TLhs bb, const AbstractTensor &_rhs) { - return BinaryAddOp(bb,_rhs.self()); -} - -template::value && - !std::is_arithmetic::value && - DIM0!=DIM1,bool>::type = 0 > -FASTOR_INLINE BinaryAddOp::value> -operator+(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinaryAddOp::value>(_lhs.self(), _rhs.self()); -} - -} // end of namespace Fastor - - -#endif // BINARY_ADD_OP_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_assignment.h b/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_assignment.h deleted file mode 100644 index cdc73fec..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_assignment.h +++ /dev/null @@ -1,156 +0,0 @@ -#ifndef BINARY_ARITHMETIC_ASSIGNMENT_H -#define BINARY_ARITHMETIC_ASSIGNMENT_H - - -#include "Fastor/expressions/binary_ops/binary_arithmetic_ops.h" -#include "Fastor/tensor/Aliasing.h" - -namespace Fastor { - -// Create assign for all binrary arithmetic ops -#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(NAME, ASSIGN_TYPE, OP_ASSIGN_TYPE)\ -template && !is_primitive_v_ && !(requires_evaluation_v || requires_evaluation_v), bool> = false >\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template && !is_primitive_v_ && (requires_evaluation_v || requires_evaluation_v), bool> = false >\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ - assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ -}\ -template && !is_primitive_v_ && !requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template && !is_primitive_v_ && requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - assign ##ASSIGN_TYPE (dst.self(), src.lhs());\ - assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ -}\ -template && is_primitive_v_ && !requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template && is_primitive_v_ && requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ - assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs());\ -}\ - -// Create assign_add, assign_sub for BinaryAddOp and BinarySubOp -#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(NAME, ASSIGN_TYPE, OP_ASSIGN_TYPE)\ -template && !is_primitive_v_ && !(requires_evaluation_v || requires_evaluation_v), bool> = false >\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template && !is_primitive_v_ && (requires_evaluation_v || requires_evaluation_v), bool> = false >\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - if (!does_alias(dst.self(),src.rhs().self())) {\ - assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ - assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ - }\ - else{\ - const Derived tmp(dst.self());\ - assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ - assign ##OP_ASSIGN_TYPE (dst.self(), tmp);\ - }\ -}\ -template && !is_primitive_v_ && !requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template && !is_primitive_v_ && requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - assign ##ASSIGN_TYPE (dst.self(), src.lhs());\ - assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\ -}\ -template && is_primitive_v_ && !requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template && is_primitive_v_ && requires_evaluation_v, bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\ - assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs());\ -}\ - - -// Create assign_add, assign_sub for BinaryMulOp and BinaryDivOp -// Create assign_mul, assign_div for all binary ops -#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(NAME, ASSIGN_TYPE)\ -template || requires_evaluation_v),bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template || requires_evaluation_v),bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - using result_type = typename Binary ##NAME ## Op::result_type;\ - const result_type a(src.self());\ - trivial_assign ##ASSIGN_TYPE (dst.self(), a);\ -}\ - -// assign -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Add, , _add) -// assign_add -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Add, _add, _add) -// assign_sub -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Add, _sub, _sub) -// assign_mul -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Add, _mul) -// assign_div -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Add, _div) - -// assign -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Sub, , _sub) -// assign_add -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Sub, _add, _sub) -// assign_sub -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Sub, _sub, _add) -// assign_mul -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Sub, _mul) -// assign_div -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Sub, _div) - - -// assign -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Mul, , _mul) -// assign_add -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _add) -// assign_sub -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _sub) -// assign_mul -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _mul) -// assign_div -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _div) - - -// assign -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Div, , _div) -// assign_add -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _add) -// assign_sub -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _sub) -// assign_mul -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _mul) -// assign_div -FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _div) - - -} // end of namespace Fastor - - -#endif // BINARY_ARITHMETIC_ASSIGNMENT_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_ops.h b/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_ops.h deleted file mode 100644 index 664692cc..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_ops.h +++ /dev/null @@ -1,229 +0,0 @@ -#ifndef BINARY_ARITHMETIC_OP_H -#define BINARY_ARITHMETIC_OP_H - -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - - -#define FASTOR_MAKE_BINARY_ARITHMETIC_OPS(OP, NAME, EVAL_TYPE) \ -template\ -struct Binary ##NAME ## Op: public AbstractTensor,DIM0> {\ - expression_t _lhs;\ - expression_t _rhs;\ -public:\ - static constexpr FASTOR_INDEX Dimension = DIM0;\ - static constexpr FASTOR_INDEX rank() {return DIM0;}\ - using scalar_type = typename scalar_type_finder>::type;\ - using simd_vector_type = binary_op_simd_vector_t< Binary ##NAME ## Op >;\ - using simd_abi_type = typename simd_vector_type::abi_type;\ - using result_type = binary_arithmetic_result_t< Binary ##NAME ## Op >;\ - FASTOR_INLINE Binary ##NAME ## Op(expression_t inlhs, expression_t inrhs) : _lhs(inlhs), _rhs(inrhs) {}\ - FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\ - template &&\ - !is_primitive_v_,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {\ - FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");\ - return _rhs.size();\ - }\ - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\ - template &&\ - !is_primitive_v_,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\ - FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");\ - return _rhs.dimension(i);\ - }\ - FASTOR_INLINE expression_t lhs() const {return _lhs;}\ - FASTOR_INLINE expression_t rhs() const {return _rhs;}\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ - return helper(i);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return _lhs.template eval(i) OP _rhs.template eval(i);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval(i);\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return _lhs.template eval(i) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\ - return helper_s(i);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ - return _lhs.template eval_s(i) OP _rhs.template eval_s(i);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i);\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ - return _lhs.template eval_s(i) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return helper(i,j);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval(i,j) OP _rhs.template eval(i,j);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval(i,j);\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval(i,j) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return helper_s(i,j);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval_s(i,j) OP _rhs.template eval_s(i,j);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i,j);\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval_s(i,j) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ - return thelper(as);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return _lhs.template teval(as) OP _rhs.template teval(as);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template teval(as);\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return _lhs.template teval(as) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE teval_s(const std::array &as) const {\ - return thelper_s(as);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ - return _lhs.template teval_s(as) OP _rhs.template teval_s(as);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template teval_s(as);\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ - return _lhs.template teval_s(as) OP (EVAL_TYPE)_rhs;\ - }\ -};\ -template &&\ - !is_primitive_v_,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ - return Binary ##NAME ## Op(_lhs.self(), _rhs.self());\ -}\ -template &&\ - is_primitive_v_,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op operator OP(const AbstractTensor &_lhs, TRhs bb) {\ - return Binary ##NAME ## Op(_lhs.self(), bb);\ -}\ -template &&\ - !is_primitive_v_,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op operator OP(TLhs bb, const AbstractTensor &_rhs) {\ - return Binary ##NAME ## Op(bb,_rhs.self());\ -}\ -template &&\ - !is_primitive_v_ &&\ - DIM0!=DIM1,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op::value>\ -operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ - return Binary ##NAME ## Op::value>(_lhs.self(), _rhs.self());\ -}\ - -// Dispatch based on type of expressions not the tensor -FASTOR_MAKE_BINARY_ARITHMETIC_OPS(+, Add, scalar_type) -FASTOR_MAKE_BINARY_ARITHMETIC_OPS(-, Sub, scalar_type) -FASTOR_MAKE_BINARY_ARITHMETIC_OPS(*, Mul, scalar_type) -// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(/, Div, scalar_type) // Dont create div as it is a special case - -// Dispatch based on the type of tensor and not the expression -// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(+, Add, U) -// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(-, Sub, U) -// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(*, Mul, U) -// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(/, Div, U) - -} - - -#endif // BINARY_ARITHMETIC_OP_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_cmp_ops.h b/noarch/include/Fastor/expressions/binary_ops/binary_cmp_ops.h deleted file mode 100644 index da277aa2..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_cmp_ops.h +++ /dev/null @@ -1,262 +0,0 @@ -#ifndef BINARY_CMP_OPS -#define BINARY_CMP_OPS - -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" - -namespace Fastor { - -#define FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(OP, NAME, EVAL_TYPE) \ -template\ -struct BinaryCmpOp##NAME: public AbstractTensor,DIM0> {\ - expression_t _lhs;\ - expression_t _rhs;\ - static constexpr FASTOR_INDEX Dimension = DIM0;\ - static constexpr FASTOR_INDEX rank() {return DIM0;}\ - using scalar_type = typename scalar_type_finder>::type;\ - using result_type = to_bool_tensor_t>>;\ - using simd_vector_type = binary_op_simd_vector_t >;\ - using simd_abi_type = typename simd_vector_type::abi_type;\ - using ABI = simd_abi::fixed_size::Size>;\ - using UU = bool /*this needs to change to U once masks are implemented*/;\ - FASTOR_INLINE BinaryCmpOp##NAME(expression_t inlhs, expression_t inrhs) : _lhs(inlhs), _rhs(inrhs) {}\ - FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();}\ - template::value,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\ - template::value,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\ - template::value &&\ - !std::is_arithmetic::value,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {\ - return _rhs.size();\ - }\ - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);}\ - template::value,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\ - template::value,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\ - template::value &&\ - !std::is_arithmetic::value,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\ - return _rhs.dimension(i);\ - }\ - FASTOR_INLINE expression_t lhs() const {return _lhs;}\ - FASTOR_INLINE expression_t rhs() const {return _rhs;}\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ - return helper(i);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return _lhs.template eval(i) OP _rhs.template eval(i);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval(i);\ - }\ - template::value &&\ - std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return _lhs.template eval(i) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE UU eval_s(FASTOR_INDEX i) const {\ - return helper_s(i);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\ - return _lhs.template eval_s(i) OP _rhs.template eval_s(i);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i);\ - }\ - template::value &&\ - std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\ - return _lhs.template eval_s(i) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return helper(i,j);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval(i,j) OP _rhs.template eval(i,j);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval(i,j);\ - }\ - template::value &&\ - std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval(i,j) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE UU eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return helper_s(i,j);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval_s(i,j) OP _rhs.template eval_s(i,j);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template eval_s(i,j);\ - }\ - template::value &&\ - std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return _lhs.template eval_s(i,j) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ - return thelper(as);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return _lhs.template teval(as) OP _rhs.template teval(as);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template teval(as);\ - }\ - template::value &&\ - std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return _lhs.template teval(as) OP (EVAL_TYPE)_rhs;\ - }\ - template\ - FASTOR_INLINE UU teval_s(const std::array &as) const {\ - return thelper_s(as);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU thelper_s(const std::array &as) const {\ - return _lhs.template teval_s(as) OP _rhs.template teval_s(as);\ - }\ - template::value &&\ - !std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU thelper_s(const std::array &as) const {\ - return (EVAL_TYPE)_lhs OP _rhs.template teval_s(as);\ - }\ - template::value &&\ - std::is_arithmetic::value,bool>::type = 0>\ - FASTOR_INLINE UU thelper_s(const std::array &as) const {\ - return _lhs.template teval_s(as) OP (EVAL_TYPE)_rhs;\ - }\ -};\ -template::value &&\ - !std::is_arithmetic::value,bool>::type = 0 >\ -FASTOR_INLINE BinaryCmpOp##NAME operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ - return BinaryCmpOp##NAME(_lhs.self(), _rhs.self());\ -}\ -template::value &&\ - std::is_arithmetic::value,bool>::type = 0 >\ -FASTOR_INLINE BinaryCmpOp##NAME operator OP(const AbstractTensor &_lhs, TRhs bb) {\ - return BinaryCmpOp##NAME(_lhs.self(), bb);\ -}\ -template::value &&\ - !std::is_arithmetic::value,bool>::type = 0 >\ -FASTOR_INLINE BinaryCmpOp##NAME operator OP(TLhs bb, const AbstractTensor &_rhs) {\ - return BinaryCmpOp##NAME(bb,_rhs.self());\ -}\ -template::value &&\ - !std::is_arithmetic::value &&\ - DIM0!=DIM1,bool>::type = 0 >\ -FASTOR_INLINE BinaryCmpOp##NAME::value>\ -operator OP(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ - return BinaryCmpOp##NAME::value>(_lhs.self(), _rhs.self());\ -}\ - - -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(== ,EQ, scalar_type) -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(!= ,NEQ,scalar_type) -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(< ,LT, scalar_type) -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(> ,GT, scalar_type) -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(<= ,LE, scalar_type) -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(>= ,GE, scalar_type) -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(&& ,AND,scalar_type) -FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(|| ,OR, scalar_type) - - - - - -#define FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(NAME, ASSIGN_TYPE)\ -template || requires_evaluation_v),bool> = false >\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const BinaryCmpOp ##NAME &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template || requires_evaluation_v),bool> = false >\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const BinaryCmpOp ##NAME &src) {\ - using lhs_type = typename get_binary_arithmetic_result_type::type;\ - using rhs_type = typename get_binary_arithmetic_result_type::type;\ - const lhs_type a(src.lhs());\ - const rhs_type b(src.rhs());\ - trivial_assign ##ASSIGN_TYPE (dst.self(), BinaryCmpOp ##NAME (a,b));\ -}\ - -#define FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS(ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(EQ, ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(NEQ, ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(LT, ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(GT, ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(LE, ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(GE, ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(AND, ASSIGN_TYPE)\ -FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(OR, ASSIGN_TYPE)\ - -FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( ) -FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _add) -FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _sub) -FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _mul) -FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _div) - - -} // end of namespace Fastor - - -#endif // BINARY_CMP_OPS diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_div_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_div_op.h deleted file mode 100644 index f9ecd195..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_div_op.h +++ /dev/null @@ -1,306 +0,0 @@ -#ifndef BINARY_DIV_OP_H -#define BINARY_DIV_OP_H - -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/expressions/expression_traits.h" - -namespace Fastor { - -// Dispatch based on type of expressions not the tensor -#define FASTOR_BD_OP_EVAL_TYPE scalar_type - -// Dispatch based on the type of tensor and not the expression -// #define FASTOR_BD_OP_EVAL_TYPE U - - -template -struct BinaryDivOp: public AbstractTensor,DIM0> { -private: - expression_t _lhs; - expression_t _rhs; -public: - - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = binary_op_simd_vector_t >; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = binary_arithmetic_result_t< BinaryDivOp >; - - FASTOR_INLINE BinaryDivOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} - - FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} - template,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} - template,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} - template && - !is_primitive_v_,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); -#endif - return _rhs.size(); - } - - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} - template,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} - template,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} - template && - !is_primitive_v_,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); -#endif - return _rhs.dimension(i); - } - constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} - constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - return helper(i); - } - - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { -#ifndef FASTOR_UNSAFE_MATH - return _lhs.template eval(i) / _rhs.template eval(i); -#else - return _lhs.template eval(i) * rcp(_rhs.template eval(i)); -#endif - } - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { -#ifndef FASTOR_UNSAFE_MATH - return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval(i); -#else - return (FASTOR_BD_OP_EVAL_TYPE)_lhs * rcp(_rhs.template eval(i)); -#endif - } - template && - is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { -#ifndef FASTOR_UNSAFE_MATH - return _lhs.template eval(i) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; -#else - return _lhs.template eval(i) * rcp(SIMDVector(_rhs)); -#endif - } - - // scalar based - template - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE eval_s(FASTOR_INDEX i) const { - return helper_s(i); - } - - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) / _rhs.template eval_s(i); - } - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const { - return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval_s(i); - } - template && - is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; - } - - // for 2D tensors - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper(i,j); - } - - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { -#ifndef FASTOR_UNSAFE_MATH - return _lhs.template eval(i,j) / _rhs.template eval(i,j); -#else - return _lhs.template eval(i,j) * rcp(_rhs.template eval(i,j)); -#endif - } - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { -#ifndef FASTOR_UNSAFE_MATH - return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval(i,j); -#else - return (FASTOR_BD_OP_EVAL_TYPE)_lhs * rcp(_rhs.template eval(i,j)); -#endif - } - template && - is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { -#ifndef FASTOR_UNSAFE_MATH - return _lhs.template eval(i,j) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; -#else - return _lhs.template eval(i,j) * rcp(SIMDVector(_rhs)); -#endif - } - - // scalar based (for 2D tensors) - template - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper_s(i,j); - } - - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) / _rhs.template eval_s(i,j); - } - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval_s(i,j); - } - template && - is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; - } - - // for nD tensors - template - FASTOR_INLINE SIMDVector teval(const std::array &as) const { - return thelper(as); - } - - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) / _rhs.template teval(as); - } - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template teval(as); - } - template && - is_primitive_v_,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; - } - - // scalar based (for nD tensors) - template - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE teval_s(const std::array &as) const { - return thelper_s(as); - } - - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) / _rhs.template teval_s(as); - } - template && - !is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array &as) const { - return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template teval_s(as); - } - template && - is_primitive_v_,bool>::type = 0> - FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) / (FASTOR_BD_OP_EVAL_TYPE)_rhs; - } -}; - -template && - !is_primitive_v_,bool>::type = 0 > -FASTOR_INLINE BinaryDivOp operator/(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinaryDivOp(_lhs.self(), _rhs.self()); -} - -#ifndef FASTOR_DISPATCH_DIV_TO_MUL_EXPR -template && - is_primitive_v_,bool>::type = 0 > -FASTOR_INLINE BinaryDivOp operator/(const AbstractTensor &_lhs, TRhs bb) { - return BinaryDivOp(_lhs.self(), bb); -} -template && - !is_primitive_v_,bool>::type = 0 > -FASTOR_INLINE BinaryDivOp operator/(TLhs bb, const AbstractTensor &_rhs) { - return BinaryDivOp(bb,_rhs.self()); -} -#else -template && - is_primitive_v_ && !std::is_integral::value,bool>::type = 0 > -FASTOR_INLINE BinaryMulOp operator/(const AbstractTensor &_lhs, TRhs bb) { - return BinaryMulOp( - _lhs.self(), - static_cast::type>(1)/static_cast::type>(bb)); -} -template && !std::is_integral::value && - !is_primitive_v_,bool>::type = 0 > -FASTOR_INLINE BinaryMulOp operator/(TLhs bb, const AbstractTensor &_rhs) { - return BinaryMulOp( - static_cast::type>(1)/static_cast::type>(bb), - _rhs.self()); -} -// Special case for integral types -template && - is_primitive_v_ && std::is_integral::value,bool>::type = 0 > -FASTOR_INLINE BinaryDivOp operator/(const AbstractTensor &_lhs, TRhs bb) { - return BinaryDivOp(_lhs.self(), bb); -} -template && std::is_integral::value && - !is_primitive_v_,bool>::type = 0 > -FASTOR_INLINE BinaryDivOp operator/(TLhs bb, const AbstractTensor &_rhs) { - return BinaryDivOp(bb,_rhs.self()); -} -#endif - -template && - !is_arithmetic_v_ && - DIM0!=DIM1,bool> = false > -FASTOR_INLINE BinaryDivOp::value> -operator/(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinaryDivOp::value>(_lhs.self(), _rhs.self()); -} - -} - -#endif // BINARY_DIV_OP_H - diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_math_ops.h b/noarch/include/Fastor/expressions/binary_ops/binary_math_ops.h deleted file mode 100644 index c37e14d9..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_math_ops.h +++ /dev/null @@ -1,261 +0,0 @@ -#ifndef BINARY_MATH_OP_H -#define BINARY_MATH_OP_H - -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - - -#define FASTOR_MAKE_BINARY_MATH_OPS(OP_NAME, SIMD_OP, OP, NAME, EVAL_TYPE) \ -template\ -struct Binary ##NAME ## Op: public AbstractTensor,DIM0> {\ - expression_t _lhs;\ - expression_t _rhs;\ -public:\ - static constexpr FASTOR_INDEX Dimension = DIM0;\ - static constexpr FASTOR_INDEX rank() {return DIM0;}\ - using scalar_type = typename scalar_type_finder>::type;\ - using simd_vector_type = binary_op_simd_vector_t< Binary ##NAME ## Op >;\ - using simd_abi_type = typename simd_vector_type::abi_type;\ - using result_type = binary_arithmetic_result_t< Binary ##NAME ## Op >;\ - FASTOR_INLINE Binary ##NAME ## Op(expression_t inlhs, expression_t inrhs) : _lhs(inlhs), _rhs(inrhs) {}\ - FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\ - template &&\ - !is_primitive_v_,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_size() const {\ - FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");\ - return _rhs.size();\ - }\ - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\ - template,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\ - template &&\ - !is_primitive_v_,bool>::type =0 >\ - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\ - FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");\ - return _rhs.dimension(i);\ - }\ - FASTOR_INLINE expression_t lhs() const {return _lhs;}\ - FASTOR_INLINE expression_t rhs() const {return _rhs;}\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ - return helper(i);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return SIMD_OP(_lhs.template eval(i), _rhs.template eval(i));\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template eval(i));\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const {\ - return SIMD_OP(_lhs.template eval(i), (EVAL_TYPE)_rhs);\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\ - return helper_s(i);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ - return OP(_lhs.template eval_s(i), _rhs.template eval_s(i));\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ - return OP((EVAL_TYPE)_lhs, _rhs.template eval_s(i));\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\ - return OP(_lhs.template eval_s(i), (EVAL_TYPE)_rhs);\ - }\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return helper(i,j);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return SIMD_OP(_lhs.template eval(i,j), _rhs.template eval(i,j));\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template eval(i,j));\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return SIMD_OP(_lhs.template eval(i,j), (EVAL_TYPE)_rhs);\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return helper_s(i,j);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return OP(_lhs.template eval_s(i,j), _rhs.template eval_s(i,j));\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return OP((EVAL_TYPE)_lhs, _rhs.template eval_s(i,j));\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return OP(_lhs.template eval_s(i,j), (EVAL_TYPE)_rhs);\ - }\ - template\ - FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ - return thelper(as);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return SIMD_OP(_lhs.template teval(as), _rhs.template teval(as));\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template teval(as));\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE SIMDVector thelper(const std::array &as) const {\ - return SIMD_OP(_lhs.template teval(as), (EVAL_TYPE)_rhs);\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE teval_s(const std::array &as) const {\ - return thelper_s(as);\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ - return OP(_lhs.template teval_s(as), _rhs.template teval_s(as));\ - }\ - template &&\ - !is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ - return OP((EVAL_TYPE)_lhs, _rhs.template teval_s(as));\ - }\ - template &&\ - is_primitive_v_,bool>::type = 0>\ - FASTOR_INLINE EVAL_TYPE thelper_s(const std::array &as) const {\ - return OP(_lhs.template teval_s(as), (EVAL_TYPE)_rhs);\ - }\ -};\ -template &&\ - !is_primitive_v_,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op OP_NAME(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ - return Binary ##NAME ## Op(_lhs.self(), _rhs.self());\ -}\ -template &&\ - is_primitive_v_,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op OP_NAME(const AbstractTensor &_lhs, TRhs bb) {\ - return Binary ##NAME ## Op(_lhs.self(), bb);\ -}\ -template &&\ - !is_primitive_v_,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op OP_NAME(TLhs bb, const AbstractTensor &_rhs) {\ - return Binary ##NAME ## Op(bb,_rhs.self());\ -}\ -template &&\ - !is_primitive_v_ &&\ - DIM0!=DIM1,bool>::type = 0 >\ -FASTOR_INLINE Binary ##NAME ## Op::value>\ -OP_NAME(const AbstractTensor &_lhs, const AbstractTensor &_rhs) {\ - return Binary ##NAME ## Op::value>(_lhs.self(), _rhs.self());\ -}\ - -// Dispatch based on type of expressions not the tensor -FASTOR_MAKE_BINARY_MATH_OPS(min, min, std::min, Min, scalar_type) -FASTOR_MAKE_BINARY_MATH_OPS(max, max, std::max, Max, scalar_type) -FASTOR_MAKE_BINARY_MATH_OPS(pow, pow, std::pow, Pow, scalar_type) -FASTOR_MAKE_BINARY_MATH_OPS(atan2, atan2, std::atan2, Atan2, scalar_type) -FASTOR_MAKE_BINARY_MATH_OPS(hypot, hypot, std::hypot, Hypot, scalar_type) - -// Dispatch based on the type of tensor and not the expression -// FASTOR_MAKE_BINARY_MATH_OPS(min, min, std::min, Min, U) -// FASTOR_MAKE_BINARY_MATH_OPS(max, max, std::max, Max, U) -// FASTOR_MAKE_BINARY_MATH_OPS(pow, pow, std::pow, Pow, U) -// FASTOR_MAKE_BINARY_MATH_OPS(atan2, atan2, std::atan2, Atan2, U) -// FASTOR_MAKE_BINARY_MATH_OPS(hypot, hypot, std::hypot, Hypot, U) - - -// Create assignment for all binary math_ops -#define FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, ASSIGN_TYPE)\ -template || requires_evaluation_v),bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template || requires_evaluation_v),bool> = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - using result_type = typename Binary ##NAME ## Op::result_type;\ - const result_type a(src.self());\ - trivial_assign ##ASSIGN_TYPE (dst.self(), a);\ -}\ - - -#define FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(NAME)\ -FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, )\ -FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _add)\ -FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _sub)\ -FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _mul)\ -FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _div)\ - -FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Min) -FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Max) -FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Pow) -FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Atan2) -FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Hypot) - -} - - -#endif // BINARY_MATH_OP_H diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_mul_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_mul_op.h deleted file mode 100644 index ab3c90f6..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_mul_op.h +++ /dev/null @@ -1,245 +0,0 @@ -#ifndef BINARY_MUL_OP_H -#define BINARY_MUL_OP_H - -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - -template -struct BinaryMulOp: public AbstractTensor,DIM0> { -private: - expression_t _lhs; - expression_t _rhs; -public: - - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = binary_op_simd_vector_t >; - using simd_abi_type = typename simd_vector_type::abi_type; - - FASTOR_INLINE BinaryMulOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} - - - FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} - template::value && - !std::is_arithmetic::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); -#endif - return _rhs.size(); - } - - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} - template::value && - !std::is_arithmetic::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); -#endif - return _rhs.dimension(i); - } - constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} - constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - return helper(i); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return _lhs.template eval(i) * _rhs.template eval(i); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return (U)_lhs * _rhs.template eval(i); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return _lhs.template eval(i) * (U)_rhs; - } - - - // scalar based - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return helper_s(i); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) * _rhs.template eval_s(i); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return (U)_lhs * _rhs.template eval_s(i); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) * (U)_rhs; - } - - // for 2D tensors - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper(i,j); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval(i,j) * _rhs.template eval(i,j); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return (U)_lhs * _rhs.template eval(i,j); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval(i,j) * (U)_rhs; - } - - // scalar based (for 2D tensors) - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper_s(i,j); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) * _rhs.template eval_s(i,j); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return (U)_lhs * _rhs.template eval_s(i,j); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) * (U)_rhs; - } - - // for nD tensors - template - FASTOR_INLINE SIMDVector teval(const std::array &as) const { - return thelper(as); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) * _rhs.template teval(as); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return (U)_lhs * _rhs.template teval(as); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) * (U)_rhs; - } - - // scalar based (for nD tensors) - template - FASTOR_INLINE U teval_s(const std::array &as) const { - return thelper_s(as); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) * _rhs.template teval_s(as); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return (U)_lhs * _rhs.template teval_s(as); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) * (U)_rhs; - } -}; - -template::value && - !std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinaryMulOp operator*(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinaryMulOp(_lhs.self(), _rhs.self()); -} -template::value && - std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinaryMulOp operator*(const AbstractTensor &_lhs, TRhs bb) { - return BinaryMulOp(_lhs.self(), bb); -} -template::value && - !std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinaryMulOp operator*(TLhs bb, const AbstractTensor &_rhs) { - return BinaryMulOp(bb,_rhs.self()); -} - - -template::value && - !std::is_arithmetic::value && - DIM0!=DIM1,bool>::type = 0 > -FASTOR_INLINE BinaryMulOp::value> -operator*(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinaryMulOp::value>(_lhs.self(), _rhs.self()); -} - -} - -#endif // BINARY_MUL_OP_H - diff --git a/noarch/include/Fastor/expressions/binary_ops/binary_sub_op.h b/noarch/include/Fastor/expressions/binary_ops/binary_sub_op.h deleted file mode 100644 index 624004bc..00000000 --- a/noarch/include/Fastor/expressions/binary_ops/binary_sub_op.h +++ /dev/null @@ -1,246 +0,0 @@ -#ifndef BINARY_SUB_OP_H -#define BINARY_SUB_OP_H - -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/expressions/expression_traits.h" - - - -namespace Fastor { - -template -struct BinarySubOp: public AbstractTensor,DIM0> { -private: - expression_t _lhs; - expression_t _rhs; -public: - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = binary_op_simd_vector_t >; - using simd_abi_type = typename simd_vector_type::abi_type; - - FASTOR_INLINE BinarySubOp(expression_t inlhs, expression_t inrhs) : _lhs((inlhs)), _rhs((inrhs)) {} - - FASTOR_INLINE FASTOR_INDEX size() const {return helper_size();} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();} - template::value && - !std::is_arithmetic::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_size() const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH"); -#endif - return _rhs.size(); - } - - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension(i);} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);} - template::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);} - template::value && - !std::is_arithmetic::value,bool>::type =0 > - FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const { -#ifndef NDEBUG - FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH"); -#endif - return _rhs.dimension(i); - } - constexpr FASTOR_INLINE expression_t lhs() const {return _lhs;} - constexpr FASTOR_INLINE expression_t rhs() const {return _rhs;} - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - return helper(i); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return _lhs.template eval(i) - _rhs.template eval(i); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return (U)_lhs - _rhs.template eval(i); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i) const { - return _lhs.template eval(i) - (U)_rhs; - } - - - // scalar based - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return helper_s(i); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) - _rhs.template eval_s(i); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return (U)_lhs - _rhs.template eval_s(i); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i) const { - return _lhs.template eval_s(i) - (U)_rhs; - } - - - // for 2D tensors - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper(i,j); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval(i,j) - _rhs.template eval(i,j); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return (U)_lhs - _rhs.template eval(i,j); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector helper(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval(i,j) - (U)_rhs; - } - - // scalar based (for 2D tensors) - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return helper_s(i,j); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) - _rhs.template eval_s(i,j); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return (U)_lhs - _rhs.template eval_s(i,j); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _lhs.template eval_s(i,j) - (U)_rhs; - } - - // for nD tensors - template - FASTOR_INLINE SIMDVector teval(const std::array &as) const { - return thelper(as); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) - _rhs.template teval(as); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return (U)_lhs - _rhs.template teval(as); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE SIMDVector thelper(const std::array &as) const { - return _lhs.template teval(as) - (U)_rhs; - } - - // scalar based (for nD tensors) - template - FASTOR_INLINE U teval_s(const std::array &as) const { - return thelper_s(as); - } - - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) - _rhs.template teval_s(as); - } - template::value && - !std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return (U)_lhs - _rhs.template teval_s(as); - } - template::value && - std::is_arithmetic::value,bool>::type = 0> - FASTOR_INLINE U thelper_s(const std::array &as) const { - return _lhs.template teval_s(as) - (U)_rhs; - } -}; - -template::value && - !std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinarySubOp operator-(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinarySubOp(_lhs.self(), _rhs.self()); -} -template::value && - std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinarySubOp operator-(const AbstractTensor &_lhs, TRhs bb) { - return BinarySubOp(_lhs.self(), bb); -} -template::value && - !std::is_arithmetic::value,bool>::type = 0 > -FASTOR_INLINE BinarySubOp operator-(TLhs bb, const AbstractTensor &_rhs) { - return BinarySubOp(bb,_rhs.self()); -} - -template::value && - !std::is_arithmetic::value && - DIM0!=DIM1,bool>::type = 0 > -FASTOR_INLINE BinarySubOp::value> -operator-(const AbstractTensor &_lhs, const AbstractTensor &_rhs) { - return BinarySubOp::value>(_lhs.self(), _rhs.self()); -} - -} - - - -#endif // BINARY_SUB_OP_H - diff --git a/noarch/include/Fastor/expressions/expression_traits.h b/noarch/include/Fastor/expressions/expression_traits.h deleted file mode 100644 index 0c7ae3c2..00000000 --- a/noarch/include/Fastor/expressions/expression_traits.h +++ /dev/null @@ -1,327 +0,0 @@ -#ifndef EXPRESSION_TRAITS_H -#define EXPRESSION_TRAITS_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include - -namespace Fastor { - -//------------------------------------------------------------------------------------------------// -template -struct is_expression { - static constexpr bool value = false; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct is_expression> { - static constexpr bool value = true; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct is_expression> { - static constexpr bool value = true; -}; - -template -static constexpr bool is_expression_v = is_expression::value; -//------------------------------------------------------------------------------------------------// - - -// Expression binder - by value or ref -//------------------------------------------------------------------------------------------------// -template -struct expression_binder_type { -#ifndef FASTOR_COPY_EXPR - using type = conditional_t_ || is_arithmetic_v_, const T, const T&>; -#else - using type = const T; -#endif -}; - -template -using expression_t = typename expression_binder_type::type; -//------------------------------------------------------------------------------------------------// - - - -template -class Tensor; - -template -class TensorMap; - -template -class SingleValueTensor; - -// traits -//----------------------------------------------------------------------------------------------------------// -template -struct is_unary_bool_op { - static constexpr bool value = false; -}; - -template -struct is_unary_bool_op> { - static constexpr bool value = true; -}; -template -struct is_unary_bool_op> { - static constexpr bool value = true; -}; -template -struct is_unary_bool_op> { - static constexpr bool value = true; -}; -template -struct is_unary_bool_op> { - static constexpr bool value = true; -}; - -template -struct is_unary_bool_op> { - static constexpr bool value = true; -}; -template -struct is_unary_bool_op> { - static constexpr bool value = true; -}; -template -struct is_unary_bool_op> { - static constexpr bool value = true; -}; - -template -static constexpr bool is_unary_bool_op_v = is_unary_bool_op::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template -struct is_binary_cmp_op { - static constexpr bool value = false; -}; -#define FASTOR_MAKE_IS_BINARY_CMP_OP(NAME) \ -template\ -struct is_binary_cmp_op> {\ - static constexpr bool value = true;\ -};\ - -FASTOR_MAKE_IS_BINARY_CMP_OP(EQ) -FASTOR_MAKE_IS_BINARY_CMP_OP(NEQ) -FASTOR_MAKE_IS_BINARY_CMP_OP(LT) -FASTOR_MAKE_IS_BINARY_CMP_OP(GT) -FASTOR_MAKE_IS_BINARY_CMP_OP(LE) -FASTOR_MAKE_IS_BINARY_CMP_OP(GE) -FASTOR_MAKE_IS_BINARY_CMP_OP(AND) -FASTOR_MAKE_IS_BINARY_CMP_OP(OR) - -template -struct is_binary_cmp_op> { - static constexpr bool value = true; -}; -template -struct is_binary_cmp_op> { - static constexpr bool value = true; -}; -template -struct is_binary_cmp_op> { - static constexpr bool value = true; -}; - -template -static constexpr bool is_binary_cmp_op_v = is_binary_cmp_op::value; -//----------------------------------------------------------------------------------------------------------// - - -/* Is boolean expresssion */ -//----------------------------------------------------------------------------------------------------------// -template -struct is_boolean_expression { - static constexpr bool value = is_unary_bool_op::value || is_binary_cmp_op::value; -}; - -template -static constexpr bool is_boolean_expression_v = is_boolean_expression::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template -struct is_tensor_view { - static constexpr bool value = false; -}; -template -struct is_tensor_view,DIMS>> { - static constexpr bool value = true; -}; -template -struct is_tensor_view,DIMS>> { - static constexpr bool value = true; -}; - -template -struct has_tensor_view { - static constexpr bool value = is_tensor_view::value ? true : false; -}; -template -struct has_tensor_view,DIMS>> { - static constexpr bool value = true; -}; -template -struct has_tensor_view,DIMS>> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_tensor_view> { - static constexpr bool value = has_tensor_view::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_tensor_view> { - static constexpr bool value = has_tensor_view::value || has_tensor_view::value; -}; - -template -static constexpr bool has_tensor_view_v = has_tensor_view::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template -struct is_tensor_fixed_view_2d { - static constexpr bool value = false; -}; -template -struct is_tensor_fixed_view_2d,Seq0,Seq1,2>> { - static constexpr bool value = true; -}; -template -struct is_tensor_fixed_view_2d,Seq0,Seq1,2>> { - static constexpr bool value = true; -}; - -template -struct has_tensor_fixed_view_2d { - static constexpr bool value = is_tensor_fixed_view_2d::value ? true : false; -}; -template -struct has_tensor_fixed_view_2d,Seq0,Seq1,2>> { - static constexpr bool value = true; -}; -template -struct has_tensor_fixed_view_2d,Seq0,Seq1,2>> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_tensor_fixed_view_2d> { - static constexpr bool value = has_tensor_fixed_view_2d::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_tensor_fixed_view_2d> { - static constexpr bool value = has_tensor_fixed_view_2d::value || has_tensor_fixed_view_2d::value; -}; - -template -static constexpr bool has_tensor_fixed_view_2d_v = has_tensor_fixed_view_2d::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template -struct is_tensor_fixed_view_nd { - static constexpr bool value = false; -}; -template -struct is_tensor_fixed_view_nd,Fseq...>> { - static constexpr bool value = true; -}; -template -struct is_tensor_fixed_view_nd,Fseq...>> { - static constexpr bool value = true; -}; - -template -struct has_tensor_fixed_view_nd { - static constexpr bool value = is_tensor_fixed_view_nd::value ? true : false; -}; -template -struct has_tensor_fixed_view_nd,Fseq...>> { - static constexpr bool value = true; -}; -template -struct has_tensor_fixed_view_nd,Fseq...>> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_tensor_fixed_view_nd> { - static constexpr bool value = has_tensor_fixed_view_nd::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_tensor_fixed_view_nd> { - static constexpr bool value = has_tensor_fixed_view_nd::value || has_tensor_fixed_view_nd::value; -}; - -template -static constexpr bool has_tensor_fixed_view_nd_v = has_tensor_fixed_view_nd::value; -//----------------------------------------------------------------------------------------------------------// - - - -//------------------------------------------------------------------------------------------------// -template -struct get_binary_arithmetic_result_type; -template -struct get_binary_arithmetic_result_type > > { - using type = T; -}; -template -struct get_binary_arithmetic_result_type > > { - using type = typename T::result_type; -}; - -template -struct binary_arithmetic_result_type; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIM0> -struct binary_arithmetic_result_type< BinaryExpr > { - using type = conditional_t_, - typename get_binary_arithmetic_result_type::type, - typename get_binary_arithmetic_result_type::type >; -}; -template -using binary_arithmetic_result_t = typename binary_arithmetic_result_type::type; -//------------------------------------------------------------------------------------------------// - - - -//------------------------------------------------------------------------------------------------// -template -struct get_binary_op_simd_vector_type; -template -struct get_binary_op_simd_vector_type > > { - using type = T; -}; -template -struct get_binary_op_simd_vector_type > > { - using type = typename T::simd_vector_type; -}; - -template -struct binary_op_simd_vector_type; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIM0> -struct binary_op_simd_vector_type< BinaryExpr > { - using type = conditional_t_, - typename get_binary_op_simd_vector_type::type, - typename get_binary_op_simd_vector_type::type >; -}; -template -using binary_op_simd_vector_t = typename binary_op_simd_vector_type::type; -//------------------------------------------------------------------------------------------------// - - - - -template -struct scalar_type_finder; - -} // end of namespace Fastor - - -#endif // EXPRESSION_TRAITS_H diff --git a/noarch/include/Fastor/expressions/expressions.h b/noarch/include/Fastor/expressions/expressions.h deleted file mode 100644 index 7623c738..00000000 --- a/noarch/include/Fastor/expressions/expressions.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef EXPRESSIONS_H -#define EXPRESSIONS_H - -#include "Fastor/expressions/binary_ops/binary_arithmetic_ops.h" -// #include "Fastor/expressions/binary_ops/binary_add_op.h" -// #include "Fastor/expressions/binary_ops/binary_sub_op.h" -// #include "Fastor/expressions/binary_ops/binary_mul_op.h" -#include "Fastor/expressions/binary_ops/binary_div_op.h" -#include "Fastor/expressions/binary_ops/binary_arithmetic_assignment.h" -#include "Fastor/expressions/binary_ops/binary_math_ops.h" -#include "Fastor/expressions/binary_ops/binary_cmp_ops.h" -#include "Fastor/expressions/unary_ops/unary_math_ops.h" -#include "Fastor/expressions/unary_ops/unary_bool_ops.h" -#include "Fastor/expressions/linalg_ops/linalg_ops.h" - -#include "Fastor/expressions/views/tensor_fixed_views_1d.h" -#include "Fastor/expressions/views/tensor_fixed_views_2d.h" -#include "Fastor/expressions/views/tensor_fixed_views_nd.h" -#include "Fastor/expressions/views/tensor_random_views.h" -#include "Fastor/expressions/views/tensor_filter_views.h" -#include "Fastor/expressions/views/tensor_diag_views.h" -#include "Fastor/expressions/views/tensor_views.h" - -#include "Fastor/expressions/views/tensor_views_assignment.h" - -#endif // EXPRESSIONS_H - diff --git a/noarch/include/Fastor/expressions/linalg_ops/binary_cross_op.h b/noarch/include/Fastor/expressions/linalg_ops/binary_cross_op.h deleted file mode 100644 index 00838ea1..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/binary_cross_op.h +++ /dev/null @@ -1,191 +0,0 @@ -#ifndef BINARY_CROSS_OP_H -#define BINARY_CROSS_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/backend/tensor_cross.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/expressions/expression_traits.h" - -namespace Fastor { - -// classic vector cross product -template = false> -FASTOR_INLINE Tensor cross(const Tensor &a, const Tensor &b) { - Tensor out; - _vector_crossproduct(a.data(),b.data(),out.data()); - return out; -} -template = false> -FASTOR_INLINE Tensor cross(const Tensor &a, const Tensor &b) { - Tensor out; - _vector_crossproduct(a.data(),b.data(),out.data()); - return out; -} - -// Tensor cross product of two 2nd order tensors -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { - Tensor out; - _crossproduct(a.data(),b.data(),out.data()); - return out; -} - -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { - Tensor out; - _crossproduct(a.data(),b.data(),out.data()); - return out; -} - -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { - // Plane strain case - Tensor out; - Tensor a3, b3; - a3(0,0) = a(0,0); - a3(0,1) = a(0,1); - a3(1,0) = a(1,0); - a3(1,1) = a(1,1); - a3(2,2) = 1; - b3(0,0) = b(0,0); - b3(0,1) = b(0,1); - b3(1,0) = b(1,0); - b3(1,1) = b(1,1); - b3(2,2) = 1; - _crossproduct(a.data(),b.data(),out.data()); - return out; -} - -// Tensor cross product of a vector with 2nd order tensor -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { - Tensor out; - _crossproduct(a.data(),b.data(),out.data()); - return out; -} - -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { - Tensor out; - _crossproduct(a.data(),b.data(),out.data()); - return out; -} - -// Tensor cross product of a 2nd order tensor with a vector -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { - Tensor out; - _crossproduct(a.data(),b.data(),out.data()); - return out; -} - -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &b, const Tensor &a) { - Tensor out; - _crossproduct(a.data(),b.data(),out.data()); - return out; -} - - -// Tensor cross product of a 3rd order tensors -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { - Tensor C; - _crossproduct(A.data(),B.data(),C.data()); - return C; -} - - -// Tensor cross product of a 4th order tensors -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { - Tensor C; - _crossproduct(A.data(),B.data(),C.data()); - return C; -} - - -// Tensor cross product of a 4th order tensor with 2nd order tensor -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { - Tensor C; - _crossproduct42(A.data(),B.data(),C.data()); - return C; -} - - -// Tensor cross product of a 2nd order tensor with 4th order tensor -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { - Tensor C; - _crossproduct24(B.data(),A.data(),C.data()); - return C; -} - - -// Tensor cross product of a 3rd order tensor with 2nd order tensor -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { - Tensor C; - _crossproduct32(A.data(),B.data(),C.data()); - return C; -} - - -// Tensor cross product of a 2nd order tensor with 3rd order tensor -template::type=0> -FASTOR_INLINE Tensor cross(const Tensor &A, const Tensor &B) { - Tensor C; - _crossproduct23(B.data(),A.data(),C.data()); - return C; -} - - -#if FASTOR_CXX_VERSION >= 2014 -// Tensor cross product for generic expressions -template && !is_tensor_v,bool> = 0 > -FASTOR_INLINE -auto -cross(const AbstractTensor &a, const AbstractTensor &b) { - using rhs_type = typename Derived1::result_type; - const rhs_type tmp_b(b); - return cross(a.self(),tmp_b); -} - -template && is_tensor_v,bool> = 0 > -FASTOR_INLINE -auto -cross(const AbstractTensor &a, const AbstractTensor &b) { - using lhs_type = typename Derived0::result_type; - const lhs_type tmp_a(a); - return cross(tmp_a,b.self()); -} - -template && !is_tensor_v,bool> = 0 > -FASTOR_INLINE -auto -cross(const AbstractTensor &a, const AbstractTensor &b) { - using lhs_type = typename Derived0::result_type; - using rhs_type = typename Derived1::result_type; - const lhs_type tmp_a(a); - const rhs_type tmp_b(b); - return cross(tmp_a,tmp_b); -} -#endif - -} // end of namespace Fastor - - -#endif // BINARY_CROSS_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/binary_matmul_op.h b/noarch/include/Fastor/expressions/linalg_ops/binary_matmul_op.h deleted file mode 100644 index ca0dbfa4..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/binary_matmul_op.h +++ /dev/null @@ -1,627 +0,0 @@ -#ifndef BINARY_MATMUL_OP_H -#define BINARY_MATMUL_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/backend/inner.h" -#include "Fastor/backend/matmul/matmul.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - -template -struct BinaryMatMulOp: public AbstractTensor,DIM0> { - using lhs_expr_type = expression_t; - using rhs_expr_type = expression_t; - using lhs_type = typename TLhs::result_type; - using rhs_type = typename TRhs::result_type; - static constexpr FASTOR_INDEX lhs_rank = lhs_type::dimension_t::value; - static constexpr FASTOR_INDEX rhs_rank = rhs_type::dimension_t::value; - static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; - static constexpr FASTOR_INDEX K = get_tensor_dimension_v<1,lhs_type>; - static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,rhs_type>; - static constexpr FASTOR_INDEX K_other = get_tensor_dimension_v<0,rhs_type>; - static constexpr FASTOR_INDEX flop_count = M*N*K; - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - // does not matter which one as matmul bypasses this - using simd_vector_type = typename TLhs::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = conditional_t_ , - conditional_t_, - Tensor // matrix-matrix - > - >; - - FASTOR_INLINE BinaryMatMulOp(lhs_expr_type inlhs, rhs_expr_type inrhs) : _lhs(inlhs), _rhs(inrhs) { - static_assert(lhs_rank <=2 && rhs_rank <=2, "EXPRESSIONS FOR MATRIX MULTIPLICATION HAVE TO BE 2-DIMENSIONAL"); - static_assert(K == K_other, "INVALID MATMUL OPERANDS. COLUMNS(A)!=ROWS(B)"); - } - - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i==0 ? M : N;} - - constexpr FASTOR_INLINE lhs_expr_type lhs() const {return _lhs;} - constexpr FASTOR_INLINE rhs_expr_type rhs() const {return _rhs;} - -private: - lhs_expr_type _lhs; - rhs_expr_type _rhs; -}; - -template &&!is_arithmetic_v_,bool> = 0 > -constexpr FASTOR_INLINE BinaryMatMulOp::value> -operator %(const AbstractTensor &lhs, const AbstractTensor &rhs) { - return BinaryMatMulOp::value>(lhs.self(), rhs.self()); -} - - - - -// helper dispatcher functions -namespace internal { -// till streaming gemm is implemented -template -FASTOR_INLINE -void _gemm(const T alpha, const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, const T beta, T * FASTOR_RESTRICT c) { - - FASTOR_ARCH_ALIGN T tmp[M*N]; - if (beta == 0) { - // non-streaming - _matmul(a,b,tmp); - for (size_t i = 0; i(a,b,tmp); - for (size_t i = 0; i -FASTOR_INLINE -void _gemm_mul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - FASTOR_ARCH_ALIGN T tmp[M*N]; - _matmul(a,b,tmp); - for (size_t i = 0; i -FASTOR_INLINE -void _gemm_div(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) { - FASTOR_ARCH_ALIGN T tmp[M*N]; - _matmul(a,b,tmp); - for (size_t i = 0; i -FASTOR_INLINE void matmul_dispatcher(const Tensor &a, const Tensor &b, Tensor &out) { - FASTOR_IF_CONSTEXPR(J==1) { - _dyadic(a.data(),b.data(),out.data()); - } - else FASTOR_IF_CONSTEXPR(I==1 && J!=1 && K==1) { - out.data()[0] = _inner(a.data(),b.data()); - } - else { - _matmul(a.data(),b.data(),out.data()); - } -} -template -FASTOR_INLINE void matmul_dispatcher(const Tensor &a, const Tensor &b, Tensor &out) { - _matmul(a.data(),b.data(),out.data()); -} -template -FASTOR_INLINE void matmul_dispatcher(const Tensor &a, const Tensor &b, Tensor &out) { - _matmul(a.data(),b.data(),out.data()); -} - -template -FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor &a, const Tensor &b, const T beta, Tensor &out) { - _gemm(alpha,a.data(),b.data(),beta,out.data()); -} -template -FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor &a, const Tensor &b, const T beta, Tensor &out) { - _gemm(alpha,a.data(),b.data(),beta,out.data()); -} -template -FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor &a, const Tensor &b, const T beta, Tensor &out) { - _gemm(alpha,a.data(),b.data(),beta,out.data()); -} - -template -FASTOR_INLINE void matmul_dispatcher_mul(const Tensor &a, const Tensor &b, Tensor &out) { - _gemm_mul(a.data(),b.data(),out.data()); -} -template -FASTOR_INLINE void matmul_dispatcher_mul(const Tensor &a, const Tensor &b, Tensor &out) { - _gemm_mul(a.data(),b.data(),out.data()); -} -template -FASTOR_INLINE void matmul_dispatcher_mul(const Tensor &a, const Tensor &b, Tensor &out) { - _gemm_mul(a.data(),b.data(),out.data()); -} - -template -FASTOR_INLINE void matmul_dispatcher_div(const Tensor &a, const Tensor &b, Tensor &out) { - _gemm_div(a.data(),b.data(),out.data()); -} -template -FASTOR_INLINE void matmul_dispatcher_div(const Tensor &a, const Tensor &b, Tensor &out) { - _gemm_div(a.data(),b.data(),out.data()); -} -template -FASTOR_INLINE void matmul_dispatcher_div(const Tensor &a, const Tensor &b, Tensor &out) { - _gemm_div(a.data(),b.data(),out.data()); -} - -} // internal - - -// For tensors -template -FASTOR_INLINE Tensor matmul(const Tensor &a, const Tensor &b) { - Tensor out; - FASTOR_IF_CONSTEXPR(J==1) { - _dyadic(a.data(),b.data(),out.data()); - } - else FASTOR_IF_CONSTEXPR(I==1 && J!=1 && K==1) { - out.data()[0] = _inner(a.data(),b.data()); - } - else { - _matmul(a.data(),b.data(),out.data()); - } - return out; -} -template -FASTOR_INLINE Tensor matmul(const Tensor &a, const Tensor &b) { - Tensor out; - _matmul(a.data(),b.data(),out.data()); - return out; -} -template -FASTOR_INLINE Tensor matmul(const Tensor &a, const Tensor &b) { - Tensor out; - _matmul(a.data(),b.data(),out.data()); - return out; -} - - -// Generic matmul function for AbstractTensor types are provided here -template && is_less_equal_v_ - && !is_tensor_v && !is_tensor_v,bool> = 0 > -FASTOR_INLINE -conditional_t_ >, - conditional_t_ >, - Tensor , - get_tensor_dimension_v<1,typename Derived1::result_type> > - > -> -matmul(const AbstractTensor &a, const AbstractTensor &b) { - using lhs_type = typename Derived0::result_type; - using rhs_type = typename Derived1::result_type; - const lhs_type tmp_a(a); - const rhs_type tmp_b(b); - return matmul(tmp_a,tmp_b); -} - -template && is_less_equal_v_ - && !is_tensor_v && is_tensor_v,bool> = 0 > -FASTOR_INLINE -conditional_t_ >, - conditional_t_ >, - Tensor , - get_tensor_dimension_v<1,typename Derived1::result_type> > - > -> -matmul(const AbstractTensor &a, const AbstractTensor &b) { - using lhs_type = typename Derived0::result_type; - const lhs_type tmp_a(a); - return matmul(tmp_a,b.self()); -} - -template && is_less_equal_v_ - && is_tensor_v && !is_tensor_v,bool> = 0 > -FASTOR_INLINE -conditional_t_ >, - conditional_t_ >, - Tensor , - get_tensor_dimension_v<1,typename Derived1::result_type> > - > -> -matmul(const AbstractTensor &a, const AbstractTensor &b) { - using rhs_type = typename Derived1::result_type; - const rhs_type tmp_b(b); - return matmul(a.self(),tmp_b); -} - - - -// triangular matmul functions -template -Tensor tmatmul(const Tensor &a, const Tensor &b) { - Tensor out; - _tmatmul(a.data(),b.data(),out.data()); - return out; -} -template -FASTOR_INLINE Tensor tmatmul(const Tensor &a, const Tensor &b) { - Tensor out; - _tmatmul(a.data(),b.data(),out.data()); - return out; -} -template -FASTOR_INLINE Tensor tmatmul(const Tensor &a, const Tensor &b) { - Tensor out; - _tmatmul(a.data(),b.data(),out.data()); - return out; -} - -#if FASTOR_CXX_VERSION >= 2014 -// tmatmul for generic expressions -template && is_less_equal_v_ - && !is_tensor_v && !is_tensor_v,bool> = 0 > -FASTOR_INLINE -decltype(auto) -tmatmul(const AbstractTensor &a, const AbstractTensor &b) { - using lhs_type = typename Derived0::result_type; - using rhs_type = typename Derived1::result_type; - const lhs_type tmp_a(a); - const rhs_type tmp_b(b); - return tmatmul(tmp_a,tmp_b); -} - -template && is_less_equal_v_ - && !is_tensor_v && is_tensor_v,bool> = 0 > -FASTOR_INLINE -decltype(auto) -tmatmul(const AbstractTensor &a, const AbstractTensor &b) { - using lhs_type = typename Derived0::result_type; - const lhs_type tmp_a(a); - return tmatmul(tmp_a,b.self()); -} - -template && is_less_equal_v_ - && is_tensor_v && !is_tensor_v,bool> = 0 > -FASTOR_INLINE -decltype(auto) -tmatmul(const AbstractTensor &a, const AbstractTensor &b) { - using rhs_type = typename Derived1::result_type; - const rhs_type tmp_b(b); - return tmatmul(a.self(),tmp_b); -} -#endif - - - - - - - - - -// assignments -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { - // dst = matmul(src.lhs().self(),src.rhs().self()); // this makes a copy for dst, compiler emits a memcpy - internal::matmul_dispatcher(src.lhs().self(),src.rhs().self(),dst.self()); -} -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { - using lhs_t = typename BinaryMatMulOp::lhs_type; - lhs_t a(src.lhs().self()); - // dst = matmul(a,src.rhs().self()); // this makes a copy for dst, compiler emits a memcpy - internal::matmul_dispatcher(a,src.rhs().self(),dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { - using rhs_t = typename BinaryMatMulOp::rhs_type; - rhs_t b(src.rhs().self()); - // dst = matmul(src.lhs().self(),b); // this makes a copy for dst, compiler emits a memcpy - internal::matmul_dispatcher(src.lhs().self(),b,dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp &src) { - using lhs_t = typename BinaryMatMulOp::lhs_type; - using rhs_t = typename BinaryMatMulOp::rhs_type; - lhs_t a(src.lhs().self()); - rhs_t b(src.rhs().self()); - // dst = matmul(a,b); // this makes a copy for dst, compiler emits a memcpy - internal::matmul_dispatcher(a,b,dst.self()); -} - - - -// assignments add -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - internal::matmul_dispatcher((T)1,src.lhs().self(),src.rhs().self(),(T)1,dst.self()); -} -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - using lhs_t = typename BinaryMatMulOp::lhs_type; - lhs_t a(src.lhs().self()); - internal::matmul_dispatcher((T)1,a,src.rhs().self(),(T)1,dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - using rhs_t = typename BinaryMatMulOp::rhs_type; - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher((T)1,src.lhs().self(),b,(T)1,dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - using lhs_t = typename BinaryMatMulOp::lhs_type; - using rhs_t = typename BinaryMatMulOp::rhs_type; - lhs_t a(src.lhs().self()); - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher((T)1,a,b,(T)1,dst.self()); -} - - -// assignments sub -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - internal::matmul_dispatcher((T)-1,src.lhs().self(),src.rhs().self(),(T)1,dst.self()); -} -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - using lhs_t = typename BinaryMatMulOp::lhs_type; - lhs_t a(src.lhs().self()); - internal::matmul_dispatcher((T)-1,a,src.rhs().self(),(T)1,dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - using rhs_t = typename BinaryMatMulOp::rhs_type; - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher((T)-1,src.lhs().self(),b,(T)1,dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp &src) { - using T = typename BinaryMatMulOp::scalar_type; - using lhs_t = typename BinaryMatMulOp::lhs_type; - using rhs_t = typename BinaryMatMulOp::rhs_type; - lhs_t a(src.lhs().self()); - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher((T)-1,a,b,(T)1,dst.self()); -} - - -// assignments mul -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { - internal::matmul_dispatcher_mul(src.lhs().self(),src.rhs().self(),dst.self()); -} -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { - using lhs_t = typename BinaryMatMulOp::lhs_type; - lhs_t a(src.lhs().self()); - internal::matmul_dispatcher_mul(a,src.rhs().self(),dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { - using rhs_t = typename BinaryMatMulOp::rhs_type; - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher_mul(src.lhs().self(),b,dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp &src) { - using lhs_t = typename BinaryMatMulOp::lhs_type; - using rhs_t = typename BinaryMatMulOp::rhs_type; - lhs_t a(src.lhs().self()); - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher_mul(a,b,dst.self()); -} - - -// assignments div -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { - internal::matmul_dispatcher_div(src.lhs().self(),src.rhs().self(),dst.self()); -} -template::lhs_expr_type>> && - is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { - using lhs_t = typename BinaryMatMulOp::lhs_type; - lhs_t a(src.lhs().self()); - internal::matmul_dispatcher_div(a,src.rhs().self(),dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { - using rhs_t = typename BinaryMatMulOp::rhs_type; - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher_div(src.lhs().self(),b,dst.self()); -} -template::lhs_expr_type>> && - !is_tensor_v::rhs_expr_type>>, bool >::type = false> -FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp &src) { - using lhs_t = typename BinaryMatMulOp::lhs_type; - using rhs_t = typename BinaryMatMulOp::rhs_type; - lhs_t a(src.lhs().self()); - rhs_t b(src.rhs().self()); - internal::matmul_dispatcher_div(a,b,dst.self()); -} - - - - - - - -// recursive greedy-like -template -FASTOR_INLINE void assign(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { - FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); - assign(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); - } - else - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); - assign(dst, BinaryMatMulOp(tmp,src.rhs().self())); - } -} - -template -FASTOR_INLINE void assign_add(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { - FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); - assign_add(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); - } - else - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); - assign_add(dst, BinaryMatMulOp(tmp,src.rhs().self())); - } -} - -template -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { - FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); - assign_sub(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); - } - else - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); - assign_sub(dst, BinaryMatMulOp(tmp,src.rhs().self())); - } -} - -template -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { - FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); - assign_mul(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); - } - else - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); - assign_mul(dst, BinaryMatMulOp(tmp,src.rhs().self())); - } -} - -template -FASTOR_INLINE void assign_div(AbstractTensor &dst, const BinaryMatMulOp, TRhs1, OtherDIM> &src) { - FASTOR_IF_CONSTEXPR(BinaryMatMulOp::flop_count > BinaryMatMulOp::flop_count) - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().rhs().self(),src.rhs().self())); - assign_div(dst, BinaryMatMulOp(src.lhs().lhs().self(),tmp)); - } - else - { - using result_t = typename BinaryMatMulOp::result_type; - result_t tmp; - assign(tmp, BinaryMatMulOp(src.lhs().lhs().self(),src.lhs().rhs().self())); - assign_div(dst, BinaryMatMulOp(tmp,src.rhs().self())); - } -} - - -} // end of namespace Fastor - - -#endif // BINARY_MATMUL_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/binary_solve_op.h b/noarch/include/Fastor/expressions/linalg_ops/binary_solve_op.h deleted file mode 100644 index 3a0fc5bc..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/binary_solve_op.h +++ /dev/null @@ -1,197 +0,0 @@ -#ifndef BINARY_SOLVE_OP_H -#define BINARY_SOLVE_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" -#include "Fastor/expressions/linalg_ops/binary_matmul_op.h" -#include "Fastor/expressions/linalg_ops/unary_inv_op.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" -#include "Fastor/expressions/linalg_ops/unary_piv_op.h" - - -namespace Fastor { - -// Solving using LU decomposition is in the LU module [unary_lu_op] -// For tensors -// Solve - no pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { - return matmul(inverse(A),b); -} -// Solve - pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { - // We need to post multiply - swap columns using row permutation vector - Tensor p; - pivot_inplace(A,p); - auto tmp(apply_pivot(A,p)); - Tensor invA = inverse(tmp); - return matmul(reconstruct_colwise(invA,p),b); - - // // matrix version - // Tensor p; - // pivot_inplace(A,p); - // auto tmp(apply_pivot(A,p)); - // Tensor invA = inverse(tmp); - // return matmul(matmul(invA,p),b); -} - -// Multiple right hand sides -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { - return matmul(inverse(A),B); -} - -// Solve - pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { - Tensor p; - pivot_inplace(A,p); - auto tmp(apply_pivot(A,p)); - Tensor invA = inverse(tmp); - return matmul(reconstruct(invA,p),B); -} - - - -// For expressions -template && is_tensor_v && DIM0==2,bool> = false> -FASTOR_INLINE -conditional_t_ - >, - Tensor< - typename TLhs::scalar_type, - get_tensor_dimension_v<0,typename TLhs::result_type>, - get_tensor_dimension_v<1,typename TRhs::result_type> - > -> -solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { - - using lhs_type = typename TLhs::result_type; - using rhs_type = typename TRhs::result_type; - constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; - constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; - constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; - - static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); - static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); - - return solve(lhs.self(),rhs.self()); -} - - -template && is_tensor_v && DIM0==2,bool> = false> -FASTOR_INLINE -conditional_t_ - >, - Tensor< - typename TLhs::scalar_type, - get_tensor_dimension_v<0,typename TLhs::result_type>, - get_tensor_dimension_v<1,typename TRhs::result_type> - > -> -solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { - - using lhs_type = typename TLhs::result_type; - using rhs_type = typename TRhs::result_type; - constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; - constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; - constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; - - static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); - static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); - - const lhs_type A(lhs.self()); - - return solve(A,rhs.self()); -} - - -template && !is_tensor_v && DIM0==2,bool> = false> -FASTOR_INLINE -conditional_t_ - >, - Tensor< - typename TLhs::scalar_type, - get_tensor_dimension_v<0,typename TLhs::result_type>, - get_tensor_dimension_v<1,typename TRhs::result_type> - > -> -solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { - - using lhs_type = typename TLhs::result_type; - using rhs_type = typename TRhs::result_type; - constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; - constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; - constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; - - static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); - static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); - - const rhs_type b(rhs.self()); - - return solve(lhs.self(),b); -} - - -template && !is_tensor_v && DIM0==2,bool> = false> -FASTOR_INLINE -conditional_t_ - >, - Tensor< - typename TLhs::scalar_type, - get_tensor_dimension_v<0,typename TLhs::result_type>, - get_tensor_dimension_v<1,typename TRhs::result_type> - > -> -solve(const AbstractTensor &lhs, const AbstractTensor &rhs) { - - using lhs_type = typename TLhs::result_type; - using rhs_type = typename TRhs::result_type; - constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>; - constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>; - constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>; - - static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX"); - static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)"); - - const lhs_type A(lhs.self()); - const rhs_type b(rhs.self()); - - return solve(A,b); -} - - -} // end of namespace Fastor - - -#endif // BINARY_SOLVE_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/linalg_computation_types.h b/noarch/include/Fastor/expressions/linalg_ops/linalg_computation_types.h deleted file mode 100644 index 8af6685f..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/linalg_computation_types.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef LINALG_COMPUTATION_TYPES_H -#define LINALG_COMPUTATION_TYPES_H - -namespace Fastor { - -// Pivot types -enum class PivType : int -{ - V = 0, /* The permutation vector */ - M, /* The permutatoin matrix */ -}; - - -// QR factorisation computation types -enum class QRCompType : int -{ - MGSR = 0, /* Modified Gram-Schmidt Row-wise */ - MGSRPiv, /* Modified Gram-Schmidt Row-wise with pivot */ - HHR /* House Holder Reflections */ -}; - -// LU factorisation computation types -enum class LUCompType : int -{ - BlockLU = 0, /* Block LU decomposition wiht no pivot */ - BlockLUPiv, /* Block LU decomposition wiht pivot */ - SimpleLU, /* Simple LU decomposition wiht no pivot */ - SimpleLUPiv, /* Simple LU decomposition wiht pivot */ -}; - -// Determinant computation type -enum class DetCompType : int -{ - Simple = 0, /* Using simple hand-optimised calculations */ - LU, /* Using LU factorisation */ - QR, /* Using QR factorisation */ -}; - -// Inverse computation type -enum class InvCompType : int -{ - SimpleInv = 0,/* Using simple hand-optimised calculations */ - SimpleInvPiv, /* Simple with pivot */ - BlockLU, /* Using block LU factorisation */ - BlockLUPiv, /* Using block LU factorisation with pivot */ - SimpleLU, /* Using simple LU factorisation */ - SimpleLUPiv, /* Using simple LU factorisation with pivot */ -}; - -// Solve computation type -enum class SolveCompType : int -{ - SimpleInv = 0, /* Using optimised inversion */ - SimpleInvPiv, /* Using optimised inversion with pivot */ - BlockLU, /* Using block LU factorisation */ - BlockLUPiv, /* Using block LU factorisation with pivot */ - SimpleLU, /* Using simple LU factorisation */ - SimpleLUPiv, /* Using simple LU factorisation with pivot */ - QR, /* Using QR factorisation */ - Chol, /* Using Cholesky factorisation */ -}; - - -} // end of namespace Fastor - -#endif // LINALG_COMPUTATION_TYPES_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/linalg_ops.h b/noarch/include/Fastor/expressions/linalg_ops/linalg_ops.h deleted file mode 100644 index e59a6835..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/linalg_ops.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef LINALG_OPS_H -#define LINALG_OPS_H - -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" - -#include "Fastor/expressions/linalg_ops/binary_matmul_op.h" -#include "Fastor/expressions/linalg_ops/unary_piv_op.h" -#include "Fastor/expressions/linalg_ops/unary_lu_op.h" -#include "Fastor/expressions/linalg_ops/binary_solve_op.h" -#include "Fastor/expressions/linalg_ops/unary_trans_op.h" -#include "Fastor/expressions/linalg_ops/unary_ctrans_op.h" -#include "Fastor/expressions/linalg_ops/unary_adj_op.h" -#include "Fastor/expressions/linalg_ops/unary_cof_op.h" -#include "Fastor/expressions/linalg_ops/unary_inv_op.h" -#include "Fastor/expressions/linalg_ops/unary_trace_op.h" -#include "Fastor/expressions/linalg_ops/unary_norm_op.h" -#include "Fastor/expressions/linalg_ops/unary_qr_op.h" -#include "Fastor/expressions/linalg_ops/unary_det_op.h" -#include "Fastor/expressions/linalg_ops/unary_svd_op.h" -#include "Fastor/expressions/linalg_ops/binary_cross_op.h" - -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -#endif // LINALG_OPS_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/linalg_traits.h b/noarch/include/Fastor/expressions/linalg_ops/linalg_traits.h deleted file mode 100644 index 3774029f..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/linalg_traits.h +++ /dev/null @@ -1,273 +0,0 @@ -#ifndef LINALG_TRAITS_H -#define LINALG_TRAITS_H - - -#include "Fastor/expressions/linalg_ops/binary_matmul_op.h" -#include "Fastor/expressions/linalg_ops/unary_trans_op.h" -#include "Fastor/expressions/linalg_ops/unary_ctrans_op.h" -#include "Fastor/expressions/linalg_ops/unary_adj_op.h" -#include "Fastor/expressions/linalg_ops/unary_cof_op.h" -#include "Fastor/expressions/linalg_ops/unary_inv_op.h" -#include "Fastor/expressions/linalg_ops/binary_solve_op.h" -#include - - -namespace Fastor { - - -// Is a binary matmul expression -//----------------------------------------------------------------------------------------------------------// -template -struct is_binary_matmul_op { - static constexpr bool value = false; -}; -template -struct is_binary_matmul_op,Tensor,DIM>> { - static constexpr bool value = true; -}; -template -struct is_binary_matmul_op> { - static constexpr bool value = true; -}; - -template -struct has_binary_matmul_op { - static constexpr bool value = is_binary_matmul_op::value ? true : false; -}; -template -struct has_binary_matmul_op,Tensor,DIM>> { - static constexpr bool value = true; -}; -template -struct has_binary_matmul_op> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_binary_matmul_op> { - static constexpr bool value = has_binary_matmul_op::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_binary_matmul_op> { - static constexpr bool value = has_binary_matmul_op::value || has_binary_matmul_op::value; -}; - -// helper -template -static constexpr bool is_binary_matmul_op_v = is_binary_matmul_op::value; -template -static constexpr bool has_binary_matmul_op_v = has_binary_matmul_op::value; -//----------------------------------------------------------------------------------------------------------// - - -// Is unary trans expression -//----------------------------------------------------------------------------------------------------------// -template -struct is_unary_trans_op { - static constexpr bool value = false; -}; -template -struct is_unary_trans_op> { - static constexpr bool value = true; -}; - -template -struct has_unary_trans_op { - static constexpr bool value = is_unary_trans_op::value ? true : false; -}; -template -struct has_unary_trans_op> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_unary_trans_op> { - static constexpr bool value = has_unary_trans_op::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_unary_trans_op> { - static constexpr bool value = has_unary_trans_op::value || has_unary_trans_op::value; -}; - -// helper -template -static constexpr bool is_unary_trans_op_v = is_unary_trans_op::value; -template -static constexpr bool has_unary_trans_op_v = has_unary_trans_op::value; -//----------------------------------------------------------------------------------------------------------// - - -// Is unary ctrans expression -//----------------------------------------------------------------------------------------------------------// -template -struct is_unary_ctrans_op { - static constexpr bool value = false; -}; -template -struct is_unary_ctrans_op> { - static constexpr bool value = true; -}; - -template -struct has_unary_ctrans_op { - static constexpr bool value = is_unary_ctrans_op::value ? true : false; -}; -template -struct has_unary_ctrans_op> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_unary_ctrans_op> { - static constexpr bool value = has_unary_ctrans_op::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_unary_ctrans_op> { - static constexpr bool value = has_unary_ctrans_op::value || has_unary_ctrans_op::value; -}; - -// helper -template -static constexpr bool is_unary_ctrans_op_v = is_unary_ctrans_op::value; -template -static constexpr bool has_unary_ctrans_op_v = has_unary_ctrans_op::value; -//----------------------------------------------------------------------------------------------------------// - - -// Is unary adj expression -//----------------------------------------------------------------------------------------------------------// -template -struct is_unary_adj_op { - static constexpr bool value = false; -}; -template -struct is_unary_adj_op> { - static constexpr bool value = true; -}; - -template -struct has_unary_adj_op { - static constexpr bool value = is_unary_adj_op::value ? true : false; -}; -template -struct has_unary_adj_op> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_unary_adj_op> { - static constexpr bool value = has_unary_adj_op::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_unary_adj_op> { - static constexpr bool value = has_unary_adj_op::value || has_unary_adj_op::value; -}; - -// helper -template -static constexpr bool is_unary_adj_op_v = is_unary_adj_op::value; -template -static constexpr bool has_unary_adj_op_v = has_unary_adj_op::value; -//----------------------------------------------------------------------------------------------------------// - -// Is unary adj expression -//----------------------------------------------------------------------------------------------------------// -template -struct is_unary_cof_op { - static constexpr bool value = false; -}; -template -struct is_unary_cof_op> { - static constexpr bool value = true; -}; - -template -struct has_unary_cof_op { - static constexpr bool value = is_unary_cof_op::value ? true : false; -}; -template -struct has_unary_cof_op> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_unary_cof_op> { - static constexpr bool value = has_unary_cof_op::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_unary_cof_op> { - static constexpr bool value = has_unary_cof_op::value || has_unary_cof_op::value; -}; - -// helper -template -static constexpr bool is_unary_cof_op_v = is_unary_cof_op::value; -template -static constexpr bool has_unary_cof_op_v = has_unary_cof_op::value; -//----------------------------------------------------------------------------------------------------------// - - -// Is unary inv expression -//----------------------------------------------------------------------------------------------------------// -template -struct is_unary_inv_op { - static constexpr bool value = false; -}; -template -struct is_unary_inv_op> { - static constexpr bool value = true; -}; - -template -struct has_unary_inv_op { - static constexpr bool value = is_unary_inv_op::value ? true : false; -}; -template -struct has_unary_inv_op> { - static constexpr bool value = true; -}; -template class UnaryExpr, typename Expr, size_t DIM> -struct has_unary_inv_op> { - static constexpr bool value = has_unary_inv_op::value; -}; -template class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS> -struct has_unary_inv_op> { - static constexpr bool value = has_unary_inv_op::value || has_unary_inv_op::value; -}; - -// helper -template -static constexpr bool is_unary_inv_op_v = is_unary_inv_op::value; -template -static constexpr bool has_unary_inv_op_v = has_unary_inv_op::value; -//----------------------------------------------------------------------------------------------------------// - - -// Is a linear algebra expression -//----------------------------------------------------------------------------------------------------------// -template -struct has_linalg_op { - static constexpr bool value = has_binary_matmul_op::value || has_unary_trans_op::value || - has_unary_ctrans_op::value || has_unary_adj_op::value || - has_unary_cof_op::value || has_unary_inv_op::value; -}; - -// helper -template -static constexpr bool has_linalg_op_v = has_linalg_op::value; -//----------------------------------------------------------------------------------------------------------// - - -// Requires immediate evaluation -//----------------------------------------------------------------------------------------------------------// -template -struct requires_evaluation { - static constexpr bool value = has_linalg_op::value; -}; - -// helper -template -static constexpr bool requires_evaluation_v = requires_evaluation::value; -//----------------------------------------------------------------------------------------------------------// - - - -} // end of namespace Fastor - - -#endif // LINALG_TRAITS_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_adj_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_adj_op.h deleted file mode 100644 index fbe19793..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_adj_op.h +++ /dev/null @@ -1,160 +0,0 @@ -#ifndef UNARY_ADJ_OP_H -#define UNARY_ADJ_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/adjoint.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - -template -struct UnaryAdjOp: public AbstractTensor,DIM0> { - using expr_type = expression_t; - using result_type = typename Expr::result_type; - static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; - static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = typename Expr::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - - FASTOR_INLINE UnaryAdjOp(expr_type inexpr) : _expr(inexpr) { - static_assert(M==N, "MATRIX MUST BE SQUARE"); - } - - FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} - - constexpr FASTOR_INLINE expr_type expr() const {return _expr;} - -private: - expr_type _expr; -}; - -template -FASTOR_INLINE UnaryAdjOp -adj(const AbstractTensor &src) { - return UnaryAdjOp(src.self()); -} - - -namespace internal { -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void adjoint_dispatcher(const Tensor &in, Tensor& out) { - _adjoint(in.data(),out.data()); -} -} // internal - - -// For tensors -template -FASTOR_INLINE Tensor adjoint(const Tensor &a) { - Tensor out; - _adjoint(a.data(),out.data()); - return out; -} - -// For high order tensors -template=3,bool>::type=0> -FASTOR_INLINE Tensor -adjoint(const Tensor &a) { - - constexpr size_t remaining_product = last_matrix_extracter, - typename std_ext::make_index_sequence::type>::remaining_product; - - constexpr size_t I = get_value::value; - constexpr size_t J = get_value::value; - static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); - - Tensor out; - T *a_data = a.data(); - T *out_data = out.data(); - - for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); - } - - return out; -} - -// Adjoint for generic expressions is provided here -template -FASTOR_INLINE -typename Derived::result_type -adjoint(const AbstractTensor &src) { - // If we are here Derived is already an expression - using result_type = typename Derived::result_type; - const result_type tmp(src.self()); - return adjoint(tmp); -} - - - - - - -// assignments -template -FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryAdjOp &src) { - using result_type = typename Expr::result_type; - const result_type& tmp = evaluate(src.expr().self()); - internal::adjoint_dispatcher(tmp,dst.self()); -} - -template -FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryAdjOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::adjoint_dispatcher(tmp,tmp_inv); - trivial_assign_add(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryAdjOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::adjoint_dispatcher(tmp,tmp_inv); - trivial_assign_sub(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryAdjOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::adjoint_dispatcher(tmp,tmp_inv); - trivial_assign_mul(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryAdjOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::adjoint_dispatcher(tmp,tmp_inv); - trivial_assign_div(dst.self(),tmp_inv); -} - - -} // end of namespace Fastor - - -#endif // UNARY_ADJ_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_cof_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_cof_op.h deleted file mode 100644 index 9bbfac20..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_cof_op.h +++ /dev/null @@ -1,160 +0,0 @@ -#ifndef UNARY_COF_OP_H -#define UNARY_COF_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/cofactor.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - -template -struct UnaryCofOp: public AbstractTensor,DIM0> { - using expr_type = expression_t; - using result_type = typename Expr::result_type; - static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; - static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = typename Expr::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - - FASTOR_INLINE UnaryCofOp(expr_type inexpr) : _expr(inexpr) { - static_assert(M==N, "MATRIX MUST BE SQUARE"); - } - - FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} - - constexpr FASTOR_INLINE expr_type expr() const {return _expr;} - -private: - expr_type _expr; -}; - -template -FASTOR_INLINE UnaryCofOp -cof(const AbstractTensor &src) { - return UnaryCofOp(src.self()); -} - - -namespace internal { -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void cofactor_dispatcher(const Tensor &in, Tensor& out) { - _cofactor(in.data(),out.data()); -} -} // internal - - -// For tensors -template -FASTOR_INLINE Tensor cofactor(const Tensor &a) { - Tensor out; - _cofactor(a.data(),out.data()); - return out; -} - -// For high order tensors -template=3,bool>::type=0> -FASTOR_INLINE Tensor -cofactor(const Tensor &a) { - - constexpr size_t remaining_product = last_matrix_extracter, - typename std_ext::make_index_sequence::type>::remaining_product; - - constexpr size_t I = get_value::value; - constexpr size_t J = get_value::value; - static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); - - Tensor out; - T *a_data = a.data(); - T *out_data = out.data(); - - for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); - } - - return out; -} - -// Cofactor for generic expressions is provided here -template -FASTOR_INLINE -typename Derived::result_type -cofactor(const AbstractTensor &src) { - // If we are here Derived is already an expression - using result_type = typename Derived::result_type; - const result_type tmp(src.self()); - return cofactor(tmp); -} - - - - - - -// assignments -template -FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryCofOp &src) { - using result_type = typename Expr::result_type; - const result_type& tmp = evaluate(src.expr().self()); - internal::cofactor_dispatcher(tmp,dst.self()); -} - -template -FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryCofOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::cofactor_dispatcher(tmp,tmp_inv); - trivial_assign_add(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryCofOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::cofactor_dispatcher(tmp,tmp_inv); - trivial_assign_sub(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryCofOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::cofactor_dispatcher(tmp,tmp_inv); - trivial_assign_mul(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryCofOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::cofactor_dispatcher(tmp,tmp_inv); - trivial_assign_div(dst.self(),tmp_inv); -} - - -} // end of namespace Fastor - - -#endif // UNARY_COF_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_ctrans_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_ctrans_op.h deleted file mode 100644 index 18d226c3..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_ctrans_op.h +++ /dev/null @@ -1,178 +0,0 @@ -#ifndef UNARY_CTRANS_OP_H -#define UNARY_CTRANS_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - -template -struct UnaryCTransOp: public AbstractTensor,DIM0> { - using expr_type = expression_t; - static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,typename Expr::result_type>; - static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,typename Expr::result_type>; - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = typename Expr::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - - FASTOR_INLINE UnaryCTransOp(expr_type inexpr) : _expr(inexpr) { - } - - FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i == 0 ? N : M;} - - constexpr FASTOR_INLINE expr_type expr() const {return _expr;} - -private: - expr_type _expr; -}; - -/* Tensor conjugate transpose returning a tensor expression */ -template -FASTOR_INLINE UnaryCTransOp -ctrans(const AbstractTensor &src) { - return UnaryCTransOp(src.self()); -} - -/* Backend implementation */ -template -FASTOR_INLINE void _ctranspose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) { - for (size_t j=0; j -FASTOR_INLINE Tensor ctranspose(const Tensor &a) { - Tensor out; - _ctranspose(a.data(),out.data()); - return out; -} - -/* Tensor conjugate transpose for higher order tensors immediately returning a tensor */ -template=3,bool>::type=0> -FASTOR_INLINE Tensor -ctranspose(const Tensor &a) { - - constexpr size_t remaining_product = last_matrix_extracter, - typename std_ext::make_index_sequence::type>::remaining_product; - - constexpr size_t I = get_value::value; - constexpr size_t J = get_value::value; - static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); - - Tensor out; - T *a_data = a.data(); - T *out_data = out.data(); - - for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); - } - - return out; -} - -// Conjagate transpose for generic expressions -template -FASTOR_INLINE -Tensor< - typename scalar_type_finder::type, - get_tensor_dimension_v<1,typename Expr::result_type>, - get_tensor_dimension_v<0,typename Expr::result_type>> -ctranspose(const AbstractTensor &src) { - // If we are here Expr is already an expression - using result_type = typename Expr::result_type; - const result_type tmp(src.self()); - return ctranspose(tmp); -} - - - - -// assignments -template -FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryCTransOp &src) { - using result_type = typename Expr::result_type; - const result_type& tmp = evaluate(src.expr().self()); - using T = typename UnaryCTransOp::scalar_type; - static constexpr size_t M = UnaryCTransOp::M; - static constexpr size_t N = UnaryCTransOp::N; - _ctranspose(tmp.data(),dst.self().data()); -} - -template -FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryCTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryCTransOp::scalar_type; - using result_t = typename UnaryCTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryCTransOp::M; - static constexpr size_t N = UnaryCTransOp::N; - _ctranspose(tmp.data(),tmp_trans.data()); - trivial_assign_add(dst.self(),tmp_trans); -} - -template -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryCTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryCTransOp::scalar_type; - using result_t = typename UnaryCTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryCTransOp::M; - static constexpr size_t N = UnaryCTransOp::N; - _ctranspose(tmp.data(),tmp_trans.data()); - trivial_assign_sub(dst.self(),tmp_trans); -} - -template -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryCTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryCTransOp::scalar_type; - using result_t = typename UnaryCTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryCTransOp::M; - static constexpr size_t N = UnaryCTransOp::N; - _ctranspose(tmp.data(),tmp_trans.data()); - trivial_assign_mul(dst.self(),tmp_trans); -} - -template -FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryCTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryCTransOp::scalar_type; - using result_t = typename UnaryCTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryCTransOp::M; - static constexpr size_t N = UnaryCTransOp::N; - _ctranspose(tmp.data(),tmp_trans.data()); - trivial_assign_div(dst.self(),tmp_trans); -} - - -} // end of namespace Fastor - - -#endif // UNARY_CTRANS_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_det_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_det_op.h deleted file mode 100644 index 2b53d147..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_det_op.h +++ /dev/null @@ -1,86 +0,0 @@ -#ifndef UNARY_DET_OP_H -#define UNARY_DET_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/backend/determinant.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -#include - -namespace Fastor { - -// Computing determinant using QR/LU decompositions are in those respective modules -// For tensors -template && DetType==DetCompType::Simple,bool> = false> -FASTOR_INLINE T determinant(const Tensor &a) { - return _det(static_cast(a.data())); -} -template && DetType == DetCompType::Simple,bool> = false> -FASTOR_INLINE T determinant(const Tensor &a) { - // Dispatch to LU - return determinant(a); -} - -// For high order tensors -template=3 && DetType == DetCompType::Simple,bool> = false> -FASTOR_INLINE -typename last_matrix_extracter, typename std_ext::make_index_sequence::type>::type -determinant(const Tensor &a) { - - using OutTensor = typename last_matrix_extracter, - typename std_ext::make_index_sequence::type>::type; - constexpr size_t remaining_product = last_matrix_extracter, - typename std_ext::make_index_sequence::type>::remaining_product; - - constexpr size_t I = get_value::value; - constexpr size_t J = get_value::value; - static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); - - OutTensor out; - T *a_data = a.data(); - T *out_data = out.data(); - - for (size_t i=0; i(static_cast(a_data+i*J*J)); - } - - return out; -} - -// Find determinant of a tensor expression - dispatches to determinant for tensors -template -FASTOR_INLINE typename Derived::scalar_type determinant(const AbstractTensor &_src) { - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out = evaluate(src); - return determinant(out); -} - - -template -FASTOR_INLINE typename Derived::scalar_type det(const AbstractTensor &_src) { - return determinant(_src.self()); -} - -template -FASTOR_INLINE typename Derived::scalar_type absdet(const AbstractTensor &_src) { - return std::abs(determinant(_src.self())); -} - -template -FASTOR_INLINE typename Derived::scalar_type logdet(const AbstractTensor &_src) { - return std::log(std::abs(determinant(_src.self()))); -} - - -} // end of namespace Fastor - -#endif // UNARY_DET_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_inv_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_inv_op.h deleted file mode 100644 index a4d32768..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_inv_op.h +++ /dev/null @@ -1,731 +0,0 @@ -#ifndef UNARY_INV_OP_H -#define UNARY_INV_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/inverse.h" -#include "Fastor/backend/lut_inverse.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" -#include "Fastor/expressions/linalg_ops/unary_piv_op.h" - - -namespace Fastor { - -template -struct UnaryInvOp: public AbstractTensor,DIM0> { - using expr_type = expression_t; - using result_type = typename Expr::result_type; - static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; - static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = typename Expr::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - - FASTOR_INLINE UnaryInvOp(expr_type inexpr) : _expr(inexpr) { - static_assert(M==N, "MATRIX MUST BE SQUARE"); - } - - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} - - constexpr FASTOR_INLINE expr_type expr() const {return _expr;} - -private: - expr_type _expr; -}; - -template -FASTOR_INLINE UnaryInvOp -inv(const AbstractTensor &src) { - return UnaryInvOp(src.self()); -} - - -// Upper triangular block matrix inversion -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -namespace internal { - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { - _inverse(in.data(),out.data()); -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - constexpr size_t N = 4UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - // Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - ut_inverse_dispatcher(a, inv_a); - - Tensor inv_d; - ut_inverse_dispatcher(d, inv_d); - - Tensor b_invd = matmul(b, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = -matmul(inv_a, b_invd); - out(fseq(),fseq<0,N>()) = 0; - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - constexpr size_t N = 8UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - // Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - ut_inverse_dispatcher(a, inv_a); - - Tensor inv_d; - ut_inverse_dispatcher(d, inv_d); - - Tensor b_invd = matmul(b, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = -matmul(inv_a, b_invd); - out(fseq(),fseq<0,N>()) = 0; - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 16UL; // start size - constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - // Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - ut_inverse_dispatcher(a, inv_a); - - Tensor inv_d; - ut_inverse_dispatcher(d, inv_d); - - Tensor b_invd = matmul(b, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = -matmul(inv_a, b_invd); - out(fseq(),fseq<0,N>()) = 0; - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 32UL; // start size - constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - // Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - ut_inverse_dispatcher(a, inv_a); - - Tensor inv_d; - ut_inverse_dispatcher(d, inv_d); - - Tensor b_invd = tmatmul(b, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = -tmatmul(inv_a, b_invd); - out(fseq(),fseq<0,N>()) = 0; - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 64UL; // start size - constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - // Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - ut_inverse_dispatcher(a, inv_a); - - Tensor inv_d; - ut_inverse_dispatcher(d, inv_d); - - Tensor b_invd = tmatmul(b, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = -tmatmul(inv_a, b_invd); - out(fseq(),fseq<0,N>()) = 0; - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void ut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 128UL; // start size - constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - // Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - ut_inverse_dispatcher(a, inv_a); - - Tensor inv_d; - ut_inverse_dispatcher(d, inv_d); - - Tensor b_invd = tmatmul(b, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = -tmatmul(inv_a, b_invd); - out(fseq(),fseq<0,N>()) = 0; - out(fseq(),fseq()) = inv_d; -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -} // internal - - - - -// Lower uni-triangular block matrix inversion -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -namespace internal { - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { - // This restricts the recursive block to lower uni-triangualar matrices - _lowunitri_inverse(in.data(),out.data()); -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - constexpr size_t N = 4UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - // Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - lut_inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor inv_d; - lut_inverse_dispatcher(d, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = 0; - out(fseq(),fseq<0,N>()) = -matmul(inv_d, c_inva); - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - constexpr size_t N = 8UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - // Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - lut_inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor inv_d; - lut_inverse_dispatcher(d, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = 0; - out(fseq(),fseq<0,N>()) = -matmul(inv_d, c_inva); - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 16UL; // start size - constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - // Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - lut_inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor inv_d; - lut_inverse_dispatcher(d, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = 0; - out(fseq(),fseq<0,N>()) = -matmul(inv_d, c_inva); - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 32UL; // start size - constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - // Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - lut_inverse_dispatcher(a, inv_a); - - Tensor c_inva = tmatmul(c, inv_a); - - Tensor inv_d; - lut_inverse_dispatcher(d, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = 0; - out(fseq(),fseq<0,N>()) = -tmatmul(inv_d, c_inva); - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 64UL; // start size - constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - // Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - lut_inverse_dispatcher(a, inv_a); - - Tensor c_inva = tmatmul(c, inv_a); - - Tensor inv_d; - lut_inverse_dispatcher(d, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = 0; - out(fseq(),fseq<0,N>()) = -tmatmul(inv_d, c_inva); - out(fseq(),fseq()) = inv_d; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lut_inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 128UL; // start size - constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - // Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - lut_inverse_dispatcher(a, inv_a); - - Tensor c_inva = tmatmul(c, inv_a); - - Tensor inv_d; - lut_inverse_dispatcher(d, inv_d); - - out(fseq<0,N>(),fseq<0,N>()) = inv_a; - out(fseq<0,N>(),fseq()) = 0; - out(fseq(),fseq<0,N>()) = -tmatmul(inv_d, c_inva); - out(fseq(),fseq()) = inv_d; -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -} // internal - - - - -// General recursive block matrix inversion -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -namespace internal { - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { - _inverse(in.data(),out.data()); -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { - - constexpr size_t N = 4UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a = inverse(a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor block_bb = inverse(static_cast>(d - matmul(c_inva, b))); - - Tensor inva_b = matmul(inv_a, b); - Tensor bb_c_inva = matmul(block_bb, c_inva); - - Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); - Tensor block_ab = -matmul(inva_b, block_bb); - Tensor block_ba = -bb_c_inva; - - out(fseq<0,N>(),fseq<0,N>()) = block_aa; - out(fseq<0,N>(),fseq()) = block_ab; - out(fseq(),fseq<0,N>()) = block_ba; - out(fseq(),fseq()) = block_bb; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { - - constexpr size_t N = 8UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor block_bb; - inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); - - Tensor inva_b = matmul(inv_a, b); - Tensor bb_c_inva = matmul(block_bb, c_inva); - - Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); - Tensor block_ab = -matmul(inva_b, block_bb); - Tensor block_ba = -bb_c_inva; - - out(fseq<0,N>(),fseq<0,N>()) = block_aa; - out(fseq<0,N>(),fseq()) = block_ab; - out(fseq(),fseq<0,N>()) = block_ba; - out(fseq(),fseq()) = block_bb; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 16UL; // start size - constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor block_bb; - inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); - - Tensor inva_b = matmul(inv_a, b); - Tensor bb_c_inva = matmul(block_bb, c_inva); - - Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); - Tensor block_ab = -matmul(inva_b, block_bb); - Tensor block_ba = -bb_c_inva; - - out(fseq<0,N>(),fseq<0,N>()) = block_aa; - out(fseq<0,N>(),fseq()) = block_ab; - out(fseq(),fseq<0,N>()) = block_ba; - out(fseq(),fseq()) = block_bb; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 32UL; // start size - constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor block_bb; - inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); - - Tensor inva_b = matmul(inv_a, b); - Tensor bb_c_inva = matmul(block_bb, c_inva); - - Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); - Tensor block_ab = -matmul(inva_b, block_bb); - Tensor block_ba = -bb_c_inva; - - out(fseq<0,N>(),fseq<0,N>()) = block_aa; - out(fseq<0,N>(),fseq()) = block_ab; - out(fseq(),fseq<0,N>()) = block_ba; - out(fseq(),fseq()) = block_bb; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 64UL; // start size - constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor block_bb; - inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); - - Tensor inva_b = matmul(inv_a, b); - Tensor bb_c_inva = matmul(block_bb, c_inva); - - Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); - Tensor block_ab = -matmul(inva_b, block_bb); - Tensor block_ba = -bb_c_inva; - - out(fseq<0,N>(),fseq<0,N>()) = block_aa; - out(fseq<0,N>(),fseq()) = block_ab; - out(fseq(),fseq<0,N>()) = block_ba; - out(fseq(),fseq()) = block_bb; -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void inverse_dispatcher(const Tensor &in, Tensor& out) { - - // constexpr size_t N = 128UL; // start size - constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size - Tensor a = in(fseq<0,N>(),fseq<0,N>()); - Tensor b = in(fseq<0,N>(),fseq()); - Tensor c = in(fseq(),fseq<0,N>()); - Tensor d = in(fseq(),fseq()); - - Tensor inv_a; - inverse_dispatcher(a, inv_a); - - Tensor c_inva = matmul(c, inv_a); - - Tensor block_bb; - inverse_dispatcher(static_cast>(d - matmul(c_inva, b)),block_bb); - - Tensor inva_b = matmul(inv_a, b); - Tensor bb_c_inva = matmul(block_bb, c_inva); - - Tensor block_aa = inv_a + matmul(inva_b, bb_c_inva); - Tensor block_ab = -matmul(inva_b, block_bb); - Tensor block_ba = -bb_c_inva; - - out(fseq<0,N>(),fseq<0,N>()) = block_aa; - out(fseq<0,N>(),fseq()) = block_ab; - out(fseq(),fseq<0,N>()) = block_ba; - out(fseq(),fseq()) = block_bb; -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -} // internal - - - -// Inversion using LU decomposition is in the LU module [unary_lu_op] -// For tensors -// SimpleInv - no pivot -template && InvType == InvCompType::SimpleInv,bool> = false> -FASTOR_INLINE Tensor inverse(const Tensor &a) { - Tensor out; - _inverse(a.data(),out.data()); - return out; -} -template && InvType == InvCompType::SimpleInv,bool> = false> -FASTOR_INLINE Tensor inverse(const Tensor &in) { - Tensor out; - internal::inverse_dispatcher(in,out); - return out; -} -// SimpleInv pivot -template = false> -FASTOR_INLINE Tensor inverse(const Tensor &in) { - Tensor out; - // We need to post multiply - swap columns using row permutation vector - Tensor P; - pivot_inplace(in,P); - auto A(apply_pivot(in,P)); - internal::inverse_dispatcher(A,out); - return reconstruct_colwise(out,P); - - // // matrix version - // Tensor P; - // pivot_inplace(in,P); - // auto A(apply_pivot(in,P)); - // internal::inverse_dispatcher(A,out); - // return matmul(out,P); -} - -// For high order tensors -template=3 && InvType == InvCompType::SimpleInv,bool> = false > -FASTOR_INLINE Tensor -inverse(const Tensor &a) { - - constexpr size_t remaining_product = last_matrix_extracter, - typename std_ext::make_index_sequence::type>::remaining_product; - - constexpr size_t I = get_value::value; - constexpr size_t J = get_value::value; - static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); - - Tensor out; - T *a_data = a.data(); - T *out_data = out.data(); - - for (size_t i=0; i(static_cast(a_data+i*J*J)); - _inverse(a_data+i*J*J,out_data+i*J*J); - } - - return out; -} - -// Inverse for generic expressions is provided here -template -FASTOR_INLINE -typename Derived::result_type -inverse(const AbstractTensor &src) { - // If we are here Derived is already an expression - using result_type = typename Derived::result_type; - const result_type tmp(src.self()); - return inverse(tmp); -} - - -// Inverse of lower uni-triangular matrices -template,bool> = false> -FASTOR_INLINE Tensor tinverse(const Tensor &in) { - Tensor out; - internal::lut_inverse_dispatcher(in,out); - return out; -} - -// Inverse of lower uni-triangular matrices -template,bool> = false> -FASTOR_INLINE Tensor tinverse(const Tensor &in) { - Tensor out; - internal::ut_inverse_dispatcher(in,out); - return out; -} -// Inverse for generic expressions of lower uni-triangular matrices is provided here -template -FASTOR_INLINE -typename Derived::result_type -tinverse(const AbstractTensor &src) { - // If we are here Derived is already an expression - using result_type = typename Derived::result_type; - const result_type tmp(src.self()); - return tinverse(tmp); -} - - - - - - -// assignments -template -FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryInvOp &src) { - using result_type = typename Expr::result_type; - const result_type& tmp = evaluate(src.expr().self()); - internal::inverse_dispatcher(tmp,dst.self()); -} - -template -FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryInvOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::inverse_dispatcher(tmp,tmp_inv); - trivial_assign_add(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryInvOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::inverse_dispatcher(tmp,tmp_inv); - trivial_assign_sub(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryInvOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::inverse_dispatcher(tmp,tmp_inv); - trivial_assign_mul(dst.self(),tmp_inv); -} - -template -FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryInvOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - result_type tmp_inv; - internal::inverse_dispatcher(tmp,tmp_inv); - trivial_assign_div(dst.self(),tmp_inv); -} - - -} // end of namespace Fastor - - -#endif // UNARY_INV_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_lu_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_lu_op.h deleted file mode 100644 index d34dc502..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_lu_op.h +++ /dev/null @@ -1,970 +0,0 @@ -#ifndef UNARY_LU_OP_H -#define UNARY_LU_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/inner.h" -#include "Fastor/backend/lufact.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" -#include "Fastor/expressions/linalg_ops/unary_piv_op.h" - - -namespace Fastor { - -namespace internal { - -/* Compile time recursive loop with inner for forward substitution of b/B given the lower unitriangular matrix L. - The following meta functions implements L * y = b for single or multiple right sides -*/ -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -template -struct forward_subs_impl { - - template - static FASTOR_INLINE void do_single_rhs(const Tensor &L, const Tensor &b, Tensor &y) { - y(from) = b(from) - _inner(&L.data()[from*M],y.data()); - forward_subs_impl::do_single_rhs(L, b, y); - } - template - static FASTOR_INLINE void do_single_rhs_pivot(const Tensor &L, const Tensor &b, const Tensor &p, Tensor &y) { - y(from) = b(p(from)) - _inner(&L.data()[from*M],y.data()); - forward_subs_impl::do_single_rhs_pivot(L, b, p, y); - } - - template - static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &L, const Tensor &B, Tensor &y, Tensor &X) { - y(from) = B(from,j) - _inner(&L.data()[from*M],y.data()); - X(from,j) = y(from); - forward_subs_impl::do_multi_rhs(j, L, B, y, X); - } - - template - static FASTOR_INLINE void do_multi_rhs_pivot(const size_t j, const Tensor &L, const Tensor &B, - const Tensor &p, Tensor &y, Tensor &X) { - y(from) = B(p(from),j) - _inner(&L.data()[from*M],y.data()); - X(from,j) = y(from); - forward_subs_impl::do_multi_rhs_pivot(j, L, B, p, y, X); - } -}; -template -struct forward_subs_impl { - - template - static FASTOR_INLINE void do_single_rhs(const Tensor &L, const Tensor &b, Tensor &y) { - y(from) = b(from) - _inner(&L.data()[from*M],y.data()); - } - template - static FASTOR_INLINE void do_single_rhs_pivot(const Tensor &L, const Tensor &b, const Tensor &p, Tensor &y) { - y(from) = b(p(from)) - _inner(&L.data()[from*M],y.data()); - } - - template - static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &L, const Tensor &B, Tensor &y, Tensor &X) { - y(from) = B(from,j) - _inner(&L.data()[from*M],y.data()); - X(from,j) = y(from); - } - template - static FASTOR_INLINE void do_multi_rhs_pivot(const size_t j, const Tensor &L, const Tensor &B, - const Tensor &p, Tensor &y, Tensor &X) { - y(from) = B(p(from),j) - _inner(&L.data()[from*M],y.data()); - X(from,j) = y(from); - } -}; - -template -FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &b) { - - Tensor y(0); - - forward_subs_impl<0,M-1>::do_single_rhs(L, b, y); -#if 0 - // The run-time loop version - // Solve for L * y = b - for (size_t i=0; i< M; ++i) { - T value = 0; - for (size_t k=0; k -FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &B) { - - // We keep a separate output tensor X from y [y is columns of X] - // to avoid strided access in X for the inner product - Tensor X; - - for (size_t j=0; j < N; ++j) { - Tensor y(0); - forward_subs_impl<0,M-1>::do_multi_rhs(j, L, B, y, X); - } -#if 0 - // The run-time loop version - X needs to be zeroed out for this version - for (size_t j=0; j < N; ++j) { - Tensor y(0); - // Solve for L * y = b - for (size_t i=0; i< M; ++i) { - T value = 0; - for (size_t k=0; k -FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &p, const Tensor &b) { - - Tensor y(0); - - forward_subs_impl<0,M-1>::do_single_rhs_pivot(L, b, p, y); -#if 0 - // The run-time loop version - // Solve for L * y = b - for (size_t i=0; i< M; ++i) { - T value = 0; - for (size_t k=0; k -FASTOR_INLINE Tensor forward_subs(const Tensor &L, const Tensor &p, const Tensor &B) { - - // We keep a separate output tensor X from y [y is columns of X] - // to avoid strided access in X for the inner product - Tensor X; - - for (size_t j=0; j < N; ++j) { - Tensor y(0); - forward_subs_impl<0,M-1>::do_multi_rhs_pivot(j, L, B, p, y, X); - } -#if 0 - // The run-time loop version - X needs to be zeroed out for this version - for (size_t j=0; j < N; ++j) { - Tensor y(0); - // Solve for L * y = b - for (size_t i=0; i< M; ++i) { - T value = 0; - for (size_t k=0; k -struct backward_subs_impl { - - template - static FASTOR_INLINE void do_single_rhs(const Tensor &U, const Tensor &y, Tensor &x) { - constexpr int idx = (int)M - from; - const T value = _inner(&U.data()[from*M+from],&x.data()[from]); - x(from) = ( y(from) - value) / U(from, from); - backward_subs_impl::do_single_rhs(U, y, x); - } - - template - static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &U, const Tensor &Y, Tensor &x, Tensor &X) { - constexpr int idx = (int)M - from; - const T value = _inner(&U.data()[from*M+from],&x.data()[from]); - x(from) = ( Y(from,j) - value ) / U(from, from); - X(from,j) = x(from); - backward_subs_impl::do_multi_rhs(j, U, Y, x, X); - } -}; - -template <> -struct backward_subs_impl<0,0> { - - template - static FASTOR_INLINE void do_single_rhs(const Tensor &U, const Tensor &y, Tensor &x) { - constexpr int idx = (int)M; - const T value = _inner(&U.data()[0*M+0],&x.data()[0]); - x(0) = ( y(0) - value) / U(0, 0); - } - - template - static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor &U, const Tensor &Y, Tensor &x, Tensor &X) { - constexpr int idx = (int)M; - const T value = _inner(&U.data()[0*M+0],&x.data()[0]); - x(0) = ( Y(0,j) - value ) / U(0, 0); - X(0,j) = x(0); - } -}; - -template -FASTOR_INLINE Tensor backward_subs(const Tensor &U, const Tensor &y) { - - // We keep a separate output tensor X from x [x is columns of X] - // to avoid strided access in X for the inner product - Tensor x(0); - backward_subs_impl::do_single_rhs(U, y, x); -#if 0 - // The run-time loop version - // Solve for of U * x = y - for (int i= int(M) - 1; i>=0; --i) { - T value = 0; - for (int k=i; k -FASTOR_INLINE Tensor backward_subs(const Tensor &U, const Tensor &Y) { - - // We keep a separate output tensor X from x [x is columns of X] - // to avoid strided access in X for the inner product - Tensor X; - - for (size_t j=0; j < N; ++j) { - Tensor x(0); - backward_subs_impl::do_multi_rhs(j, U, Y, x, X); - } -#if 0 - // The run-time loop version - X needs to be zeroed out for this version - Tensor X(0); - for (size_t j=0; j < 1; ++j) { - // Solve for of U * x = y - for (int i= int(M) - 1; i>=0; --i) { - T value = 0; - for (int k=i; k && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lu_simple_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - _lufact(A.data(),L.data(),U.data()); -} - -template ,bool> = false> -FASTOR_INLINE void lu_simple_dispatcher(const Tensor& A1, Tensor& L, Tensor& U) { - L.fill(0); - U.fill(0); - for (size_t j = 0; j < M; ++j) { - L(j, j) = 1; - for (size_t i = 0; i <= j; ++i) { - T value = A1(i, j); - for (size_t k = 0; k < i; ++k) { - value -= L(i, k) * U(k, j); - } - U(i, j) = value; - } - for (size_t i = j; i < M; ++i) { - T value = A1(i, j); - for (size_t k = 0; k < j; ++k) { - value -= L(i, k) * U(k, j); - } - value /= U(j, j); - L(i, j) = value; - } - } -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - -// Recursive non-modifying LU using matmul/outer product -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -/* This in essence implements the following loop but with static views - as dynamic views cannot be dispatched to matmul or a tensor cannot be - constructed to be sent to matmul - - for(size_t j=0; j -struct recursive_lu_impl { - template - static FASTOR_INLINE void Do(Tensor &L, Tensor &U) { - L(fseq(),fix) = U(fseq(),fix) / U.data()[from*N+from]; - U(fseq(),fix) = 0; - U(fseq(),fseq()) -= matmul(L(fseq(),fix), U(fix,fseq())); - recursive_lu_impl::Do(L, U); - } -}; -template -struct recursive_lu_impl { - template - static FASTOR_INLINE void Do(Tensor &L, Tensor &U) { - L(fseq(),fix) = U(fseq(),fix) / U.data()[from*N+from]; - U(fseq(),fix) = 0; - U(fseq(),fseq()) -= matmul(L(fseq(),fix), U(fix,fseq())); - } -}; - -template ,bool> = false> -FASTOR_INLINE void recursive_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - L(0,0) = 1; U(0,0) = A(0,0); -} -template ,bool> = false> -FASTOR_INLINE void recursive_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - // Requires M >=2 - U = A; - L.eye2(); - recursive_lu_impl<0,M-2>::Do(L, U); -} - - -#if 0 -// Recursive in-place LU using matmul/outer product -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -/* This in essence implements the following loop but with static views - as dynamic views cannot be dispatched to matmul or a tensor cannot be - constructed to be sent to matmul - - for(size_t j=0; j -struct recursive_lu_impl_inplace { - template - static FASTOR_INLINE void Do(Tensor &LU) { - LU(fseq(),fix) /= LU.data()[from*N+from]; - LU(fseq(),fseq()) -= matmul(LU(fseq(),fix), LU(fix,fseq())); - recursive_lu_impl_inplace::Do(LU); - } -}; -template -struct recursive_lu_impl_inplace { - template - static FASTOR_INLINE void Do(Tensor &LU) { - LU(fseq(),fix) /= LU.data()[from*N+from]; - LU(fseq(),fseq()) -= matmul(LU(fseq(),fix), LU(fix,fseq())); - } -}; - - -template ,bool> = false> -FASTOR_INLINE void recursive_inplace_lu_dispatcher(Tensor& A) { - return; -} -template ,bool> = false> -FASTOR_INLINE void recursive_inplace_lu_dispatcher(Tensor& A) { - // Requires M >=2 - recursive_lu_impl_inplace<0,M-2>::Do(A); -} - -template ,bool> = false> -FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& LU) { - LU(0,0) = A(0,0); -} -template ,bool> = false> -FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& LU) { - // Requires M >=2 - LU =A; - recursive_lu_impl_inplace<0,M-2>::Do(LU); -} - -template ,bool> = false> -FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - L(0,0) = 1; U(0,0) = A(0,0); -} -template ,bool> = false> -FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - // Requires M >=2 - Tensor LU(A); - recursive_lu_impl_inplace<0,M-2>::Do(LU); - - // Extracting L and U after this recursive decomposition is done - // almost beats the purpose performance wise - for (size_t i=0; i && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - _lufact(A.data(),L.data(),U.data()); -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - recursive_lu_dispatcher(A, L, U); -} - -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - - // We will compute LU decomposition block-wise assuming that A11 and A22 are invertible - // - // [A11 A12] [L11 0] [U11 U12] - // [A21 A22] [L21 L22] [0 U22] - // - // This results in - // - // A11 = L11 * U11 - // A12 = L11 * U12 - // A21 = L21 * U11 - // A22 = L21 * U12 - L22 * U22 - // - // Hence we need to do LU factorisation once for A11 and once for A22 - - // This is to avoid odd sizes for instance for size 35 we would - // want to do 35 = 16 + 19 rather than 35 = 32 + 3 if the start size was 32 - constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size - Tensor A11 = A(fseq<0,N>(),fseq<0,N>()); - Tensor A12 = A(fseq<0,N>(),fseq()); - Tensor A21 = A(fseq(),fseq<0,N>()); - Tensor A22 = A(fseq(),fseq()); - - Tensor L11(0), U11(0); - lu_block_dispatcher(A11, L11, U11); - - // Solve for U12 = {L11}^(-1)*A12 - Tensor U12 = tmatmul(tinverse(L11),A12); - // Solve for L21 = A21*{U11}^(-1) - Tensor L21 = tmatmul(A21,tinverse(U11)); - - Tensor S = A22 - matmul(L21,U12); - - Tensor L22(0), U22(0); - lu_block_dispatcher(S, L22, U22); - - L(fseq<0,N>(),fseq<0,N>()) = L11; - // L(fseq<0,N>(),fseq()) = 0; - L(fseq(),fseq<0,N>()) = L21; - L(fseq(),fseq()) = L22; - - U(fseq<0,N>(),fseq<0,N>()) = U11; - U(fseq<0,N>(),fseq()) = U12; - // U(fseq(),fseq<0,N>()) = 0; - U(fseq(),fseq()) = U22; -} - -// Conditional dispatch -namespace useless { -template && is_less_equal_v_,bool> = false> -FASTOR_INLINE void lu_block_simple_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - lu_block_dispatcher(A, L, U); - return; -} -template ,bool> = false> -FASTOR_INLINE void lu_block_simple_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - // lu_simple_dispatcher(A, L, U); - recursive_lu_dispatcher(A, L, U); - return; -} -} // useless - -/* For sizes greater than 64 we tile differently to avoid too many recursions -*/ -template ,bool> = false> -FASTOR_INLINE void lu_block_dispatcher(const Tensor& A, Tensor& L, Tensor& U) { - - // We will compute LU decomposition block-wise assuming that A11 and A22 are invertible - // - // [A11 A12] [L11 0] [U11 U12] - // [A21 A22] [L21 L22] [0 U22] - // - // This results in - // - // A11 = L11 * U11 - // A12 = L11 * U12 - // A21 = L21 * U11 - // A22 = L21 * U12 - L22 * U22 - // - // Hence we need to do LU factorisation once for A11 and once for A22 - - // This is to avoid odd sizes for instance for size 65 we would - // want to do 65 = 32 + 33 rather than 65 = 64 + 1 if the start size was 64 - constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size - Tensor A11 = A(fseq<0,N>(),fseq<0,N>()); - Tensor A12 = A(fseq<0,N>(),fseq()); - Tensor A21 = A(fseq(),fseq<0,N>()); - Tensor A22 = A(fseq(),fseq()); - - Tensor L11(0), U11(0); - useless::lu_block_simple_dispatcher(A11, L11, U11); - - // Solve for U12 = {L11}^(-1)*A12 - Tensor U12 = tmatmul(tinverse(L11),A12); - // Ideally use forward_subs but its iterative nature makes it less efficient than tmatmul - // Tensor U12 = forward_subs(L11,A12); - // Solve for L21 = A21*{U11}^(-1) - Tensor L21 = tmatmul(A21,tinverse(U11)); - // Not quite performant as we can't avoid the matmul here - // Tensor I; I.eye2(); - // Tensor L21 = matmul(A21, backward_subs(U11, I)); - - Tensor S = A22 - matmul(L21,U12); - - Tensor L22(0), U22(0); - useless::lu_block_simple_dispatcher(S, L22, U22); - - L(fseq<0,N>(),fseq<0,N>()) = L11; - // L(fseq<0,N>(),fseq()) = 0; - L(fseq(),fseq<0,N>()) = L21; - L(fseq(),fseq()) = L22; - - U(fseq<0,N>(),fseq<0,N>()) = U11; - U(fseq<0,N>(),fseq()) = U12; - // U(fseq(),fseq<0,N>()) = 0; - U(fseq(),fseq()) = U22; -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -} // internal - - -/* Block LU factorisation overloads */ -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -// BlockLU - no pivot -template && LUType == LUCompType::BlockLU,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U) { - L.fill(0); - U.fill(0); - internal::lu_block_dispatcher(src.self(),L,U); -} -template && LUType == LUCompType::BlockLU,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U) { - L.fill(0); - U.fill(0); - typename Expr::result_type tmp(src.self()); - internal::lu_block_dispatcher(tmp,L,U); -} - -// BlockLU - vector pivot -template && LUType == LUCompType::BlockLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - L.fill(0); - U.fill(0); - pivot_inplace(src.self(),P); - auto A(apply_pivot(src.self(),P)); - internal::lu_block_dispatcher(A,L,U); -} -template && LUType == LUCompType::BlockLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - L.fill(0); - U.fill(0); - typename Expr::result_type A(src.self()); - pivot_inplace(A,P); - // Modify A as A is a temporary anyway - apply_pivot_inplace(A,P); - internal::lu_block_dispatcher(A,L,U); -} - -// BlockLU - matrix pivot -template && LUType == LUCompType::BlockLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - L.fill(0); - U.fill(0); - pivot_inplace(src.self(),P); - auto A(apply_pivot(src.self(),P)); - internal::lu_block_dispatcher(A,L,U); -} -template && LUType == LUCompType::BlockLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - L.fill(0); - U.fill(0); - typename Expr::result_type A(src.self()); - pivot_inplace(A,P); - // Modify A as A is a temporary anyway - apply_pivot_inplace(A,P); - internal::lu_block_dispatcher(A,L,U); -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - - -/* Simple LU factorisation overloads */ -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -// SimpleLU - no pivot -template && LUType == LUCompType::SimpleLU,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U) { - internal::lu_simple_dispatcher(src.self(),L,U); -} -template && LUType == LUCompType::SimpleLU,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U) { - typename Expr::result_type tmp(src.self()); - internal::lu_simple_dispatcher(tmp,L,U); -} - -// SimpleLU - vector pivot -template && LUType == LUCompType::SimpleLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - pivot_inplace(src.self(),P); - auto A(apply_pivot(src.self(),P)); - internal::lu_simple_dispatcher(A,L,U); -} -template && LUType == LUCompType::SimpleLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - typename Expr::result_type A(src.self()); - pivot_inplace(A,P); - // Modify A as A is a temporary anyway - apply_pivot_inplace(A,P); - internal::lu_simple_dispatcher(A,L,U); -} - -// SimpleLU - matrix pivot -template && LUType == LUCompType::SimpleLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - pivot_inplace(src.self(),P); - auto A(apply_pivot(src.self(),P)); - internal::lu_simple_dispatcher(A,L,U); -} -template && LUType == LUCompType::SimpleLUPiv,bool> = false> -FASTOR_INLINE -void -lu(const AbstractTensor &src, Tensor& L, Tensor& U, Tensor& P) { - typename Expr::result_type A(src.self()); - pivot_inplace(A,P); - // Modify A as A is a temporary anyway - apply_pivot_inplace(A,P); - internal::lu_simple_dispatcher(A,L,U); -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - - - - - - - -// Inversion using LU -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -namespace internal { - -template -FASTOR_INLINE Tensor get_lu_inverse(const Tensor &L, const Tensor &U) { - // We will solve for multiple RHS [B = I] - // Loop over columns of B = I - Tensor I; I.eye2(); - Tensor Y = forward_subs(L, I); - Tensor X = backward_subs(U, Y); - return X; -} - -template -FASTOR_INLINE Tensor get_lu_inverse(Tensor &L, const Tensor &U, const Tensor &p) { - // We will solve for multiple RHS [B = I] - // Loop over columns of B = I - Tensor I; I.eye2(); - Tensor Y = forward_subs(L, p, I); - Tensor X = backward_subs(U, Y); - return X; -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -} // internal - - -// SimpleLU -template = false> -FASTOR_INLINE Tensor inverse(const Tensor &A) { - Tensor L, U; - lu(A, L, U); - return internal::get_lu_inverse(L, U); -} - -// BlockLU -template = false> -FASTOR_INLINE Tensor inverse(const Tensor &A) { - Tensor L, U; - lu(A, L, U); - return internal::get_lu_inverse(L, U); -} - -// SimpleLUPiv -template = false> -FASTOR_INLINE Tensor inverse(const Tensor &A) { - Tensor L, U; - Tensor p; - lu(A, L, U, p); - return internal::get_lu_inverse(L, U, p); -} - -// BlockLUPiv -template = false> -FASTOR_INLINE Tensor inverse(const Tensor &A) { - Tensor L, U; - Tensor p; - lu(A, L, U, p); - return internal::get_lu_inverse(L, U, p); -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - - - - - - - - -// Solving linear system of equations using LU -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -namespace internal { - -template -FASTOR_INLINE Tensor get_lu_solve(const Tensor &L, const Tensor &U, const Tensor &b) { - Tensor y = forward_subs(L, b); - Tensor x = backward_subs(U, y); - return x; -} - -template -FASTOR_INLINE Tensor get_lu_solve(Tensor &L, const Tensor &U, const Tensor &p, const Tensor &b) { - Tensor y = forward_subs(L, p, b); - Tensor x = backward_subs(U, y); - return x; -} - -// Multiple RHS -template -FASTOR_INLINE Tensor get_lu_solve(const Tensor &L, const Tensor &U, const Tensor &B) { - Tensor Y = forward_subs(L, B); - Tensor X = backward_subs(U, Y); - return X; -} - -template -FASTOR_INLINE Tensor get_lu_solve(const Tensor &L, const Tensor &U, const Tensor &p, const Tensor &B) { - Tensor Y = forward_subs(L, p, B); - Tensor X = backward_subs(U, Y); - return X; -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -} // internal - - -// Single RHS -// SimpleLU - no pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { - Tensor L, U; - lu(A, L, U); - return internal::get_lu_solve(L, U, b); -} - -// SimpleLU - pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { - Tensor L, U; - Tensor p; - lu(A, L, U, p); - return internal::get_lu_solve(L, U, p, b); -} - -// BlockLU - no pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { - Tensor L, U; - lu(A, L, U); - return internal::get_lu_solve(L, U, b); -} - -// BlockLU - pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &b) { - Tensor L, U; - Tensor p; - lu(A, L, U, p); - return internal::get_lu_solve(L, U, p, b); -} - -// Multiple RHS -// SimpleLU - no pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { - Tensor L, U; - lu(A, L, U); - return internal::get_lu_solve(L, U, B); -} - -// SimpleLU - pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { - Tensor L, U; - Tensor p; - lu(A, L, U, p); - return internal::get_lu_solve(L, U, p, B); -} - -// SimpleLU - no pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { - Tensor L, U; - lu(A, L, U); - return internal::get_lu_solve(L, U, B); -} - -// SimpleLU - pivot -template = false> -FASTOR_INLINE Tensor solve(const Tensor &A, const Tensor &B) { - Tensor L, U; - Tensor p; - lu(A, L, U, p); - return internal::get_lu_solve(L, U, p, B); -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - - - - - - - - - -// Computing determinant using LU -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -template = false> -FASTOR_INLINE T determinant(const Tensor &A) { - int nswaps = internal::count_swaps(A) % 2UL == 0 ? 1 : -1; - Tensor L, U; - Tensor p; - lu(A, L, U, p); - return product(diag(U)) * nswaps; -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - -} // end of namespace Fastor - - -#endif // UNARY_LU_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_norm_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_norm_op.h deleted file mode 100644 index bcd83f71..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_norm_op.h +++ /dev/null @@ -1,106 +0,0 @@ -#ifndef UNARY_NORM_OP_H -#define UNARY_NORM_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/backend/norm.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -namespace Fastor { - -// For tensors -template,bool> = false> -FASTOR_INLINE T norm(const T &a) { - return std::abs(a); -} -template -FASTOR_INLINE T norm(const Tensor &a) { - if (sizeof...(Rest) == 0) - return *a.data(); - return _norm::value>(a.data()); -} - -// For generic expressions -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type norm(const AbstractTensor &_src) { - - const Derived &src = _src.self(); - using T = typename Derived::scalar_type; - using V = choose_best_simd_vector_t; - T _scal=0; -#ifdef FASTOR_AVX512_IMPL - V omm0, omm1, omm2, omm3, omm4, omm5, omm6, omm7; -#else - V omm0, omm1, omm2, omm3; -#endif - FASTOR_INDEX i = 0; - - // With AVX utilises all the 16 registers but hurts the performance - // due to spill if eval has created temporary registers so only - // activated for AVX512 -#ifdef FASTOR_AVX512_IMPL - for (; i < ROUND_DOWN(src.size(),8*V::Size); i+=8*V::Size) { - const auto smm0 = src.template eval(i); - const auto smm1 = src.template eval(i+V::Size); - const auto smm2 = src.template eval(i+2*V::Size); - const auto smm3 = src.template eval(i+3*V::Size); - const auto smm4 = src.template eval(i+4*V::Size); - const auto smm5 = src.template eval(i+5*V::Size); - const auto smm6 = src.template eval(i+6*V::Size); - const auto smm7 = src.template eval(i+7*V::Size); - omm0 = fmadd(smm0,smm0,omm0); - omm1 = fmadd(smm1,smm1,omm1); - omm2 = fmadd(smm2,smm2,omm2); - omm3 = fmadd(smm3,smm3,omm3); - omm4 = fmadd(smm4,smm4,omm4); - omm5 = fmadd(smm5,smm5,omm5); - omm6 = fmadd(smm6,smm6,omm6); - omm7 = fmadd(smm7,smm7,omm7); - } -#endif - for (; i < ROUND_DOWN(src.size(),4*V::Size); i+=4*V::Size) { - const auto smm0 = src.template eval(i); - const auto smm1 = src.template eval(i+V::Size); - const auto smm2 = src.template eval(i+2*V::Size); - const auto smm3 = src.template eval(i+3*V::Size); - omm0 = fmadd(smm0,smm0,omm0); - omm1 = fmadd(smm1,smm1,omm1); - omm2 = fmadd(smm2,smm2,omm2); - omm3 = fmadd(smm3,smm3,omm3); - } - for (; i < ROUND_DOWN(src.size(),2*V::Size); i+=2*V::Size) { - const auto smm0 = src.template eval(i); - const auto smm1 = src.template eval(i+V::Size); - omm0 = fmadd(smm0,smm0,omm0); - omm1 = fmadd(smm1,smm1,omm1); - } - for (; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { - const auto smm0 = src.template eval(i); - omm0 = fmadd(smm0,smm0,omm0); - } - for (; i < src.size(); ++i) { - const auto smm0 = src.template eval_s(i); - _scal += smm0*smm0; - } -#ifdef FASTOR_AVX512_IMPL - return sqrts( (omm0 + omm1 + omm2 + omm3 + omm4 + omm5 + omm6 + omm7).sum() + _scal); -#else - return sqrts( (omm0 + omm1 + omm2 + omm3).sum() + _scal); -#endif -} - - -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type norm(const AbstractTensor &_src) { - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out(src); - return norm(out); -} - -} // end of namespace Fastor - -#endif // UNARY_NORM_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_piv_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_piv_op.h deleted file mode 100644 index 42c5b12b..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_piv_op.h +++ /dev/null @@ -1,409 +0,0 @@ -#ifndef UNARY_PIV_OP_H -#define UNARY_PIV_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" - -#include - - -namespace Fastor { - -namespace internal { - -template -FASTOR_INLINE size_t count_swaps(const Tensor& A) { - size_t count = 0; - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A(i, j)) > std::abs(A(max_index, j))) - max_index = i; - } - if (j != max_index) - count++; - } - return count; -} - -} - - -template = false> -FASTOR_INLINE Tensor pivot(const Tensor& A) { - Tensor perm; - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A(i, j)) > std::abs(A(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } - Tensor P(0); - for (size_t i = 0; i < M; ++i) - P(i, perm(i)) = 1; - return P; -} - -template = false> -FASTOR_INLINE Tensor pivot(const Tensor& A) { - Tensor perm; - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A(i, j)) > std::abs(A(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } - return perm; -} - - -// For generic expressions -template, bool> = false> -FASTOR_INLINE -Tensor, - get_tensor_dimension_v<1,typename Derived::result_type> -> -pivot(const AbstractTensor& src) { - typename Derived::result_type tmp(src.self()); - return pivot(tmp); -} -template, bool> = false> -FASTOR_INLINE -Tensor, - get_tensor_dimension_v<1,typename Derived::result_type> -> -pivot(const AbstractTensor& src) { - - static_assert(DIM==2, "TENSOR MUST BE SQUARE FOR PIVOT COMPUTATION"); - - using T = typename Derived::scalar_type; - using result_type = typename Derived::result_type; - constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; - constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; - - const Derived &A = src.self(); - - Tensor perm; - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } - Tensor P(0); - for (size_t i = 0; i < M; ++i) - P(i, perm(i)) = 1; - return P; -} - -template, bool> = false> -FASTOR_INLINE -Tensor -> -pivot(const AbstractTensor& src) { - typename Derived::result_type tmp(src.self()); - return pivot(tmp); -} -template, bool> = false> -FASTOR_INLINE -Tensor -> -pivot(const AbstractTensor& src) { - - static_assert(DIM==2, "TENSOR MUST BE SQUARE FOR PIVOT COMPUTATION"); - - using T = typename Derived::scalar_type; - using result_type = typename Derived::result_type; - constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>; - constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>; - - const Derived &A = src.self(); - - Tensor perm; - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } - return perm; -} - -/* In place versions - Given an evaluated pivot tensor populates it -*/ -template -FASTOR_INLINE void pivot_inplace(const Tensor& A, Tensor& perm) { - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A(i, j)) > std::abs(A(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } -} - -template -FASTOR_INLINE void pivot_inplace(const Tensor& A, Tensor &P) { - Tensor perm; - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A(i, j)) > std::abs(A(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } - P.fill(0); - for (size_t i = 0; i < M; ++i) - P(i, perm(i)) = 1; -} - -template, bool> = false> -FASTOR_INLINE -void -pivot_inplace(const AbstractTensor& src, Tensor &perm) { - typename Derived::result_type tmp(src.self()); - pivot_inplace(tmp,perm); -} -template, bool> = false> -FASTOR_INLINE -void -pivot_inplace(const AbstractTensor& src, Tensor &perm) { - - static_assert(DIM==2, "TENSOR MUST BE 2D FOR PIVOT COMPUTATION"); - - using T = typename Derived::scalar_type; - - const Derived &A = src.self(); - - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } -} - -template, bool> = false> -FASTOR_INLINE -void -pivot_inplace(const AbstractTensor& src, Tensor &P) { - typename Derived::result_type tmp(src.self()); - pivot_inplace(tmp,P); -} -template, bool> = false> -FASTOR_INLINE -void -pivot_inplace(const AbstractTensor& src, Tensor &P) { - - static_assert(DIM==2, "TENSOR MUST BE 2D FOR PIVOT COMPUTATION"); - - const Derived &A = src.self(); - - Tensor perm; - perm.iota(); - for (size_t j = 0; j < M; ++j) { - size_t max_index = j; - for (size_t i = j; i < M; ++i) { - // std::abs is necessary only for complex valued numbers - if (std::abs(A.template eval_s(i, j)) > std::abs(A.template eval_s(max_index, j))) - max_index = i; - } - if (j != max_index) - std::swap(perm(j), perm(max_index)); - } - for (size_t i = 0; i < M; ++i) - P(i, perm(i)) = 1; -} - - - - -/* Apply a pivot on a tensor/matrix - Applying pivot can only work on evaluated tensors and not expression - as non-evaluated expression do not have storage/data -*/ -template -FASTOR_INLINE Tensor apply_pivot(const Tensor& A, const Tensor& P) { - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - if (P(i) != i) { - std::copy_n(&A.data()[P(i)*N],N,©A.data()[i*N]); - } - } - return copyA; -} -template -FASTOR_INLINE Tensor apply_pivot(const Tensor& A, const Tensor& P) { - // The output tensor is just matmul(P,A), but we are going to avoid - // the call to matmul - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1)); - size_t p = std::distance(&P.data()[i*N],it); - // Swap row p and i - if (p != i) { - std::copy_n(&A.data()[p*N],N,©A.data()[i*N]); - } - } - return copyA; -} - -template -FASTOR_INLINE void apply_pivot_inplace(Tensor& A, const Tensor& P) { - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - if (P(i) != i) { - std::copy_n(©A.data()[P(i)*N],N,&A.data()[i*N]); - } - } -} -template -FASTOR_INLINE void apply_pivot_inplace(Tensor& A, const Tensor& P) { - // The output tensor is just matmul(P,A), but we are going to avoid - // the call to matmul - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1)); - size_t p = std::distance(&P.data()[i*N],it); - // Swap row p and i - if (p != i) { - std::copy_n(©A.data()[p*N],N,&A.data()[i*N]); - } - } -} - - - - -/* Reconstructing the matrix back from a pivoted factorisation - Reconstructs from LU/QR/QL etc -*/ -template -FASTOR_INLINE Tensor reconstruct(const Tensor& L, Tensor& U) { - return matmul(L,U); -} - -/* Reconstructing the matrix back from a pivoted factorisation - Reconstructs from an PLU decomposition given P, L, U where P is - integral permutation vector - A = {P}^(-1)*L*U -*/ -template -FASTOR_INLINE Tensor reconstruct(const Tensor& L, Tensor& U, const Tensor& P) { - Tensor A = matmul(L,U); - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - if (P(i) != i) { - std::copy_n(©A.data()[i*N],N,&A.data()[P(i)*N]); - } - } - return A; -} - -/* Reconstructing the matrix back from a pivoted factorisation - Reconstructs from an PLU decomposition given P, L, U - A = {P}^(-1)*L*U -*/ -template -FASTOR_INLINE Tensor reconstruct(const Tensor& L, Tensor& U, const Tensor& P) { - // To avoid computing the inverse of P in {P}^(-1)*L*U - Tensor A = matmul(L,U); - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1)); - size_t p = std::distance(&P.data()[i*N],it); - if (p != i) { - std::copy_n(©A.data()[i*N],N,&A.data()[p*N]); - } - } - return A; -} - - -/* Reconstructing the matrix back from a pivot only -*/ -template -FASTOR_INLINE Tensor reconstruct(const Tensor& A, const Tensor& P) { - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - if (P(i) != i) { - std::copy_n(&A.data()[i*N],N,©A.data()[P(i)*N]); - } - } - return copyA; -} - -/* Reconstructing the matrix back from a pivot only - column-wise or rather post multiplication - used for computing inverse -*/ -template -FASTOR_INLINE Tensor reconstruct_colwise(const Tensor& A, const Tensor& P) { - Tensor copyA(A); - for (size_t i=0; i< M; ++i) { - if (P(i) != i) { - copyA(all,P(i)) = A(all,i); - } - } - return copyA; -} - - - -} // end of namespace Fastor - - -#endif // UNARY_PIV_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_qr_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_qr_op.h deleted file mode 100644 index f83a90fd..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_qr_op.h +++ /dev/null @@ -1,198 +0,0 @@ -#ifndef UNARY_QR_OP_H -#define UNARY_QR_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" -#include "Fastor/expressions/linalg_ops/unary_piv_op.h" - - -namespace Fastor { - -namespace internal { - -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -/* Modified Gram-Schmidt Row-wise [MGSR] QR factorisation - that provides numerical stabilitiy. - This simple implementation provided for now is just for - convenience and not tuned for performance. Given that Tensor - types have fixed dimensions the compiler typically does a good - job - - See: - Thomas Jakobs et. al. "Performance and energy consumption of the SIMD - Gram–Schmidt process for vector orthogonalization" - for implementation details including AVX/AVX512 implementation. A straight- - forward explicit vectorisation of this does not yield good performance for - row-major tensors, as step 1-2 need gather instructions and step 3-4 require - dynamic masking to gain performance -*/ -template -FASTOR_INLINE void qr_mgsr_dispatcher(const Tensor &A0, Tensor& Q, Tensor& R) { - - // Copy incoming tensor as we will modify A - Tensor A(A0); - // Zero out - R.fill(0); - - for (size_t i=0; i< N; ++i) { - // step 1 - T R_ii = 0; - for (size_t k=0; k< M; ++k) { - R_ii += A(k,i)*A(k,i); - } - R_ii = sqrts(R_ii); - R(i,i) = R_ii; - - // step 2 - for (size_t k=0; k< M; ++k) { - Q(k,i) = A(k,i) / R_ii; - } - - // step 3 - for (size_t k=0; k< M; ++k) { - for (size_t j=i+1; j && QRType == QRCompType::MGSR,bool> = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R) { - internal::qr_mgsr_dispatcher(src.self(),Q,R); -} - -template && QRType == QRCompType::MGSR,bool> = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R) { - Tensor A(src.self()); - internal::qr_mgsr_dispatcher(A,Q,R); -} - -// QR MGSR - vector pivot -template && QRType == QRCompType::MGSRPiv,bool> = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { - pivot_inplace(src.self(),P); - auto A(apply_pivot(src.self(),P)); - internal::qr_mgsr_dispatcher(A,Q,R); -} - -template && QRType == QRCompType::MGSRPiv,bool> = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { - typename Expr::result_type A(src.self()); - pivot_inplace(A,P); - // Modify A as A is a temporary anyway - apply_pivot_inplace(A,P); - internal::qr_mgsr_dispatcher(A,Q,R); -} - -// QR MGSR - matrix pivot -template && QRType == QRCompType::MGSRPiv,bool> = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { - pivot_inplace(src.self(),P); - auto A(apply_pivot(src.self(),P)); - internal::qr_mgsr_dispatcher(A,Q,R); -} - -template && QRType == QRCompType::MGSRPiv,bool> = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { - typename Expr::result_type A(src.self()); - pivot_inplace(A,P); - // Modify A as A is a temporary anyway - apply_pivot_inplace(A,P); - internal::qr_mgsr_dispatcher(A,Q,R); -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -// QR householder reflections etc -template = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R) { - static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET"); -} -template = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { - static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET"); -} -template = false> -FASTOR_INLINE -void -qr(const AbstractTensor &src, Tensor &Q, Tensor &R, Tensor &P) { - static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET"); -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - - - - -// Computing determinant using QR -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// -template = false> -FASTOR_INLINE T determinant(const Tensor &a) { - Tensor Q, R; - qr(a, Q, R); - return product(diag(R)); -} -//-----------------------------------------------------------------------------------------------------------// -//-----------------------------------------------------------------------------------------------------------// - - - -} // end of namespace Fastor - - -#endif // UNARY_QR_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_svd_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_svd_op.h deleted file mode 100644 index ce1ba59d..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_svd_op.h +++ /dev/null @@ -1,163 +0,0 @@ -#ifndef UNARY_SVD_OP_H -#define UNARY_SVD_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/inner.h" -#include "Fastor/backend/lufact.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_computation_types.h" -#include "Fastor/expressions/linalg_ops/unary_det_op.h" -#include "Fastor/expressions/linalg_ops/unary_trans_op.h" - - -namespace Fastor { - -// SVD -template = false > -FASTOR_INLINE void svd(const Tensor &A, Tensor &U, Tensor &S, Tensor &V) { - - constexpr T Epsilon_v = std::numeric_limits::epsilon(); - - const T f00 = A(0, 0); - const T f01 = A(0, 1); - const T f10 = A(1, 0); - const T f11 = A(1, 1); - - // If matrix is diagonal, SVD is trivial - if (std::abs(f01 - f10) < Epsilon_v && std::abs(f01) < Epsilon_v) - { - // Compute U - U(0,0) = f00 < 0 ? -1. : 1.; - U(0,1) = 0.; - U(1,0) = 0.; - U(1,1) = f11 < 0. ? -1. : 1.; - - // Compute S - S(0,0) = std::abs(f00); - S(0,1) = 0; - S(1,0) = 0; - S(1,1) = std::abs(f11); - - // Compute V - V.eye2(); - } - // Otherwise, we need to compute A^T*A - else - { - T j = f00 * f00 + f01 * f01; - T k = f10 * f10 + f11 * f11; - T v_c = f00 * f10 + f01 * f11; - // Check to see if A^T*A is diagonal - if (std::abs(v_c) < Epsilon_v) - { - // Compute S - T s1 = std::sqrt(j); - T s2 = std::abs(j - k) < Epsilon_v ? s1 : std::sqrt(k); - S(0,0) = s1; - S(0,1) = 0; - S(1,0) = 0; - S(1,1) = s2; - - // Compute U - U.eye2(); - - // Compute V - V(0,0) = f00 / s1; - V(0,1) = f10 / s2; - V(1,0) = f01 / s1; - V(1,1) = f11 / s2; - } - // Otherwise, solve quadratic equation for eigenvalues - else - { - T jmk = j - k; - T jpk = j + k; - T root = std::sqrt(jmk * jmk + 4. * v_c * v_c); - T eig1 = (jpk + root) * 0.5; - T eig2 = (jpk - root) * 0.5; - - // Compute S - T s1 = std::sqrt(eig1); - T s2 = std::abs(root) < Epsilon_v ? s1 : ( eig2 > 0 ? std::sqrt(eig2) : Epsilon_v); - S(0,0) = s1; - S(0,1) = 0; - S(1,0) = 0; - S(1,1) = s2; - - // Compute U - use eigenvectors of A^T*A as U - T v_s = eig1 - j; - T len = std::max(std::sqrt(v_s * v_s + v_c * v_c), Epsilon_v); - v_c /= len; - v_s /= len; - - U(0,0) = v_c; - U(0,1) = -v_s; - U(1,0) = v_s; - U(1,1) = v_c; - - // Compute V - as A * U / s - const T cc = (f00 * v_c + f10 * v_s) / s1; - const T cs = (f01 * v_c + f11 * v_s) / s1; - if (std::abs(s2) > Epsilon_v) - { - V(0,0) = cc; - V(0,1) = (f10* v_c - f00 * v_s) / s2; - V(1,0) = cs; - V(1,1) = (f11 * v_c - f01 * v_s) / s2; - } - else - { - V(0,0) = cc; - V(0,1) = cs; - V(1,0) = cs; - V(1,1) = -cc; - } - } - } -} - - - -// Signed SVD -template -FASTOR_INLINE void ssvd(const Tensor &A, Tensor &U, Tensor &S, Tensor &V) { - - // Same as above but avoiding the L matrix - svd(A, U, S, V); - - // See where to pull the reflection out of - const T detU = determinant(U); - const T detV = determinant(V); - - if (detU >= 0 && detV >= 0) - { - // No reflection svd == svd_rv, return - return; - } - - Tensor L = matmul(U, transpose(V)); - const T lastColumn = determinant(L); - - if (detU < 0 && detV > 0) - { - U(all, M - 1) *= lastColumn; - } - else if (detU > 0 && detV < 0) - { - V(all, M - 1) *= lastColumn; - } - - // Push the reflection to the diagonal - S(M - 1, M - 1) *= lastColumn; -} -//-----------------------------------------------------------------------------------------------------------// - -} // end of namespace Fastor - - -#endif // UNARY_SVD_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_trace_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_trace_op.h deleted file mode 100644 index 95a62781..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_trace_op.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef UNARY_TRACE_OP_H -#define UNARY_TRACE_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -namespace Fastor { - -// For tensors -template -FASTOR_INLINE T trace(const Tensor &a) { - return _trace(static_cast(a.data())); -} - -// For high order tensors -template=3,bool>::type=0> -FASTOR_INLINE -typename last_matrix_extracter, typename std_ext::make_index_sequence::type>::type -trace(const Tensor &a) { - - using OutTensor = typename last_matrix_extracter, - typename std_ext::make_index_sequence::type>::type; - constexpr size_t remaining_product = last_matrix_extracter, - typename std_ext::make_index_sequence::type>::remaining_product; - - constexpr size_t I = get_value::value; - constexpr size_t J = get_value::value; - static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); - - OutTensor out; - T *a_data = a.data(); - T *out_data = out.data(); - - for (size_t i=0; i(static_cast(a_data+i*J*J)); - } - - return out; -} - -// Trace for generic expressions -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type trace(const AbstractTensor &_src) { - - const Derived& src = _src.self(); - using T = typename Derived::scalar_type; - using tensor_type = typename Derived::result_type; - constexpr size_t M = get_tensor_dimensions::dims[0]; - constexpr size_t N = get_tensor_dimensions::dims[1]; - static_assert(DIMS==2,"TENSOR EXPRESSION SHOULD BE UNIFORM (SQUARE)"); - static_assert(M==N,"TENSOR EXPRESSION SHOULD BE TWO DIMENSIONAL"); - - FASTOR_INDEX i; - T _scal=0; - for (i = 0; i < M; ++i) { - _scal += src.template eval_s(i*(N+1)); - } - return _scal; -} - -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type trace(const AbstractTensor &_src) { - - using result_type = typename Derived::result_type; - const result_type out(_src.self()); - return trace(out); -} - -} // end of namespace Fastor - -#endif // UNARY_TRACE_OP_H diff --git a/noarch/include/Fastor/expressions/linalg_ops/unary_trans_op.h b/noarch/include/Fastor/expressions/linalg_ops/unary_trans_op.h deleted file mode 100644 index 20858785..00000000 --- a/noarch/include/Fastor/expressions/linalg_ops/unary_trans_op.h +++ /dev/null @@ -1,173 +0,0 @@ -#ifndef UNARY_TRANS_OP_H -#define UNARY_TRANS_OP_H - -#include "Fastor/meta/meta.h" -#include "Fastor/backend/transpose/transpose.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Aliasing.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/TensorTraits.h" -#include "Fastor/expressions/expression_traits.h" - - -namespace Fastor { - -template -struct UnaryTransOp: public AbstractTensor,DIM0> { - using expr_type = expression_t; - static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,typename Expr::result_type>; - static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,typename Expr::result_type>; - static constexpr FASTOR_INDEX Dimension = DIM0; - static constexpr FASTOR_INDEX rank() {return DIM0;} - using scalar_type = typename scalar_type_finder>::type; - using simd_vector_type = typename Expr::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - - FASTOR_INLINE UnaryTransOp(expr_type inexpr) : _expr(inexpr) { - } - - FASTOR_INLINE FASTOR_INDEX size() const {return M*N;} - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i == 0 ? N : M;} - - constexpr FASTOR_INLINE expr_type expr() const {return _expr;} - -private: - expr_type _expr; -}; - - -/* Tensor transpose returning a tensor expression */ -template -FASTOR_INLINE UnaryTransOp -trans(const AbstractTensor &src) { - return UnaryTransOp(src.self()); -} - - -/* Tensor transpose immediately returning a tensor */ -template -FASTOR_INLINE Tensor transpose(const Tensor &a) { - Tensor out; - _transpose(a.data(),out.data()); - return out; -} - -/* Tensor transpose for higher order tensors immediately returning a tensor */ -template=3,bool>::type=0> -FASTOR_INLINE Tensor -transpose(const Tensor &a) { - - constexpr size_t remaining_product = last_matrix_extracter, - typename std_ext::make_index_sequence::type>::remaining_product; - - constexpr size_t I = get_value::value; - constexpr size_t J = get_value::value; - static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME"); - - Tensor out; - T *a_data = a.data(); - T *out_data = out.data(); - - for (size_t i=0; i(a_data+i*J*J,out_data+i*J*J); - } - - return out; -} - -// Transpose for generic expressions -template -FASTOR_INLINE -Tensor< - typename scalar_type_finder::type, - get_tensor_dimension_v<1,typename Expr::result_type>, - get_tensor_dimension_v<0,typename Expr::result_type>> -transpose(const AbstractTensor &src) { - // If we are here Expr is already an expression - using result_type = typename Expr::result_type; - const result_type tmp(src.self()); - return transpose(tmp); -} - - - - -// assignments -template -FASTOR_INLINE void assign(AbstractTensor &dst, const UnaryTransOp &src) { - using result_type = typename Expr::result_type; - const result_type& tmp = evaluate(src.expr().self()); - using T = typename UnaryTransOp::scalar_type; - static constexpr size_t M = UnaryTransOp::M; - static constexpr size_t N = UnaryTransOp::N; - _transpose(tmp.data(),dst.self().data()); -} - -template -FASTOR_INLINE void assign_add(AbstractTensor &dst, const UnaryTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryTransOp::scalar_type; - using result_t = typename UnaryTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryTransOp::M; - static constexpr size_t N = UnaryTransOp::N; - _transpose(tmp.data(),tmp_trans.data()); - trivial_assign_add(dst.self(),tmp_trans); -} - -template -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const UnaryTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryTransOp::scalar_type; - using result_t = typename UnaryTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryTransOp::M; - static constexpr size_t N = UnaryTransOp::N; - _transpose(tmp.data(),tmp_trans.data()); - trivial_assign_sub(dst.self(),tmp_trans); -} - -template -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const UnaryTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryTransOp::scalar_type; - using result_t = typename UnaryTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryTransOp::M; - static constexpr size_t N = UnaryTransOp::N; - _transpose(tmp.data(),tmp_trans.data()); - trivial_assign_mul(dst.self(),tmp_trans); -} - -template -FASTOR_INLINE void assign_div(AbstractTensor &dst, const UnaryTransOp &src) { - using result_type = typename Expr::result_type; - // no copies if expr is a tensor - const result_type& tmp = evaluate(src.expr().self()); - // one copy for the inverse - using T = typename UnaryTransOp::scalar_type; - using result_t = typename UnaryTransOp::result_type; - result_t tmp_trans; - static constexpr size_t M = UnaryTransOp::M; - static constexpr size_t N = UnaryTransOp::N; - _transpose(tmp.data(),tmp_trans.data()); - trivial_assign_div(dst.self(),tmp_trans); -} - - -} // end of namespace Fastor - - -#endif // UNARY_TRANS_OP_H diff --git a/noarch/include/Fastor/expressions/unary_ops/unary_bool_ops.h b/noarch/include/Fastor/expressions/unary_ops/unary_bool_ops.h deleted file mode 100644 index 0ce51021..00000000 --- a/noarch/include/Fastor/expressions/unary_ops/unary_bool_ops.h +++ /dev/null @@ -1,101 +0,0 @@ -#ifndef UNARY_BOOL_OP_H -#define UNARY_BOOL_OP_H - - -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/simd_math/simd_math.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" -#include "Fastor/expressions/expression_traits.h" - -namespace Fastor { - -// All unary bool ops -#define FASTOR_MAKE_UNARY_BOOL_OPS(OP_NAME, SIMD_OP, SCALAR_OP, STRUCT_NAME, EVAL_TYPE)\ -template\ -struct Unary ##STRUCT_NAME ## Op: public AbstractTensor,DIM0> {\ -private:\ - expression_t _expr;\ -public:\ - using scalar_type = typename Expr::scalar_type;\ - using simd_vector_type = typename Expr::simd_vector_type;\ - using simd_abi_type = typename simd_vector_type::abi_type;\ - using result_type = to_bool_tensor_t;\ - using result_scalar_type = typename result_type::scalar_type;\ - using result_simd_abi_type = typename result_type::simd_abi_type;\ - using result_simd_vector_type = typename result_type::simd_vector_type;\ - using UU = bool /*this needs to change to U once masks are implemented*/;\ - using ABI = simd_abi::fixed_size::Size>;\ - static constexpr FASTOR_INDEX Dimension = DIM0;\ - static constexpr FASTOR_INDEX rank() {return DIM0;}\ - FASTOR_INLINE FASTOR_INDEX size() const {return _expr.size();}\ - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _expr.dimension(i);}\ - Unary ##STRUCT_NAME ## Op(expression_t inexpr) : _expr(inexpr) {}\ - FASTOR_INLINE expression_t expr() const {return _expr;}\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ - return SIMD_OP(_expr.template eval(i));\ - }\ - template\ - FASTOR_INLINE UU eval_s(FASTOR_INDEX i) const {\ - return SCALAR_OP(_expr.template eval_s(i));\ - }\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return SIMD_OP(_expr.template eval(i,j));\ - }\ - template\ - FASTOR_INLINE UU eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return SCALAR_OP(_expr.template eval_s(i,j));\ - }\ - template\ - FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ - return SIMD_OP(_expr.template teval(as));\ - }\ - template\ - FASTOR_INLINE UU teval_s(const std::array &as) const {\ - return SCALAR_OP(_expr.template teval_s(as));\ - }\ -};\ -template::value,bool>::type = 0 >\ -FASTOR_INLINE Unary ##STRUCT_NAME ## Op OP_NAME(const AbstractTensor &_expr) {\ - return Unary ##STRUCT_NAME ## Op(_expr.self());\ -}\ - -FASTOR_MAKE_UNARY_BOOL_OPS(operator!, ! , ! , Not , scalar_type) -FASTOR_MAKE_UNARY_BOOL_OPS(isinf , isinf , std::isinf , Isinf , scalar_type) -FASTOR_MAKE_UNARY_BOOL_OPS(isnan , isnan , std::isnan , Isnan , scalar_type) -FASTOR_MAKE_UNARY_BOOL_OPS(isfinite, isfinite, std::isfinite, Isfinite, scalar_type) - - - - -#define FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\ -template,bool>::type = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ - using result_type = typename OtherDerived::result_type;\ - const result_type tmp(src.expr().self());\ - trivial_assign ##ASSIGN_TYPE (dst.self(), OP(tmp));\ -}\ -template,bool>::type = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ - -// only assignment -FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(! , Not , ) -FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isinf , Isinf , ) -FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isnan , Isnan , ) -FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isfinite, Isfinite, ) - - - - -} // end of namespace Fastor - - -#endif // UNARY_BOOL_OP_H - diff --git a/noarch/include/Fastor/expressions/unary_ops/unary_math_ops.h b/noarch/include/Fastor/expressions/unary_ops/unary_math_ops.h deleted file mode 100644 index 98af613c..00000000 --- a/noarch/include/Fastor/expressions/unary_ops/unary_math_ops.h +++ /dev/null @@ -1,209 +0,0 @@ -#ifndef UNARY_MATH_OP_H -#define UNARY_MATH_OP_H - - -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/simd_math/simd_math.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" -#include "Fastor/expressions/expression_traits.h" - -namespace Fastor { - -// All unary math ops -#define FASTOR_MAKE_UNARY_MATH_OPS(OP_NAME, SIMD_OP, SCALAR_OP, STRUCT_NAME, EVAL_TYPE)\ -template\ -struct Unary ##STRUCT_NAME ## Op: public AbstractTensor,DIM0> {\ -private:\ - expression_t _expr;\ -public:\ - using scalar_type = typename Expr::scalar_type;\ - using simd_vector_type = typename Expr::simd_vector_type;\ - using simd_abi_type = typename simd_vector_type::abi_type;\ - using result_type = typename Expr::result_type;\ - static constexpr FASTOR_INDEX Dimension = DIM0;\ - static constexpr FASTOR_INDEX rank() {return DIM0;}\ - FASTOR_INLINE FASTOR_INDEX size() const {return _expr.size();}\ - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _expr.dimension(i);}\ - Unary ##STRUCT_NAME ## Op(expression_t inexpr) : _expr(inexpr) {}\ - FASTOR_INLINE expression_t expr() const {return _expr;}\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const {\ - return SIMD_OP(_expr.template eval(i));\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\ - return SCALAR_OP(_expr.template eval_s(i));\ - }\ - template\ - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return SIMD_OP(_expr.template eval(i,j));\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\ - return SCALAR_OP(_expr.template eval_s(i,j));\ - }\ - template\ - FASTOR_INLINE SIMDVector teval(const std::array &as) const {\ - return SIMD_OP(_expr.template teval(as));\ - }\ - template\ - FASTOR_INLINE EVAL_TYPE teval_s(const std::array &as) const {\ - return SCALAR_OP(_expr.template teval_s(as));\ - }\ -};\ -template::value,bool>::type = 0 >\ -FASTOR_INLINE Unary ##STRUCT_NAME ## Op OP_NAME(const AbstractTensor &_expr) {\ - return Unary ##STRUCT_NAME ## Op(_expr.self());\ -}\ - - -FASTOR_MAKE_UNARY_MATH_OPS(operator+, , , Add, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(operator-, -, -, Sub, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(abs, abs, std::abs, Abs, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(sqrt, sqrt, sqrts, Sqrt, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(cbrt, cbrt, std::cbrt, Cbrt, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(exp, exp, std::exp, Exp, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(exp2, exp2, std::exp2, Exp2, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(expm1, expm1, std::expm1, Expm1, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(log, log, std::log, Log, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(log10, log10, std::log10, Log10, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(log2, log2, std::log2, Log2, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(log1p, log1p, std::log1p, Log1p, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(sin, sin, std::sin, Sin, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(cos, cos, std::cos, Cos, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(tan, tan, std::tan, Tan, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(asin, asin, std::asin, Asin, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(acos, acos, std::acos, Acos, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(atan, atan, std::atan, Atan, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(sinh, sinh, std::sinh, Sinh, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(cosh, cosh, std::cosh, Cosh, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(tanh, tanh, std::tanh, Tanh, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(asinh, asinh, std::asinh, Asinh, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(acosh, acosh, std::acosh, Acosh, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(atanh, atanh, std::atanh, Atanh, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(erf, erf, std::erf, Erf, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(tgamma, tgamma, std::tgamma, Tgamma, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(lgamma, lgamma, std::lgamma, Lgamma, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(ceil, ceil, std::ceil, Ceil, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(round, round, std::round, Round, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(floor, floor, std::floor, Floor, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(trunc, trunc, std::trunc, Trunc, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(conj, conj, std::conj, Conj, scalar_type) -FASTOR_MAKE_UNARY_MATH_OPS(arg , arg , std::arg , Arg , scalar_type) - - - - -#define FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\ -template,bool>::type = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ - assign ##ASSIGN_TYPE (dst.self(), src.expr().self());\ - trivial_assign(dst.self(), OP(dst.self()));\ -}\ -template,bool>::type = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ - -// only assignment -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT( , Add, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(-, Sub, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(abs, Abs, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sqrt, Sqrt, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cbrt, Cbrt, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(exp, Exp, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(exp2, Exp2, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(expm1, Expm1, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log, Log, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log10, Log10, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log2, Log2, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log1p, Log1p, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sin, Sin, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cos, Cos, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tan, Tan, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(asin, Asin, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(acos, Acos, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(atan, Atan, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sinh, Sinh, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cosh, Cosh, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tanh, Tanh, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(asinh, Asinh, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(acosh, Acosh, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(atanh, Atanh, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(erf, Erf, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tgamma, Tgamma, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(lgamma, Lgamma, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(ceil, Ceil, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(round, Round, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(floor, Floor, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(trunc, Trunc, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(conj, Conj, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(arg , Arg , ) - - - -// arithmetic assignments -#define FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\ -template,bool>::type = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ - using result_type = typename Unary ##NAME ## Op::result_type;\ - const result_type tmp(src.expr().self());\ - trivial_assign ##ASSIGN_TYPE (dst.self(), OP(tmp));\ -}\ -template,bool>::type = false>\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ - -#define FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT( , Add, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(-, Sub, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(abs, Abs, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sqrt, Sqrt, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cbrt, Cbrt, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(exp, Exp, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(exp2, Exp2, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(expm1, Expm1, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log, Log, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log10, Log10, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log2, Log2, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log1p, Log1p, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sin, Sin, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cos, Cos, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tan, Tan, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(asin, Asin, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(acos, Acos, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(atan, Atan, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sinh, Sinh, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cosh, Cosh, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tanh, Tanh, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(asinh, Asinh, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(acosh, Acosh, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(atanh, Atanh, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(erf, Erf, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tgamma, Tgamma, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(lgamma, Lgamma, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(ceil, Ceil, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(round, Round, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(floor, Floor, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(trunc, Trunc, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(conj, Conj, ASSIGN_TYPE)\ -FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(arg , Arg , ASSIGN_TYPE)\ - -// FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, ) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _add) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _sub) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _mul) -FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _div) - -} // end of namespace Fastor - - -#endif // UNARY_MATH_OP_H - diff --git a/noarch/include/Fastor/expressions/views/tensor_diag_views.h b/noarch/include/Fastor/expressions/views/tensor_diag_views.h deleted file mode 100644 index 8b9d8237..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_diag_views.h +++ /dev/null @@ -1,317 +0,0 @@ -#ifndef TENSOR_DIAG_VIEWS_H -#define TENSOR_DIAG_VIEWS_H - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/TensorMap.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -namespace Fastor { - -template class TensorType, typename T, size_t M, size_t N, size_t DIM> -struct TensorDiagViewExpr,DIM> : public AbstractTensor,DIM>,DIM> { -private: - TensorType& _expr; - bool _does_alias = false; - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - using V = SIMDVector; - static constexpr FASTOR_INDEX Dimension = 1; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return 1;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;} - constexpr const TensorType& expr() const {return _expr;} - - FASTOR_INLINE TensorViewExpr,2>& noalias() { - _does_alias = true; - return *this; - } - - FASTOR_INLINE TensorDiagViewExpr(Tensor &_ex) : _expr(_ex) { - static_assert(M==N, "MATRIX MUST BE SQUARE FOR DIAGONAL VIEW"); - } - - // Copy assignment - //---------------------------------------------------------------------------------// - FASTOR_INLINE void operator=(const TensorDiagViewExpr,DIM>& other_src) { - // Diagonal views don't alias -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - // Get around recursive inlining issue - const result_type tmp(other_src); - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i,i); - } - } - - // AbstractTensor binders - //---------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) = other_src.template eval_s(i); - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) += other_src.template eval_s(i); - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) -= other_src.template eval_s(i); - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) *= other_src.template eval_s(i); - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorDiagViewExpr,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) /= other_src.template eval_s(i); - } - } - - // Scalar binders - //---------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) = num; - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) += num; - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) -= num; - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) *= num; - } - } - - template && !is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - T inum = T(1) / T(num); - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) *= inum; - } - } - template && is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - for (FASTOR_INDEX i = 0; i < M; ++i) { - _expr(i,i) /= num; - } - } - - // Evaluators - //---------------------------------------------------------------------------------// - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - V _vec; - std::array inds; - for (auto j=0; j - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - return _expr(idx,idx); - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - V _vec; - vector_setter(_vec,_expr.data(),i*N+j); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr(i,j); - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - V _vec; - vector_setter(_vec,_expr.data(),as[0]*N+as[1]); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.eval_s(as); - } -}; - - - -// Actual definition of diag function -// Note that like other views the input can only be an evaluated tensor -template -TensorDiagViewExpr,2> diag(Tensor &a) { - return TensorDiagViewExpr,2>(a); -} - -template -TensorDiagViewExpr,2> diag(TensorMap &a) { - return TensorDiagViewExpr,2>(a); -} - - -} // end of namespace Fastor - - -#endif diff --git a/noarch/include/Fastor/expressions/views/tensor_filter_views.h b/noarch/include/Fastor/expressions/views/tensor_filter_views.h deleted file mode 100644 index 900f5661..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_filter_views.h +++ /dev/null @@ -1,281 +0,0 @@ -#ifndef TENSOR_FILTER_VIEWS_H -#define TENSOR_FILTER_VIEWS_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - - -namespace Fastor { - -template -struct TensorFilterViewExpr,Tensor,DIMS>: - public AbstractTensor,Tensor,DIMS>,DIMS> { -private: - Tensor &_expr; - const Tensor &fl_expr; - bool _does_alias = false; - - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = DIMS; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return DIMS;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return pack_prod::value;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return fl_expr.dimension(i);} - constexpr const Tensor& expr() const {return _expr;} - - FASTOR_INLINE TensorFilterViewExpr,Tensor,DIMS>& noalias() { - _does_alias = true; - return *this; - } - - constexpr FASTOR_INLINE TensorFilterViewExpr(Tensor &_ex, const Tensor &_it) : _expr(_ex), fl_expr(_it) { - static_assert(sizeof...(Rest)==DIMS, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - } - - // In place operators - //------------------------------------------------------------------------------------// - void operator=(const TensorFilterViewExpr,Tensor,DIMS> &src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - } - } - } - - - template,bool> = false> - void operator=(const AbstractTensor &src) { - const typename Derived::result_type& tmp = evaluate(src.self()); - this->operator=(tmp); - } - template,bool> = false> - void operator=(const AbstractTensor &src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - } - } - } - - template,bool> = false> - void operator+=(const AbstractTensor &src) { - const typename Derived::result_type& tmp = evaluate(src.self()); - this->operator+=(tmp); - } - template,bool> = false> - void operator+=(const AbstractTensor &src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - } - } - } - - template,bool> = false> - void operator-=(const AbstractTensor &src) { - const typename Derived::result_type& tmp = evaluate(src.self()); - this->operator-=(tmp); - } - template,bool> = false> - void operator-=(const AbstractTensor &src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - } - } - } - - template,bool> = false> - void operator*=(const AbstractTensor &src) { - const typename Derived::result_type& tmp = evaluate(src.self()); - this->operator*=(tmp); - } - template,bool> = false> - void operator*=(const AbstractTensor &src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - } - } - } - - template,bool> = false> - void operator/=(const AbstractTensor &src) { - const typename Derived::result_type& tmp = evaluate(src.self()); - this->operator/=(tmp); - } - template,bool> = false> - void operator/=(const AbstractTensor &src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - } - } - } - - //------------------------------------------------------------------------------------// - template,bool> = false> - void operator=(U num) { - T tnum = (T)num; - for (FASTOR_INDEX i = 0; i ,bool> = false> - void operator+=(U num) { - T tnum = (T)num; - for (FASTOR_INDEX i = 0; i ,bool> = false> - void operator-=(U num) { - T tnum = (T)num; - for (FASTOR_INDEX i = 0; i ,bool> = false> - void operator*=(U num) { - T tnum = (T)num; - for (FASTOR_INDEX i = 0; i && !is_integral_v_,bool> = false> - void operator/=(U num) { - T tnum = T(1)/(T)num; - for (FASTOR_INDEX i = 0; i && is_integral_v_,bool> = false> - void operator/=(U num) { - T tnum = (T)num; - for (FASTOR_INDEX i = 0; i - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - constexpr size_t _Stride = SIMDVector::Size; - SIMDVector _vec; - U inds[_Stride]; - for (FASTOR_INDEX j=0; j<_Stride; ++j) - inds[j] = fl_expr.eval_s(i+j) ? _expr.eval_s(i+j) : 0; - _vec.load(inds,false); - return _vec; - } - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return fl_expr.eval_s(i) ? _expr.eval_s(i) : 0; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { - constexpr size_t _Stride = SIMDVector::Size; - SIMDVector _vec; - U inds[_Stride]; - for (FASTOR_INDEX j=0; j<_Stride; ++j) - inds[j] = fl_expr.eval_s(i,k+j) ? _expr.eval_s(i,k+j) : 0; - _vec.load(inds,false); - return _vec; - } - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX k) const { - return fl_expr.eval_s(i,k) ? _expr.eval_s(i,k) : 0; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - constexpr size_t _Stride = SIMDVector::Size; - SIMDVector _vec; - U inds[_Stride]; - for (FASTOR_INDEX j=0; j<_Stride; ++j) - inds[j] = fl_expr.teval_s(as) ? _expr.teval_s(as) : 0; - _vec.load(inds,false); - return _vec; - } - - template - FASTOR_INLINE U teval_s(const std::array& as) const { - return fl_expr.teval_s(as) ? _expr.teval_s(as) : 0; - } - //------------------------------------------------------------------------------------// - -}; - -} // end of namespace Fastor - - -#endif // TENSOR_FILTER_VIEWS_H diff --git a/noarch/include/Fastor/expressions/views/tensor_fixed_views_1d.h b/noarch/include/Fastor/expressions/views/tensor_fixed_views_1d.h deleted file mode 100644 index f606772d..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_fixed_views_1d.h +++ /dev/null @@ -1,681 +0,0 @@ -#ifndef TENSOR_FIXED_VIEWS_1D_H -#define TENSOR_FIXED_VIEWS_1D_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - - -namespace Fastor { - - -// 1D const fixed views -//----------------------------------------------------------------------------------------------// -template -struct TensorConstFixedViewExpr1D,fseq,1> : - public AbstractTensor,fseq,1>,1> { -private: - const Tensor &_expr; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor::value>; - static constexpr FASTOR_INDEX Dimension = 1; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INLINE FASTOR_INDEX size() { return range_detector::value;} - static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) {return range_detector::value;} - constexpr const Tensor& expr() const {return _expr;} - - constexpr FASTOR_INLINE TensorConstFixedViewExpr1D(const Tensor &_ex) : _expr(_ex) {} - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),S0*i+F0,S0); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr.data()[S0*i+F0]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),S0*(i+j)+F0,S0); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[S0*(i+j)+F0]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),S0*as[0]+F0,S0); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[S0*as[0]+F0]; - } -}; - - - - - -// 1D non-const fixed views -//----------------------------------------------------------------------------------------------// -template -struct TensorFixedViewExpr1D,fseq,1> : - public AbstractTensor,fseq,1>,1> { -private: - Tensor &_expr; - bool _does_alias = false; - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor::value>; - static constexpr FASTOR_INDEX Dimension = 1; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return range_detector::value;} - static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) {return range_detector::value;} - constexpr const Tensor& expr() const {return _expr;} - - FASTOR_INLINE TensorFixedViewExpr1D,fseq,1>& noalias() { - _does_alias = true; - return *this; - } - - constexpr FASTOR_INLINE TensorFixedViewExpr1D(Tensor &_ex) : _expr(_ex) {} - - - //----------------------------------------------------------------------------------// - FASTOR_HINT_INLINE void operator=(const TensorFixedViewExpr1D,fseq,1> &other_src) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); - // Assign other to temporary - tmp = other_src; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - _vec_other.store(&_data[i+F0],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[S0*(i+j) + F0] = _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - //----------------------------------------------------------------------------------// - - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i (i); - } - return; - } - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i (i); - _vec_other.store(&_data[i+F0],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[S0*(i+j) + F0] = _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) + other_src.template eval(i); - _vec.store(&_data[i+F0],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[S0*(i+j) + F0] += _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) - other_src.template eval(i); - _vec.store(&_data[i+F0],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[S0*(i+j) + F0] -= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) * other_src.template eval(i); - _vec.store(&_data[i+F0],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[S0*(i+j) + F0] *= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr1D,fseq,1>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) / other_src.template eval(i); - _vec.store(&_data[i+F0],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[S0*(i+j) + F0] /= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - //----------------------------------------------------------------------------------// - - - // Scalar binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _data[S0*(i+j) + F0] = _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) + _vec_other; - _vec.store(&_data[i+F0],false); - } - for (; i ::Size; ++j) { - _data[S0*(i+j) + F0] += _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) - _vec_other; - _vec.store(&_data[i+F0],false); - } - for (; i ::Size; ++j) { - _data[S0*(i+j) + F0] -= _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) * _vec_other; - _vec.store(&_data[i+F0],false); - } - for (; i ::Size; ++j) { - _data[S0*(i+j) + F0] *= _vec_other[j]; - } - } - for (; i && !is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - T inum = T(1) / static_cast(num); - simd_vector_type _vec_other(inum); - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) * _vec_other; - _vec.store(&_data[i+F0],false); - } - for (; i ::Size; ++j) { - _data[S0*(i+j) + F0] *= _vec_other[j]; - } - } - for (; i && is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR (S0 == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) / _vec_other; - _vec.store(&_data[i+F0],false); - } - for (; i ::Size; ++j) { - _data[S0*(i+j) + F0] /= _vec_other[j]; - } - } - for (; i - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),S0*i+F0,S0); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr.data()[S0*i+F0]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),S0*(i+j)+F0,S0); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[S0*(i+j)+F0]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),S0*as[0]+F0,S0); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[S0*as[0]+F0]; - } -}; - - -} - - -#endif // TENSOR_FIXED_VIEWS_1D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_fixed_views_2d.h b/noarch/include/Fastor/expressions/views/tensor_fixed_views_2d.h deleted file mode 100644 index d56a6adc..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_fixed_views_2d.h +++ /dev/null @@ -1,1121 +0,0 @@ -#ifndef TENSOR_FIXED_VIEWS_2D_H -#define TENSOR_FIXED_VIEWS_2D_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -namespace Fastor { - - -// 2D const fixed views -//----------------------------------------------------------------------------------------------// -template -struct TensorConstFixedViewExpr2D,fseq,fseq,2> : - public AbstractTensor,fseq,fseq,2>,2> { -private: - const Tensor &_expr; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor::value, range_detector::value>; - static constexpr FASTOR_INDEX Dimension = 2; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INLINE FASTOR_INDEX size() { - return range_detector::value*range_detector::value; - } - static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) { - return i==0 ? range_detector::value : range_detector::value; - } - constexpr const Tensor& expr() const {return _expr;} - static constexpr FASTOR_INDEX Padding = F0*N+F1; - - constexpr FASTOR_INLINE TensorConstFixedViewExpr2D(const Tensor &_ex) : _expr(_ex) {} - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) { - auto it = (idx+j) / range_detector::value, jt = (idx+j) % range_detector::value; - inds[j] = S0*it*N+S1*jt + Padding; - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - auto it = idx / range_detector::value, jt = idx % range_detector::value; - auto ind = S0*it*N+S1*jt + Padding; - return _expr.data()[ind]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*i*N+j + Padding, false); - else vector_setter(_vec,_expr.data(),S0*i*N+S1*j + Padding,S1); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[S0*i*N+S1*j + Padding]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*as[0]*N+as[1] + Padding, false); - else vector_setter(_vec,_expr.data(),S0*as[0]*N+S1*as[1] + Padding,S1); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[S0*as[0]*N+S1*as[1] + Padding]; - } -}; - - - - - -// 2D non-const fixed views -//----------------------------------------------------------------------------------------------// -template -struct TensorFixedViewExpr2D,fseq,fseq,2> : - public AbstractTensor,fseq,fseq,2>,2> { - -private: - Tensor &_expr; - bool _does_alias = false; - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor::value, range_detector::value>; - static constexpr FASTOR_INDEX Dimension = 2; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INLINE FASTOR_INDEX size() { - return range_detector::value*range_detector::value; - } - static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) { - return i==0 ? range_detector::value : range_detector::value; - } - constexpr const Tensor& expr() const {return _expr;} - static constexpr FASTOR_INDEX Padding = F0*N+F1; - - FASTOR_INLINE TensorFixedViewExpr2D,fseq,fseq,2>& noalias() { - _does_alias = true; - return *this; - } - - constexpr FASTOR_INLINE TensorFixedViewExpr2D(Tensor &_ex) : _expr(_ex) {} - - //----------------------------------------------------------------------------------// - FASTOR_HINT_INLINE void operator=(const TensorFixedViewExpr2D,fseq,fseq,2> &other_src) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other_src; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i,j); - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j (i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i (i,j); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j (i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(i,j); - } - } -#endif - } - } - - // AbstractTensor binders - [equal order] - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i) { - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(i,j); - } - } - return; - } - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i (i,j); - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j (i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i (i,j); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j (i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) + other_src.template eval(i,j); - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j (i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) + other_src.template eval(i,j); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j (i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) += other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) - other_src.template eval(i,j); - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j (i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) - other_src.template eval(i,j); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j (i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) -= other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) * other_src.template eval(i,j); - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j (i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) * other_src.template eval(i,j); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j (i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) *= other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) / other_src.template eval(i,j); - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j (i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) / other_src.template eval(i,j); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j (i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) /= other_src.template eval_s(i,j); - } - } -#endif - } - } - //----------------------------------------------------------------------------------// - - - // AbstractTensor binders for other nth rank tensors - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(counter); - counter++; - } - } - return; - } - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i (counter); - _vec.store(&_data[S0*i*N+j+Padding],false); - counter += Stride; - } - for (; j (counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i (counter); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - counter += Stride; - } - for (; j (counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) = other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) + other_src.template eval(counter); - _vec.store(&_data[S0*i*N+j+Padding],false); - counter += Stride; - } - for (; j (counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) + other_src.template eval(counter); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - counter += Stride; - } - for (; j (counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) += other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) - other_src.template eval(counter); - _vec.store(&_data[S0*i*N+j+Padding],false); - counter += Stride; - } - for (; j (counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) - other_src.template eval(counter); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - counter += Stride; - } - for (; j (counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) -= other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) * other_src.template eval(counter); - _vec.store(&_data[S0*i*N+j+Padding],false); - counter += Stride; - } - for (; j (counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) * other_src.template eval(counter); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - counter += Stride; - } - for (; j (counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) *= other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#ifndef FASTOR_NO_ALIAS - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExpr2D,fseq,fseq,2>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) / other_src.template eval(counter); - _vec.store(&_data[S0*i*N+j+Padding],false); - counter += Stride; - } - for (; j (counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i template eval(i,j) / other_src.template eval(counter); - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - counter += Stride; - } - for (; j (counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) /= other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - //----------------------------------------------------------------------------------// - - - // Scalar binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { - T *FASTOR_RESTRICT _data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) = num; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { - T *FASTOR_RESTRICT _data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) + _vec_other; - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j template eval(i,j) + _vec_other; - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) += num; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { - T *FASTOR_RESTRICT _data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) - _vec_other; - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j template eval(i,j) - _vec_other; - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) -= num; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { - T *FASTOR_RESTRICT _data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) * _vec_other; - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j template eval(i,j) * _vec_other; - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) *= num; - } - } -#endif - } - } - - template && !is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - T *FASTOR_RESTRICT _data = _expr.data(); - T inum = T(1)/num; - SIMDVector _vec_other(static_cast(inum)); - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) * _vec_other; - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j template eval(i,j) * _vec_other; - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) *= inum; - } - } -#endif - } - } - template && is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - T *FASTOR_RESTRICT _data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - FASTOR_IF_CONSTEXPR (S1==1) { - for (FASTOR_INDEX i = 0; i template eval(i,j) / _vec_other; - _vec.store(&_data[S0*i*N+j+Padding],false); - } - for (; j template eval(i,j) / _vec_other; - data_setter(_data,_vec,S0*i*N+S1*j+Padding,S1); - } - for (; j ::value; i++) { - for (FASTOR_INDEX j = 0; j ::value; j++) { - _expr(S0*i+F0,S1*j+F1) /= num; - } - } -#endif - } - } - //----------------------------------------------------------------------------------// - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) { - auto it = (idx+j) / range_detector::value, jt = (idx+j) % range_detector::value; - inds[j] = S0*it*N+S1*jt + Padding; - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - auto it = idx / range_detector::value, jt = idx % range_detector::value; - auto ind = S0*it*N+S1*jt + Padding; - return _expr.data()[ind]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*i*N+j + Padding, false); - else vector_setter(_vec,_expr.data(),S0*i*N+S1*j + Padding,S1); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[S0*i*N+S1*j + Padding]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - FASTOR_IF_CONSTEXPR (S1==1) _vec.load(_expr.data()+S0*as[0]*N+as[1] + Padding, false); - else vector_setter(_vec,_expr.data(),S0*as[0]*N+S1*as[1] + Padding,S1); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[S0*as[0]*N+S1*as[1] + Padding]; - } -}; - - -} - - -#endif // TENSOR_FIXED_VIEWS_2D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_fixed_views_nd.h b/noarch/include/Fastor/expressions/views/tensor_fixed_views_nd.h deleted file mode 100644 index e40cbe27..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_fixed_views_nd.h +++ /dev/null @@ -1,1645 +0,0 @@ -#ifndef TENSOR_FIXED_VIEWS_ND_H -#define TENSOR_FIXED_VIEWS_ND_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - - -namespace Fastor { - - -// Generic const fixed tensor views based on sequences/slices -//----------------------------------------------------------------------------------------------// -template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> -struct TensorConstFixedViewExprnD,Fseqs...> : - public AbstractTensor,Fseqs...>,sizeof...(Fseqs)> { - -private: - const TensorType &_expr; - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; - static constexpr std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; -public: - using scalar_type = T; - using simd_vector_type = typename TensorType::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using dimension_helper = get_fixed_sequence_pack_dimensions< - TensorType, - typename std_ext::make_index_sequence::type, - Fseqs...>; - using result_type = typename dimension_helper::type; - using V = simd_vector_type; - // Get nth to_positive type - template using get_nth_pt = get_nth_type...>; - static constexpr FASTOR_INDEX Dimension = sizeof...(Fseqs); - static constexpr FASTOR_INDEX DIMS = sizeof...(Fseqs); - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - constexpr FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} - constexpr FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return dimension_helper::Size;} - static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) {return dimension_helper::dims[i];} - constexpr const TensorType& expr() const {return _expr;} -private: - static constexpr std::array _dims = dimension_helper::dims; - static constexpr bool _is_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step==1) ? true : false; - static constexpr bool _is_strided_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step!=1) ? true : false; -public: - - constexpr FASTOR_INLINE TensorConstFixedViewExprnD(const TensorType &_ex) : _expr(_ex) {} - //----------------------------------------------------------------------------------------------// - - // Evals - //----------------------------------------------------------------------------------------------// - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - - SIMDVector _vec; - std::array as = {}; - std::array inds; - for (auto j=0; j::Do(j,inds, as); - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - - std::array as = {}; - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - return _expr.eval_s(ind); - } - - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { - - idx += j; - V _vec; - std::array as = {}; - - std::array inds; - for (auto j=0; j::Do(j, inds, as); - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { - idx += j; - std::array as = {}; - - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - - return _expr.eval_s(ind); - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - FASTOR_IF_CONSTEXPR (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); - else FASTOR_IF_CONSTEXPR (_is_strided_vectorisable) { - V _vec; - vector_setter(_vec,_expr.data(),ind,get_nth_pt::_step); - return _vec; - } - else { - - V _vec; - std::array inds = {}; - std::array as_ = as; - for (auto j=0; j::Do(j, inds, as); - - for(int jt = (int)DIMS-1; jt>=0; jt--) - { - as_[jt] +=1; - if(as_[jt]<_dims[jt]) - break; - else - as_[jt]=0; - } - } - - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - } - - template - FASTOR_INLINE U teval_s(const std::array& as) const { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - return _expr.eval_s(ind); - } - //----------------------------------------------------------------------------------------------// - - - - //----------------------------------------------------------------------------------------------// -private: - template - struct get_index_v { - template - static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { - inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - get_index_v::Do(j, inds, as); - } - }; - template - struct get_index_v { - template - static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { - inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - } - }; - - template - struct get_index_s { - template - static FASTOR_INLINE void Do(int &ind, const std::array& as) { - ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - get_index_s::Do(ind, as); - } - }; - template - struct get_index_s { - template - static FASTOR_INLINE void Do(int &ind, const std::array& as) { - ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - } - }; - //----------------------------------------------------------------------------------------------// -}; - - -template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> -constexpr std::array TensorConstFixedViewExprnD,Fseqs...>::products_; - -template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> -constexpr std::array TensorConstFixedViewExprnD,Fseqs...>::_dims; - - - - - - - -//----------------------------------------------------------------------------------------------// -// Generic non-const fixed tensor views based on sequences/slices -//----------------------------------------------------------------------------------------------// -template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> -struct TensorFixedViewExprnD,Fseqs...> : - public AbstractTensor,Fseqs...>,sizeof...(Fseqs)> { - -private: - TensorType &_expr; - bool _does_alias = false; - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; - static constexpr std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; -public: - using scalar_type = T; - using simd_vector_type = typename TensorType::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using dimension_helper = get_fixed_sequence_pack_dimensions< - TensorType, - typename std_ext::make_index_sequence::type, - Fseqs...>; - using result_type = typename dimension_helper::type; - using V = simd_vector_type; - // Get nth to_positive type - template using get_nth_pt = get_nth_type...>; - static constexpr FASTOR_INDEX Dimension = sizeof...(Fseqs); - static constexpr FASTOR_INDEX DIMS = sizeof...(Fseqs); - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - constexpr FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} - constexpr FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return dimension_helper::Size;} - static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) {return dimension_helper::dims[i];} - constexpr const TensorType& expr() const {return _expr;} -private: - static constexpr std::array _dims = dimension_helper::dims; - static constexpr bool _is_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step==1) ? true : false; - static constexpr bool _is_strided_vectorisable = !is_same_v_ && _dims[DIMS-1] % V::Size == 0 && (get_nth_type::_step!=1) ? true : false; -public: - - FASTOR_INLINE TensorFixedViewExprnD,Fseqs...>& noalias() { - _does_alias = true; - return *this; - } - - constexpr FASTOR_INLINE TensorFixedViewExprnD(TensorType &_ex) : _expr(_ex) {} - //----------------------------------------------------------------------------------------------// - - // View evalution operators - // Copy assignment operators [Needed in addition to generic AbstractTensor overload] - //----------------------------------------------------------------------------------// - FASTOR_HINT_INLINE void operator=(const TensorFixedViewExprnD,Fseqs...> &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - // constexpr FASTOR_INDEX stride = V::Size; - V _vec; - while(counter < total) - { - int ind = 0; - // for(int it = 0; it< DIMS; it++) { - // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - // } - get_index_s<0,DIMS-1>::Do(ind, as); - _vec = other.template teval(as); - _vec.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - // for(int it = 0; it< DIMS; it++) { - // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - // } - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] = other.template teval_s(as); - // print(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] += 1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - //----------------------------------------------------------------------------------------------// - - // AbstractTensor binders [equal order] - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - while(counter < total) - { - int ind = 0; - // for(int it = 0; it< DIMS; it++) { - // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - // } - get_index_s<0,DIMS-1>::Do(ind, as); - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - // for(int it = 0; it< DIMS; it++) { - // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - // } - get_index_s<0,DIMS-1>::Do(ind, as); - // _data[ind] = other_src.template eval_s(counter); - _data[ind] = other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - - // // Generic vectorised version that takes care of the remainder scalar ops - // using V=SIMDVector; - // while(counter < total) - // { - // int ind = 0; - // for(int it = 0; it< DIMS; it++) { - // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - // } - // get_index_s<0,DIMS-1>::Do(ind, as); - // if (_dims[DIMS-1] - as[DIMS-1] % V::Size == 0) { - // // V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - // _vec.store(&_data[ind],false); - // counter+=V::Size; - // } - // else { - // // _data[ind] = other_src.template eval_s(counter); - // _data[ind] = other_src.template teval_s(as); - // counter++; - // } - - // for(jt = DIMS-1; jt>=0; jt--) - // { - // if (jt == _dims.size()-1) as[jt]+=V::Size; - // else as[jt] +=1; - // if(as[jt]<_dims[jt]) - // break; - // else - // as[jt]=0; - // } - // if(jt<0) - // break; - // } - } - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out += _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // _data[ind] += other_src.template eval_s(counter); - _data[ind] += other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out -= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // _data[ind] -= other_src.template eval_s(counter); - _data[ind] -= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out *= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // _data[ind] *= other_src.template eval_s(counter); - _data[ind] *= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out /= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - // _data[ind] /= other_src.template eval_s(counter); - _data[ind] /= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - //----------------------------------------------------------------------------------------------// - - - // AbstractTensor binders [non-equal orders] - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] = other_src.template eval_s(counter); - // _data[ind] = other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out += _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] += other_src.template eval_s(counter); - // _data[ind] += other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out -= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] -= other_src.template eval_s(counter); - // _data[ind] -= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out *= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] *= other_src.template eval_s(counter); - // _data[ind] *= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorFixedViewExprnD,Fseqs...>(tmp_this_tensor); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out /= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] /= other_src.template eval_s(counter); - // _data[ind] /= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - // scalar binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _vec.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] = (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _vec_out.load(&_data[ind],false); - _vec_out += _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] += (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _vec_out.load(&_data[ind],false); - _vec_out -= _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] -= (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _vec_out.load(&_data[ind],false); - _vec_out *= _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] *= (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _vec_out.load(&_data[ind],false); - _vec_out /= _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - _data[ind] /= (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - //----------------------------------------------------------------------------------// - - // Evals - //----------------------------------------------------------------------------------------------// - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - - SIMDVector _vec; - std::array as = {}; - std::array inds; - for (auto j=0; j::Do(j,inds, as); - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - - std::array as = {}; - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - return _expr.eval_s(ind); - } - - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { - - idx += j; - V _vec; - std::array as = {}; - - std::array inds; - for (auto j=0; j::Do(j, inds, as); - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { - idx += j; - std::array as = {}; - - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - - return _expr.eval_s(ind); - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - FASTOR_IF_CONSTEXPR (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); - else FASTOR_IF_CONSTEXPR (_is_strided_vectorisable) { - V _vec; - vector_setter(_vec,_expr.data(),ind,get_nth_pt::_step); - return _vec; - } - else { - - V _vec; - std::array inds = {}; - std::array as_ = as; - for (auto j=0; j::Do(j, inds, as); - - for(int jt = (int)DIMS-1; jt>=0; jt--) - { - as_[jt] +=1; - if(as_[jt]<_dims[jt]) - break; - else - as_[jt]=0; - } - } - - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - } - - template - FASTOR_INLINE U teval_s(const std::array& as) const { - int ind = 0; - get_index_s<0,DIMS-1>::Do(ind, as); - return _expr.eval_s(ind); - } - //----------------------------------------------------------------------------------------------// - - - - //----------------------------------------------------------------------------------------------// -private: - template - struct get_index_v { - template - static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { - inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - get_index_v::Do(j, inds, as); - } - }; - template - struct get_index_v { - template - static FASTOR_INLINE void Do(size_t j, std::array &inds, const std::array& as) { - inds[j] += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - } - }; - - template - struct get_index_s { - template - static FASTOR_INLINE void Do(int &ind, const std::array& as) { - ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - get_index_s::Do(ind, as); - } - }; - template - struct get_index_s { - template - static FASTOR_INLINE void Do(int &ind, const std::array& as) { - ind += products_[from]*as[from]*get_nth_pt::_step + get_nth_pt::_first*products_[from]; - } - }; - //----------------------------------------------------------------------------------------------// -}; - - -template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> -constexpr std::array TensorFixedViewExprnD,Fseqs...>::products_; - -template class TensorType, typename T, size_t ...Rest, typename ... Fseqs> -constexpr std::array TensorFixedViewExprnD,Fseqs...>::_dims; - - - - - - - -} // end of namespace Fastor - - -#endif // TENSOR_FIXED_VIEWS_ND_H diff --git a/noarch/include/Fastor/expressions/views/tensor_random_views.h b/noarch/include/Fastor/expressions/views/tensor_random_views.h deleted file mode 100644 index 1aa795a0..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_random_views.h +++ /dev/null @@ -1,1152 +0,0 @@ -#ifndef TENSOR_RANDOM_VIEWS_H -#define TENSOR_RANDOM_VIEWS_H - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - - -namespace Fastor { - - -// Const versions -//----------------------------------------------------------------------------------// -template -struct TensorConstRandomViewExpr,Tensor,1>: - public AbstractTensor,Tensor,1>,1> { -private: - const Tensor &_expr; - const Tensor &it_expr; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = 1; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return 1;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return IterSize;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return IterSize;} - constexpr const Tensor& expr() const {return _expr;} - - constexpr FASTOR_INLINE TensorConstRandomViewExpr(const Tensor &_ex, const Tensor &_it) : _expr(_ex), it_expr(_it) {} - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr.data()[it_expr.data()[i]]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { - i += k; - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr(it_expr(i+j)); - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[as[0]+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[it_expr.data()[as[0]]]; - } -}; - - -//----------------------------------------------------------------------------------// -template -struct TensorConstRandomViewExpr,Tensor,DIMS>: - public AbstractTensor,Tensor,DIMS>,DIMS> { -private: - const Tensor &_expr; - // The index tensor needs to be copied for the purpose of indexing a - // multidimensional tensor with multiple 1D or any other lower other - // tensors. This issue could be solved by providing another type parameter/flag - // such as CopyIndex to the class which would use std::conditional to choose - // between "const Tensor &" and "Tensor" - - // const Tensor &it_expr; - Tensor it_expr; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = DIMS; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return DIMS;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return pack_prod::value;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return it_expr.dimension(i);} - constexpr const Tensor& expr() const {return _expr;} - - constexpr FASTOR_INLINE TensorConstRandomViewExpr(const Tensor &_ex, - const Tensor &_it) : _expr(_ex), it_expr(_it) { - static_assert(sizeof...(Rest)==DIMS && sizeof...(IterSizes)==DIMS, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr.data()[it_expr.data()[i]]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { - i += k; - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[it_expr.data()[i+j]]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - int i = std::accumulate(as.begin(), as.end(), 0); - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - FASTOR_INLINE U teval_s(const std::array& as) const { - int i = std::accumulate(as.begin(), as.end(), 0); - return _expr.data()[it_expr.data()[i]]; - } - -}; - -//----------------------------------------------------------------------------------// - - - - - -template -struct TensorRandomViewExpr,Tensor,1>: - public AbstractTensor,Tensor,1>,1> { -private: - Tensor &_expr; - const Tensor &it_expr; - bool _does_alias = false; - - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = 1; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return 1;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return IterSize;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return IterSize;} - constexpr const Tensor& expr() const {return _expr;} - - FASTOR_INLINE TensorRandomViewExpr,Tensor,1>& noalias() { - _does_alias = true; - return *this; - } - - constexpr FASTOR_INLINE TensorRandomViewExpr(Tensor &_ex, const Tensor &_it) : _expr(_ex), it_expr(_it) {} - - - // View evalution operators - // Copy assignment operators - //----------------------------------------------------------------------------------// - FASTOR_HINT_INLINE void operator=(const TensorRandomViewExpr,Tensor,1> &other_src) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other_src; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - for (auto j=0; j::Size; ++j) { - _expr(it_expr(i+j)) = _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - - // AbstractTensor binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i (i); - } - return; - } - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _expr(it_expr(i+j)) = _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _expr(it_expr(i+j)) += _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _expr(it_expr(i+j)) -= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _expr(it_expr(i+j)) *= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,1>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _expr(it_expr(i+j)) /= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - //----------------------------------------------------------------------------------// - - - // scalar binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(static_cast(num)); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _expr(it_expr(i+j)) = _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(static_cast(num)); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _expr(it_expr(i+j)) += _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(static_cast(num)); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _expr(it_expr(i+j)) -= _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(static_cast(num)); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _expr(it_expr(i+j)) *= _vec_other[j]; - } - } - for (; i && !is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - T inum = T(1)/(T)num; - SIMDVector _vec_other(inum); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _expr(it_expr(i+j)) *= _vec_other[j]; - } - } - for (; i && is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(static_cast(num)); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _expr(it_expr(i+j)) /= _vec_other[j]; - } - } - for (; i - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr(i+j); - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr(it_expr(i)); - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { - i += k; - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr(i+j); - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr(it_expr(i+j)); - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[as[0]+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[it_expr.data()[as[0]]]; - } - -}; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -template -struct TensorRandomViewExpr,Tensor,DIMS>: - public AbstractTensor,Tensor,DIMS>,DIMS> { -private: - Tensor &_expr; - // The index tensor needs to be copied for the purpose of indexing a - // multidimensional tensor with multiple 1D or any other lower other - // tensors. - - // const Tensor &it_expr; - Tensor it_expr; - bool _does_alias = false; - - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = DIMS; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return DIMS;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return pack_prod::value;} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return it_expr.dimension(i);} - constexpr const Tensor& expr() const {return _expr;}; - - FASTOR_INLINE TensorRandomViewExpr,Tensor,DIMS>& noalias() { - _does_alias = true; - return *this; - } - - constexpr FASTOR_INLINE TensorRandomViewExpr(Tensor &_ex, const Tensor &_it) : _expr(_ex), it_expr(_it) { - static_assert(sizeof...(Rest)==DIMS && sizeof...(IterSizes)==DIMS, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - } - - // Copy assignment - //------------------------------------------------------------------------------------// - FASTOR_HINT_INLINE void operator=(const TensorRandomViewExpr,Tensor,DIMS> &other_src) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other_src; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i); - for (auto j=0; j::Size; ++j) { - _data[it_expr.data()[i+j]] = _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - // AbstractTensor overloads - //------------------------------------------------------------------------------------// - template - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i (i); - } - return; - } - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[it_expr.data()[i+j]] = _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[it_expr.data()[i+j]] += _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[it_expr.data()[i+j]] -= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[it_expr.data()[i+j]] *= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - template - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorRandomViewExpr,Tensor,DIMS>(tmp_this_tensor,it_expr); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j::Size; ++j) { - _data[it_expr.data()[i+j]] /= _vec_other[j]; - } - } - for (; i (i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - - // Scalar binders - //------------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(num); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _data[it_expr.data()[i+j]] = _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(num); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _data[it_expr.data()[i+j]] += _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(num); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _data[it_expr.data()[i+j]] -= _vec_other[j]; - } - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other(num); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _data[it_expr.data()[i+j]] *= _vec_other[j]; - } - } - for (; i && !is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - T inum = T(1.)/(T)num; - SIMDVector _vec_other(inum); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _data[it_expr.data()[i+j]] *= _vec_other[j]; - } - } - for (; i && is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - T *_data = _expr.data(); -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - SIMDVector _vec_other((T)num); - FASTOR_INDEX i; - for (i = 0; i ::Size; ++j) { - _data[it_expr.data()[i+j]] /= _vec_other[j]; - } - } - for (; i - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr.data()[it_expr.data()[i]]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX k) const { - i += k; - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[it_expr.data()[i+j]]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - int i = std::accumulate(as.begin(), as.end(), 0); - SIMDVector _vec; - std::array::Size> inds; - for (FASTOR_INDEX j=0; j::Size; ++j) - inds[j] = it_expr.data()[i+j]; - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - FASTOR_INLINE U teval_s(const std::array& as) const { - int i = std::accumulate(as.begin(), as.end(), 0); - return _expr.data()[it_expr.data()[i]]; - } -}; - - - - -} - - - - -#endif // TENSOR_RANDOM_VIEWS_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views.h b/noarch/include/Fastor/expressions/views/tensor_views.h deleted file mode 100644 index 08688a9e..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_views.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef TENSOR_VIEWS_H -#define TENSOR_VIEWS_H - -#include "Fastor/expressions/views/tensor_views_1d.h" -#include "Fastor/expressions/views/tensor_views_2d.h" -#include "Fastor/expressions/views/tensor_views_nd.h" - -#include "Fastor/expressions/views/tensor_views_assignment.h" - -#endif // TENSOR_VIEWS_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_1d.h b/noarch/include/Fastor/expressions/views/tensor_views_1d.h deleted file mode 100644 index 6289517f..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_views_1d.h +++ /dev/null @@ -1,739 +0,0 @@ -#ifndef TENSOR_VIEWS_1D_H -#define TENSOR_VIEWS_1D_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -namespace Fastor { - - - -// 1D const views -//-------------------------------------------------------------------------------------// -template -struct TensorConstViewExpr,1>: public AbstractTensor,1>,1> { -private: - const Tensor &_expr; - seq _seq; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using V = simd_vector_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = 1; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return 1;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq.size();} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return _seq.size();} - constexpr const Tensor& expr() const {return _expr;} - - FASTOR_INLINE TensorConstViewExpr(const Tensor &_ex, const seq &_s) : _expr(_ex), _seq(_s) { - if (_seq._last < 0) _seq._last += N + /*including the end point*/ 1; - if (_seq._first < 0) _seq._first += N + /*including the end point*/ 1; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),i*_seq._step+_seq._first,_seq._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr.data()[i*_seq._step+_seq._first]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),(i+j)*_seq._step+_seq._first,_seq._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[(i+j)*_seq._step+_seq._first]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),as[0]*_seq._step+_seq._first,_seq._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[as[0]*_seq._step+_seq._first]; - } -}; - - - - - - - - - - - - - - - -// 1D non-const views -//-------------------------------------------------------------------------------------// -template -struct TensorViewExpr,1>: public AbstractTensor,1>,1> { -private: - Tensor &_expr; - seq _seq; - bool _does_alias = false; - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;} -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using V = simd_vector_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = 1; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return 1;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq.size();} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return _seq.size();} - constexpr const Tensor& expr() const {return _expr;} - - FASTOR_INLINE TensorViewExpr,1>& noalias() { - _does_alias = true; - return *this; - } - - FASTOR_INLINE TensorViewExpr(Tensor &_ex, const seq &_s) : _expr(_ex), _seq(_s) { - if (_seq._last < 0) _seq._last += N + /*including the end point*/ 1; - if (_seq._first < 0) _seq._first += N + /*including the end point*/ 1; - } - - // View evalution operators - // Copy assignment operators [Needed in addition to generic AbstractTensor overload] - //----------------------------------------------------------------------------------// - FASTOR_HINT_INLINE void operator=(const TensorViewExpr,1> &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for this 1D case for loop unnecessary - for (FASTOR_INDEX i=0; i(i); - _vec.store(&_data[i+_seq._first],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j(i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - //----------------------------------------------------------------------------------// - - // AbstractTensor binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i (i); - } - return; - } - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i (i); - _vec.store(&_data[i+_seq._first],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j(i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) + other_src.template eval(i); - _vec.store(&_data[i+_seq._first],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j(i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) - other_src.template eval(i); - _vec.store(&_data[i+_seq._first],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j(i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) * other_src.template eval(i); - _vec.store(&_data[i+_seq._first],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j(i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,1>(tmp_this_tensor,_seq); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) / other_src.template eval(i); - _vec.store(&_data[i+_seq._first],false); - } - for (; i (i); - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - FASTOR_INDEX i; - for (i = 0; i (i); - for (auto j=0; j(i); - } -#else - for (FASTOR_INDEX i = 0; i (i); - } -#endif - } - } - //----------------------------------------------------------------------------------// - - // scalar binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i ,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) + _vec_other; - _vec.store(&_data[i+_seq._first],false); - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) - _vec_other; - _vec.store(&_data[i+_seq._first],false); - } - for (; i ,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) * _vec_other; - _vec.store(&_data[i+_seq._first],false); - } - for (; i && !is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - T inum = T(1) / static_cast(num); - simd_vector_type _vec_other(inum); - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) * _vec_other; - _vec.store(&_data[i+_seq._first],false); - } - for (; i && is_integral_v_,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - simd_vector_type _vec_other(static_cast(num)); - T *FASTOR_RESTRICT _data = _expr.data(); - if (_seq._step == 1) { - FASTOR_INDEX i; - for (i = 0; i template eval(i) / _vec_other; - _vec.store(&_data[i+_seq._first],false); - } - for (; i - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),i*_seq._step+_seq._first,_seq._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const { - return _expr.data()[i*_seq._step+_seq._first]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),(i+j)*_seq._step+_seq._first,_seq._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr.data()[(i+j)*_seq._step+_seq._first]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),as[0]*_seq._step+_seq._first,_seq._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr.data()[as[0]*_seq._step+_seq._first]; - } -}; - - -} // end of namespace Fastor - - -#endif // TENSOR_VIEWS_1D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_2d.h b/noarch/include/Fastor/expressions/views/tensor_views_2d.h deleted file mode 100644 index 70e01f00..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_views_2d.h +++ /dev/null @@ -1,1163 +0,0 @@ -#ifndef TENSOR_VIEWS_2D_H -#define TENSOR_VIEWS_2D_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -namespace Fastor { - - -// 2D non-const views -//-------------------------------------------------------------------------------------// -template -struct TensorConstViewExpr,2>: public AbstractTensor,2>,2> { -private: - const Tensor& _expr; - seq _seq0; - seq _seq1; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = 2; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return 2;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq0.size()*_seq1.size();} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i==0 ? _seq0.size() : _seq1.size();} - constexpr const Tensor& expr() const {return _expr;} - - FASTOR_INLINE TensorConstViewExpr(const Tensor &_ex, seq _s0, seq _s1) : _expr(_ex), _seq0(std::move(_s0)), _seq1(std::move(_s1)) { - if (_seq0._last < 0 && _seq0._first >= 0) {_seq0._last += M + 1;} - else if (_seq0._last==0 && _seq0._first==-1) {_seq0._first=M-1; _seq0._last=M;} - else if (_seq0._last < 0 && _seq0._first < 0) {_seq0._first += M +1; _seq0._last += M+1;} - if (_seq1._last < 0 && _seq1._first >= 0) {_seq1._last += N + 1;} - else if (_seq1._last==0 && _seq1._first==-1) {_seq1._first=N-1; _seq1._last=N;} - else if (_seq1._last < 0 && _seq1._first < 0) {_seq1._first += N +1; _seq1._last += N+1;} -#ifndef NDEBUG - FASTOR_ASSERT(_seq0._last <= M && _seq0._first - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - SIMDVector _vec; - std::array::Size> inds; - for (auto j=0; j::Size; ++j) { - auto it = (idx+j) / _seq1.size(), jt = (idx+j) % _seq1.size(); - inds[j] = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - auto it = idx / _seq1.size(), jt = idx % _seq1.size(); - auto ind = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; - return _expr.data()[ind]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*i*N+j + _seq0._first*N + _seq1._first,false); - else vector_setter(_vec,_expr.data(),_seq0._step*i*N+_seq1._step*j + _seq0._first*N + _seq1._first,_seq1._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first); - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*as[0]*N+as[1] + _seq0._first*N + _seq1._first,false); - else vector_setter(_vec,_expr.data(),_seq0._step*as[0]*N+_seq1._step*as[1] + _seq0._first*N + _seq1._first,_seq1._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr(_seq0._step*as[0]+_seq0._first,_seq1._step*as[1]+_seq1._first); - } -}; - - - - - - - - - - - - - -// 2D non-const views -//-------------------------------------------------------------------------------------// -template -struct TensorViewExpr,2>: public AbstractTensor,2>,2> { -private: - Tensor& _expr; - seq _seq0; - seq _seq1; - bool _does_alias = false; - // std::array _dims; - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; -public: - using scalar_type = T; - using simd_vector_type = typename Tensor::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - static constexpr FASTOR_INDEX Dimension = 2; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr FASTOR_INDEX rank() {return 2;} - constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq0.size()*_seq1.size();} - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i==0 ? _seq0.size() : _seq1.size();} - constexpr const Tensor& expr() const {return _expr;} - - FASTOR_INLINE TensorViewExpr,2>& noalias() { - _does_alias = true; - return *this; - } - - FASTOR_INLINE TensorViewExpr(Tensor &_ex, seq _s0, seq _s1) : - _expr(_ex), _seq0(std::move(_s0)), _seq1(std::move(_s1)) { - - if (_seq0._last < 0 && _seq0._first >= 0) {_seq0._last += M + 1;} - else if (_seq0._last==0 && _seq0._first==-1) {_seq0._first=M-1; _seq0._last=M;} - else if (_seq0._last < 0 && _seq0._first < 0) {_seq0._first += M +1; _seq0._last += M+1;} - if (_seq1._last < 0 && _seq1._first >= 0) {_seq1._last += N + 1;} - else if (_seq1._last==0 && _seq1._first==-1) {_seq1._first=N-1; _seq1._last=N;} - else if (_seq1._last < 0 && _seq1._first < 0) {_seq1._first += N +1; _seq1._last += N+1;} -#ifndef NDEBUG - FASTOR_ASSERT(_seq0._last <= M && _seq0._first,2> &other_src) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other_src; - // assign temporary to this - this->operator=(tmp); - return; - // Alternatively one could do, but slower - // _does_alias = false; - // // Evaluate this into a temporary - // auto tmp_this_tensor = get_tensor(); - // tmp_this_tensor(_seq0,_seq1) = other_src; - // // assign temporary to this - // this->operator=(tmp_this_tensor(_seq0,_seq1)); - // return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i(i,j); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) = other_src.template eval_s(i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j (i,j); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); - } - } -#endif - } - } - //----------------------------------------------------------------------------------// - - - - // AbstractTensor binders - equal order - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); - } - } - return; - } - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j (i,j); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) = other_src.template eval_s(i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j (i,j); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) + other_src.template eval(i,j); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) += other_src.template eval_s(i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) + other_src.template eval(i,j); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) - other_src.template eval(i,j); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) -= other_src.template eval_s(i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) - other_src.template eval(i,j); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) * other_src.template eval(i,j); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= other_src.template eval_s(i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) * other_src.template eval(i,j); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(i,j); - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; itemplate eval(i,j) / other_src.template eval(i,j); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) /= other_src.template eval_s(i,j); - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) / other_src.template eval(i,j); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(i,j); - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(i,j); - } - } -#endif - } - } - //----------------------------------------------------------------------------------// - - - // AbstractTensor binders [non-equal order] - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - FASTOR_IF_CONSTEXPR(is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(counter); - counter++; - } - } - return; - } - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j (counter); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) = other_src.template eval_s(counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j (counter); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) = other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) + other_src.template eval(counter); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) += other_src.template eval_s(counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) + other_src.template eval(counter); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) - other_src.template eval(counter); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) -= other_src.template eval_s(counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) - other_src.template eval(counter); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) * other_src.template eval(counter); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= other_src.template eval_s(counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) * other_src.template eval(counter); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,2>(tmp_this_tensor,_seq0,_seq1); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *FASTOR_RESTRICT _data = _expr.data(); - FASTOR_INDEX counter = 0; - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) / other_src.template eval(counter); - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) /= other_src.template eval_s(counter); - counter++; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) / other_src.template eval(counter); - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - counter+=Stride; - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(counter); - counter++; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= other_src.template eval_s(counter); - counter++; - } - } -#endif - } - } - //----------------------------------------------------------------------------------// - - - // Scalar binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_INLINE void operator=(U num) { - T *_data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j ,bool> = false> - FASTOR_INLINE void operator+=(U num) { - T *_data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) + _vec_other; - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) += num; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) + _vec_other; - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += num; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) += num; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - T *_data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) - _vec_other; - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) -= num; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) - _vec_other; - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= num; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) -= num; - } - } -#endif - } - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - T *_data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) * _vec_other; - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= num; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) * _vec_other; - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= num; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= num; - } - } -#endif - } - } - - template && !is_integral_v_,bool> = false> - FASTOR_INLINE void operator/=(U num) { - T *_data = _expr.data(); - T inum = T(1.0)/T(num); - SIMDVector _vec_other((inum)); - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) * _vec_other; - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) *= inum; - } - } - } -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - else { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) * _vec_other; - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= inum; - } - } - } -#else - else { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) *= inum; - } - } - } -#endif - } - template && is_integral_v_,bool> = false> - FASTOR_INLINE void operator/=(U num) { - T *_data = _expr.data(); - SIMDVector _vec_other(static_cast(num)); - if (_seq1._step == 1) { - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) / _vec_other; - _vec.store(&_data[(_seq0._step*i+_seq0._first)*N+j+_seq1._first],false); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,j+_seq1._first) /= num; - } - } - } - else { -#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - FASTOR_INDEX j; - for (j = 0; j template eval(i,j) / _vec_other; - data_setter(_data,_vec,(_seq0._step*i+_seq0._first)*N+_seq1._step*j+_seq1._first,_seq1._step); - } - for (; j <_seq1.size(); ++j) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= num; - } - } -#else - for (FASTOR_INDEX i = 0; i <_seq0.size(); i++) { - for (FASTOR_INDEX j = 0; j <_seq1.size(); j++) { - _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first) /= num; - } - } -#endif - } - } - //----------------------------------------------------------------------------------// - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - SIMDVector _vec; - std::array::Size> inds; - for (auto j=0; j::Size; ++j) { - auto it = (idx+j) / _seq1.size(), jt = (idx+j) % _seq1.size(); - inds[j] = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - auto it = idx / _seq1.size(), jt = idx % _seq1.size(); - auto ind = _seq0._step*it*N+_seq1._step*jt + _seq0._first*N + _seq1._first; - return _expr.data()[ind]; - } - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*i*N+j + _seq0._first*N + _seq1._first,false); - else vector_setter(_vec,_expr.data(),_seq0._step*i*N+_seq1._step*j + _seq0._first*N + _seq1._first,_seq1._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _expr(_seq0._step*i+_seq0._first,_seq1._step*j+_seq1._first); - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - SIMDVector _vec; - if (_seq1._step==1) _vec.load(_expr.data()+_seq0._step*as[0]*N+as[1] + _seq0._first*N + _seq1._first,false); - else vector_setter(_vec,_expr.data(),_seq0._step*as[0]*N+_seq1._step*as[1] + _seq0._first*N + _seq1._first,_seq1._step); - return _vec; - } - - template - constexpr FASTOR_INLINE U teval_s(const std::array& as) const { - return _expr(_seq0._step*as[0]+_seq0._first,_seq1._step*as[1]+_seq1._first); - } -}; - - -} // end of namespace Fastor - - -#endif // TENSOR_VIEWS_2D_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_assignment.h b/noarch/include/Fastor/expressions/views/tensor_views_assignment.h deleted file mode 100644 index ebefc27c..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_views_assignment.h +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef TENSOR_VIEWS_ASSIGNMENT_H -#define TENSOR_VIEWS_ASSIGNMENT_H - - -#include "Fastor/expressions/views/tensor_views.h" -#include "Fastor/expressions/views/tensor_fixed_views_1d.h" -#include "Fastor/expressions/views/tensor_fixed_views_2d.h" -#include "Fastor/expressions/views/tensor_random_views.h" - - -namespace Fastor { - -#define FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(ASSIGN_TYPE)\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstFixedViewExpr1D& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFixedViewExpr1D& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstFixedViewExpr2D& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFixedViewExpr2D& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstFixedViewExprnD& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFixedViewExprnD& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstViewExpr& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorViewExpr& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorConstRandomViewExpr& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorRandomViewExpr& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorFilterViewExpr& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ -template\ -FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor &dst, const TensorDiagViewExpr& src) {\ - trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\ -}\ - -FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT( ) -FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_add) -FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_sub) -FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_mul) -FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_div) - -} // end of namespace Fastor - - -#endif // TENSOR_VIEWS_ASSIGNMENT_H diff --git a/noarch/include/Fastor/expressions/views/tensor_views_nd.h b/noarch/include/Fastor/expressions/views/tensor_views_nd.h deleted file mode 100644 index 05818d27..00000000 --- a/noarch/include/Fastor/expressions/views/tensor_views_nd.h +++ /dev/null @@ -1,1680 +0,0 @@ -#ifndef TENSOR_VIEWS_ND_H -#define TENSOR_VIEWS_ND_H - - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/expressions/linalg_ops/linalg_traits.h" - -namespace Fastor { - - - -// Generic const tensor views based on sequences/slices -//----------------------------------------------------------------------------------------------// -template class TensorType, typename T, size_t DIMS, size_t ... Rest> -struct TensorConstViewExpr,DIMS>: public AbstractTensor,DIMS>,DIMS> { -private: - const TensorType &_expr; - std::array _seqs; - std::array _dims; - bool _is_vectorisable; - bool _is_strided_vectorisable; -public: - using scalar_type = T; - using simd_vector_type = typename TensorType::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = TensorType; - static constexpr FASTOR_INDEX Dimension = DIMS; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; - - FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} - FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} - FASTOR_INLINE FASTOR_INDEX size() const { - int sizer = 1; - for (auto &_seq: _seqs) sizer *= _seq.size(); - return sizer; - } - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _seqs[i].size();} - constexpr const TensorType& expr() const {return _expr;} - - FASTOR_INLINE TensorConstViewExpr(const TensorType &_ex, std::array _s) : _expr(_ex), _seqs(std::move(_s)) { - static_assert(DIMS==sizeof...(Rest),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - auto counter = 0; - for (auto &_seq: _seqs) { - if (_seq._last < 0 && _seq._first>=0) { - _seq._last += _expr.dimension(counter) + 1; - } - // take care of scalar indexing with -1 - else if (_seq._last == 0 && _seq._first==-1) { - auto dim = _expr.dimension(counter); - _seq._first = dim-1; - _seq._last = dim; - } - else if (_seq._last < 0 && _seq._first < 0) { - auto dim = _expr.dimension(counter); - _seq._first += dim + 1; - _seq._last += dim + 1; - } -#ifndef NDEBUG - FASTOR_ASSERT(_seq._last <= _expr.dimension(counter) && _seq._first<_expr.dimension(counter),"INDEX OUT OF BOUNDS"); -#endif - counter++; - } - - for (FASTOR_INDEX i=0; i && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step==1) ? true : false; - _is_strided_vectorisable = !is_same_v_ && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step!=1) ? true : false; - } - - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - - SIMDVector _vec; - std::array as = {}; - std::array::Size> inds; - for (auto j=0; j::Size; ++j) { - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( int(idx+j) / remaining ) % _dims[n]; - } - inds[j] = 0; - for(int it = 0; it< DIMS; it++) { - inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - - std::array as = {}; - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - - return _expr.data()[ind]; - } - - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { - - idx += j; - SIMDVector _vec; - std::array as = {}; - - std::array::Size> inds; - for (auto j=0; j::Size; ++j) { - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( int(idx+j) / remaining ) % _dims[n]; - } - inds[j] = 0; - for(int it = 0; it< DIMS; it++) { - inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { - idx += j; - std::array as = {}; - - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - - return _expr.data()[ind]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - if (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); - else if (_is_strided_vectorisable) { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),ind,_seqs[DIMS-1]._step); - return _vec; - } - else { - // return eval(ind); - - SIMDVector _vec; - std::array::Size> inds; - std::array as_ = as; - for (auto j=0; j::Size; ++j) { - int _sum = 0; - for(int it = 0; it< DIMS; it++) { - _sum += products_[it]*as_[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - inds[j] = _sum; - - for(int jt = (int)DIMS-1; jt>=0; jt--) - { - as_[jt] +=1; - if(as_[jt]<_dims[jt]) - break; - else - as_[jt]=0; - } - } - - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - } - - template - FASTOR_INLINE U teval_s(const std::array& as) const { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - return _expr.data()[ind]; - } -}; -//----------------------------------------------------------------------------------------------// - - - - - - -// Generic non-const tensor views based on sequences/slices -//----------------------------------------------------------------------------------------------// -template class TensorType, typename T, size_t DIMS, size_t ... Rest> -struct TensorViewExpr,DIMS>: public AbstractTensor,DIMS>,DIMS> { -private: - TensorType &_expr; - std::array _seqs; - bool _does_alias = false; - std::array _dims; - bool _is_vectorisable; - bool _is_strided_vectorisable; - - constexpr FASTOR_INLINE Tensor get_tensor() const {return _expr;}; - constexpr FASTOR_INLINE std::array get_sequences() const {return _seqs;} - -public: - using scalar_type = T; - using simd_vector_type = typename TensorType::simd_vector_type; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = TensorType; - static constexpr FASTOR_INDEX Dimension = DIMS; - static constexpr FASTOR_INDEX Stride = simd_vector_type::Size; - static constexpr std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; - - FASTOR_INLINE bool is_vectorisable() const {return _is_vectorisable;} - FASTOR_INLINE bool is_strided_vectorisable() const {return _is_strided_vectorisable;} - FASTOR_INLINE FASTOR_INDEX size() const { - int sizer = 1; - for (auto &_seq: _seqs) sizer *= _seq.size(); - return sizer; - } - constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _seqs[i].size();} - constexpr const TensorType& expr() const {return _expr;} - - FASTOR_INLINE TensorViewExpr,DIMS>& noalias() { - _does_alias = true; - return *this; - } - - TensorViewExpr(TensorType &_ex, std::array _s) : _expr(_ex), _seqs(std::move(_s)) { - static_assert(DIMS==sizeof...(Rest),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - auto counter = 0; - for (auto &_seq: _seqs) { - if (_seq._last < 0 && _seq._first>=0) { - _seq._last += _expr.dimension(counter) + 1; - } - // take care of scalar indexing with -1 - else if (_seq._last == 0 && _seq._first==-1) { - auto dim = _expr.dimension(counter); - _seq._first = dim-1; - _seq._last = dim; - } - else if (_seq._last < 0 && _seq._first < 0) { - auto dim = _expr.dimension(counter); - _seq._first += dim + 1; - _seq._last += dim + 1; - } -#ifndef NDEBUG - FASTOR_ASSERT(_seq._last <= _expr.dimension(counter) && _seq._first<_expr.dimension(counter),"INDEX OUT OF BOUNDS"); -#endif - counter++; - } - - for (FASTOR_INDEX i=0; i && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step==1) ? true : false; - _is_strided_vectorisable = !is_same_v_ && _seqs[DIMS-1].size() % SIMDVector::Size == 0 && (_seqs[DIMS-1]._step!=1) ? true : false; - } - - // View evalution operators - // Copy assignment operators [Needed in addition to generic AbstractTensor overload] - //----------------------------------------------------------------------------------// - FASTOR_HINT_INLINE void operator=(const TensorViewExpr,DIMS> &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif -#ifndef NDEBUG - FASTOR_ASSERT(other.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - // constexpr FASTOR_INDEX stride = V::Size; - V _vec; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _vec = other.template teval(as); - _vec.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] = other.template teval_s(as); - // print(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] += 1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - //----------------------------------------------------------------------------------// - - // AbstractTensor binders [equal order] - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // _data[ind] = other_src.template eval_s(counter); - _data[ind] = other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - - // // Generic vectorised version that takes care of the remainder scalar ops - // using V=SIMDVector; - // while(counter < total) - // { - // int ind = 0; - // for(int it = 0; it< DIMS; it++) { - // ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - // } - // if (_dims[DIMS-1] - as[DIMS-1] % V::Size == 0) { - // // V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - // _vec.store(&_data[ind],false); - // counter+=V::Size; - // } - // else { - // // _data[ind] = other_src.template eval_s(counter); - // _data[ind] = other_src.template teval_s(as); - // counter++; - // } - - // for(jt = DIMS-1; jt>=0; jt--) - // { - // if (jt == _dims.size()-1) as[jt]+=V::Size; - // else as[jt] +=1; - // if(as[jt]<_dims[jt]) - // break; - // else - // as[jt]=0; - // } - // if(jt<0) - // break; - // } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out += _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // _data[ind] += other_src.template eval_s(counter); - _data[ind] += other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out -= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // _data[ind] -= other_src.template eval_s(counter); - _data[ind] -= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out *= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // _data[ind] *= other_src.template eval_s(counter); - _data[ind] *= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // V _vec = other_src.template eval(counter); - V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out /= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - // _data[ind] /= other_src.template eval_s(counter); - _data[ind] /= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - //----------------------------------------------------------------------------------// - - // AbstractTensor binders [non-equal orders] - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else - { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] = other_src.template eval_s(counter); - // _data[ind] = other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator+=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator+=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out += _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] += other_src.template eval_s(counter); - // _data[ind] += other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator-=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator-=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out -= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] -= other_src.template eval_s(counter); - // _data[ind] -= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator*=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator*=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out *= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] *= other_src.template eval_s(counter); - // _data[ind] *= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { - const typename Derived::result_type& tmp = evaluate(other.self()); - this->operator/=(tmp); - } - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(const AbstractTensor &other) { -#if !(FASTOR_NO_ALIAS) - if (_does_alias) { - _does_alias = false; - // Evaluate this into a temporary - auto tmp_this_tensor = get_tensor(); - auto tmp = TensorViewExpr,DIMS>(tmp_this_tensor,get_sequences()); - // Assign other to temporary - tmp = other; - // assign temporary to this - this->operator/=(tmp); - return; - } -#endif - const Derived& other_src = other.self(); -#ifndef NDEBUG - FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - V _vec = other_src.template eval(counter); - // V _vec = other_src.template teval(as); - _vec_out.load(&_data[ind],false); - _vec_out /= _vec; - _vec_out.store(&_data[ind],false); - - counter+=V::Size; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] /= other_src.template eval_s(counter); - // _data[ind] /= other_src.template teval_s(as); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - // scalar binders - //----------------------------------------------------------------------------------// - template,bool> = false> - FASTOR_HINT_INLINE void operator=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _vec.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] = (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator+=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _vec_out.load(&_data[ind],false); - _vec_out += _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] += (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator-=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _vec_out.load(&_data[ind],false); - _vec_out -= _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] -= (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator*=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _vec_out.load(&_data[ind],false); - _vec_out *= _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] *= (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - - template,bool> = false> - FASTOR_HINT_INLINE void operator/=(U num) { - - T *_data = _expr.data(); - std::array as = {}; - int total = size(); - int jt, counter = 0; - - if (_is_vectorisable) { - using V=SIMDVector; - constexpr FASTOR_INDEX stride = V::Size; - V _vec = (T)num; - V _vec_out; - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _vec_out.load(&_data[ind],false); - _vec_out /= _vec; - _vec_out.store(&_data[ind],false); - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - if (jt == _dims.size()-1) as[jt]+=stride; - else as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - else { - while(counter < total) - { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*(as[it]*_seqs[it]._step + _seqs[it]._first); - } - _data[ind] /= (T)num; - - counter++; - for(jt = DIMS-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]<_dims[jt]) - break; - else - as[jt]=0; - } - if(jt<0) - break; - } - } - } - //----------------------------------------------------------------------------------// - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx) const { - - SIMDVector _vec; - std::array as = {}; - std::array::Size> inds; - for (auto j=0; j::Size; ++j) { - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( int(idx+j) / remaining ) % _dims[n]; - } - inds[j] = 0; - for(int it = 0; it< DIMS; it++) { - inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const { - - std::array as = {}; - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - - return _expr.data()[ind]; - } - - - template - FASTOR_INLINE SIMDVector eval(FASTOR_INDEX idx, FASTOR_INDEX j) const { - - idx += j; - SIMDVector _vec; - std::array as = {}; - - std::array::Size> inds; - for (auto j=0; j::Size; ++j) { - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( int(idx+j) / remaining ) % _dims[n]; - } - inds[j] = 0; - for(int it = 0; it< DIMS; it++) { - inds[j] += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - } - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - - - template - FASTOR_INLINE U eval_s(FASTOR_INDEX idx, FASTOR_INDEX j) const { - - idx += j; - std::array as = {}; - int remaining = size(); - for (int n = 0; n < DIMS; ++n) { - remaining /= _dims[n]; - as[n] = ( (int)idx / remaining ) % _dims[n]; - } - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - - return _expr.data()[ind]; - } - - template - FASTOR_INLINE SIMDVector teval(const std::array& as) const { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - if (_is_vectorisable) return SIMDVector(&_expr.data()[ind],false); - else if (_is_strided_vectorisable) { - SIMDVector _vec; - vector_setter(_vec,_expr.data(),ind,_seqs[DIMS-1]._step); - return _vec; - } - else { - // return eval(ind); - - SIMDVector _vec; - std::array::Size> inds; - std::array as_ = as; - for (auto j=0; j::Size; ++j) { - int _sum = 0; - for(int it = 0; it< DIMS; it++) { - _sum += products_[it]*as_[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - inds[j] = _sum; - - for(int jt = (int)DIMS-1; jt>=0; jt--) - { - as_[jt] +=1; - if(as_[jt]<_dims[jt]) - break; - else - as_[jt]=0; - } - } - - vector_setter(_vec,_expr.data(),inds); - return _vec; - } - } - - template - FASTOR_INLINE U teval_s(const std::array& as) const { - int ind = 0; - for(int it = 0; it< DIMS; it++) { - ind += products_[it]*as[it]*_seqs[it]._step + _seqs[it]._first*products_[it]; - } - return _expr.data()[ind]; - } -}; - - -template class TensorType, typename T, size_t DIMS, size_t ... Rest> -constexpr std::array TensorViewExpr,DIMS>::products_; - - -} // end of namespace Fastor - - -#endif // TENSOR_VIEWS_ND_H diff --git a/noarch/include/Fastor/meta/einsum_meta.h b/noarch/include/Fastor/meta/einsum_meta.h deleted file mode 100644 index ebe348cd..00000000 --- a/noarch/include/Fastor/meta/einsum_meta.h +++ /dev/null @@ -1,1133 +0,0 @@ -#ifndef EINSUM_META_H -#define EINSUM_META_H - -#include "Fastor/meta/meta.h" -#include "Fastor/config/config.h" -#include "Fastor/tensor/Tensor.h" -#include - -namespace Fastor { - -//namespace detail { - - -template -struct Index; - - -// Find return type of tensor contraction -//------------------------------------------------------------------------------------------------------------// -// is ind[i] unique in ind? -template -constexpr bool is_uniq(const int (&ind)[N], size_t i, size_t cur = 0){ - return cur == N ? true : - (cur == i || ind[cur] != ind[i]) ? is_uniq(ind, i, cur + 1) : false; -} - -// For every i where ind[i] == index, is dim[i] == dimension? -template -constexpr bool check_all_eq(int index, int dimension, - const int (&ind)[N], const int (&dim)[N], size_t cur = 0) { - return cur == N ? true : - (ind[cur] != index || dim[cur] == dimension) ? - check_all_eq(index, dimension, ind, dim, cur + 1) : false; -} - -// if position i should be contracted away, return 1001001, otherwise return dim[i]. -// triggers a compile-time error on mismatch. -template -constexpr int calc(size_t i, const int (&ind)[N], const int (&dim)[N]){ - return is_uniq(ind, i) ? dim[i] : - check_all_eq(ind[i], dim[i], ind, dim) ? 1001001 : throw "dimension mismatch"; -} - -// if position i should be contracted away, return 1001001, otherwise return ind[i]. -// triggers a compile-time error on mismatch. -template -constexpr int calc_idx(size_t i, const int (&ind)[N], const int (&dim)[N]){ - return is_uniq(ind, i) ? ind[i] : - check_all_eq(ind[i], dim[i], ind, dim) ? 1001001 : throw "dimension mismatch"; -} - -//Now we need a way to get rid of the 1001001s: -template -struct concat_ { using type = Ind; }; -template -struct concat_, Index, Inds...> - : concat_, Inds...> {}; - -template -struct filter_ - : concat_, Index>::type...> {}; - -template -using filter_t = typename filter_::type; - - -//Use them: -template -struct contraction_impl; - -template class Derived, typename T, size_t... Ind, size_t... Dim, size_t... Seq> -struct contraction_impl, Derived, std_ext::index_sequence>{ - static constexpr int ind[sizeof...(Ind)] = { Ind... }; - static constexpr int dim[sizeof...(Dim)] = { Dim... }; - static constexpr int result[sizeof...(Seq)] = {calc(Seq, ind, dim)...}; - - template - static auto unpack_helper(Index) -> Derived; - - using type = decltype(unpack_helper(typename filter_<1001001, result[Seq]...>::type{})); - - // Get the indices instead of values - static constexpr int result2[sizeof...(Seq)] = {calc_idx(Seq, ind, dim)...}; - template - static auto unpack_helper2(Index) -> Index; - using indices = decltype(unpack_helper2(typename filter_<1001001, result2[Seq]...>::type{})); -}; -//------------------------------------------------------------------------------------------------------------// - - - - -// Generate product of dimensions e.g. products_a, products_b and products_out -//------------------------------------------------------------------------------------------------------------// -// products generator -template -constexpr int products(const size_t (&seq)[N], int i = N-1) { - return i == (N-1) ? seq[N-1] : products(seq, i+1)*seq[i]; -} - -template -constexpr int shifter(const size_t (&seq)[N], int i) { - return i < N-1 ? seq[i+1] : shifter(seq, i-1); -} - -template -constexpr int zeroer(const size_t (&seq)[N], int i) { - return i == N-1 ? 0 : seq[i]; -} - - -template -struct nprods; - -template -struct nprods,std_ext::index_sequence> { - constexpr static size_t vals[sizeof...(Rest)] = {Rest...}; - static constexpr size_t pvals[sizeof...(Rest)] = {products(vals,ss)...}; - static constexpr size_t svals[sizeof...(Rest)] = {shifter(pvals,ss)...}; - static constexpr std::array values = {zeroer(svals,ss)...}; -}; - -template -constexpr std::array nprods,std_ext::index_sequence>::values; -//------------------------------------------------------------------------------------------------------------// - - -// For views only -//------------------------------------------------------------------------------------------------------------// -template -constexpr int _oner(const size_t (&seq)[N], int i) { - return i == N-1 ? 1 : seq[i]; -} - -template -struct nprods_views; - -template -struct nprods_views,std_ext::index_sequence> { - constexpr static size_t vals[sizeof...(Rest)] = {Rest...}; - static constexpr size_t pvals[sizeof...(Rest)] = {products(vals,ss)...}; - static constexpr size_t svals[sizeof...(Rest)] = {shifter(pvals,ss)...}; - static constexpr std::array values = {_oner(svals,ss)...}; -}; - -template -constexpr std::array nprods_views,std_ext::index_sequence>::values; -//------------------------------------------------------------------------------------------------------------// - - - - - - -// Auxilary meta functions for tensor contraction -//------------------------------------------------------------------------------------------------------------// -// this is a meta-function equivalent to numpy's "where" -template -constexpr int find_index(const size_t (&ind)[N], int num, size_t i=0){ - return (i==N) ? N : (static_cast(ind[i])==num ? i : find_index(ind,num,i+1)); -} -template -constexpr int find_index(const std::array &ind, int num, size_t i=0){ - return (i==N) ? N : (static_cast(ind[i])==num ? i : find_index(ind,num,i+1)); -} - -// check if a given value is ind1 and not ind0 -template -constexpr bool check_diff(const size_t (&ind0)[M], const size_t (&ind1)[N], int num){ - return (find_index(ind0,num) == static_cast(M)) & (find_index(ind1,num) < static_cast(N)); -} - -// this is a meta-function somewhat equivalent to numpy's "setdiff1d" -// if a given value is in ind1 and not in ind0, then it returns the index in to the array ind1 such that -// ind1[index] = value (num) -template -constexpr int find_index_diff(const size_t (&ind0)[M], const size_t (&ind1)[N], int num){ - return check_diff(ind0,ind1,num) ? find_index(ind1,num) : N; -} - -// based on index from find_index_diff retrieve the actual value -template -constexpr int retrieve_value(const size_t (&ind0)[M], const size_t (&ind1)[N], const int (&nums1)[N], size_t i=0){ - return find_index_diff(ind0,ind1,ind1[i]) == static_cast(N) - ? 1 : nums1[find_index(ind1,ind1[i])]; -} - - -// does an array ind contain a number num (bool equivalent of find_index) -template -constexpr bool contains(const size_t (&ind)[N], int num){ - return find_index(ind,num)!=N; -} - - -template -struct put_dims_in_Index; - -template class Derived, size_t ... Rest, typename T> -struct put_dims_in_Index> { - using type = Index; -}; -//------------------------------------------------------------------------------------------------------------// - - -//! Checks vectorisability and returns a stride and a type -//------------------------------------------------------------------------------------------------------------// -template -struct is_vectorisable; - -template -struct is_vectorisable,Index,Tensor> { - template using _vec_size = internal::get_simd_vector_size>; - static constexpr size_t fastest_changing_index = get_value::value; - static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; - static constexpr bool does_2nd_tensor_disappear = ((int)no_of_unique::value == (int)sizeof...(Idx0) - (int)sizeof...(Idx1)); - static constexpr bool last_index_contracted = contains(idx,get_value::value); - static constexpr bool is_reducible = does_2nd_tensor_disappear && last_index_contracted; - static constexpr bool value = (!last_index_contracted) && (fastest_changing_index % _vec_size::value==0); - static constexpr bool sse_vectorisability = (!last_index_contracted) && - (fastest_changing_index % _vec_size::value==0 && fastest_changing_index % _vec_size::value!=0); - static constexpr bool avx_vectorisability = (!last_index_contracted) && - (fastest_changing_index % _vec_size::value==0 && fastest_changing_index % _vec_size::value==0); - static constexpr int stride = (avx_vectorisability ? _vec_size::value : (sse_vectorisability ? _vec_size::value : 1)); - - using type = typename std::conditional, - typename std::conditional,SIMDVector>::type>::type; -}; - - -template -struct is_vectorisable,Index,Tensor> { - static constexpr size_t fastest_changing_index = get_value::value; - static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; - static constexpr bool does_2nd_tensor_disappear = ((int)no_of_unique::value == (int)sizeof...(Idx0) - (int)sizeof...(Idx1)); - static constexpr bool last_index_contracted = contains(idx,get_value::value); - static constexpr bool is_reducible = does_2nd_tensor_disappear && last_index_contracted; - static constexpr bool value = (!last_index_contracted) && (fastest_changing_index % 4==0); - static constexpr bool sse_vectorisability = (!last_index_contracted) && (fastest_changing_index % 4==0 && fastest_changing_index % 8!=0); - static constexpr bool avx_vectorisability = (!last_index_contracted) && (fastest_changing_index % 4==0 && fastest_changing_index % 8==0); - static constexpr int stride = (avx_vectorisability ? 8 : (sse_vectorisability ? 4 : 1)); - - using type = typename std::conditional, - typename std::conditional,SIMDVector>::type>::type; -}; - -template -struct is_vectorisable,Index,Tensor> { - static constexpr size_t fastest_changing_index = get_value::value; - static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; - static constexpr bool does_2nd_tensor_disappear = ((int)no_of_unique::value == (int)sizeof...(Idx0) - (int)sizeof...(Idx1)); - static constexpr bool last_index_contracted = contains(idx,get_value::value); - static constexpr bool is_reducible = does_2nd_tensor_disappear && last_index_contracted; - static constexpr bool value = (!last_index_contracted) && (fastest_changing_index % 2==0); - static constexpr bool sse_vectorisability = (!last_index_contracted) && (fastest_changing_index % 2==0 && fastest_changing_index % 4!=0); - static constexpr bool avx_vectorisability = (!last_index_contracted) && (fastest_changing_index % 2==0 && fastest_changing_index % 4==0); - static constexpr int stride = (avx_vectorisability ? 4 : (sse_vectorisability ? 2 : 1)); - - using type = typename std::conditional, - typename std::conditional,SIMDVector>::type>::type; -}; - -//------------------------------------------------------------------------------------------------------------// - - -//! Checks reducible vectorisability and returns a stride and a type (use this for working on general strides) -//------------------------------------------------------------------------------------------------------------// -template -struct is_reducibly_vectorisable; - -template -struct is_reducibly_vectorisable,Tensor> { - template using _vec_size = internal::get_simd_vector_size>; - static constexpr size_t fastest_changing_index = get_value::value; - static constexpr bool value = (fastest_changing_index % _vec_size::value==0); - static constexpr bool sse_vectorisability = (fastest_changing_index % _vec_size::value==0 && - fastest_changing_index % _vec_size::value!=0); - static constexpr bool avx_vectorisability = (fastest_changing_index % _vec_size::value==0 && - fastest_changing_index % _vec_size::value==0); - static constexpr int stride = (avx_vectorisability ? _vec_size::value : - (sse_vectorisability ? _vec_size::value : 1)); - - using type = typename std::conditional, - typename std::conditional,SIMDVector>::type>::type; -}; -//------------------------------------------------------------------------------------------------------------// - - - - -//! Check if indices in Einstein summation appear more than twice -//------------------------------------------------------------------------------------------------------------// -template -struct einsum_index_checker; - -template -struct einsum_index_checker> { - static constexpr bool value = no_more_than_two::value; -}; - -//template -//struct einsum_index_checker,Index> { -// static constexpr bool value = no_more_than_two::value; -//}; -//------------------------------------------------------------------------------------------------------------// - - - - - - - - - -//------------------------------------------------------------------------------------------------------------// -template -struct get_resuling_tensor; - -template -struct get_resuling_tensor,Index,Tensor,Tensor> { - using type = typename contraction_impl, Tensor, - typename std_ext::make_index_sequence::type>::type; -}; - - -template -struct get_resuling_index; - -template -struct get_resuling_index,Index,Tensor,Tensor> { - using type = typename contraction_impl, Tensor, - typename std_ext::make_index_sequence::type>::indices; -}; -//------------------------------------------------------------------------------------------------------------// - - - -// How many nested loops should be set up -//------------------------------------------------------------------------------------------------------------// -template -struct no_of_loops_to_set; - -template -struct no_of_loops_to_set,Index,Tensor,Tensor,std_ext::index_sequence> { - - using index_temp = apply_typelist_t, - uniq_t>>; - - static constexpr size_t concat_idx[sizeof...(Idx0)+sizeof...(Idx1)] = {Idx0...,Idx1...}; - static constexpr size_t concat_nums[sizeof...(Idx0)+sizeof...(Idx1)] = {Rest0...,Rest1...}; - static constexpr std::array idx_in_concat = {find_index(concat_idx,index_temp::values[ss])...}; - static constexpr std::array dims = {concat_nums[idx_in_concat[ss]]...}; - static constexpr int value = pack_prod::value; - - using type = Tensor; - using indices = Index; - - using dims_type = Index; -}; - -template -constexpr std::array -no_of_loops_to_set,Index,Tensor,Tensor,std_ext::index_sequence>::dims; -//------------------------------------------------------------------------------------------------------------// - - - - - - -// Find indices from the number of loops to set up e.g. idx_a, idx_b and idx_out -//------------------------------------------------------------------------------------------------------------// -template -struct IndexFirstTensor; - -template -struct IndexFirstTensor,Index,Tensor,Tensor,std_ext::index_sequence> { - - using index_temp = typename no_of_loops_to_set,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::value>::type>::indices; - - static constexpr size_t idx[sizeof...(Idx0)] = {Idx0...}; - static constexpr std::array - indices = {find_index(index_temp::values, idx[ss])...}; - - using type = Tensor; - -}; - -template -constexpr std::array -IndexFirstTensor,Index,Tensor,Tensor,std_ext::index_sequence>::indices; - - -template -struct IndexSecondTensor; - -template -struct IndexSecondTensor,Index,Tensor,Tensor,std_ext::index_sequence> { - - using index_temp = typename no_of_loops_to_set,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::value>::type>::indices; - - static constexpr size_t idx[sizeof...(Idx1)] = {Idx1...}; - static constexpr std::array - indices = {find_index(index_temp::values, idx[ss])...}; - - using type = Tensor; - -}; - -template -constexpr std::array -IndexSecondTensor,Index,Tensor,Tensor,std_ext::index_sequence>::indices; - - - -template -struct IndexResultingTensor; - -template -struct IndexResultingTensor,Index,Tensor,Tensor,std_ext::index_sequence> { - - using index_temp = typename no_of_loops_to_set,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::value>::type>::indices; - - using resulting_index_0 = typename get_resuling_index,Index, - Tensor,Tensor>::type; - - static constexpr std::array - indices = {find_index(index_temp::values, resulting_index_0::values[ss])...}; - - using type = Tensor; -}; - -template -constexpr std::array -IndexResultingTensor,Index,Tensor,Tensor,std_ext::index_sequence>::indices; -//------------------------------------------------------------------------------------------------------------// - - - -//------------------------------------------------------------------------------------------------------------// -// This is for pure inner reduction and not permuted reduction -template -struct is_pair_reduction; - -template -struct is_pair_reduction,Index> { - static constexpr bool value = is_same_v_,Index>; -}; - -// helper -template -constexpr bool is_pair_reduction_v = is_pair_reduction::value; - - -// Reduction for a single tensor -template -struct is_single_reduction; - -template -struct is_single_reduction, Tensor> { - using resulting_tensor = typename contraction_impl, Tensor, - typename std_ext::make_index_sequence::type>::type; - static constexpr bool value = resulting_tensor::dimension_t::value == 0; -}; - -// helper -template -constexpr bool is_single_reduction_v = is_single_reduction::value; -//------------------------------------------------------------------------------------------------------------// - - - -//--------------------------------------------------------------------------------------------------------------// -template -constexpr int contain_prod(const size_t (&ind)[N], const size_t (&sseq)[N], size_t num){ - return (ind[num]==0) ? static_cast(sseq[num]) : -1; -} - -template -constexpr int last_indices_prod(const int (&sseq)[N], int num){ - return num > 0 ? ( (sseq[num-1]!=-1) ? sseq[num-1]*last_indices_prod(sseq,num-1) : 1) : 1; -} - - -template -struct general_stride_finder; - -template -struct general_stride_finder,Index,Tensor,Tensor, std_ext::index_sequence> { - - using OutIndices = typename contraction_impl, Tensor, - typename std_ext::make_index_sequence::type>::indices; - - static constexpr size_t b_idx[sizeof...(Idx1)] = {Idx1...}; - static constexpr size_t b_dim[sizeof...(Rest1)] = {Rest1...}; -// static constexpr std::array container_idx = {contains(OutIndices::values,b_idx[ss])...}; - static constexpr size_t container_idxx[sizeof...(Rest1)] = {contains(OutIndices::values,b_idx[ss])...}; -// static constexpr std::array container_dim = {contain_prod(container_idxx,b_dim,ss)...}; - static constexpr int container_dim[sizeof...(Rest1)] = {contain_prod(container_idxx,b_dim,ss)...}; - static constexpr int value = last_indices_prod(container_dim,sizeof...(Rest1)); -}; - -//template -//constexpr std::array -//general_stride_finder,Index,Tensor,Tensor,std_ext::index_sequence>::container_idx; -//template -//constexpr std::array -//general_stride_finder,Index,Tensor,Tensor,std_ext::index_sequence>::container_dim; -//--------------------------------------------------------------------------------------------------------------------// - - - - - - -namespace internal { -//--------------------------------------------------------------------------------------------------------------------// -//! Given the einsum indices of two tensors, matches their indices to see -//! if it is a genearalised matrix-vector multiplication. If either the indices -//! of the first tensor or the second tensor disappear while matching indices -//! from the end, then it is a genearalised matrix-vector multiplication. -//! Note that this function matches indices from the end in that it can detect -//! generalised matrix-vector product of the forms or (it can detect swapping) -//! but it cannot detect or which is vector-matrix product -template -constexpr -inline bool match_indices_from_end(const T (&ind0)[N0], - const T (&ind1)[N1], T num0=N0-1, T num1=N1-1) { - return ind1[num1] == ind0[num0] ? ( num1 == 0 ? ind1[num1] == ind0[num0] : - (num0 == 0 ? ind1[num1] == ind0[num0] : match_indices_from_end(ind0, ind1, num0 - 1, num1 - 1))) : - false; -} - -//! Same as above but gives the index up to which the higher order tensor (generalised matrix) -//! matches the lower order tensor (generalised vector) from the end. The index is for higher -//! higher order tensor counting from the start including the index itself for instance -//! will return 0 (i.e. index 0 does not match) and will return 1 (i.e. indices 0 and 1 do no match) -//! The function in essence gives the remainder indices that don't match the vector -//! -//! This function only works (gives the correct index) if the accompanying boolean -//! function (match_indices_from_end) is true i.e. it only works if we have a true -//! generalised matrix-vector product otherwise gives an incorrect index -template -constexpr -inline T match_indices_from_end_index(const T (&ind0)[N0], - const T (&ind1)[N1], T num0=N0-1, T num1=N1-1) { - return ind1[num1] == ind0[num0] ? ( num1 == 0 ? (ind1[num1] == ind0[num0] ? num0-1 : num0) : - num0 == 0 ? (ind1[num1] == ind0[num0] ? num1-1 : num1) : - match_indices_from_end_index(ind0, ind1, num0 - 1, num1 - 1) ) : num0-1; -} - - -//! Given the einsum indices of two tensors, matches their indices to see -//! if it is a genearalised vector-matrix multiplication. If either the indices -//! of the first tensor or the second tensor disappear while matching indices -//! from the beggining, then it is a genearalised vector-matrix multiplication. -//! Note that this function matches indices from the beggingin in that it can detect -//! generalised vector-matrix product of the forms or (it can detect swapping) -//! but it cannot detect or which is matrix-vector product -template -constexpr -inline bool match_indices_from_start(const T (&ind0)[N0], - const T (&ind1)[N1], T num0=0, T num1=0) { - return ind1[num1] == ind0[num0] ? ( num1 == N1-1 ? ind1[num1] == ind0[num0] : - (num0 == N0-1 ? ind1[num1] == ind0[num0] : match_indices_from_start(ind0, ind1, num0 + 1, num1 + 1))) : - false; -} - -//! Same as above but gives the index up to which the higher order tensor (generalised matrix) -//! matches the lower order tensor (generalised vector) from the start. The index is for higher -//! higher order tensor counting from the end excluding the index itself for instance -//! will return 1 (only 0 matches) and will return 2 (i.e. indices 0 and 1 match) -//! The function in essence gives the remainder indices that don't match the vector -//! -//! function (match_indices_from_start) is true i.e. it only works if we have a true -//! generalised vector-matrix product otherwise gives an incorrect index -template -constexpr -inline T match_indices_from_start_index(const T (&ind0)[N0], - const T (&ind1)[N1], T num0=0, T num1=0) { - return ind1[num1] == ind0[num0] ? ( num1 == N1-1 ? (ind1[num1] == ind0[num0] ? num0+1 : num0) : - num0 == N0-1 ? (ind1[num1] == ind0[num0] ? num1+1 : num1) : - match_indices_from_start_index(ind0, ind1, num0 + 1, num1 + 1) ) : num0; -} - - -//! Given the einsum indices of two tensors, matches their indices to see -//! if it is a standard genearalised matrix-matrix multiplication. -//! By standard we mean . This function cannot detect transposed cases of -//! gemm such as and so on. -//! Note that this function matches indices from the two ends but it also detects -//! as gemm. So it should be used in conjunction with is_vector_matrix/is_matrix_vec -//! and is_inner. Look at the corresponding struct that uses it -template -constexpr -inline bool match_indices_from_two_ends(const T (&ind0)[N0], - const T (&ind1)[N1], int ncontracted, T num0=N0-1, T num1=0) { - return ncontracted != 0 ? - (ncontracted == 1 ? (ind1[num1] == ind0[num0 - ncontracted + 1] ? true : false) - : (ind1[num1] == ind0[num0 - ncontracted + 1] ? - match_indices_from_two_ends(ind0, ind1, ncontracted-1, num0, num1 + 1) : - false)) : - false; -} - - - - -template -struct is_generalised_matrix_vector; - -template -struct is_generalised_matrix_vector,Index > { - static constexpr size_t which_one_is_vector = sizeof...(Idx0) > sizeof...(Idx1) ? 1 : 0; - static constexpr size_t idx0[sizeof...(Idx0)] = {Idx0...}; - static constexpr size_t idx1[sizeof...(Idx1)] = {Idx1...}; - static constexpr bool value = match_indices_from_end(idx0, idx1) && sizeof...(Idx0) != sizeof...(Idx1); - static constexpr size_t matches_up_to = match_indices_from_end_index(idx0, idx1); -}; - - -template -struct is_generalised_vector_matrix; - -template -struct is_generalised_vector_matrix,Index > { - static constexpr size_t which_one_is_vector = sizeof...(Idx0) > sizeof...(Idx1) ? 1 : 0; - static constexpr size_t idx0[sizeof...(Idx0)] = {Idx0...}; - static constexpr size_t idx1[sizeof...(Idx1)] = {Idx1...}; - static constexpr bool value = match_indices_from_start(idx0, idx1) && sizeof...(Idx0) != sizeof...(Idx1); - static constexpr size_t matches_up_to = match_indices_from_start_index(idx0, idx1); -}; - - -template -struct is_generalised_matrix_matrix; - -template -struct is_generalised_matrix_matrix,Index > { - static constexpr bool is_mat_vec = is_generalised_matrix_vector,Index>::value; - static constexpr bool is_vec_mat = is_generalised_vector_matrix,Index>::value; - static constexpr int ncontracted = sizeof...(Idx0) + sizeof...(Idx1) - no_of_unique::value; - static constexpr bool is_inner = sizeof...(Idx0) == sizeof...(Idx1) && no_of_unique::value == sizeof...(Idx1); - static constexpr size_t idx0[sizeof...(Idx0)] = {Idx0...}; - static constexpr size_t idx1[sizeof...(Idx1)] = {Idx1...}; - static constexpr bool value = !is_mat_vec && !is_vec_mat && !is_inner && match_indices_from_two_ends(idx0, idx1, ncontracted); -}; -//--------------------------------------------------------------------------------------------------------------------// -} // namespace internal - - - - - - - -// A complete tensor contraction meta-engine -//--------------------------------------------------------------------------------------------------------------// -//--------------------------------------------------------------------------------------------------------------// -#if FASTOR_CXX_VERSION >= 2017 -template -inline constexpr std::array find_remaining(const std::array &maxes_out, int total) { - std::array remainings = {}; - remainings[0] = total / maxes_out[0]; - for (int i=1; i -inline constexpr std::array,(size_t)total> cartesian_product_2(const std::array &maxes_out) { - std::array,(size_t)total> as_all = {}; - for (int i=0; i -constexpr int find_remaining(const int (&maxes_out)[N], int remaining, int i) { - return i==0 ? remaining/maxes_out[0] : find_remaining(maxes_out,remaining,i-1) / maxes_out[i]; -} - -template -constexpr int cartesian_product_single(const int (&maxes_out)[N], int remaining, int i, int n=0) { - return (i/(find_remaining(maxes_out,remaining,n))) % maxes_out[n]; -} - -template -struct gen_single_cartesian_product; - -template -struct gen_single_cartesian_product,std_ext::index_sequence> { - static constexpr int vals[sizeof...(Rest)] = {Rest...}; - static constexpr std::array values = {cartesian_product_single(vals,pack_prod::value,I,ss)...}; -}; - -template -constexpr std::array gen_single_cartesian_product,std_ext::index_sequence>::values; - - -template -constexpr std::array all_cartesian_product() { - return gen_single_cartesian_product,typename std_ext::make_index_sequence::type>::values; -} - - -template -struct cartesian_product; - -template -struct cartesian_product,std_ext::index_sequence> { - static constexpr std::array,sizeof...(ss)> values = {all_cartesian_product()...}; - -}; - -template -constexpr std::array,sizeof...(ss)> cartesian_product,std_ext::index_sequence>::values; - - -template -constexpr int get_indices(const std::array &products, - const std::array& idx, - const std::array,All> &as_all, - int i, - int it) { - return it==0 ? as_all[i][idx[static_cast(N)-1]] + products[0]*as_all[i][idx[0]] : - products[it]*as_all[i][idx[it]]+get_indices(products,idx,as_all,i,it-1); -} - -// Blowing compilation time and memory usage 101 -//--------------------------------------------------------------------------------------------------------------// -template -struct contract_meta_engine; - -template -struct contract_meta_engine,Index,Tensor,Tensor,std_ext::index_sequence> { - - using OutTensor = typename contraction_impl, Tensor, - typename std_ext::make_index_sequence::type>::type; - using OutIndices = typename contraction_impl, Tensor, - typename std_ext::make_index_sequence::type>::indices; - - static constexpr int a_dim = sizeof...(Rest0); - static constexpr int b_dim = sizeof...(Rest1); - static constexpr int out_dim = OutTensor::Dimension; - static constexpr int total = sizeof...(ss); - - static constexpr auto& idx_a = IndexFirstTensor,Index, Tensor,Tensor, - typename std_ext::make_index_sequence::type>::indices; - static constexpr auto& idx_b = IndexSecondTensor,Index, Tensor,Tensor, - typename std_ext::make_index_sequence::type>::indices; - static constexpr auto& idx_out = IndexResultingTensor,Index, Tensor,Tensor, - typename std_ext::make_index_sequence::type>::indices; - - static constexpr int uniques = no_of_unique::value; - using uniques_type = typename std_ext::make_index_sequence::type; - static constexpr auto& maxes_out = no_of_loops_to_set,Index,Tensor,Tensor, - uniques_type>::dims; - - using maxes_out_type = typename no_of_loops_to_set,Index,Tensor,Tensor, - uniques_type>::type; - - static constexpr std::array products_a = nprods,typename std_ext::make_index_sequence::type>::values; - static constexpr std::array products_b = nprods,typename std_ext::make_index_sequence::type>::values; - using Index_with_dims = typename put_dims_in_Index::type; - static constexpr std::array products_out = nprods::type>::values; - - // Generate the cartesian product - static constexpr auto& as_all = cartesian_product::type>::values; - // Alternatively you can pass the ss... directly into cartesian_product but that does not change anything in terms of - // memory usage or compilation time - //using maxes_out_indices = typename no_of_loops_to_set,Index,Tensor,Tensor, - // uniques_type>::indices; - //static constexpr std::array,total> as_all = {all_cartesian_product()...}; - - static constexpr std::array index_a = {get_indices(products_a,idx_a,as_all,ss,a_dim-1)...}; - static constexpr std::array index_b = {get_indices(products_b,idx_b,as_all,ss,b_dim-1)...}; - static constexpr std::array index_out = {get_indices(products_out,idx_out,as_all,ss,out_dim-1)...}; -}; - -template -constexpr std::array -contract_meta_engine,Index, -Tensor,Tensor, -std_ext::index_sequence>::index_a; - -template -constexpr std::array -contract_meta_engine,Index, -Tensor,Tensor, -std_ext::index_sequence>::index_b; - -template -constexpr std::array -contract_meta_engine,Index, -Tensor,Tensor, -std_ext::index_sequence>::index_out; - -//--------------------------------------------------------------------------------------------------------------// -//--------------------------------------------------------------------------------------------------------------// - - - - - - - - -// The followings are generic implemenations that work for any type of complex tensor network -//------------------------------------------------------------------------------------------------------------// -//------------------------------------------------------------------------------------------------------------// -//------------------------------------------------------------------------------------------------------------// - - -// Find how many loops needs to be set. Works for any complex tensor network -//------------------------------------------------------------------------------------------------------------// -// This is equivalent to no_of_loops_to_set_up but more generic (works for arbitrary tensor networks) -template -struct loop_setter; - -template -struct loop_setter,Tensor,std_ext::index_sequence> { - - using index_temp = apply_typelist_t, - uniq_t>>; - - static constexpr size_t concat_idx[sizeof...(Idx)] = {Idx...}; - static constexpr size_t concat_nums[sizeof...(Rest)] = {Rest...}; - static constexpr std::array idx_in_concat = {find_index(concat_idx,index_temp::values[ss])...}; - static constexpr std::array dims = {concat_nums[idx_in_concat[ss]]...}; - static constexpr int value = pack_prod::value; - - using type = Tensor; - using indices = Index; - - using dims_type = Index; -}; - -template -constexpr std::array -loop_setter,Tensor,std_ext::index_sequence>::dims; -//------------------------------------------------------------------------------------------------------------// - - -// Get indices of every individual tensor in the network. Works for any complex tensor network -//------------------------------------------------------------------------------------------------------------// -template -struct IndexTensors; - -template class Derived0, - template class Derived1, - typename T, size_t... Idx, size_t... Idx_t, size_t ...Rest, size_t ...Rest_t, size_t ... ss> -struct IndexTensors,Derived0,Index,Derived1,std_ext::index_sequence> { - - using index_temp = typename loop_setter,Tensor, - typename std_ext::make_index_sequence::value>::type>::indices; - - static constexpr size_t idx[sizeof...(Idx_t)] = {Idx_t...}; - static constexpr std::array - indices = {find_index(index_temp::values, idx[ss])...}; - - using type = Tensor; - -}; - -template class Derived0, - template class Derived1, - typename T, size_t... Idx, size_t... Idx_t, size_t ...Rest, size_t ...Rest_t, size_t ... ss> -constexpr std::array -IndexTensors,Derived0,Index,Derived1,std_ext::index_sequence>::indices; -//------------------------------------------------------------------------------------------------------------// - - - -namespace internal { -//------------------------------------------------------------------------------------------------------------// -template -struct meta_argmin_wrapper; -template -struct meta_argmin_wrapper> { - static constexpr int value = meta_argmin::value; -}; -template -struct meta_argmin_wrapper> { - static constexpr int value = idx; -}; - - -// Complete compile-time sorting algorithm: Does not work if there a duplicate entries in a pack -template -struct meta_sort; -template -struct meta_sort> { - static constexpr int least_value_idx = meta_argmin_wrapper>::value; - static constexpr int least_value = get_value::value; - using reduced_seq = typename filter_::type; - using new_seq = typename concat_,typename meta_sort::new_seq>::type; -}; -template -struct meta_sort> { - using reduced_seq = Index; - using new_seq = Index; -}; - - -// Complete compile-time arg-sorting algorithm: Does not work if there a duplicate entries in a pack -template -struct meta_argsort; -template -struct meta_argsort,Index> { - static constexpr int least_value_idx = meta_argmin_wrapper>::value; - static constexpr int least_value = get_value::value; - using reduced_seq = typename filter_::type; - using new_seq = typename concat_,typename meta_sort::new_seq>::type; - - static constexpr int least_index = get_value::value; - - using reduced_argseq = typename filter_::type; - using new_argseq = typename concat_,typename meta_argsort::new_argseq>::type; -}; -template -struct meta_argsort,Index> { - using reduced_seq = Index; - using new_seq = Index; - - using reduced_argseq = Index; - using new_argseq = Index; -}; -//------------------------------------------------------------------------------------------------------------// - - -// Permutation functions -//------------------------------------------------------------------------------------------------------------// -template -constexpr size_t count_less(const size_t (&seq)[N], size_t i, size_t cur = 0) { - return cur == N ? 0 : (count_less(seq, i, cur + 1) + (seq[cur] < i ? 1 : 0)); -} - -/* Check if a compile time array is sequential */ -template -constexpr bool is_sequential(const size_t (&seq)[N], size_t i=0) { - return i+1 == N ? true : ( seq[i] + 1 == seq[i+1] ? is_sequential(seq, i+1) : false ) ; -} -template -constexpr bool is_sequential(const std::array &seq, size_t i=0) { - return i+1 == N ? true : ( seq[i] + 1 == seq[i+1] ? is_sequential(seq, i+1) : false ) ; -} - -// permutation helper class -template -struct permute_impl; - -template -struct permute_impl, Tensor, std_ext::index_sequence> { - constexpr static size_t lst[sizeof...(ls)] = { ls... }; - constexpr static size_t fvals[sizeof...(ls)] = {fs...}; - using resulting_tensor = Tensor; - using resulting_index = typename meta_argsort,Index>::new_argseq; - using maxes_out_type = Index,Index>::new_argseq::values[ss]]...>; - static constexpr bool requires_permutation = !(is_same_v_> && - is_sequential(resulting_index::values)); -}; - -// permute helper class -template -struct new_permute_impl; - -template -struct new_permute_impl, Tensor, std_ext::index_sequence> { - constexpr static size_t lst[sizeof...(ls)] = { ls... }; - constexpr static size_t fvals[sizeof...(ls)] = {fs...}; - using resulting_tensor = Tensor; - constexpr static size_t aranger[sizeof...(ss)] = { ss... }; - using resulting_index = Index; - static constexpr bool requires_permutation = !(is_same_v_> && - is_sequential(resulting_index::values)); -}; -//------------------------------------------------------------------------------------------------------------// - - -//------------------------------------------------------------------------------------------------------------// -#if FASTOR_CXX_VERSION >= 2017 -template -constexpr std::array get_floor_map(const std::array &idx) { - std::array out = {}; - for (size_t i=0; i -constexpr std::array find_permuation(const std::array &idx0, const std::array &idx1) { - std::array out = {}; - for (size_t i=0; i,OIndex>(a)]. - In the above example the resulting index from einsum is already Index but the user decides to force this - nevertheless using OIndex. Now if permuation is simply performed using OIndex the final output would - be incorrect since permuation only works with what indices it is given and does not have the knowledge of the context. - This meta function is responsible for creating a mapping between einsum and permutation. It takes the resulting index - of einsum and the permutation index [the output index OIndex] and figures out how the tensor should be permuted. -*/ -template -struct permute_mapped_index_impl; - -// // specialisation for when the input and output indices are the same - the no permutation case -// template -// struct permute_mapped_index_impl,Index,std_ext::index_sequence> { -// using resulting_index = Index; -// }; - -template -struct permute_mapped_index_impl,Index,std_ext::index_sequence> { - constexpr static size_t einsum_idx[sizeof...(Idx0)] = { Idx0... }; - constexpr static size_t to_be_permuted_idx[sizeof...(Idx1)] = { Idx1... }; - /* Given that the resulting index from einsum can be discontinuous like Index<5,8,2> we need to first create a continuous - Index starting from zero. This is done through [get_floor_map] function. We floor both resulting einsum and output - indices and we then call the [find_permuation] function to know how many swaps/permutations are necessary going from - the resulting einsum index to the output index - */ - static constexpr std::array argsort_idx0 = meta_argsort,Index>::new_argseq::values; - static constexpr std::array argsort_idx1 = meta_argsort,Index>::new_argseq::values; - static constexpr std::array mapped_idx0 = get_floor_map(argsort_idx0); - static constexpr std::array mapped_idx1 = get_floor_map(argsort_idx1); - static constexpr std::array mapped_idx = find_permuation(mapped_idx0,mapped_idx1); - using resulting_index = Index; -}; - -/* Provided for debugging */ -template -constexpr std::array -permute_mapped_index_impl,Index,std_ext::index_sequence>::argsort_idx0; -template -constexpr std::array -permute_mapped_index_impl,Index,std_ext::index_sequence>::argsort_idx1; -template -constexpr std::array -permute_mapped_index_impl,Index,std_ext::index_sequence>::mapped_idx0; -template -constexpr std::array -permute_mapped_index_impl,Index,std_ext::index_sequence>::mapped_idx1; -template -constexpr std::array -permute_mapped_index_impl,Index,std_ext::index_sequence>::mapped_idx; - - -template -struct permute_mapped_index { - using resulting_index = typename permute_mapped_index_impl::type>::resulting_index; -}; - -template -using permute_mapped_index_t = typename permute_mapped_index::resulting_index; - -#endif // CXX 2017 -//------------------------------------------------------------------------------------------------------------// - -} // internal - - -template -struct requires_permutation; -template -struct requires_permutation, Tensor> { - using _permute_impl = internal::permute_impl, Tensor, - typename std_ext::make_index_sequence::type>; - static constexpr bool value = _permute_impl::requires_permutation; -}; - -// helper -template -constexpr bool requires_permutation_v = requires_permutation::value; - - -template -struct requires_permute; -template -struct requires_permute, Tensor> { - using _permute_impl = internal::new_permute_impl, Tensor, - typename std_ext::make_index_sequence::type>; - static constexpr bool value = _permute_impl::requires_permutation; -}; - -// helper -template -constexpr bool requires_permute_v = requires_permute::value; -//------------------------------------------------------------------------------------------------------------// - - -// einsum helper to extract the resulting index and the resulting tensor -//------------------------------------------------------------------------------------------------------------// -template -struct permute_helper; - -template -struct permute_helper> { - using resulting_index = typename internal::new_permute_impl, - typename std_ext::make_index_sequence::type>::resulting_index; - using resulting_tensor = typename internal::new_permute_impl, - typename std_ext::make_index_sequence::type>::resulting_tensor; -}; -//------------------------------------------------------------------------------------------------------------// - - - -} - -#endif // EINSUM_META_H - diff --git a/noarch/include/Fastor/meta/meta.h b/noarch/include/Fastor/meta/meta.h deleted file mode 100644 index f8fcf7da..00000000 --- a/noarch/include/Fastor/meta/meta.h +++ /dev/null @@ -1,344 +0,0 @@ -#ifndef META_H_ -#define META_H_ - - -#include -#include - - -namespace Fastor { - -//----------------------------------------------------------------------------------------------------------// -template< bool B, class T = void > -using enable_if_t_ = typename std::enable_if::type; - -template< bool B, class T = void > -using disable_if_t_ = typename std::enable_if::type; - -template< bool B, class T, class F > -using conditional_t_ = typename std::conditional::type; - -template< class T, class U > -constexpr bool is_same_v_ = std::is_same::value; - -template< class T > -constexpr bool is_fundamental_v_ = std::is_fundamental::value; - -template< class T > -constexpr bool is_arithmetic_v_ = std::is_arithmetic::value; - -template< class T > -constexpr bool is_integral_v_ = std::is_integral::value; - -template< class T > -constexpr bool is_floating_v_ = std::is_floating_point::value; - -template< class T > -constexpr bool is_array_v_ = std::is_array::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -// If BLAS compatible type -template< class T > struct is_numeric { static constexpr bool value = false; }; -template<> struct is_numeric { static constexpr bool value = true; }; -template<> struct is_numeric { static constexpr bool value = true; }; -template<> struct is_numeric> { static constexpr bool value = true; }; -template<> struct is_numeric> { static constexpr bool value = true; }; - -template< class T > -constexpr bool is_numeric_v_ = is_numeric::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -// If complex type -template< class T > struct is_complex { static constexpr bool value = false; }; -template< class T > struct is_complex> { static constexpr bool value = true; }; - -template< class T > -constexpr bool is_complex_v_ = is_complex::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -// If a type is std::is_fundamental + std::complex + any type that specialises this trait class -// This class is provided because the behaviour of any code that specialises std::is_fundamental/arithmetic/ -// integral/floating_point is undefined -template< class T > struct is_primitive { - static constexpr bool value = std::is_fundamental::value || is_numeric::value; -}; - -template< class T > -constexpr bool is_primitive_v_ = is_primitive::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -// Checks if all parameters in a variadic parameter pack are arithmetic -template -struct is_arithmetic_pack; -template -struct is_arithmetic_pack { - static constexpr bool value = std::is_arithmetic::value && is_arithmetic_pack::value; -}; -template -struct is_arithmetic_pack { static constexpr bool value = std::is_arithmetic::value; }; - -template -static constexpr bool is_arithmetic_pack_v = is_arithmetic_pack::value; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -// Remove cv-qualified and their refs -template struct remove_cvref_ { using type = T; }; -template struct remove_cvref_ { using type = typename remove_cvref_::type; }; -template struct remove_cvref_ { using type = typename remove_cvref_::type; }; -template struct remove_cvref_ { using type = typename remove_cvref_::type; }; -template struct remove_cvref_ { using type = typename remove_cvref_::type; }; -template struct remove_cvref_ { using type = typename remove_cvref_::type; }; - -template -using remove_cv_ref_t = typename remove_cvref_::type; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template struct remove_all { using type = T; }; -template struct remove_all { using type = typename remove_all::type; }; -template struct remove_all { using type = typename remove_all::type; }; -template struct remove_all { using type = typename remove_all::type; }; -template struct remove_all { using type = typename remove_all::type; }; -template struct remove_all { using type = typename remove_all::type; }; -template struct remove_all { using type = typename remove_all::type; }; -template struct remove_all { using type = typename remove_all::type; }; -template struct remove_all { using type = typename remove_all::type; }; - -template -using remove_all_t = typename remove_all::type; -//----------------------------------------------------------------------------------------------------------// - - -// Sum/multiply reduce all elements of a variadic pack -//----------------------------------------------------------------------------------------------------------// -// Sum -template struct pack_add; -template -struct pack_add { static const size_t value = Head+pack_add::value;}; -template<> struct pack_add<> { static const size_t value = 0;}; - -// Multiply -template struct pack_prod; -template -struct pack_prod { static const size_t value = Head*pack_prod::value;}; -template<> struct pack_prod<> { static const size_t value = 1;}; - -// Multiply first n elements -template struct prod_nel; -template -struct prod_nel { static const size_t value = prod_nel::value;}; -template -struct prod_nel<1, First, Rest...> { static const size_t value = pack_prod::value;}; - -//! Partial product of a sequence up to an index up_to -template -constexpr -inline T partial_prod(const T (&ind)[N], T up_to, T num=0) { - return num == up_to ? ind[num] : ind[num]*partial_prod(ind, up_to, num+1); -} - -//! Partial product of a sequence up to an index num from the end -template -constexpr -inline T partial_prod_reverse(const T (&ind)[N], T up_to, T num=N-1) { - return num == up_to ? ind[num] : ind[num]*partial_prod_reverse(ind, up_to, num-1); -} - -template -constexpr T size_proder_(T one){return one.size();} -template -constexpr T size_proder_(T one, T two){return one.size()*two.size();} -template -constexpr T size_proder_(T one, T two, Ts ... ts) {return _proder_(_proder_(one,two),ts...);} -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template -struct get_value; -template -struct get_value { - static const size_t value = get_value::value; -}; -template -struct get_value<1,First,Rest...> { - static const size_t value = First; -}; -template -// Work around to avoid compiler errors -struct get_value { - static const size_t value = 0; -}; - -template -constexpr inline auto get_index(Args&&... as) --> decltype(std::get(std::forward_as_tuple(std::forward(as)...))) { - return std::get(std::forward_as_tuple(std::forward(as)...)); -} - -template -using get_nth_type = typename std::tuple_element>::type; - -template -struct get_all { - static const size_t indices[sizeof...(Rest)]; -}; -//----------------------------------------------------------------------------------------------------------// - - -// comparitors -//----------------------------------------------------------------------------------------------------------// -template -struct is_equal { - static constexpr bool value = I == J; -}; -template -static constexpr bool is_equal_v_ = is_equal::value; - -template -struct is_less { - static constexpr bool value = (I < J); -}; -template -static constexpr bool is_less_v_ = is_less::value; - -template -struct is_less_equal { - static constexpr bool value = I <= J; -}; -template -static constexpr bool is_less_equal_v_ = is_less_equal::value; - -template -struct is_greater { - static constexpr bool value = I > J; -}; -template -static constexpr bool is_greater_v_ = is_greater::value; - -template -struct is_greater_equal { - static constexpr bool value = I >= J; -}; -template -static constexpr bool is_greater_equal_v_ = is_greater_equal::value; -//----------------------------------------------------------------------------------------------------------// - - -// min-max -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// -template -struct meta_min; -template -struct meta_min { - static constexpr size_t pval = meta_min::value; - static const size_t value = (pval <= meta_min::value) ? - pval : meta_min::value; -}; -template -struct meta_min { - static const size_t value = (m<=n) ? m : n; -}; - -template -struct meta_max; -template -struct meta_max { - static constexpr size_t pval = meta_max::value; - static const size_t value = (pval >= meta_max::value) ? - pval : meta_max::value; -}; -template -struct meta_max { - static const size_t value = (m>=n) ? m : n; -}; - -namespace internal { -// namespace to avoid clash with MSVC macros -template constexpr FASTOR_INLINE T min_(const T a, const T b) {return a < b ? a : b;} -template constexpr FASTOR_INLINE T max_(const T a, const T b) {return a > b ? a : b;} -} -//----------------------------------------------------------------------------------------------------------// - -//----------------------------------------------------------------------------------------------------------// -template -struct meta_argmin; -template -struct meta_argmin { - static constexpr size_t pval = meta_min::value; - static const size_t value = (pval <= meta_min::value) ? - meta_argmin::value : meta_argmin::value+1; -}; -template -struct meta_argmin { - static const size_t value = (m -struct meta_argmax; -template -struct meta_argmax { - static constexpr size_t pval = meta_max::value; - static const size_t value = (pval >= meta_max::value) ? - meta_argmax::value : meta_argmax::value+1; -}; -template -struct meta_argmax { - static const size_t value = (m>n) ? 0 : 1; -}; -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// - - -// square/cube -//----------------------------------------------------------------------------------------------------------// -namespace internal { -template struct meta_square { static constexpr size_t value = Val*Val;}; -template struct meta_cube { static constexpr size_t value = Val*Val*Val;}; -} -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -//square root of integers at compile time, use like meta_sqrt<36>::ret -template Y))) > - // use ?: instead of || just to shut up a gcc 4.3 warning -class meta_sqrt -{ - enum { - MidX = (InfX+SupX)/2, - TakeInf = MidX*MidX > Y ? 1 : 0, - NewInf = int(TakeInf) ? InfX : int(MidX), - NewSup = int(TakeInf) ? int(MidX) : SupX -}; - public: - enum { ret = meta_sqrt::ret }; -}; - -template -class meta_sqrt -{ - public: enum { ret = (SupX*SupX <= Y) ? SupX : InfX }; -}; -//----------------------------------------------------------------------------------------------------------// - - -} // end of namespace Fastor - -#endif // META_H_ diff --git a/noarch/include/Fastor/meta/opmin_meta.h b/noarch/include/Fastor/meta/opmin_meta.h deleted file mode 100644 index 24738d4a..00000000 --- a/noarch/include/Fastor/meta/opmin_meta.h +++ /dev/null @@ -1,620 +0,0 @@ -#ifndef OPMIN_META_H -#define OPMIN_META_H - -#ifndef FASTOR_DONT_PERFORM_OP_MIN - -#include "tensor_meta.h" -#include "einsum_meta.h" - -namespace Fastor { - - -// Cost model for by-pair tensor contraction -//------------------------------------------------------------------------------------------------------------// -template -struct pair_flop_cost; - -template -struct pair_flop_cost,Index,Tensor,Tensor,std_ext::index_sequence> { - - static constexpr size_t ind0[sizeof...(Idx0)] = {Idx0... }; - static constexpr size_t ind1[sizeof...(Idx1)] = {Idx1... }; - static constexpr int nums1[sizeof...(Rest1)] = {Rest1... }; - static constexpr size_t cost_tensor0 = pack_prod::value; - static constexpr size_t remaining_cost = pack_prod::value; - static constexpr size_t value = cost_tensor0*remaining_cost; - - using resulting_tensor = typename get_resuling_tensor,Index, - Tensor,Tensor>::type; - using resulting_index = typename get_resuling_index,Index, - Tensor,Tensor>::type; -}; - - -template -struct pair_flop_cost,Index<>,Tensor,Tensor,std_ext::index_sequence> { - static constexpr size_t value = pack_prod::value; -}; - -template -struct pair_flop_cost,Index,Tensor,Tensor,std_ext::index_sequence> { - static constexpr size_t value = pack_prod::value; -}; - -template -struct pair_flop_cost,Index<>,Tensor,Tensor,std_ext::index_sequence> { - static constexpr size_t value = 1; -}; -//------------------------------------------------------------------------------------------------------------// - - - -// Cost of triplet tensor contraction (single evaluation) -//------------------------------------------------------------------------------------------------------------// -template -struct single_evaluation_triplet_cost; - -template -struct single_evaluation_triplet_cost,Index,Index, - Tensor,Tensor,Tensor> { - - using concat_tensor_01 = typename no_of_loops_to_set,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::value>::type>::type; - - using concat_index_01 = typename no_of_loops_to_set,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::value>::type>::indices; - - static constexpr size_t value = pair_flop_cost,concat_tensor_01,Tensor, - typename std_ext::make_index_sequence::type>::value; -}; -//------------------------------------------------------------------------------------------------------------// - - - - -// Cost model for triplet tensor network contraction -//------------------------------------------------------------------------------------------------------------// -template -struct triplet_flop_cost; - -template -struct triplet_flop_cost,Index,Index, - Tensor,Tensor,Tensor> { - - // Perform depth-first search - //--------------------------------------------------------------------- - // first two tensors contracted first - using resulting_tensor_0 = typename get_resuling_tensor,Index, - Tensor,Tensor>::type; - using resulting_index_0 = typename get_resuling_index,Index, - Tensor,Tensor>::type; - - static constexpr size_t flop_count_01_0 = pair_flop_cost,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_01_1 = pair_flop_cost,resulting_tensor_0,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_01 = flop_count_01_0 + flop_count_01_1; - - - // first and last tensors contracted first - using resulting_tensor_1 = typename get_resuling_tensor,Index, - Tensor,Tensor>::type; - using resulting_index_1 = typename get_resuling_index,Index, - Tensor,Tensor>::type; - - static constexpr size_t flop_count_02_0 = pair_flop_cost,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_02_1 = pair_flop_cost,resulting_tensor_1,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_02 = flop_count_02_0 + flop_count_02_1; - - - // second and last tensors contracted first - using resulting_tensor_2 = typename get_resuling_tensor,Index, - Tensor,Tensor>::type; - using resulting_index_2 = typename get_resuling_index,Index, - Tensor,Tensor>::type; - - static constexpr size_t flop_count_12_0 = pair_flop_cost,Index,Tensor,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_12_1 = pair_flop_cost,resulting_tensor_2,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_12 = flop_count_12_0 + flop_count_12_1; - - static constexpr size_t flop_count_012 = single_evaluation_triplet_cost,Index,Index, - Tensor,Tensor,Tensor>::value; - - static constexpr size_t min_cost = meta_min::value; - static constexpr int which_variant = meta_argmin::value; - - // this is the overall resulting tensor and index from overall triplet contraction - using resulting_tensor = typename std::conditional< - which_variant==0, - typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type - >::type - >::type; - - using resulting_index = typename std::conditional< - which_variant==0, - typename get_resuling_index,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type - >::type - >::type; - -}; -//------------------------------------------------------------------------------------------------------------// - - - - - -// Cost model for quartet tensor network contraction -//------------------------------------------------------------------------------------------------------------// -template -struct quartet_flop_cost; - -template -struct quartet_flop_cost,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor> { - - // Perform depth-first search - //--------------------------------------------------------------------- - // first three tensors contracted first - using triplet_cost_012 = triplet_flop_cost,Index,Index, - Tensor,Tensor,Tensor>; - - using resulting_tensor_0 = typename triplet_cost_012::resulting_tensor; - using resulting_index_0 = typename triplet_cost_012::resulting_index; - - static constexpr size_t flop_count_012 = triplet_cost_012::min_cost; - static constexpr size_t flop_count_012_3 = pair_flop_cost,resulting_tensor_0,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_0 = flop_count_012 + flop_count_012_3; - - - // first, second and last tensors contracted first - using triplet_cost_013 = triplet_flop_cost,Index,Index, - Tensor,Tensor,Tensor>; - - using resulting_tensor_1 = typename triplet_cost_013::resulting_tensor; - using resulting_index_1 = typename triplet_cost_013::resulting_index; - - static constexpr size_t flop_count_013 = triplet_cost_013::min_cost; - static constexpr size_t flop_count_013_2 = pair_flop_cost,resulting_tensor_1,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_1 = flop_count_013 + flop_count_013_2; - - - // first, third and last tensors contracted first - using triplet_cost_023 = triplet_flop_cost,Index,Index, - Tensor,Tensor,Tensor>; - - using resulting_tensor_2 = typename triplet_cost_023::resulting_tensor; - using resulting_index_2 = typename triplet_cost_023::resulting_index; - - static constexpr size_t flop_count_023 = triplet_cost_023::min_cost; - static constexpr size_t flop_count_023_1 = pair_flop_cost,resulting_tensor_2,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_2 = flop_count_023 + flop_count_023_1; - - - // last three tensors contracted first - using triplet_cost_123 = triplet_flop_cost,Index,Index, - Tensor,Tensor,Tensor>; - - using resulting_tensor_3 = typename triplet_cost_123::resulting_tensor; - using resulting_index_3 = typename triplet_cost_123::resulting_index; - - static constexpr size_t flop_count_123 = triplet_cost_123::min_cost; - static constexpr size_t flop_count_123_0 = pair_flop_cost,resulting_tensor_3,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_3 = flop_count_123 + flop_count_123_0; - - static constexpr size_t min_cost = meta_min::value; - static constexpr int which_variant = meta_argmin::value; - - - // this is the overall resulting tensor and index from overall quartet contraction - using resulting_tensor = typename std::conditional< - which_variant==0, - typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename std::conditional< - which_variant==2, - typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type, - typename get_resuling_tensor,resulting_index_3,Tensor,resulting_tensor_3>::type - >::type - >::type - >::type; - - using resulting_index = typename std::conditional< - which_variant==0, - typename get_resuling_index,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename std::conditional< - which_variant==2, - typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type, - typename get_resuling_index,resulting_index_3,Tensor,resulting_tensor_3>::type - >::type - >::type - >::type; - -}; -//------------------------------------------------------------------------------------------------------------// - - - - - - -// Cost model for quintet tensor contraction -//------------------------------------------------------------------------------------------------------------// -template -struct quintet_flop_cost; - -template -struct quintet_flop_cost,Index,Index,Index, Index, - Tensor,Tensor,Tensor,Tensor,Tensor> { - - // Perform depth-first search - //--------------------------------------------------------------------- - // first four tensors contracted first - using quartet_cost_0123 = quartet_flop_cost,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_0 = typename quartet_cost_0123::resulting_tensor; - using resulting_index_0 = typename quartet_cost_0123::resulting_index; - - static constexpr size_t flop_count_0123 = quartet_cost_0123::min_cost; - static constexpr size_t flop_count_0123_4 = pair_flop_cost,resulting_tensor_0,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_0 = flop_count_0123 + flop_count_0123_4; - - - // 1st, 2nd, 3rd, 5th tensors contracted first - using quartet_cost_0124 = quartet_flop_cost,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_1 = typename quartet_cost_0124::resulting_tensor; - using resulting_index_1 = typename quartet_cost_0124::resulting_index; - - static constexpr size_t flop_count_0124 = quartet_cost_0124::min_cost; - static constexpr size_t flop_count_0124_3 = pair_flop_cost,resulting_tensor_1,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_1 = flop_count_0124 + flop_count_0124_3; - - - // 1st, 2nd, 4th, 5th tensors contracted first - using quartet_cost_0134 = quartet_flop_cost,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_2 = typename quartet_cost_0134::resulting_tensor; - using resulting_index_2 = typename quartet_cost_0134::resulting_index; - - static constexpr size_t flop_count_0134 = quartet_cost_0134::min_cost; - static constexpr size_t flop_count_0134_2 = pair_flop_cost,resulting_tensor_2,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_2 = flop_count_0134 + flop_count_0134_2; - - - // 1st, 3rd, 4th, 5th tensors contracted first - using quartet_cost_0234 = quartet_flop_cost,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_3 = typename quartet_cost_0234::resulting_tensor; - using resulting_index_3 = typename quartet_cost_0234::resulting_index; - - static constexpr size_t flop_count_0234 = quartet_cost_0234::min_cost; - static constexpr size_t flop_count_0234_1 = pair_flop_cost,resulting_tensor_3,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_3 = flop_count_0234 + flop_count_0234_1; - - - // last four tensors contracted first - using quartet_cost_1234 = quartet_flop_cost,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_4 = typename quartet_cost_1234::resulting_tensor; - using resulting_index_4 = typename quartet_cost_1234::resulting_index; - - static constexpr size_t flop_count_1234 = quartet_cost_1234::min_cost; - static constexpr size_t flop_count_1234_0 = pair_flop_cost,resulting_tensor_4,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_4 = flop_count_1234 + flop_count_1234_0; - - - static constexpr size_t min_cost = meta_min::value; - static constexpr int which_variant = meta_argmin::value; - - - // this is the overall resulting tensor and index from overall quintet contraction - using resulting_tensor = typename std::conditional< - which_variant==0, - typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename std::conditional< - which_variant==2, - typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type, - typename std::conditional< - which_variant==3, - typename get_resuling_tensor,resulting_index_3,Tensor,resulting_tensor_3>::type, - typename get_resuling_tensor,resulting_index_4,Tensor,resulting_tensor_4>::type - >::type - >::type - >::type - >::type; - - using resulting_index = typename std::conditional< - which_variant==0, - typename get_resuling_index,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename std::conditional< - which_variant==2, - typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type, - typename std::conditional< - which_variant==3, - typename get_resuling_index,resulting_index_3,Tensor,resulting_tensor_3>::type, - typename get_resuling_index,resulting_index_4,Tensor,resulting_tensor_4>::type - >::type - >::type - >::type - >::type; - -}; -//------------------------------------------------------------------------------------------------------------// - - - - -// Cost model for sixtet tensor contraction -//------------------------------------------------------------------------------------------------------------// -template -struct sixtet_flop_cost; - -template -struct sixtet_flop_cost,Index,Index,Index, Index, Index, - Tensor,Tensor,Tensor,Tensor,Tensor,Tensor> { - - // Perform depth-first search - //--------------------------------------------------------------------- - // first 5 tensors contracted first - using quintet_cost_01234 = quintet_flop_cost,Index,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_0 = typename quintet_cost_01234::resulting_tensor; - using resulting_index_0 = typename quintet_cost_01234::resulting_index; - - static constexpr size_t flop_count_01234 = quintet_cost_01234::min_cost; - static constexpr size_t flop_count_01234_5 = pair_flop_cost,resulting_tensor_0,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_0 = flop_count_01234 + flop_count_01234_5; - - - // 1st, 2nd, 3rd, 4th, 6th tensors contracted first - using quintet_cost_01235 = quintet_flop_cost,Index,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_1 = typename quintet_cost_01235::resulting_tensor; - using resulting_index_1 = typename quintet_cost_01235::resulting_index; - - static constexpr size_t flop_count_01235 = quintet_cost_01235::min_cost; - static constexpr size_t flop_count_01235_4 = pair_flop_cost,resulting_tensor_1,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_1 = flop_count_01235 + flop_count_01235_4; - - - // 1st, 2nd, 3rd, 5th, 6th tensors contracted first - using quintet_cost_01245 = quintet_flop_cost,Index,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_2 = typename quintet_cost_01245::resulting_tensor; - using resulting_index_2 = typename quintet_cost_01245::resulting_index; - - static constexpr size_t flop_count_01245 = quintet_cost_01235::min_cost; - static constexpr size_t flop_count_01245_3 = pair_flop_cost,resulting_tensor_2,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_2 = flop_count_01245 + flop_count_01245_3; - - - // 1st, 2nd, 4th, 5th, 6th tensors contracted first - using quintet_cost_01345 = quintet_flop_cost,Index,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_3 = typename quintet_cost_01345::resulting_tensor; - using resulting_index_3 = typename quintet_cost_01345::resulting_index; - - static constexpr size_t flop_count_01345 = quintet_cost_01235::min_cost; - static constexpr size_t flop_count_01345_2 = pair_flop_cost,resulting_tensor_3,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_3 = flop_count_01345 + flop_count_01345_2; - - - // 1st, 3rd, 4th, 5th, 6th tensors contracted first - using quintet_cost_02345 = quintet_flop_cost,Index,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_4 = typename quintet_cost_02345::resulting_tensor; - using resulting_index_4 = typename quintet_cost_02345::resulting_index; - - static constexpr size_t flop_count_02345 = quintet_cost_01235::min_cost; - static constexpr size_t flop_count_02345_1 = pair_flop_cost,resulting_tensor_4,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_4 = flop_count_02345 + flop_count_02345_1; - - - // last 5 tensors contracted first - using quintet_cost_12345 = quintet_flop_cost,Index,Index,Index,Index, - Tensor,Tensor,Tensor,Tensor,Tensor>; - - using resulting_tensor_5 = typename quintet_cost_12345::resulting_tensor; - using resulting_index_5 = typename quintet_cost_12345::resulting_index; - - static constexpr size_t flop_count_12345 = quintet_cost_01235::min_cost; - static constexpr size_t flop_count_12345_0 = pair_flop_cost,resulting_tensor_5,Tensor, - typename std_ext::make_index_sequence::type>::value; - - static constexpr size_t flop_count_5 = flop_count_12345 + flop_count_12345_0; - - - static constexpr size_t min_cost = meta_min::value; - static constexpr int which_variant = meta_argmin::value; - - - // this is the overall resulting tensor and index from overall sextet contraction - using resulting_tensor = typename std::conditional< - which_variant==0, - typename get_resuling_tensor,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_tensor,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename std::conditional< - which_variant==2, - typename get_resuling_tensor,resulting_index_2,Tensor,resulting_tensor_2>::type, - typename std::conditional< - which_variant==3, - typename get_resuling_tensor,resulting_index_3,Tensor,resulting_tensor_3>::type, - typename std::conditional< - which_variant==4, - typename get_resuling_tensor,resulting_index_4,Tensor,resulting_tensor_4>::type, - typename get_resuling_tensor,resulting_index_5,Tensor,resulting_tensor_5>::type - >::type - >::type - >::type - >::type - >::type; - - using resulting_index = typename std::conditional< - which_variant==0, - typename get_resuling_index,resulting_tensor_0,Tensor>::type, - typename std::conditional< - which_variant==1, - typename get_resuling_index,resulting_index_1,Tensor,resulting_tensor_1>::type, - typename std::conditional< - which_variant==2, - typename get_resuling_index,resulting_index_2,Tensor,resulting_tensor_2>::type, - typename std::conditional< - which_variant==3, - typename get_resuling_index,resulting_index_3,Tensor,resulting_tensor_3>::type, - typename std::conditional< - which_variant==4, - typename get_resuling_index,resulting_index_4,Tensor,resulting_tensor_4>::type, - typename get_resuling_index,resulting_index_5,Tensor,resulting_tensor_5>::type - >::type - >::type - >::type - >::type - >::type; - -}; -//------------------------------------------------------------------------------------------------------------// - - - - - -// einsum helper to extract the resulting index and the resulting tensor -//------------------------------------------------------------------------------------------------------------// -template -struct einsum_helper; - -template -struct einsum_helper> { - using resulting_index = typename contraction_impl, - typename std_ext::make_index_sequence::type>::indices; - using resulting_tensor = typename contraction_impl, - typename std_ext::make_index_sequence::type>::type; -}; - -template -struct einsum_helper { - using resulting_index = typename get_resuling_index::type; - using resulting_tensor = typename get_resuling_tensor::type; -}; - -template -struct einsum_helper { - using cost_model = triplet_flop_cost; - using resulting_index = typename cost_model::resulting_index; - using resulting_tensor = typename cost_model::resulting_tensor; -}; - -template -struct einsum_helper { - using cost_model = quartet_flop_cost; - using resulting_index = typename cost_model::resulting_index; - using resulting_tensor = typename cost_model::resulting_tensor; -}; - -template -struct einsum_helper { - using cost_model = quintet_flop_cost; - using resulting_index = typename cost_model::resulting_index; - using resulting_tensor = typename cost_model::resulting_tensor; -}; - -template -struct einsum_helper { - using cost_model = sixtet_flop_cost; - using resulting_index = typename cost_model::resulting_index; - using resulting_tensor = typename cost_model::resulting_tensor; -}; -//------------------------------------------------------------------------------------------------------------// - - -} // end of namespace Fastor - - -#endif // FASTOR_DONT_PERFORM_OP_MIN - - -#endif // OPMIN_META_H diff --git a/noarch/include/Fastor/meta/tensor_meta.h b/noarch/include/Fastor/meta/tensor_meta.h deleted file mode 100644 index 5f86c674..00000000 --- a/noarch/include/Fastor/meta/tensor_meta.h +++ /dev/null @@ -1,209 +0,0 @@ -#ifndef TENSOR_META_H -#define TENSOR_META_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" - -namespace Fastor { - -//----------------------------------------------------------------------------------------------------------// -// UpLoType -namespace UpLoType { -struct General {}; -struct Lower {}; -struct UniLower {}; -struct StrictlyLower {}; -struct Upper {}; -struct UniUpper {}; -struct StrictlyUpper {}; -struct Diagonal {}; -struct BiDiagonal {}; -struct TriDiagonal {}; -struct BlockDiagonal {}; -struct Symmetric {}; -struct SymmetricPositiveDefinite {}; -} -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template -struct is_unique : std::integral_constant {}; - -template -struct is_unique : std::integral_constant::value> {}; - -template -struct no_of_unique : std::integral_constant {}; - -template -struct no_of_unique : std::integral_constant::value + no_of_unique::value> {}; - - - -//----------------------------------------------------------------------------------------------------------// -// Note that Intel's ICC 2017 does not support make_index_sequence and the following -// version also seems faster than c++14's built-in (for clang) on Linux systems -namespace std_ext // back port to c++11 -{ - template - struct index_sequence - { - using type = index_sequence; - using value_type = size_t; - static constexpr std::size_t size() { return sizeof...(Ints); } - }; - - // -------------------------------------------------------------- - - template - struct _merge_and_renumber; - - template - struct _merge_and_renumber, index_sequence> - : index_sequence - { }; - - // -------------------------------------------------------------- - - template - struct make_index_sequence - : _merge_and_renumber::type, - typename make_index_sequence::type> - { }; - - template<> struct make_index_sequence<0> : index_sequence<> { }; - template<> struct make_index_sequence<1> : index_sequence<0> { }; -} -//----------------------------------------------------------------------------------------------------------// - - - - -//----------------------------------------------------------------------------------------------------------// -template struct typelist { }; -template -using typelist_c = typelist...>; - -template struct concat; -template <> struct concat<> { using type = typelist<>; }; -template struct concat> { using type = typelist; }; -template -struct concat, typelist, Args...> : concat, Args...> { }; - -template struct filter_out; -template struct filter_out> - : concat< conditional_t_, typelist<>, typelist>...> { }; - -template -using filter_out_t = typename filter_out::type; - - - -template -struct uniq; - -template -using uniq_t = typename uniq::type; - -template <> -struct uniq> { - using type = typelist<>; -}; - -template -struct uniq> - : concat, uniq_t>>> -{ }; - -template -using size_t_ = std::integral_constant; - -template struct length; -template using length_t = typename length::type; -template -struct length> -: size_t_ -{ }; - -template -using no_of_uniques = length_t...>>>; -//----------------------------------------------------------------------------------------------------------// - - -//////////////// -template class Z> -struct quote_c { - template - using apply = Z; -}; - -template -struct apply_typelist; - -template -using apply_typelist_t = typename apply_typelist::type; - - -template -struct apply_typelist> { - using type = typename MFC::template apply; -}; -//----------------------------------------------------------------------------------------------------------// - - - -// Check if indices appear more than twice [for Einstein summation] -//----------------------------------------------------------------------------------------------------------// -namespace useless { - -template -constexpr const T& ct_max(T const& t1, T const& t2) { - return t1 < t2 ? t2 : t1; -} - - -template -struct count__; - -template -struct count__ { - static constexpr size_t value = std::integral_constant::value; -}; - -template -struct count__ { - static constexpr size_t value = std::integral_constant::value>::value; -}; - -template -struct count__ { - static constexpr size_t value = count__::value; -}; - -template -struct max_count; - -template <> -struct max_count<> { - static constexpr size_t value = std::integral_constant::value; -}; - -template -struct max_count { - static constexpr size_t value = std::integral_constant::value, - max_count::value)>::value; -}; - -} // useless - -template -struct no_more_than_two { - static constexpr size_t value = std::integral_constant::value <= 2>::value; -}; -//----------------------------------------------------------------------------------------------------------// - - -} - -#endif // TENSOR_META_H diff --git a/noarch/include/Fastor/simd_math/simd_math.h b/noarch/include/Fastor/simd_math/simd_math.h deleted file mode 100644 index 9ac07e04..00000000 --- a/noarch/include/Fastor/simd_math/simd_math.h +++ /dev/null @@ -1,549 +0,0 @@ -#ifndef SIMD_MATH_H -#define SIMD_MATH_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include - -// SHUT GCC6 -Wignored-attributes WARNINGS -#ifdef __GNUC__ -#if __GNUC__==6 -#pragma GCC diagnostic ignored "-Wignored-attributes" -#endif -#endif - -namespace Fastor { - -// minimum -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::min(((T*)&a)[i],((T*)&b)[i]); } - return out; -} -template -FASTOR_INLINE SIMDVector min(const SIMDVector &a, T b) { - return min(a,SIMDVector(b)); -} -template -FASTOR_INLINE SIMDVector min(T a, const SIMDVector &b) { - return min(SIMDVector(a),b); -} -#ifdef FASTOR_SSE2_IMPL -#ifdef FASTOR_SSE4_1_IMPL -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm_min_epi32(a.value,b.value); -} -#endif -#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm_min_epi64(a.value,b.value); -} -#endif -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm_min_ps(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm_min_pd(a.value,b.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -#ifdef FASTOR_AVX2_IMPL -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm256_min_epi32(a.value,b.value); -} -#endif -#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm256_min_epi64(a.value,b.value); -} -#endif -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm256_min_ps(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm256_min_pd(a.value,b.value); -} -#endif -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm512_min_epi32(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm512_min_epi64(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm512_min_ps(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector min(const SIMDVector &a, const SIMDVector &b) { - return _mm512_min_pd(a.value,b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// maximum -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::max(((T*)&a)[i],((T*)&b)[i]); } - return out; -} -template -FASTOR_INLINE SIMDVector max(const SIMDVector &a, T b) { - return max(a,SIMDVector(b)); -} -template -FASTOR_INLINE SIMDVector max(T a, const SIMDVector &b) { - return max(SIMDVector(a),b); -} -#ifdef FASTOR_SSE2_IMPL -#ifdef FASTOR_SSE4_1_IMPL -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm_max_epi32(a.value,b.value); -} -#endif -#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm_max_epi64(a.value,b.value); -} -#endif -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm_max_ps(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm_max_pd(a.value,b.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -#ifdef FASTOR_AVX2_IMPL -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm256_max_epi32(a.value,b.value); -} -#endif -#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL) -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm256_max_epi64(a.value,b.value); -} -#endif -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm256_max_ps(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm256_max_pd(a.value,b.value); -} -#endif -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm512_max_epi32(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm512_max_epi64(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm512_max_ps(a.value,b.value); -} -template<> -FASTOR_INLINE SIMDVector max(const SIMDVector &a, const SIMDVector &b) { - return _mm512_max_pd(a.value,b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// ceil -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::ceil(((T*)&a)[i]);} - return out; -} -#ifdef FASTOR_SSE4_1_IMPL -template<> -FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { - return _mm_ceil_ps(a.value); -} -template<> -FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { - return _mm_ceil_pd(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { - return _mm256_ceil_ps(a.value); -} -template<> -FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { - return _mm256_ceil_pd(a.value); -} -#endif -// Part of SVML -// #ifdef FASTOR_AVX512F_IMPL -// template<> -// FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { -// return _mm512_ceil_ps(a.value); -// } -// template<> -// FASTOR_INLINE SIMDVector ceil(const SIMDVector &a) { -// return _mm512_ceil_pd(a.value); -// } -// #endif -//----------------------------------------------------------------------------------------------------------// - - -// round -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector round(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::round(((T*)&a)[i]);} - return out; -} -#ifdef FASTOR_SSE4_1_IMPL -template<> -FASTOR_INLINE SIMDVector round(const SIMDVector &a) { - return _mm_round_ps(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); -} -template<> -FASTOR_INLINE SIMDVector round(const SIMDVector &a) { - return _mm_round_pd(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector round(const SIMDVector &a) { - return _mm256_round_ps(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); -} -template<> -FASTOR_INLINE SIMDVector round(const SIMDVector &a) { - return _mm256_round_pd(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) ); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// floor -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::floor(((T*)&a)[i]);} - return out; -} -#ifdef FASTOR_SSE4_1_IMPL -template<> -FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { - return _mm_floor_ps(a.value); -} -template<> -FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { - return _mm_floor_pd(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { - return _mm256_floor_ps(a.value); -} -template<> -FASTOR_INLINE SIMDVector floor(const SIMDVector &a) { - return _mm256_floor_pd(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - - -// remaining math functions from STL -//----------------------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------------------// - -template -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::exp(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::exp2(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::expm1(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log10(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log2(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::log1p(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i = 0; i < SIMDVector::Size; i++) { ((T*)&out)[i] = std::pow(((T*)&a)[i], ((T*)&b)[i]);} - return out; -} -template -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, T b) { - return pow(a, SIMDVector(b)); -} -template -FASTOR_INLINE SIMDVector pow(T a, const SIMDVector &b) { - return pow(SIMDVector(a),b); -} - -template -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::cbrt(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::sin(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::cos(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::tan(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::asin(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::acos(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::atan(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i = 0; i < SIMDVector::Size; i++) { ((T*)&out)[i] = std::atan2(((T*)&a)[i], ((T*)&b)[i]);} - return out; -} -template -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, T b) { - return atan2(a, SIMDVector(b)); -} -template -FASTOR_INLINE SIMDVector atan2(T a, const SIMDVector &b) { - return atan2(SIMDVector(a),b); -} - -template -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::sinh(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::cosh(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::tanh(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::asinh(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::acosh(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::atanh(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::erf(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::tgamma(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::lgamma(((T*)&a)[i]);} - return out; -} - -template -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i = 0; i < SIMDVector::Size; i++) { ((T*)&out)[i] = std::hypot(((T*)&a)[i], ((T*)&b)[i]);} - return out; -} -template -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, T b) { - return hypot(a, SIMDVector(b)); -} -template -FASTOR_INLINE SIMDVector hypot(T a, const SIMDVector &b) { - return hypot(SIMDVector(a),b); -} - -template -FASTOR_INLINE SIMDVector trunc(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((T*)&out)[i] = std::trunc(((T*)&a)[i]);} - return out; -} -//----------------------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------------------// - - - -// Boolean arithmetic -// ! or not -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector::Size>> operator!(const SIMDVector &a) { - SIMDVector::Size>> out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = !(((T*)&a)[i]); } - return out; -} -//----------------------------------------------------------------------------------------------------------// - - -// isinf/nan/finite -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector::Size>> isinf(const SIMDVector &a) { - SIMDVector::Size>> out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = std::isinf(((T*)&a)[i]); } - return out; -} - -template -FASTOR_INLINE SIMDVector::Size>> isnan(const SIMDVector &a) { - SIMDVector::Size>> out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = std::isnan(((T*)&a)[i]); } - return out; -} - -template -FASTOR_INLINE SIMDVector::Size>> isfinite(const SIMDVector &a) { - SIMDVector::Size>> out; - for (FASTOR_INDEX i=0; i::Size; i++) { ((bool*)&out)[i] = std::isfinite(((T*)&a)[i]); } - return out; -} -//----------------------------------------------------------------------------------------------------------// - - -} // end of namespace Fastor - - -// Include all backends -#include "Fastor/simd_math/sleef_backend.h" - -#endif // SIMD_MATH_H diff --git a/noarch/include/Fastor/simd_math/sleef_backend.h b/noarch/include/Fastor/simd_math/sleef_backend.h deleted file mode 100644 index 87378652..00000000 --- a/noarch/include/Fastor/simd_math/sleef_backend.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef SLEEF_BACKEND_H -#define SLEEF_BACKEND_H - -#include "Fastor/simd_math/sleef_backend_u10.h" -#include "Fastor/simd_math/sleef_backend_u35.h" - -#endif // SLEEF_BACKEND_H diff --git a/noarch/include/Fastor/simd_math/sleef_backend_u10.h b/noarch/include/Fastor/simd_math/sleef_backend_u10.h deleted file mode 100644 index 98f68009..00000000 --- a/noarch/include/Fastor/simd_math/sleef_backend_u10.h +++ /dev/null @@ -1,932 +0,0 @@ -#ifndef SLEEF_BACKEND_U10_H -#define SLEEF_BACKEND_U10_H - -#if defined(FASTOR_USE_SLEEF_U10) || defined(FASTOR_USE_SLEEF) - -#include - -namespace Fastor { - -// exp -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// exp2 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2f4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2d2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2f8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2d4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2f16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2d8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// expm1 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1f4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1d2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1f8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1d4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1f16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1d8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log10 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10f4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10d2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10f8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10d4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10f16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10d8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log2 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2f4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2d2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2f8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2d4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2f16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2d8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log1p -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// pow - other variants are automatically taken care off through the generic overloads -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powf4_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powd2_u10(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powf8_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powd4_u10(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powf16_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powd8_u10(a.value, b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// cbrt -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// sin -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sinf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sind2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sinf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sind4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sinf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sind8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// cos -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// tan -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tanf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tand2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tanf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tand4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tanf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tand8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// asin -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asinf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asind2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asinf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asind4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asinf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asind8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// acos -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// atan -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atanf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atand2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atanf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atand4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atanf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atand8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// atan2 - other variants are automatically taken care off through the generic overloads -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2f4_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2d2_u10(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2f8_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2d4_u10(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2f16_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2d8_u10(a.value, b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// sinh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// cosh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// tanh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// asinh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// acosh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// atanh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// erf -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erff4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erfd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erff8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erfd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erff16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erfd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// tgamma -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammaf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammad2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammaf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammad4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammaf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammad8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// lgamma -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammaf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammad2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammaf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammad4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammaf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammad8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// hypot - other variants are automatically taken care off through the generic overloads -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf4_u05(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd2_u05(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX2_IMPL -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf8_u05avx2(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd4_u05avx2(a.value, b.value); -} -#elif defined(FASTOR_AVX_IMPL) -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf8_u05avx(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd4_u05avx(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf16_u05avx512f(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd8_u05avx512f(a.value, b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - -} // end of namespace Fastor - -#endif // FASTOR_USE_SLEEF_U10 - -#endif // SLEEF_BACKEND_U10_H diff --git a/noarch/include/Fastor/simd_math/sleef_backend_u35.h b/noarch/include/Fastor/simd_math/sleef_backend_u35.h deleted file mode 100644 index d337e01c..00000000 --- a/noarch/include/Fastor/simd_math/sleef_backend_u35.h +++ /dev/null @@ -1,933 +0,0 @@ -#ifndef SLEEF_BACKEND_U35_H -#define SLEEF_BACKEND_U35_H - -// Not all functions have u35 variant implemented, so falling back to u10 in such cases -#ifdef FASTOR_USE_SLEEF_U35 - -#include - -namespace Fastor { - -// exp -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp(const SIMDVector &a) { - return Sleef_expd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// exp2 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2f4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2d2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2f8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2d4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2f16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector exp2(const SIMDVector &a) { - return Sleef_exp2d8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// expm1 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1f4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1d2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1f8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1d4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1f16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector expm1(const SIMDVector &a) { - return Sleef_expm1d8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logd2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logd4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector log(const SIMDVector &a) { - return Sleef_logd8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log10 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10f4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10d2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10f8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10d4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10f16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log10(const SIMDVector &a) { - return Sleef_log10d8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log2 -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2f4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2d2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2f8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2d4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2f16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector log2(const SIMDVector &a) { - return Sleef_log2d8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// log1p -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector log1p(const SIMDVector &a) { - return Sleef_log1pd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// pow - other variants are automatically taken care off through the generic overloads -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powf4_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powd2_u10(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powf8_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powd4_u10(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powf16_u10(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector pow(const SIMDVector &a, const SIMDVector &b) { - return Sleef_powd8_u10(a.value, b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// cbrt -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtd2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtd4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cbrt(const SIMDVector &a) { - return Sleef_cbrtd8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// sin -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sinf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sind2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sinf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sind4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sinf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector sin(const SIMDVector &a) { - return Sleef_sind8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// cos -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosd2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosd4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cos(const SIMDVector &a) { - return Sleef_cosd8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// tan -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tanf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tand2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tanf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tand4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tanf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector tan(const SIMDVector &a) { - return Sleef_tand8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// asin -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asinf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asind2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asinf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asind4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asinf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector asin(const SIMDVector &a) { - return Sleef_asind8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// acos -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosd2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosd4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector acos(const SIMDVector &a) { - return Sleef_acosd8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// atan -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atanf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atand2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atanf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atand4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atanf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector atan(const SIMDVector &a) { - return Sleef_atand8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// atan2 - other variants are automatically taken care off through the generic overloads -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2f4_u35(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2d2_u35(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2f8_u35(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2d4_u35(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2f16_u35(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector atan2(const SIMDVector &a, const SIMDVector &b) { - return Sleef_atan2d8_u35(a.value, b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// sinh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhd2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhd4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector sinh(const SIMDVector &a) { - return Sleef_sinhd8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// cosh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshd2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshd4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector cosh(const SIMDVector &a) { - return Sleef_coshd8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// tanh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhf4_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhd2_u35(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhf8_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhd4_u35(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhf16_u35(a.value); -} -template<> -FASTOR_INLINE SIMDVector tanh(const SIMDVector &a) { - return Sleef_tanhd8_u35(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// asinh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector asinh(const SIMDVector &a) { - return Sleef_asinhd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// acosh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector acosh(const SIMDVector &a) { - return Sleef_acoshd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// atanh -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector atanh(const SIMDVector &a) { - return Sleef_atanhd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// erf -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erff4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erfd2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erff8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erfd4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erff16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector erf(const SIMDVector &a) { - return Sleef_erfd8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// tgamma -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammaf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammad2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammaf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammad4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammaf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector tgamma(const SIMDVector &a) { - return Sleef_tgammad8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// lgamma -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammaf4_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammad2_u10(a.value); -} -#endif -#ifdef FASTOR_AVX_IMPL -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammaf8_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammad4_u10(a.value); -} -#endif -#ifdef FASTOR_AVX512_IMPL -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammaf16_u10(a.value); -} -template<> -FASTOR_INLINE SIMDVector lgamma(const SIMDVector &a) { - return Sleef_lgammad8_u10(a.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - - -// hypot - other variants are automatically taken care off through the generic overloads -//----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf4_u05(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd2_u05(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX2_IMPL -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf8_u05avx2(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd4_u05avx2(a.value, b.value); -} -#elif defined(FASTOR_AVX_IMPL) -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf8_u05avx(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd4_u05avx(a.value, b.value); -} -#endif -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotf16_u05avx512f(a.value, b.value); -} -template<> -FASTOR_INLINE SIMDVector hypot(const SIMDVector &a, const SIMDVector &b) { - return Sleef_hypotd8_u05avx512f(a.value, b.value); -} -#endif -//----------------------------------------------------------------------------------------------------------// - -} // end of namespace Fastor - -#endif // FASTOR_USE_SLEEF_U35 - -#endif // FASTOR_USE_SLEEF_U35 diff --git a/noarch/include/Fastor/simd_vector/SIMDVector.h b/noarch/include/Fastor/simd_vector/SIMDVector.h deleted file mode 100644 index a4b8e53d..00000000 --- a/noarch/include/Fastor/simd_vector/SIMDVector.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef SIMDVECTOR_H -#define SIMDVECTOR_H - -#include "Fastor/simd_vector/simd_vector_base.h" -#include "Fastor/simd_vector/simd_vector_scalar.h" -#include "Fastor/simd_vector/simd_vector_float.h" -#include "Fastor/simd_vector/simd_vector_double.h" -#include "Fastor/simd_vector/simd_vector_int32.h" -#include "Fastor/simd_vector/simd_vector_int64.h" -#include "Fastor/simd_vector/simd_vector_complex_scalar.h" -#include "Fastor/simd_vector/simd_vector_complex_float.h" -#include "Fastor/simd_vector/simd_vector_complex_double.h" -#include "Fastor/simd_vector/simd_vector_common.h" - -#endif // SIMDVECTOR_H - diff --git a/noarch/include/Fastor/simd_vector/extintrin.h b/noarch/include/Fastor/simd_vector/extintrin.h deleted file mode 100644 index 0a19c73f..00000000 --- a/noarch/include/Fastor/simd_vector/extintrin.h +++ /dev/null @@ -1,1255 +0,0 @@ -#ifndef EXT_INTRIN_H -#define EXT_INTRIN_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include - - -namespace Fastor { - -// Macros for immediate construction -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -#define ZEROPS (_mm_set1_ps(0.f)) -#define ZEROPD (_mm_set1_pd(0.0)) -#define MZEROPS (_mm_set1_ps(-0.f)) -#define MZEROPD (_mm_set1_pd(-0.0)) -#define ONEPS (_mm_set1_ps(1.f)) -#define ONEPD (_mm_set1_pd(1.0)) -#define HALFPS (_mm_set1_ps(0.5f)) -#define HALFPD (_mm_set1_pd(0.5)) -#define TWOPS (_mm_set1_ps(2.0f)) -#define TOWPD (_mm_set1_pd(2.0)) -#endif -#ifdef FASTOR_AVX_IMPL -#define VZEROPS (_mm256_set1_ps(0.f)) -#define VZEROPD (_mm256_set1_pd(0.0)) -#define MVZEROPS (_mm256_set1_ps(-0.f)) -#define MVZEROPD (_mm256_set1_pd(-0.0)) -#define VONEPS (_mm256_set1_ps(1.f)) -#define VONEPD (_mm256_set1_pd(1.0)) -#define VHALFPS (_mm256_set1_ps(0.5f)) -#define VHALFPD (_mm256_set1_pd(0.5)) -#define VTWOPS (_mm256_set1_ps(2.0f)) -#define VTOWPD (_mm256_set1_pd(2.0)) -#endif -//----------------------------------------------------------------------------------------------------------------// - - -// Mask load the 3 lower parts -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE __m128 _mm_loadl3_ps(const float *arr) { -#ifdef FASTOR_HAS_AVX512_MASKS - return _mm_mask_load_ps(ZEROPS, (__mmask8)0x07, arr); -#elif defined(FASTOR_AVX_IMPL) - __m128i mask = _mm_set_epi32(0,-1,-1,-1); - return _mm_maskload_ps(arr,(__m128i) mask); -#else - __m128i xy = _mm_loadl_epi64((const __m128i*)arr); - __m128 z = _mm_load_ss(&arr[2]); - return _mm_movelh_ps(_mm_castsi128_ps(xy), z); -#endif -} - -FASTOR_INLINE __m128 _mm_loadul3_ps(const float *arr) { -#ifdef FASTOR_HAS_AVX512_MASKS - return _mm_mask_loadu_ps(ZEROPS, (__mmask8)0x07, arr); -#elif defined(FASTOR_AVX_IMPL) - // AVX maskloads apparently have no alignment requirement - __m128i mask = _mm_set_epi32(0,-1,-1,-1); - return _mm_maskload_ps(arr,(__m128i) mask); -#else - __m128 x = _mm_load_ss( arr ); - __m128 y = _mm_load_ss(&arr[1]); - __m128 z = _mm_load_ss(&arr[2]); - __m128 xy = _mm_movelh_ps(x, y); - return _mm_shuffle_ps(xy, z, _MM_SHUFFLE(2, 0, 2, 0)); -#endif -} -#endif - -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE __m256d _mm256_loadl3_pd(const double *arr) { -#ifdef FASTOR_HAS_AVX512_MASKS - return _mm256_mask_load_pd(VZEROPD, (__mmask8)0x07, arr); -#else - __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); - return _mm256_maskload_pd(arr,(__m256i) mask); -#endif -} - -FASTOR_INLINE __m256d _mm256_loadul3_pd(const double *arr) { -#ifdef FASTOR_HAS_AVX512_MASKS - return _mm256_mask_loadu_pd(VZEROPD, (__mmask8)0x07, arr); -#else - // AVX maskloads apparently have no alignment requirement - __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); - return _mm256_maskload_pd(arr,(__m256i) mask); - // __m128d xy = _mm_loadu_pd(arr); - // __m128d z = _mm_load_sd(&arr[2]); - // __m256d vec = _mm256_castpd128_pd256(xy); - // return _mm256_insertf128_pd(vec, z,0x1); -#endif -} -#endif - -// Mask store the 3 lower parts -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE void _mm_storel3_ps(float *arr, __m128 value) { -#ifdef FASTOR_HAS_AVX512_MASKS - _mm_mask_store_ps(arr, (__mmask8)0x07, value); -#elif defined(FASTOR_AVX_IMPL) - __m128i mask = _mm_set_epi32(0,-1,-1,-1); - _mm_maskstore_ps(arr, (__m128i)mask, value); -#else - _mm_storel_pi((__m64*)arr, value); - _mm_store_ss(&arr[2],_mm_shuffle_ps(value,value,0x2)); -#endif -} - -FASTOR_INLINE void _mm_storeul3_ps(float *arr, __m128 value) { -#ifdef FASTOR_HAS_AVX512_MASKS - _mm_mask_storeu_ps(arr, (__mmask8)0x07, value); -#elif defined(FASTOR_AVX_IMPL) - __m128i mask = _mm_set_epi32(0,-1,-1,-1); - _mm_maskstore_ps(arr, (__m128i)mask, value); -#else - _mm_storel_pi((__m64*)arr, value); - _mm_store_ss(&arr[2],_mm_shuffle_ps(value,value,0x2)); -#endif -} -#endif - -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE void _mm256_storel3_pd(double *arr, __m256d value) { -#ifdef FASTOR_HAS_AVX512_MASKS - _mm256_mask_store_pd(arr, (__mmask8)0x07, value); -#else - __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); - _mm256_maskstore_pd(arr, (__m256i)mask, value); -#endif -} - -FASTOR_INLINE void _mm256_storeul3_pd(double *arr, __m256d value) { -#ifdef FASTOR_HAS_AVX512_MASKS - _mm256_mask_storeu_pd(arr, (__mmask8)0x07, value); -#else - // AVX maskloads apparently have no alignment requirement - __m256i mask = _mm256_set_epi64x(0,-1,-1,-1); - _mm256_maskstore_pd(arr, (__m256i)mask, value); - // _mm_storeu_pd(arr , _mm256_castpd256_pd128(value) ); - // _mm_store_sd (arr+2, _mm256_extractf128_pd (value,0x1)); -#endif -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - - -//! Horizontal summation/multiplication of registers -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE int _mm_sum_epi32(__m128i a) { - // W/O HADD: IVY 5 - HW 5 - SKY 6 - // __m128i c = _mm_hadd_epi32(a,a); // SSSE3 one extra op - __m128i c = _mm_add_epi32(a,_mm_shuffle_epi32(a,_MM_SHUFFLE(2,3,0,1))); - __m128i d = _mm_add_epi32(c,_mm_shuffle_epi32(c,_MM_SHUFFLE(0,1,2,3))); - return _mm_cvtsi128_si32(d); -} - -FASTOR_INLINE int _mm_prod_epi32(__m128i a) { - // IVY 13 - HW 13 - SKY 12 - __m128i c = _mm_mul_epu32(a,_mm_shuffle_epi32(a,_MM_SHUFFLE(2,3,0,1))); - __m128i d = _mm_mul_epu32(c,_mm_shuffle_epi32(c,_MM_SHUFFLE(2,2,2,2))); - return _mm_cvtsi128_si32(d); -} -#endif - -#ifdef FASTOR_USE_HADD -#ifdef FASTOR_SSSE3_IMPL -FASTOR_INLINE float _mm_sum_ps(__m128 a) { - // 10 OPS - float sum32; - __m128 sum = _mm_hadd_ps(a, a); - _mm_store_ss(&sum32,_mm_hadd_ps(sum, sum)); - return sum32; -} -FASTOR_INLINE double _mm_sum_pd(__m128d a) { - // 5 OPS - double sum64; - _mm_store_sd(&sum64,_mm_hadd_pd(a, a)); - return sum64; -} -#endif -#else -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE float _mm_sum_ps(__m128 a) { - // 8 OPS -#ifdef FASTOR_SSE3_IMPL - __m128 shuf = _mm_movehdup_ps(a); -#else - __m128 shuf = _mm_shuffle_ps(a,a, _MM_SHUFFLE(3,3,1,1)); -#endif - __m128 sums = _mm_add_ps(a, shuf); - shuf = _mm_movehl_ps(shuf, sums); - sums = _mm_add_ss(sums, shuf); - return _mm_cvtss_f32(sums); -} -FASTOR_INLINE double _mm_sum_pd(__m128d a) { - // 4 OPS - __m128 shuftmp= _mm_movehl_ps(ZEROPS, _mm_castpd_ps(a)); - __m128d shuf = _mm_castps_pd(shuftmp); - return _mm_cvtsd_f64(_mm_add_sd(a, shuf)); -} - -FASTOR_INLINE float _mm_prod_ps(__m128 a) { - // 12 OPS -#ifdef FASTOR_SSE3_IMPL - __m128 shuf = _mm_movehdup_ps(a); -#else - __m128 shuf = _mm_shuffle_ps(a,a, _MM_SHUFFLE(3,3,1,1)); -#endif - __m128 prods = _mm_mul_ps(a, shuf); - shuf = _mm_movehl_ps(shuf, prods); - prods = _mm_mul_ss(prods, shuf); - return _mm_cvtss_f32(prods); -} -FASTOR_INLINE double _mm_prod_pd(__m128d a) { - // 6 OPS - __m128 shuftmp= _mm_movehl_ps(ZEROPS, _mm_castpd_ps(a)); - __m128d shuf = _mm_castps_pd(shuftmp); - return _mm_cvtsd_f64(_mm_mul_sd(a, shuf)); -} -#endif -#endif - -#ifdef FASTOR_AVX_IMPL - -FASTOR_INLINE float _mm256_sum_ps(__m256 a) { -#ifdef FASTOR_USE_HADD - // IVY 14 OPS - HW 16 OPS - __m256 sum = _mm256_hadd_ps(a, a); - sum = _mm256_hadd_ps(sum, sum); - __m128 result = _mm_add_ps(_mm256_castps256_ps128(sum),_mm256_extractf128_ps(sum, 0x1)); - return _mm_cvtss_f32(result); -#else - // IVY 14 OPS - return _mm_sum_ps(_mm_add_ps(_mm256_castps256_ps128(a),_mm256_extractf128_ps(a,0x1))); -#endif -} -FASTOR_INLINE double _mm256_sum_pd(__m256d a) { -#ifdef FASTOR_USE_HADD - // IVY 9 OPS - HW - 11 OPS - __m256d sum = _mm256_hadd_pd(a, a); -#else - // IVY 8 OPS - HW 10 OPS - SKY 11 OPS (BUT 2 PARALLEL ADDS SO POTENTIALLY 7OPS) - __m256d sum = _mm256_add_pd(a, _mm256_shuffle_pd(a,a,0x5)); -#endif - __m128d result = _mm_add_sd(_mm256_castpd256_pd128(sum),_mm256_extractf128_pd(sum, 0x1)); - return _mm_cvtsd_f64(result); -} - - -FASTOR_INLINE float _mm256_prod_ps(__m256 a) { - // ~ IVY 30 OPS - HW 32 OPS - return _mm_prod_ps(_mm256_castps256_ps128(a))*_mm_prod_ps(_mm256_extractf128_ps(a, 0x1)); -} -FASTOR_INLINE double _mm256_prod_pd(__m256d a) { - // IVY 12 OPS - HW - 14 OPS - __m256d sum = _mm256_mul_pd(a, _mm256_shuffle_pd(a,a,0x5)); - __m128d sum_high = _mm256_extractf128_pd(sum, 0x1); - __m128d result = _mm_mul_sd(sum_high, _mm256_castpd256_pd128(sum)); - return _mm_cvtsd_f64(result); -} - -#endif -//----------------------------------------------------------------------------------------------------------------// - - - -//! Reversing a register -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE __m128 _mm_reverse_ps(__m128 a) { - // 1OP - return _mm_shuffle_ps(a,a,_MM_SHUFFLE(0,1,2,3)); -} -FASTOR_INLINE __m128d _mm_reverse_pd(__m128d a) { - // 1OP - return _mm_shuffle_pd(a,a,0x1); -} -FASTOR_INLINE __m128i _mm_reverse_epi32(__m128i v) { - // 1 OP - return _mm_shuffle_epi32(v, 0x1b); -} -FASTOR_INLINE __m128i _mm_reverse_epi64(__m128i v) { - // 1 OP - return _mm_castpd_si128(_mm_reverse_pd(_mm_castsi128_pd(v))); -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE __m256 _mm256_reverse_ps(__m256 a) { - // IVY 2OPS / HW 4OPS - __m256 r1 = _mm256_permute2f128_ps(a,a,0x1); - return _mm256_shuffle_ps(r1,r1,27); -} -FASTOR_INLINE __m256d _mm256_reverse_pd(__m256d a) { - // IVY 2OPS / HW 4OPS - __m256d r1 = _mm256_permute2f128_pd(a,a,0x1); - return _mm256_shuffle_pd(r1,r1,5); -} -FASTOR_INLINE __m256i _mm256_reverse_epi32(__m256i v) { - // IVY 2OPS / HW 4OPS - return _mm256_castps_si256(_mm256_reverse_ps(_mm256_castsi256_ps(v))); - /* - // 8 OPS - __m128i lo = _mm_shuffle_epi32(_mm256_castsi256_si128(_a)); - __m128i hi = _mm_shuffle_epi32(_mm256_extractf128_si256(_a,1)); - __m256i out = _mm256_castsi128_si256(lo); - out = _mm256_insertf128_si256(out,hi,1); - return out; - */ -} -FASTOR_INLINE __m256i _mm256_reverse_epi64(__m256i v) { - // IVY 2OPS / HW 4OPS - return _mm256_castpd_si256(_mm256_reverse_pd(_mm256_castsi256_pd(v))); -} -#endif -#ifdef FASTOR_AVX512F_IMPL -FASTOR_INLINE __m512 _mm512_reverse_ps(__m512 a) { - // SKY/ICE 3 OPS - const __m512i reverse_mask = _mm512_set_epi32(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); - return _mm512_permutexvar_ps(reverse_mask, a); -} -FASTOR_INLINE __m512d _mm512_reverse_pd(__m512d a) { - // SKY/ICE 3 OPS - const __m512i reverse_mask = _mm512_set_epi64(0,1,2,3,4,5,6,7); - return _mm512_permutexvar_pd(reverse_mask, a); -} -FASTOR_INLINE __m512i _mm512_reverse_epi32(__m512i a) { - // SKY/ICE 3 OPS - const __m512i reverse_mask = _mm512_set_epi32(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15); - return _mm512_permutexvar_epi32(reverse_mask, a); -} -FASTOR_INLINE __m512i _mm512_reverse_epi64(__m512i a) { - // SKY/ICE 3 OPS - const __m512i reverse_mask = _mm512_set_epi64(0,1,2,3,4,5,6,7); - return _mm512_permutexvar_epi64(reverse_mask, a); -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - - -//! Bit shifting - for extracting values and so on -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE __m128 _mm_shift1_ps(__m128 a) { - // 1OP - return _mm_castsi128_ps(_mm_slli_si128(_mm_castps_si128(a), 4)); -} -FASTOR_INLINE __m128 _mm_shift2_ps(__m128 a) { - // 1OP - return _mm_shuffle_ps(ZEROPS, a, 0x40); -} -FASTOR_INLINE __m128 _mm_shift3_ps(__m128 a) { - // 2OPS - __m128 shift2 = _mm_shuffle_ps(ZEROPS, a, 0x40); - return _mm_shuffle_ps(ZEROPS,shift2,_MM_SHUFFLE(2,0,2,0)); -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE __m256 _mm256_shift1_ps(__m256 a) { - // IVY 3OPS / HW 5OPS - __m256 r1 = _mm256_permute_ps(a,_MM_SHUFFLE(2,1,0,3)); - __m256 r2 = _mm256_permute2f128_ps(r1,r1,41); - return _mm256_blend_ps(r1,r2,0x11); -} -FASTOR_INLINE __m256 _mm256_shift2_ps(__m256 a) { - // IVY 3OPS / HW 5OPS - __m256 r1 = _mm256_permute_ps(a,_MM_SHUFFLE(1,0,3,2)); - __m256 r2 = _mm256_permute2f128_ps(r1,r1,41); - return _mm256_blend_ps(r1,r2,0x33); -} -FASTOR_INLINE __m256 _mm256_shift3_ps(__m256 a) { - // IVY 1OPS / HW 3OPS - return _mm256_permute2f128_ps(a,a,41); -} -FASTOR_INLINE __m256 _mm256_shift4_ps(__m256 a) { - // IVY 1OPS / HW 3OPS - return _mm256_permute2f128_ps(a,a,42); -} -FASTOR_INLINE __m256 _mm256_shift5_ps(__m256 a) { - // IVY 2 OPS - HW 4 OPS - __m128 r1 = _mm_shift1_ps(_mm256_castps256_ps128(a)); - return _mm256_insertf128_ps(VZEROPS,r1,0x1); -} -FASTOR_INLINE __m256 _mm256_shift6_ps(__m256 a) { - // IVY 2 OPS - HW 4 OPS - __m128 r1 = _mm_shift2_ps(_mm256_castps256_ps128(a)); - return _mm256_insertf128_ps(VZEROPS,r1,0x1); -} -FASTOR_INLINE __m256 _mm256_shift7_ps(__m256 a) { - // IVY 2 OPS - HW 4 OPS - __m128 r1 = _mm_shift3_ps(_mm256_castps256_ps128(a)); - return _mm256_insertf128_ps(VZEROPS,r1,0x1); -} -#endif -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE __m128d _mm_shift1_pd(__m128d a) { - // 1OP - return _mm_shuffle_pd(ZEROPD,a,0x1); -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE __m256d _mm256_shift1_pd(__m256d a) { - // IVY - 4 OPS / HW - 8 OPS - __m128d r1 = _mm256_castpd256_pd128(a); - __m128d r2 = _mm256_extractf128_pd(a,0x1); - __m128d r3 = _mm_shuffle_pd(r1,r2,0x1); - __m256d r4 = _mm256_castpd128_pd256(_mm_shift1_pd(r1)); - return _mm256_insertf128_pd(r4,r3,0x1); -} -FASTOR_INLINE __m256d _mm256_shift2_pd(__m256d a) { - // IVY - 1OP / HW - 3OPS - return _mm256_permute2f128_pd(a,a,8); -} -FASTOR_INLINE __m256d _mm256_shift3_pd(__m256d a) { - // IVY - 2OPS / HW - 4OPS - __m256d r1 = _mm256_castpd128_pd256(_mm_shift1_pd(_mm256_castpd256_pd128(a))); - return _mm256_permute2f128_pd(r1,r1,0x1); -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - -// Negation -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -// Change sign of a register - all one cycle -FASTOR_INLINE __m128 _mm_neg_ps(__m128 a) { - return _mm_xor_ps(a, MZEROPS); -} -FASTOR_INLINE __m128d _mm_neg_pd(__m128d a) { - return _mm_xor_pd(a, MZEROPD); -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE __m256 _mm256_neg_ps(__m256 a) { - return _mm256_xor_ps(a, MVZEROPS); -} -FASTOR_INLINE __m256d _mm256_neg_pd(__m256d a) { - return _mm256_xor_pd(a, MVZEROPD); -} -#endif -#ifdef FASTOR_AVX512F_IMPL -FASTOR_INLINE __m512 _mm512_neg_ps(__m512 a) { -#ifdef FASTOR_AVX512DQ_IMPL - return _mm512_xor_ps(a, _mm512_set1_ps(-0.f)); -#else - return _mm512_sub_ps(_mm512_set1_ps(0.f),a); -#endif -} -FASTOR_INLINE __m512d _mm512_neg_pd(__m512d a) { -#ifdef FASTOR_AVX512DQ_IMPL - return _mm512_xor_pd(a, _mm512_set1_pd(-0.0)); -#else - return _mm512_sub_pd(_mm512_set1_pd(0.0),a); -#endif -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - -// Absolute values -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -// Absolute value of a register - all one cycle -FASTOR_INLINE __m128 _mm_abs_ps(__m128 x) { - static const __m128 sign_mask = _mm_set1_ps(-0.f); // -0.f = 1 << 31 - return _mm_andnot_ps(sign_mask, x); -} -FASTOR_INLINE __m128d _mm_abs_pd(__m128d x) { - static const __m128d sign_mask = _mm_set1_pd(-0.); // -0. = 1 << 63 - return _mm_andnot_pd(sign_mask, x); // !sign_mask & x -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE __m256 _mm256_abs_ps(__m256 x) { - static const __m256 sign_mask = _mm256_set1_ps(-0.f); // -0.f = 1 << 31 - return _mm256_andnot_ps(sign_mask, x); -} -FASTOR_INLINE __m256d _mm256_abs_pd(__m256d x) { - static const __m256d sign_mask = _mm256_set1_pd(-0.); // -0. = 1 << 63 - return _mm256_andnot_pd(sign_mask, x); // !sign_mask & x -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - -// Horizontal max -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -// maximum value in a register - horizontal max -FASTOR_INLINE float _mm_hmax_ps(__m128 a) { - // 8OPS - __m128 max0 = _mm_max_ps(a,_mm_reverse_ps(a)); - __m128 tmp = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); - return _mm_cvtss_f32(_mm_max_ps(max0,tmp)); -} -FASTOR_INLINE double _mm_hmax_pd(__m128d a) { - // 4OPS - return _mm_cvtsd_f64(_mm_max_pd(a,_mm_reverse_pd(a))); -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE float _mm256_hmax_ps(__m256 a) { - // IVY 18OPS / HW 24 OPS - __m128 lo = _mm256_castps256_ps128(a); - __m128 max0 = _mm_max_ps(lo,_mm_reverse_ps(lo)); - __m128 tmp0 = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); - __m128 max_lo = _mm_max_ps(max0,tmp0); - - __m128 hi = _mm256_extractf128_ps(a,0x1); - __m128 max1 = _mm_max_ps(hi,_mm_reverse_ps(hi)); - __m128 tmp1 = _mm_shuffle_ps(max1,max1,_MM_SHUFFLE(0,0,0,1)); - __m128 max_hi = _mm_max_ps(max1,tmp1); - - return _mm_cvtss_f32(_mm_max_ps(max_lo,max_hi)); -} -FASTOR_INLINE double _mm256_hmax_pd(__m256d a) { - // IVY 9OPS / HW 11 OPS - __m256d max0 = _mm256_max_pd(a,_mm256_reverse_pd(a)); - __m256d tmp = _mm256_shuffle_pd(max0,max0,_MM_SHUFFLE(0,0,0,1)); - return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_max_pd(max0,tmp))); -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - -// Horizontal min -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE float _mm_hmin_ps(__m128 a) { - // 8OPS - __m128 max0 = _mm_min_ps(a,_mm_reverse_ps(a)); - __m128 tmp = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); - return _mm_cvtss_f32(_mm_min_ps(max0,tmp)); -} -FASTOR_INLINE double _mm_hmin_pd(__m128d a) { - // 4OPS - return _mm_cvtsd_f64(_mm_min_pd(a,_mm_reverse_pd(a))); -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE float _mm256_hmin_ps(__m256 a) { - // IVY 18OPS / HW 24 OPS - __m128 lo = _mm256_castps256_ps128(a); - __m128 max0 = _mm_min_ps(lo,_mm_reverse_ps(lo)); - __m128 tmp0 = _mm_shuffle_ps(max0,max0,_MM_SHUFFLE(0,0,0,1)); - __m128 max_lo = _mm_min_ps(max0,tmp0); - - __m128 hi = _mm256_extractf128_ps(a,0x1); - __m128 max1 = _mm_min_ps(hi,_mm_reverse_ps(hi)); - __m128 tmp1 = _mm_shuffle_ps(max1,max1,_MM_SHUFFLE(0,0,0,1)); - __m128 max_hi = _mm_min_ps(max1,tmp1); - - return _mm_cvtss_f32(_mm_min_ps(max_lo,max_hi)); -} -FASTOR_INLINE double _mm256_hmin_pd(__m256d a) { - // IVY 9OPS / HW 11 OPS - __m256d max0 = _mm256_min_pd(a,_mm256_reverse_pd(a)); - __m256d tmp = _mm256_shuffle_pd(max0,max0,_MM_SHUFFLE(0,0,0,1)); - return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_min_pd(max0,tmp))); -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - -// Indexing a register -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE float _mm_get0_ps(__m128 a) { - // NO OP - return _mm_cvtss_f32(a); -} -FASTOR_INLINE float _mm_get1_ps(__m128 a) { - // 1 OP - return _mm_cvtss_f32(_mm_shuffle_ps(a,a,_MM_SHUFFLE(0,0,0,1))); -} -FASTOR_INLINE float _mm_get2_ps(__m128 a) { - // 1 OP - return _mm_cvtss_f32(_mm_shuffle_ps(a,a,_MM_SHUFFLE(0,0,0,2))); -} -FASTOR_INLINE float _mm_get3_ps(__m128 a) { - // 1 OP - return _mm_cvtss_f32(_mm_shuffle_ps(a,a,_MM_SHUFFLE(0,0,0,3))); -} -FASTOR_INLINE double _mm_get0_pd(__m128d a) { - // NO OP - return _mm_cvtsd_f64(a); -} -FASTOR_INLINE double _mm_get1_pd(__m128d a) { - // 1 OP - return _mm_cvtsd_f64(_mm_shuffle_pd(a,a,_MM_SHUFFLE2(0,1))); -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE float _mm256_get0_ps(__m256 a) { - // NO OP - return _mm_cvtss_f32(_mm256_castps256_ps128(a)); -} -FASTOR_INLINE float _mm256_get1_ps(__m256 a) { - // 1 OP - __m128 lower = _mm256_castps256_ps128(a); - return _mm_cvtss_f32(_mm_shuffle_ps(lower,lower,_MM_SHUFFLE(0,0,0,1))); -} -FASTOR_INLINE float _mm256_get2_ps(__m256 a) { - // 1 OP - __m128 lower = _mm256_castps256_ps128(a); - return _mm_cvtss_f32(_mm_shuffle_ps(lower,lower,_MM_SHUFFLE(0,0,0,2))); -} -FASTOR_INLINE float _mm256_get3_ps(__m256 a) { - // NO OP - __m128 lower = _mm256_castps256_ps128(a); - return _mm_cvtss_f32(_mm_shuffle_ps(lower,lower,_MM_SHUFFLE(0,0,0,3))); -} -FASTOR_INLINE float _mm256_get4_ps(__m256 a) { - // IVY 1OP / HW 3OPS - return _mm_cvtss_f32(_mm256_extractf128_ps(a,0x1)); -} -FASTOR_INLINE float _mm256_get5_ps(__m256 a) { - // IVY 2OPS/ HW 4OPS - __m128 higher = _mm256_extractf128_ps(a,0x1); - return _mm_cvtss_f32(_mm_shuffle_ps(higher,higher,_MM_SHUFFLE(0,0,0,1))); -} -FASTOR_INLINE float _mm256_get6_ps(__m256 a) { - // IVY 2OPS/ HW 4OPS - __m128 higher = _mm256_extractf128_ps(a,0x1); - return _mm_cvtss_f32(_mm_shuffle_ps(higher,higher,_MM_SHUFFLE(0,0,0,2))); -} -FASTOR_INLINE float _mm256_get7_ps(__m256 a) { - // IVY 2OPS/ HW 4OPS - __m128 higher = _mm256_extractf128_ps(a,0x1); - return _mm_cvtss_f32(_mm_shuffle_ps(higher,higher,_MM_SHUFFLE(0,0,0,3))); -} -FASTOR_INLINE double _mm256_get0_pd(__m256d a) { - // NO OP - return _mm_cvtsd_f64(_mm256_castpd256_pd128(a)); -} -FASTOR_INLINE double _mm256_get1_pd(__m256d a) { - // 1 OP - __m128d lower = _mm256_castpd256_pd128(a); - return _mm_cvtsd_f64(_mm_shuffle_pd(lower,lower,_MM_SHUFFLE2(0,1))); -} -FASTOR_INLINE double _mm256_get2_pd(__m256d a) { - // IVY 1OP / HW 3OPS - return _mm_cvtsd_f64(_mm256_extractf128_pd(a,0x1)); -} -FASTOR_INLINE double _mm256_get3_pd(__m256d a) { - // IVY 2OPS / HW 4OPS - __m128d higher = _mm256_extractf128_pd(a,0x1); - return _mm_cvtsd_f64(_mm_shuffle_pd(higher,higher,_MM_SHUFFLE2(0,1))); -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - - -// Integral arithmetics that are not available pre AVX2 -//----------------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE __m128i _mm_mul_epi32x(__m128i a, __m128i b) -{ -#ifdef FASTOR_SSE4_1_IMPL - return _mm_mullo_epi32(a, b); -#else // SSE2 - __m128i a13 = _mm_shuffle_epi32(a, 0xF5); // (-,a3,-,a1) - __m128i b13 = _mm_shuffle_epi32(b, 0xF5); // (-,b3,-,b1) - __m128i prod02 = _mm_mul_epu32(a, b); // (-,a2*b2,-,a0*b0) - __m128i prod13 = _mm_mul_epu32(a13, b13); // (-,a3*b3,-,a1*b1) - __m128i prod01 = _mm_unpacklo_epi32(prod02, prod13); // (-,-,a1*b1,a0*b0) - __m128i prod23 = _mm_unpackhi_epi32(prod02, prod13); // (-,-,a3*b3,a2*b2) - return _mm_unpacklo_epi64(prod01, prod23); // (ab3,ab2,ab1,ab0) -#endif - - -} -#endif - -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE __m128i _mm_mul_epi64(__m128i _a, __m128i _b) { - __m128i out; - for (FASTOR_INDEX i=0; i<2; i++) { - ((int64_t*)&out)[i] = (((int64_t*)&_a)[i])*(((int64_t*)&_b)[i]); - } - return out; -} -#endif - -#ifdef FASTOR_AVX_IMPL -// #ifndef FASTOR_AVX2_IMPL -FASTOR_INLINE __m256i _mm256_add_epi32x(__m256i _a, __m256i _b) { - __m128i low_a = _mm256_castsi256_si128(_a); - __m128i high_a = _mm256_extractf128_si256(_a,1); - __m128i low_b = _mm256_castsi256_si128(_b); - __m128i high_b = _mm256_extractf128_si256(_b,1); - __m128i low = _mm_add_epi32(low_a,low_b); - __m128i high = _mm_add_epi32(high_a,high_b); - __m256i out = _mm256_castsi128_si256(low); - out = _mm256_insertf128_si256(out,high,1); - return out; -} - -FASTOR_INLINE __m256i _mm256_sub_epi32x(__m256i _a, __m256i _b) { - __m128i low_a = _mm256_castsi256_si128(_a); - __m128i high_a = _mm256_extractf128_si256(_a,1); - __m128i low_b = _mm256_castsi256_si128(_b); - __m128i high_b = _mm256_extractf128_si256(_b,1); - __m128i low = _mm_sub_epi32(low_a,low_b); - __m128i high = _mm_sub_epi32(high_a,high_b); - __m256i out = _mm256_castsi128_si256(low); - out = _mm256_insertf128_si256(out,high,1); - return out; -} - -FASTOR_INLINE __m256i _mm256_mul_epi32x(__m256i _a, __m256i _b) { - __m128i low_a = _mm256_castsi256_si128(_a); - __m128i high_a = _mm256_extractf128_si256(_a,0x1); - __m128i low_b = _mm256_castsi256_si128(_b); - __m128i high_b = _mm256_extractf128_si256(_b,0x1); - __m128i low = _mm_mul_epi32x(low_a,low_b); - __m128i high = _mm_mul_epi32x(high_a,high_b); - __m256i out = _mm256_castsi128_si256(low); - out = _mm256_insertf128_si256(out,high,0x1); - return out; -} - -// 64bit -FASTOR_INLINE __m256i _mm256_add_epi64x(__m256i _a, __m256i _b) { - __m128i low_a = _mm256_castsi256_si128(_a); - __m128i high_a = _mm256_extractf128_si256(_a,1); - __m128i low_b = _mm256_castsi256_si128(_b); - __m128i high_b = _mm256_extractf128_si256(_b,1); - __m128i low = _mm_add_epi64(low_a,low_b); - __m128i high = _mm_add_epi64(high_a,high_b); - __m256i out = _mm256_castsi128_si256(low); - out = _mm256_insertf128_si256(out,high,1); - return out; -} - -FASTOR_INLINE __m256i _mm256_sub_epi64x(__m256i _a, __m256i _b) { - __m128i low_a = _mm256_castsi256_si128(_a); - __m128i high_a = _mm256_extractf128_si256(_a,1); - __m128i low_b = _mm256_castsi256_si128(_b); - __m128i high_b = _mm256_extractf128_si256(_b,1); - __m128i low = _mm_sub_epi64(low_a,low_b); - __m128i high = _mm_sub_epi64(high_a,high_b); - __m256i out = _mm256_castsi128_si256(low); - out = _mm256_insertf128_si256(out,high,1); - return out; -} -// #else -// Note that these instruction work on alternating bytes -// FASTOR_INLINE __m256i _mm256_add_epi32x(__m256i _a, __m256i _b) { -// return _mm256_add_epi32(_a,_b); -// } -// FASTOR_INLINE __m256i _mm256_sub_epi32x(__m256i _a, __m256i _b) { -// return _mm256_sub_epi32(_a,_b); -// } -// FASTOR_INLINE __m256i _mm256_mul_epi32x(__m256i _a, __m256i _b) { -// return _mm256_mul_epi32(_a,_b); -// } -// FASTOR_INLINE __m256i _mm256_add_epi64x(__m256i _a, __m256i _b) { -// return _mm256_add_epi64(_a,_b); -// } -// FASTOR_INLINE __m256i _mm256_sub_epi64x(__m256i _a, __m256i _b) { -// return _mm256_sub_epi64(_a,_b); -// } -// #endif - -FASTOR_INLINE __m256i _mm256_div_epi32x(__m256i _a, __m256i _b) { - // YIELDS INCORRECT - int *a_data = (int*) &_a; - int *b_data = (int*) &_b; - FASTOR_ARCH_ALIGN int out_data[8]; - for (int i=0; i<8; ++i) - out_data[i] = a_data[i]/b_data[i]; - __m256i out = _mm256_setzero_si256(); - _mm256_store_si256((__m256i*)out_data,out); - return out; -} - -FASTOR_INLINE __m256i _mm256_mul_epi64x(__m256i _a, __m256i _b) { - __m128i low_a = _mm256_castsi256_si128(_a); - __m128i high_a = _mm256_extractf128_si256(_a,0x1); - __m128i low_b = _mm256_castsi256_si128(_b); - __m128i high_b = _mm256_extractf128_si256(_b,0x1); - __m128i low = _mm_mul_epi64(low_a,low_b); - __m128i high = _mm_mul_epi64(high_a,high_b); - __m256i out = _mm256_castsi128_si256(low); - out = _mm256_insertf128_si256(out,high,0x1); - return out; -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - - -//! Some further auxilary functions C++ only -//----------------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------------// -#if defined(__cplusplus) -#ifdef FASTOR_SSE2_IMPL -static FASTOR_INLINE __m128d _add_pd(__m128d a) { - // IVY 4 OPS - __m128 shuftmp= _mm_movehl_ps(ZEROPS, _mm_castpd_ps(a)); - __m128d shuf = _mm_castps_pd(shuftmp); - return _mm_add_sd(a, shuf); -} -#endif - -#ifdef FASTOR_USE_HADD // hadd is beneficial here and the flag is used in the opposite way -#ifdef FASTOR_AVX_IMPL -static FASTOR_INLINE __m128d _add_pd(__m256d a) { - // IVY 12 OPS / HW 14 OPS - __m128d sum_low = _add_pd(_mm256_castpd256_pd128(a)); - __m128d sum_high = _add_pd(_mm256_extractf128_pd(sum_low, 0x1)); - return _mm_add_pd(sum_high, sum_low); -} -#endif -#else -#ifdef FASTOR_AVX_IMPL -static FASTOR_INLINE __m128d _add_pd(__m256d a) { - // IVY 9 OPS / HW 11 OPS - __m256d sum_low = _mm256_hadd_pd(a, a); - __m128d sum_high = _mm256_extractf128_pd(sum_low, 0x1); - return _mm_add_pd(sum_high, _mm256_castpd256_pd128(sum_low)); -} -#endif -#endif -#ifdef FASTOR_SSE3_IMPL -FASTOR_INLINE __m128 _add_ps(__m128 a) { - // 8 OPS - __m128 shuf = _mm_movehdup_ps(a); // line up elements 3,1 with 2,0 - __m128 sums = _mm_add_ps(a, shuf); - shuf = _mm_movehl_ps(shuf, sums); // high half -> low half - sums = _mm_add_ss(sums, shuf); - return sums; -} -#endif -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE __m128 _add_ps(__m256 a) { - // IVY 20 OPS / HW 22 OPS - __m128 sum_low = _add_ps(_mm256_castps256_ps128(a)); - __m128 sum_high = _add_ps(_mm256_extractf128_ps(a,0x1)); - return _mm_add_ss(sum_low,sum_high); -} -#endif - -// horizontal add_sub -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE __m128 _addsub_ps(__m128 a) { - // 8 OPS - // If a = [a0 a1 a2 a3] this function returns (a1+a3)-(a0+a2) - // Note that only the first element of __m128 corresponds to this - __m128 shuf = _mm_shuffle_ps(a,a,_MM_SHUFFLE(1,0,3,2)); - __m128 sums = _mm_add_ps(a, shuf); - shuf = _mm_shuffle_ps(sums,sums,_MM_SHUFFLE(2,3,0,1)); - return _mm_sub_ps(shuf, sums); -} - -FASTOR_INLINE __m128 _mulsub_ps(__m128 a) { - // 10 OPS - // If a = [a0 a1 a2 a3] this function returns (a1*a3)-(a0+a2) - // Note that only the first element of __m128 corresponds to this - __m128 shuf = _mm_shuffle_ps(a,a,_MM_SHUFFLE(1,0,3,2)); - __m128 muls = _mm_mul_ps(a, shuf); - shuf = _mm_shuffle_ps(muls,muls,_MM_SHUFFLE(2,3,0,1)); - return _mm_sub_ps(shuf, muls); -} - -FASTOR_INLINE __m128d _hsub_pd(__m128d a) { - // horizontal sub, returns a[0] - a[1] - // 4 OPS - return _mm_sub_sd(a,_mm_shuffle_pd(a,a,0x1)); -} -#endif - -#ifdef FASTOR_AVX_IMPL -// Similar to SSE4 _mm_dp_pd for dot product -FASTOR_INLINE __m128d _mm256_dp_pd(__m256d __X, __m256d __Y) { - return _add_pd(_mm256_mul_pd(__X, __Y)); -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - -// Arrangments for std::complex after load / before store -//----------------------------------------------------------------------------------------------------------------// -// complex single -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE void arrange_from_load(__m128& value_r, __m128& value_i, __m128 lo, __m128 hi) { - value_r = _mm_shuffle_ps(lo, hi, _MM_SHUFFLE(2, 0, 2, 0)); - value_i = _mm_shuffle_ps(lo, hi, _MM_SHUFFLE(3, 1, 3, 1)); -} - -FASTOR_INLINE void arrange_for_store(__m128 &lo, __m128 &hi, __m128 value_r, __m128 value_i) { - lo = _mm_unpacklo_ps(value_r,value_i); - hi = _mm_unpackhi_ps(value_r,value_i); -} -#endif - -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE void arrange_from_load(__m256& value_r, __m256& value_i, __m256 lo, __m256 hi) { -#ifdef FASTOR_AVX2_IMPL - value_r = _mm256_castpd_ps(_mm256_permute4x64_pd( - _mm256_castps_pd(_mm256_shuffle_ps(lo, hi, _MM_SHUFFLE(2, 0, 2, 0))),_MM_SHUFFLE(3, 1, 2, 0))); - value_i = _mm256_castpd_ps(_mm256_permute4x64_pd( - _mm256_castps_pd(_mm256_shuffle_ps(lo, hi, _MM_SHUFFLE(3, 1, 3, 1))),_MM_SHUFFLE(3, 1, 2, 0))); -#else - __m128 tmp0 = _mm256_castps256_ps128(lo); - __m128 tmp1 = _mm256_extractf128_ps(lo, 0x1); - __m128 tmp2 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(2, 0, 2, 0)); - __m128 tmp3 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(3, 1, 3, 1)); - value_r = _mm256_castps128_ps256(tmp2); - value_i = _mm256_castps128_ps256(tmp3); - tmp0 = _mm256_castps256_ps128(hi); - tmp1 = _mm256_extractf128_ps(hi, 0x1); - tmp2 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(2, 0, 2, 0)); - tmp3 = _mm_shuffle_ps(tmp0, tmp1, _MM_SHUFFLE(3, 1, 3, 1)); - value_r = _mm256_insertf128_ps(value_r, tmp2, 0x1); - value_i = _mm256_insertf128_ps(value_i, tmp3, 0x1); -#endif -} - -FASTOR_INLINE void arrange_for_store(__m256 &lo, __m256 &hi, __m256 value_r, __m256 value_i) { - __m256 tmp0 = _mm256_unpacklo_ps(value_r, value_i); - __m256 tmp1 = _mm256_unpackhi_ps(value_r, value_i); - lo = _mm256_permute2f128_ps(tmp1, tmp0, 0x2); - hi = _mm256_permute2f128_ps(tmp0, tmp1, 0x1); - hi = _mm256_insertf128_ps(hi,_mm256_extractf128_ps(tmp1,0x1),0x1); -} -#endif - -#ifdef FASTOR_AVX512F_IMPL -FASTOR_INLINE void arrange_from_load(__m512& value_r, __m512& value_i, __m512 lo, __m512 hi) { - // Define to help with immediate construction - most likely - // the compiler will emit [vmovdqa32] - #define FT_LOAD512SIDXR _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30) - #define FT_LOAD512SIDXI _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31) - value_r = _mm512_permutex2var_ps(lo, FT_LOAD512SIDXR, hi); - value_i = _mm512_permutex2var_ps(lo, FT_LOAD512SIDXI, hi); -} - -FASTOR_INLINE void arrange_for_store(__m512 &lo, __m512 &hi, __m512 value_r, __m512 value_i) { - // Define to help with immediate construction - most likely - // the compiler will emit [vmovdqa32] - #define FT_STORE512SIDXR _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23) - #define FT_STORE512SIDXI _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31) - lo = _mm512_permutex2var_ps(value_r, FT_STORE512SIDXR, value_i); - hi = _mm512_permutex2var_ps(value_r, FT_STORE512SIDXI, value_i); -} -#endif - -// complex double -#ifdef FASTOR_SSE2_IMPL -FASTOR_INLINE void arrange_from_load(__m128d& value_r, __m128d& value_i, __m128d lo, __m128d hi) { - value_r = _mm_shuffle_pd(lo, hi, _MM_SHUFFLE2(0, 0)); - value_i = _mm_shuffle_pd(lo, hi, _MM_SHUFFLE2(1, 1)); -} - -FASTOR_INLINE void arrange_for_store(__m128d &lo, __m128d &hi, __m128d value_r, __m128d value_i) { - lo = _mm_shuffle_pd(value_r, value_i, _MM_SHUFFLE2(0, 0)); - hi = _mm_shuffle_pd(value_r, value_i, _MM_SHUFFLE2(1, 1)); -} -#endif - -#ifdef FASTOR_AVX_IMPL -FASTOR_INLINE void arrange_from_load(__m256d& value_r, __m256d& value_i, __m256d lo, __m256d hi) { -#ifdef FASTOR_AVX2_IMPL - value_r = _mm256_permute4x64_pd(_mm256_unpacklo_pd(lo, hi), _MM_SHUFFLE(3, 1, 2, 0)); - value_i = _mm256_permute4x64_pd(_mm256_unpackhi_pd(lo, hi), _MM_SHUFFLE(3, 1, 2, 0)); -#else - __m256d tmp0 = _mm256_unpacklo_pd(lo, hi); - __m256d tmp1 = _mm256_unpackhi_pd(lo, hi); - __m128d tmp2 = _mm256_castpd256_pd128(tmp0); - __m128d tmp3 = _mm256_extractf128_pd (tmp0,0x1); - value_r = _mm256_castpd128_pd256( _mm_unpacklo_pd(tmp2,tmp3)); - value_r = _mm256_insertf128_pd (value_r,_mm_unpackhi_pd(tmp2,tmp3),0x1); - tmp2 = _mm256_castpd256_pd128(tmp1); - tmp3 = _mm256_extractf128_pd (tmp1,0x1); - value_i = _mm256_castpd128_pd256( _mm_unpacklo_pd(tmp2,tmp3)); - value_i = _mm256_insertf128_pd (value_i,_mm_unpackhi_pd(tmp2,tmp3),0x1); -#endif -} - -FASTOR_INLINE void arrange_for_store(__m256d &lo, __m256d &hi, __m256d value_r, __m256d value_i) { - __m256d tmp0 = _mm256_unpacklo_pd(value_r, value_i); - __m256d tmp1 = _mm256_unpackhi_pd(value_r, value_i); - lo = _mm256_permute2f128_pd(tmp1, tmp0, 0x2); - hi = _mm256_permute2f128_pd(tmp0, tmp1, 0x1); - hi = _mm256_insertf128_pd(hi,_mm256_extractf128_pd(tmp1,0x1),0x1); -} -#endif - -#ifdef FASTOR_AVX512F_IMPL -FASTOR_INLINE void arrange_from_load(__m512d& value_r, __m512d& value_i, __m512d lo, __m512d hi) { - // Define to help with immediate construction - most likely - // the compiler will emit [vmovdqa64] - #define FT_LOAD512DIDXR _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14) - #define FT_LOAD512DIDXI _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15) - value_r = _mm512_permutex2var_pd(lo, FT_LOAD512DIDXR, hi); - value_i = _mm512_permutex2var_pd(lo, FT_LOAD512DIDXI, hi); -} - -FASTOR_INLINE void arrange_for_store(__m512d &lo, __m512d &hi, __m512d value_r, __m512d value_i) { - // Define to help with immediate construction - most likely - // the compiler will emit [vmovdqa64] - #define FT_STORE512DIDXR _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11) - #define FT_STORE512DIDXI _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15) - lo = _mm512_permutex2var_pd(value_r, FT_STORE512DIDXR, value_i); - hi = _mm512_permutex2var_pd(value_r, FT_STORE512DIDXI, value_i); -} -#endif -//----------------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------------// -// Additional math functions for scalars -> the name sqrts is to remove ambiguity with libm sqrt -template,bool> = false> -FASTOR_INLINE T sqrts(T a) {return std::sqrt(a);} -#ifdef FASTOR_SSE2_IMPL -template<> -FASTOR_INLINE float sqrts(float a) {return _mm_cvtss_f32(_mm_sqrt_ps(_mm_set1_ps(a)));} -template<> -FASTOR_INLINE double sqrts(double a) {return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_set1_pd(a)));} -#endif - -//----------------------------------------------------------------------------------------------------------------// -#endif -//----------------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------------// - - - -//----------------------------------------------------------------------------------------------------------------// -// helper functions for going from array to mask and vice-versa -// used when AVX512 masking is available -template = false> -FASTOR_INLINE uint8_t array_to_mask(const int (&b)[N]) -{ - uint8_t c = 0; - for (int i=0; i < N; ++i) { - if (b[i] == -1) { - c |= 1 << (N - i - 1); - } - } - return c; -} -template = false> -FASTOR_INLINE uint16_t array_to_mask(const int (&b)[N]) -{ - uint16_t c = 0; - for (int i=0; i < N; ++i) { - if (b[i] == -1) { - c |= 1 << (N - i - 1); - } - } - return c; -} -template = false> -FASTOR_INLINE uint32_t array_to_mask(const int (&b)[N]) -{ - uint32_t c = 0; - for (int i=0; i < N; ++i) { - if (b[i] == -1) { - c |= 1 << (N - i - 1); - } - } - return c; -} -template = false> -FASTOR_INLINE uint64_t array_to_mask(const int (&b)[N]) -{ - uint64_t c = 0; - for (int i=0; i < N; ++i) { - if (b[i] == -1) { - c |= 1 << (N - i - 1); - } - } - return c; -} - - -template = false> -FASTOR_INLINE void mask_to_array(uint8_t c, int (&b)[N]) -{ - for (int i=0; i < N; ++i) - b[i] = (c & (1 << (N - i -1))) != 0; - - // set bits need to be -1 not 1 - for (int i=0; i < N; ++i) - b[i] *= -1; -} -template = false> -FASTOR_INLINE void mask_to_array(uint16_t c, int (&b)[N]) -{ - for (int i=0; i < N; ++i) - b[i] = (c & (1 << (N - i -1))) != 0; - - // set bits need to be -1 not 1 - for (int i=0; i < N; ++i) - b[i] *= -1; -} -template = false> -FASTOR_INLINE void mask_to_array(uint32_t c, int (&b)[N]) -{ - for (int i=0; i < N; ++i) - b[i] = (c & (1 << (N - i -1))) != 0; - - // set bits need to be -1 not 1 - for (int i=0; i < N; ++i) - b[i] *= -1; -} -template = false> -FASTOR_INLINE void mask_to_array(uint64_t c, int (&b)[N]) -{ - for (int i=0; i < N; ++i) - b[i] = (c & (1 << (N - i -1))) != 0; - - // set bits need to be -1 not 1 - for (int i=0; i < N; ++i) - b[i] *= -1; -} -//----------------------------------------------------------------------------------------------------------------// - - - -// Splitting a mask in to two - used for masking complex SIMD vectors -//----------------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------------// -/* Moves "from" bit of c1 to "to" bit of c2 */ -FASTOR_INLINE uint8_t move_bit(uint8_t c1, int from, uint8_t c2, int to) -{ - int bit; - /* Get the source bit as 0/1 value */ - bit = (c1 >> from) & 1; - /* clear destination bit */ - c2 &= ~(1 << to); - /* set destination bit */ - return (uint8_t)(c2 | (bit << to)); -} - -/* Moves "from" bit of c1 to "to" bit of c2 */ -FASTOR_INLINE uint16_t move_bit(uint16_t c1, int from, uint16_t c2, int to) -{ - int bit; - /* Get the source bit as 0/1 value */ - bit = (c1 >> from) & 1; - /* clear destination bit */ - c2 &= ~(1 << to); - /* set destination bit */ - return (uint16_t)(c2 | (bit << to)); -} -//----------------------------------------------------------------------------------------------------------------// - -// Splitting a mask into 2 parts when only 2 bits of the incoming mask are set -template = false> -FASTOR_INLINE void split_mask(uint8_t mask, uint8_t& mask0, uint8_t& mask1) { - mask0 = 0; /* clear all bits */ - mask0 = move_bit(mask, 0, mask0, 0); - mask0 = move_bit(mask, 0, mask0, 1); - - mask1 = 0; /* clear all bits */ - mask1 = move_bit(mask, 1, mask1, 0); - mask1 = move_bit(mask, 1, mask1, 1); -} - -// Splitting a mask into 2 parts when only 4 bits of the incoming mask are set -template = false> -FASTOR_INLINE void split_mask(uint8_t mask, uint8_t& mask0, uint8_t& mask1) { - mask0 = 0; /* clear all bits */ - mask0 = move_bit(mask, 0, mask0, 0); - mask0 = move_bit(mask, 0, mask0, 1); - mask0 = move_bit(mask, 1, mask0, 2); - mask0 = move_bit(mask, 1, mask0, 3); - - mask1 = 0; /* clear all bits */ - mask1 = move_bit(mask, 2, mask1, 0); - mask1 = move_bit(mask, 2, mask1, 1); - mask1 = move_bit(mask, 3, mask1, 2); - mask1 = move_bit(mask, 3, mask1, 3); -} - -// Splitting a mask into 2 parts when all 8 bits of the incoming mask are set -template = false> -FASTOR_INLINE void split_mask(uint8_t mask, uint8_t& mask0, uint8_t& mask1) { - mask0 = 0; /* clear all bits */ - mask0 = move_bit(mask, 0, mask0, 0); - mask0 = move_bit(mask, 0, mask0, 1); - mask0 = move_bit(mask, 1, mask0, 2); - mask0 = move_bit(mask, 1, mask0, 3); - mask0 = move_bit(mask, 2, mask0, 4); - mask0 = move_bit(mask, 2, mask0, 5); - mask0 = move_bit(mask, 3, mask0, 6); - mask0 = move_bit(mask, 3, mask0, 7); - - mask1 = 0; /* clear all bits */ - mask1 = move_bit(mask, 4, mask1, 0); - mask1 = move_bit(mask, 4, mask1, 1); - mask1 = move_bit(mask, 5, mask1, 2); - mask1 = move_bit(mask, 5, mask1, 3); - mask1 = move_bit(mask, 6, mask1, 4); - mask1 = move_bit(mask, 6, mask1, 5); - mask1 = move_bit(mask, 7, mask1, 6); - mask1 = move_bit(mask, 7, mask1, 7); -} - -// Splitting a mask into 2 parts when all 16 bits of the incoming mask are set -template = false> -FASTOR_INLINE void split_mask(uint16_t mask, uint16_t& mask0, uint16_t& mask1) { - mask0 = 0; /* clear all bits */ - mask0 = move_bit(mask, 0, mask0, 0); - mask0 = move_bit(mask, 0, mask0, 1); - mask0 = move_bit(mask, 1, mask0, 2); - mask0 = move_bit(mask, 1, mask0, 3); - mask0 = move_bit(mask, 2, mask0, 4); - mask0 = move_bit(mask, 2, mask0, 5); - mask0 = move_bit(mask, 3, mask0, 6); - mask0 = move_bit(mask, 3, mask0, 7); - mask0 = move_bit(mask, 4, mask0, 8); - mask0 = move_bit(mask, 4, mask0, 9); - mask0 = move_bit(mask, 5, mask0, 10); - mask0 = move_bit(mask, 5, mask0, 11); - mask0 = move_bit(mask, 6, mask0, 12); - mask0 = move_bit(mask, 6, mask0, 13); - mask0 = move_bit(mask, 7, mask0, 14); - mask0 = move_bit(mask, 7, mask0, 15); - - mask1 = 0; /* clear all bits */ - mask1 = move_bit(mask, 8, mask1, 0); - mask1 = move_bit(mask, 8, mask1, 1); - mask1 = move_bit(mask, 9, mask1, 2); - mask1 = move_bit(mask, 9, mask1, 3); - mask1 = move_bit(mask, 10, mask1, 4); - mask1 = move_bit(mask, 10, mask1, 5); - mask1 = move_bit(mask, 11, mask1, 6); - mask1 = move_bit(mask, 11, mask1, 7); - mask1 = move_bit(mask, 12, mask1, 8); - mask1 = move_bit(mask, 12, mask1, 9); - mask1 = move_bit(mask, 13, mask1, 10); - mask1 = move_bit(mask, 13, mask1, 11); - mask1 = move_bit(mask, 14, mask1, 12); - mask1 = move_bit(mask, 14, mask1, 13); - mask1 = move_bit(mask, 15, mask1, 14); - mask1 = move_bit(mask, 15, mask1, 15); -} -//----------------------------------------------------------------------------------------------------------------// - -} // end of namespace Fastor - - -#endif // EXT_INTRIN_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_abi.h b/noarch/include/Fastor/simd_vector/simd_vector_abi.h deleted file mode 100644 index b0530455..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_abi.h +++ /dev/null @@ -1,225 +0,0 @@ -#ifndef FASTOR_SIMD_VECTOR_ABI -#define FASTOR_SIMD_VECTOR_ABI - -#include "Fastor/meta/meta.h" -#include "Fastor/config/config.h" -#include -#include - -namespace Fastor { - -namespace simd_abi { - -struct scalar {}; -struct sse {}; -struct avx {}; -struct avx512 {}; -struct mic {}; -template struct fixed_size {}; - -#ifndef FASTOR_DONT_VECTORISE -#if defined(FASTOR_AVX512_IMPL) -using native = simd_abi::avx512; -#elif defined(FASTOR_AVX_IMPL) -using native = simd_abi::avx; -#elif defined(FASTOR_SSE2_IMPL) -using native = simd_abi::sse; -#else -using native = simd_abi::scalar; -#endif -#else -using native = simd_abi::scalar; -#endif - -} - -// Definition of DEFAULT_ABI is here -//--------------------------------------------------------------------------------------------------------------// -#define DEFAULT_ABI simd_abi::native -//--------------------------------------------------------------------------------------------------------------// - - -//--------------------------------------------------------------------------------------------------------------// -namespace internal { - -template -struct get_simd_vector_size; -template class __svec, typename T, typename ABI> -struct get_simd_vector_size<__svec> { - static constexpr size_t bitsize = std::is_same::value - ? FASTOR_AVX512_BITSIZE : (std::is_same::value - ? FASTOR_AVX_BITSIZE : (std::is_same::value - ? FASTOR_SSE_BITSIZE : sizeof(T)*8)); - - // Size should be at least 1UL - static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL; -}; -// Specialisation for fixed size simd vectors -template class __svec, typename T, size_t N> -struct get_simd_vector_size<__svec > > { - static constexpr size_t bitsize = N*8UL; - static constexpr size_t value = N; -}; -// Specialisation for complex simd vectors -template class __svec, typename ABI> -struct get_simd_vector_size<__svec,ABI>> { - using T = float; - static constexpr size_t bitsize = std::is_same::value - ? FASTOR_AVX512_BITSIZE : (std::is_same::value - ? FASTOR_AVX_BITSIZE : (std::is_same::value - ? FASTOR_SSE_BITSIZE : sizeof(T)*8)); - - // Size should be at least 1UL - static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL; -}; -template class __svec, typename ABI> -struct get_simd_vector_size<__svec,ABI>> { - using T = double; - static constexpr size_t bitsize = std::is_same::value - ? FASTOR_AVX512_BITSIZE : (std::is_same::value - ? FASTOR_AVX_BITSIZE : (std::is_same::value - ? FASTOR_SSE_BITSIZE : sizeof(T)*8)); - - // Size should be at least 1UL - static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL; -}; - - -template -struct get_half_simd_type; -template class __svec, typename T, typename ABI> -struct get_half_simd_type<__svec> { - // If not a half of simd we give back the actual incoming type - using actual_type = __svec; - using type = typename std::conditional< std::is_same::value, __svec, - typename std::conditional< std::is_same::value, __svec, actual_type>::type - >::type; -}; - - -template -struct get_quarter_simd_type; -template class __svec, typename T, typename ABI> -struct get_quarter_simd_type<__svec> { - // If not a quarter of simd we give back the actual incoming type - using actual_type = __svec; - using type = typename std::conditional< std::is_same::value, __svec, actual_type>::type; -}; - - -template -struct is_exact_multiple_of_smaller_simd; -template class __svec, typename T, typename ABI, size_t N> -struct is_exact_multiple_of_smaller_simd<__svec,N> { - // If not a fraction of simd we give back the actual incoming type - using actual_type = __svec; - // if N is half simd which=2, if it is a 1/4th which=4, else which=1 - static constexpr int which = get_simd_vector_size<__svec>::value / N == 2UL ? 2UL - : (get_simd_vector_size<__svec>::value / N == 4UL ? 4UL : 1UL); - static constexpr bool value = which != 1UL && !std::is_same::value ? true : false; - - static constexpr bool is_half_of_avx512 = std::is_same::value && which==2UL; - static constexpr bool is_half_of_avx = std::is_same::value && which==2UL; - static constexpr bool is_4th_of_avx512 = std::is_same::value && which==4UL; - // static constexpr bool is_4th_of_avx = std::is_same::value && which==4UL; - - static constexpr bool is_half_type = is_half_of_avx512 || is_half_of_avx; - static constexpr bool is_quarter_type = is_4th_of_avx512; - - using half_type = typename std::conditional< is_half_of_avx512, __svec, - typename std::conditional< is_half_of_avx, __svec, actual_type>::type - >::type; - using quarter_type = typename std::conditional< is_4th_of_avx512, __svec, actual_type>::type; - - using type = typename std::conditional::type - >::type; -}; - -template -struct choose_best_simd_type; -template class __svec, typename TT, typename ABI, size_t N> -struct choose_best_simd_type<__svec,N> { - using T = remove_cv_ref_t; - using actual_type = __svec; - static constexpr size_t _vec_size = get_simd_vector_size<__svec>::value; - // For exact fractions simd gets proper speed up for instance for matmul - using exact_multiple_t = typename is_exact_multiple_of_smaller_simd<__svec,N>::type; - static constexpr bool is_exact_multiple = is_exact_multiple_of_smaller_simd<__svec,N>::value; - -#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS) - using size_based_type = typename std::conditional::type; -#else - using size_based_type = typename std::conditional::value, typename get_half_simd_type<__svec>::type, actual_type - >::type - >::type; -#endif - - // using size_based_type = typename std::conditional,N>::type, - // typename std::conditional::value, typename get_quarter_simd_type<__svec>::type, - // typename std::conditional::value, typename get_half_simd_type<__svec>::type, actual_type - // >::type - // >::type - // >::type; - - // // For other fractions masking might be a better idea than, hence this special logic for remainder using is_less. - // // For no special logic use the above case - // using size_based_type = typename std::conditional,N>::type, - // typename std::conditional::value && is_greater<_vec_size % N,1UL>::value, - // typename get_quarter_simd_type<__svec>::type, - // typename std::conditional::value && is_greater<_vec_size % N,1UL>::value, - // typename get_half_simd_type<__svec>::type, actual_type - // >::type - // >::type - // >::type; - - using type = typename std::conditional< std::is_same::value || - std::is_same::value || - std::is_same>::value || - std::is_same>::value || - std::is_same::value || - std::is_same::value, - size_based_type, - __svec - >::type; -}; - -} // end of namesapce internal - - -// This is specifically for matmul and other backend tensor kernels that need -// to choose/switch between best simd types for utmost performance -template -using choose_best_simd_t = typename internal::choose_best_simd_type<__svec,N>::type; -//--------------------------------------------------------------------------------------------------------------// - - -// For switching between complex and non-complex implementation of simd we need different return types -//--------------------------------------------------------------------------------------------------------------// -template -struct get_simd_cmplx_value_type; -template -struct get_simd_cmplx_value_type > > { - using type = T; -}; -template -struct get_simd_cmplx_value_type > > { - using type = typename T::value_type; -}; - -template -struct simd_cmplx_value_type; -template class __svec, typename T, typename ABI> -struct simd_cmplx_value_type< __svec > { - using type = typename get_simd_cmplx_value_type::type; -}; -template -using simd_cmplx_value_t = typename simd_cmplx_value_type::type; -//--------------------------------------------------------------------------------------------------------------// - - - -} // end of namesapce Fastor - -#endif // FASTOR_SIMD_VECTOR_ABI diff --git a/noarch/include/Fastor/simd_vector/simd_vector_base.h b/noarch/include/Fastor/simd_vector/simd_vector_base.h deleted file mode 100644 index bf0325a8..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_base.h +++ /dev/null @@ -1,391 +0,0 @@ -#ifndef SIMD_VECTOR_BASE_H -#define SIMD_VECTOR_BASE_H - -#include "Fastor/meta/meta.h" -#include "Fastor/config/config.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/simd_vector_abi.h" - -#include -#include - - -namespace Fastor { - -/* The default SIMDVector class that falls back to scalar implementation -* if SIMD types are not available or if vectorisation is disallowed -*/ -//--------------------------------------------------------------------------------------------------------------------// -template -struct SIMDVector { - using T = remove_cv_ref_t; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - using vector_type = SIMDVector; - using value_type = T[Size]; - using scalar_value_type = T; - using abi_type = ABI; - - FASTOR_INLINE SIMDVector() : value{} {} - FASTOR_INLINE SIMDVector(T num) { std::fill(value, value+Size, num); } - FASTOR_INLINE SIMDVector(const SIMDVector &a) { std::copy(a.value,a.value+a.Size,value); } - FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) { std::copy(data,data+Size,value); unused(Aligned); } - - FASTOR_INLINE SIMDVector operator=(T num) { std::fill(value, value+Size, num); return *this;} - FASTOR_INLINE SIMDVector operator=(const SIMDVector &a) { std::copy(a.value,a.value+a.Size,value); return *this; }; - - FASTOR_INLINE void load(const T *data, bool Aligned=true ) { std::copy(data,data+Size,value); unused(Aligned);} - FASTOR_INLINE void store(T *data, bool Aligned=true ) const { std::copy(value,value+Size,data); unused(Aligned);} - - FASTOR_INLINE void aligned_load(const T *data) { std::copy(data,data+Size,value); } - FASTOR_INLINE void aligned_store(T *data) const { std::copy(value,value+Size,data);} - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - std::fill(value, value+Size, 0); - for (FASTOR_INDEX i=0; i,bool> = false> - FASTOR_INLINE SIMDVector,ABI> real() const { - simd_cmplx_value_t arr[Size]; - for (FASTOR_INDEX i=0; i,ABI> out(arr,false); - return out; - } - template,bool> = false> - FASTOR_INLINE SIMDVector,ABI> imag() const { - simd_cmplx_value_t arr[Size]; - for (FASTOR_INDEX i=0; i,ABI> out(arr,false); - return out; - } - - FASTOR_INLINE void set(T num) { - for (FASTOR_INDEX i=0; i - FASTOR_INLINE void set(U first, Args ... args) { - static_assert(sizeof...(args)+1==Size,"CANNOT SET VECTOR WITH SPECIFIED NUMBER OF VALUES DUE TO ABI CONSIDERATION"); - T arr[Size] = {first,args...}; - std::reverse_copy(arr, arr+Size, value); - } - FASTOR_INLINE void set_sequential(T num0) { - for (FASTOR_INDEX i=0; i &a) { - for (FASTOR_INDEX i=0; i &a) { - for (FASTOR_INDEX i=0; i &a) { - for (FASTOR_INDEX i=0; i &a) { - for (FASTOR_INDEX i=0; i shift(FASTOR_INDEX i) { - SIMDVector out; - std::fill(out.value,out.value+out.Size,static_cast(0)); - std::copy(value,value+Size, out.value+i); - return out; - } - FASTOR_INLINE T sum() { - T quan = 0; - for (FASTOR_INDEX i=0; i reverse() { - SIMDVector out; - std::copy(value,value+Size,out.value); - std::reverse(out.value,out.value+Size); - return out; - } - // This is for comatibility with complex simd vectors - template,bool> = false> - FASTOR_INLINE SIMDVector,ABI> magnitude() { - simd_cmplx_value_t arr[Size]; - for (FASTOR_INDEX i=0; i,ABI> out(arr,false); - return out; - } - template,bool> = false> - FASTOR_INLINE SIMDVector,ABI> norm() { - simd_cmplx_value_t arr[Size]; - for (FASTOR_INDEX i=0; i,ABI> out(arr,false); - return out; - } - FASTOR_INLINE T minimum() { - T quan = 0; - for (FASTOR_INDEX i=0; iquan) - quan = value[i]; - return quan; - } - - FASTOR_INLINE T dot(const SIMDVector &other) { - T quan = 0; - for (FASTOR_INDEX i=0; i - FASTOR_INLINE SIMDVector cast() { - SIMDVector out; - for (FASTOR_INDEX i=0; i(value[i]); - } - return out; - } - -#ifdef FASTOR_ZERO_INITIALISE - FASTOR_ARCH_ALIGN T value[Size] = {}; -#else - FASTOR_ARCH_ALIGN T value[Size]; -#endif -}; - -template -constexpr FASTOR_INDEX SIMDVector::Size; - -template -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - os << "["; - for (FASTOR_INDEX i=0; i -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] + b.value[i]; - return out; -} -template -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, T b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] + b; - return out; -} -template -FASTOR_INLINE SIMDVector operator+(T a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a + b.value[i]; - return out; -} -template -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -template -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] - b.value[i]; - return out; -} -template -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, T b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] - b; - return out; -} -template -FASTOR_INLINE SIMDVector operator-(T a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a - b.value[i]; - return out; -} -template -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = -b.value[i]; - return out; -} - -template -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] * b.value[i]; - return out; -} -template -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, T b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] * b; - return out; -} -template -FASTOR_INLINE SIMDVector operator*(T a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a * b.value[i]; - return out; -} - -template -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] / b.value[i]; - return out; -} -template -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, T b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a.value[i] / b; - return out; -} -template -FASTOR_INLINE SIMDVector operator/(T a, const SIMDVector &b) { - SIMDVector out; - for (FASTOR_INDEX i=0; i::Size; ++i) - out.value[i] = a / b.value[i]; - return out; -} - -template -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - SIMDVector out; - for (FASTOR_INDEX i=0; i,bool> = false> -FASTOR_INLINE SIMDVector conj(const SIMDVector &a) { - T arr[SIMDVector::Size]; - for (FASTOR_INDEX i=0UL; i::Size; ++i) { - arr[i] = std::conj(a[i]); - } - return SIMDVector(arr,false); -} -template,bool> = false> -FASTOR_INLINE SIMDVector arg(const SIMDVector &a) { - T arr[SIMDVector::Size]; - for (FASTOR_INDEX i=0UL; i::Size; ++i) { - arr[i] = std::arg(a[i]); - } - return SIMDVector(arr,false); -} - -} // end of namespace Fastor - -#endif // SIMD_VECTOR_H - diff --git a/noarch/include/Fastor/simd_vector/simd_vector_common.h b/noarch/include/Fastor/simd_vector/simd_vector_common.h deleted file mode 100644 index 908c6772..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_common.h +++ /dev/null @@ -1,1177 +0,0 @@ -#ifndef SIMD_VECTOR_COMMON_H -#define SIMD_VECTOR_COMMON_H - -#include "Fastor/meta/meta.h" -#include "Fastor/simd_vector/simd_vector_base.h" -#include "Fastor/simd_vector/simd_vector_scalar.h" -#include "Fastor/simd_vector/simd_vector_float.h" -#include "Fastor/simd_vector/simd_vector_double.h" -#include "Fastor/simd_vector/simd_vector_int32.h" -#include "Fastor/simd_vector/simd_vector_int64.h" -#include "Fastor/simd_vector/simd_vector_complex_scalar.h" -#include "Fastor/simd_vector/simd_vector_complex_float.h" -#include "Fastor/simd_vector/simd_vector_complex_double.h" - - -namespace Fastor { - -/* Common functions for all SIMDVector types -*/ - -// This is for generic use in tensor expressions that need a uniform simd type -// between all of them -//----------------------------------------------------------------------------------------------------------// -template -struct choose_best_simd_vector { - using T = remove_cv_ref_t; - using type = typename std::conditional< std::is_same::value || - std::is_same::value || - std::is_same>::value || - std::is_same>::value || - std::is_same::value || - std::is_same::value, - SIMDVector, - SIMDVector - >::type; -}; - -// helper function -template -using choose_best_simd_vector_t = typename choose_best_simd_vector::type; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -// Get architecture native size/width of the SIMDVector - for instance for [T=double and AVX512] returns 8 -template -static constexpr size_t native_simd_size_v = internal::get_simd_vector_size>::value; -// Get Fastor supported size/width of the SIMDVector -template -static constexpr size_t simd_size_v = internal::get_simd_vector_size>::value; -//----------------------------------------------------------------------------------------------------------// - - -// Broadcasting vectorisation on general strides [gather operations] -//---------------------------------------------------------------------------------------------------------------- -// 1 word scalar -template>::bitsize==8,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int ) { - vec.set(data[idx]); -} -// 1 word in a 2 - for compatibility -template>::bitsize==16,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+general_stride],data[idx]); -} -// 1 word in a 4 - for compatibility -template>::bitsize==32,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} -// 1 word in a 8 - for compatibility -template>::bitsize==64,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+7*general_stride],data[idx+6*general_stride], - data[idx+5*general_stride],data[idx+4*general_stride], - data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} -// 1 word - sse -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+15*general_stride],data[idx+14*general_stride], - data[idx+13*general_stride],data[idx+12*general_stride], - data[idx+11*general_stride],data[idx+10*general_stride], - data[idx+9*general_stride],data[idx+8*general_stride], - data[idx+7*general_stride],data[idx+6*general_stride], - data[idx+5*general_stride],data[idx+4*general_stride], - data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} -// 1 word avx -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+31*general_stride],data[idx+30*general_stride], - data[idx+29*general_stride],data[idx+28*general_stride], - data[idx+27*general_stride],data[idx+26*general_stride], - data[idx+25*general_stride],data[idx+24*general_stride], - data[idx+23*general_stride],data[idx+22*general_stride], - data[idx+21*general_stride],data[idx+20*general_stride], - data[idx+19*general_stride],data[idx+18*general_stride], - data[idx+17*general_stride],data[idx+16*general_stride], - data[idx+15*general_stride],data[idx+14*general_stride], - data[idx+13*general_stride],data[idx+12*general_stride], - data[idx+11*general_stride],data[idx+10*general_stride], - data[idx+9 *general_stride],data[idx+8 *general_stride], - data[idx+7 *general_stride],data[idx+6 *general_stride], - data[idx+5 *general_stride],data[idx+4 *general_stride], - data[idx+3 *general_stride],data[idx+2 *general_stride], - data[idx+general_stride] ,data[idx]); -} -// 1 word avx512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+63*general_stride],data[idx+62*general_stride], - data[idx+61*general_stride],data[idx+60*general_stride], - data[idx+59*general_stride],data[idx+58*general_stride], - data[idx+57*general_stride],data[idx+56*general_stride], - data[idx+55*general_stride],data[idx+54*general_stride], - data[idx+53*general_stride],data[idx+52*general_stride], - data[idx+51*general_stride],data[idx+50*general_stride], - data[idx+49*general_stride],data[idx+48*general_stride], - data[idx+47*general_stride],data[idx+46*general_stride], - data[idx+45*general_stride],data[idx+44*general_stride], - data[idx+43*general_stride],data[idx+42*general_stride], - data[idx+41*general_stride],data[idx+40*general_stride], - data[idx+39*general_stride],data[idx+38*general_stride], - data[idx+37*general_stride],data[idx+36*general_stride], - data[idx+35*general_stride],data[idx+34*general_stride], - data[idx+33*general_stride],data[idx+32*general_stride], - data[idx+31*general_stride],data[idx+30*general_stride], - data[idx+29*general_stride],data[idx+28*general_stride], - data[idx+27*general_stride],data[idx+26*general_stride], - data[idx+25*general_stride],data[idx+24*general_stride], - data[idx+23*general_stride],data[idx+22*general_stride], - data[idx+21*general_stride],data[idx+20*general_stride], - data[idx+19*general_stride],data[idx+18*general_stride], - data[idx+17*general_stride],data[idx+16*general_stride], - data[idx+15*general_stride],data[idx+14*general_stride], - data[idx+13*general_stride],data[idx+12*general_stride], - data[idx+11*general_stride],data[idx+10*general_stride], - data[idx+9 *general_stride],data[idx+8 *general_stride], - data[idx+7 *general_stride],data[idx+6 *general_stride], - data[idx+5 *general_stride],data[idx+4 *general_stride], - data[idx+3 *general_stride],data[idx+2 *general_stride], - data[idx+general_stride] ,data[idx]); -} - -// 4 word scalar -template>::bitsize==32,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int ) { - vec.set(data[idx]); -} -// 4 word in an 8 - for compatibility -template>::bitsize==64,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+general_stride],data[idx]); -} -// 4 word SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+3*general_stride],data[idx+2*general_stride],data[idx+general_stride],data[idx]); -} -// 4 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+7*general_stride],data[idx+6*general_stride], - data[idx+5*general_stride],data[idx+4*general_stride], - data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} -// 4 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+15*general_stride],data[idx+14*general_stride], - data[idx+13*general_stride],data[idx+12*general_stride], - data[idx+11*general_stride],data[idx+10*general_stride], - data[idx+9*general_stride],data[idx+8*general_stride], - data[idx+7*general_stride],data[idx+6*general_stride], - data[idx+5*general_stride],data[idx+4*general_stride], - data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} - -// 8 word scalar -template>::bitsize==64,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int ) { - vec.set(data[idx]); -} -// 8 word SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+general_stride],data[idx]); -} -// 8 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} -// 8 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+7*general_stride],data[idx+6*general_stride], - data[idx+5*general_stride],data[idx+4*general_stride], - data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} - -// // 16 word scalar -// template>::bitsize==64,bool>::type=0> -// FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { -// vec.set(data[idx]); -// } -// 16 word scalar/SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+general_stride],data[idx]); -} -// 16 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} -// 16 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, int idx, int general_stride) { - vec.set(data[idx+7*general_stride],data[idx+6*general_stride], - data[idx+5*general_stride],data[idx+4*general_stride], - data[idx+3*general_stride],data[idx+2*general_stride], - data[idx+general_stride],data[idx]); -} -//---------------------------------------------------------------------------------------------------------------- - - -// [Gather operations], when strides are not constant (i.e totally random) -//---------------------------------------------------------------------------------------------------------------- -// 4 word scalar -template>::bitsize==32,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { - vec.set(data[a[0]]); -} -// 4 word in an 8 - for compatibility -template>::bitsize==64,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { - vec.set(data[a[1]],data[a[0]]); -} -// 4 word SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[3]],data[a[2]],data[a[1]],data[a[0]]); -} -// 4 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[7]],data[a[6]],data[a[5]],data[a[4]], - data[a[3]],data[a[2]],data[a[1]],data[a[0]]); -} -// 4 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[15]],data[a[14]],data[a[13]],data[a[12]], - data[a[11]],data[a[10]],data[a[9]],data[a[8]], - data[a[7]],data[a[6]],data[a[5]],data[a[4]], - data[a[3]],data[a[2]],data[a[1]],data[a[0]]); -} - -// 8 word scalar -template>::bitsize==64,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { - vec.set(data[a[0]]); -} -// 8 word SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[1]],data[a[0]]); -} -// 8 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[3]],data[a[2]],data[a[1]],data[a[0]]); -} -// 8 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[7]],data[a[6]],data[a[5]],data[a[4]], - data[a[3]],data[a[2]],data[a[1]],data[a[0]]); -} - -// 16 word scalar/SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array &a) { - vec.set(data[a[1]],data[a[0]]); -} -// 16 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[3]],data[a[2]],data[a[1]],data[a[0]]); -} -// 16 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void vector_setter(SIMDVector &vec, const T *data, const std::array a) { - vec.set(data[a[7]],data[a[6]],data[a[5]],data[a[4]], - data[a[3]],data[a[2]],data[a[1]],data[a[0]]); -} -//---------------------------------------------------------------------------------------------------------------- -//---------------------------------------------------------------------------------------------------------------- - - - - -// Scatter operations -//---------------------------------------------------------------------------------------------------------------- -// 4 word scalar -template>::bitsize==32,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int ) { - data[idx] = vec.value; -} -// 4 word in an 8 - for compatibility -template>::bitsize==64,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; -} -// 4 word SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; - data[idx+2*general_stride] = vec[2]; - data[idx+3*general_stride] = vec[3]; -} -// 4 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; - data[idx+2*general_stride] = vec[2]; - data[idx+3*general_stride] = vec[3]; - data[idx+4*general_stride] = vec[4]; - data[idx+5*general_stride] = vec[5]; - data[idx+6*general_stride] = vec[6]; - data[idx+7*general_stride] = vec[7]; -} -// 4 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride=1) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; - data[idx+2*general_stride] = vec[2]; - data[idx+3*general_stride] = vec[3]; - data[idx+4*general_stride] = vec[4]; - data[idx+5*general_stride] = vec[5]; - data[idx+6*general_stride] = vec[6]; - data[idx+7*general_stride] = vec[7]; - data[idx+8*general_stride] = vec[8]; - data[idx+9*general_stride] = vec[9]; - data[idx+10*general_stride] = vec[10]; - data[idx+11*general_stride] = vec[11]; - data[idx+12*general_stride] = vec[12]; - data[idx+13*general_stride] = vec[13]; - data[idx+14*general_stride] = vec[14]; - data[idx+15*general_stride] = vec[15]; -} - -// 8 word scalar -template>::bitsize==64,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int ) { - data[idx] = vec.value; -} -// 8 word SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; -} -// 8 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; - data[idx+2*general_stride] = vec[2]; - data[idx+3*general_stride] = vec[3]; -} -// 8 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; - data[idx+2*general_stride] = vec[2]; - data[idx+3*general_stride] = vec[3]; - data[idx+4*general_stride] = vec[4]; - data[idx+5*general_stride] = vec[5]; - data[idx+6*general_stride] = vec[6]; - data[idx+7*general_stride] = vec[7]; -} - -// 16 word scalar/SSE -template>::bitsize==128,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int ) { - data[idx] = vec.value; -} -// 16 word AVX -template>::bitsize==256,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; -} -// 16 word AVX 512 -template>::bitsize==512,bool>::type=0> -FASTOR_INLINE void data_setter(T *FASTOR_RESTRICT data, const SIMDVector &vec, Int idx, int general_stride) { - data[idx] = vec[0]; - data[idx+general_stride] = vec[1]; - data[idx+2*general_stride] = vec[2]; - data[idx+3*general_stride] = vec[3]; -} -//---------------------------------------------------------------------------------------------------------------- - - - - - - - - - -// Mask load and store operations implemented as free functions till SIMDVector finds a proper -// companion masked vector. Note that these require at least AVX even the SSE variants -//---------------------------------------------------------------------------------------------------------------- -template -FASTOR_INLINE V -maskload(const typename V::scalar_value_type * FASTOR_RESTRICT a, const int (&maska)[V::Size]) { - // masked array is reversed like other intrinsics - typename V::scalar_value_type val_out[V::Size] = {}; // zero out the rest - for (FASTOR_INDEX i=0; i -FASTOR_INLINE SIMDVector,simd_abi::avx> -maskload,simd_abi::avx>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[4]) { - // Split the mask in to a higher and lower part - we need two masks for this - __m256i mask0 = _mm256_set_epi64x(maska[2],maska[2],maska[3],maska[3]); - __m256d lo = _mm256_maskload_pd(reinterpret_cast(a ), (__m256i) mask0); - __m256i mask1 = _mm256_set_epi64x(maska[0],maska[0],maska[1],maska[1]); - __m256d hi = _mm256_maskload_pd(reinterpret_cast(a+2), (__m256i) mask1); - __m256d value_r, value_i; - arrange_from_load(value_r, value_i, lo, hi); - return SIMDVector,simd_abi::avx>(value_r,value_i); -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx> -maskload,simd_abi::avx>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[8]) { - // Split the mask in to a higher and lower part - we need two masks for this - __m256i mask0 = _mm256_set_epi32(maska[4],maska[4],maska[5],maska[5],maska[6],maska[6],maska[7],maska[7]); - __m256 lo = _mm256_maskload_ps(reinterpret_cast(a ), (__m256i) mask0); - __m256i mask1 = _mm256_set_epi32(maska[0],maska[0],maska[1],maska[1],maska[2],maska[2],maska[3],maska[3]); - __m256 hi = _mm256_maskload_ps(reinterpret_cast(a+4), (__m256i) mask1); - __m256 value_r, value_i; - arrange_from_load(value_r, value_i, lo, hi); - return SIMDVector,simd_abi::avx>(value_r,value_i); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const double * FASTOR_RESTRICT a, const int (&maska)[4]) { - __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); - return _mm256_maskload_pd(a,(__m256i) mask); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const Int64 * FASTOR_RESTRICT a, const int (&maska)[4]) { - __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); - return _mm256_maskload_epi64(reinterpret_cast(a),(__m256i) mask); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const float * FASTOR_RESTRICT a, const int (&maska)[8]) { - __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); - return _mm256_maskload_ps(a,(__m256i) mask); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const int * FASTOR_RESTRICT a, const int (&maska)[8]) { - __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); - return _mm256_maskload_epi32(a,(__m256i) mask); -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> -maskload,simd_abi::sse>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[2]) { - // Split the mask in to a higher and lower part - we need two masks for this - __m128i mask0 = _mm_set_epi64x(maska[1],maska[1]); - __m128d lo = _mm_maskload_pd(reinterpret_cast(a ), (__m128i) mask0); - __m128i mask1 = _mm_set_epi64x(maska[0],maska[0]); - __m128d hi = _mm_maskload_pd(reinterpret_cast(a+1), (__m128i) mask1); - __m128d value_r, value_i; - arrange_from_load(value_r, value_i, lo, hi); - return SIMDVector,simd_abi::sse>(value_r,value_i); -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> -maskload,simd_abi::sse>>(const std::complex * FASTOR_RESTRICT a, const int (&maska)[4]) { - // Split the mask in to a higher and lower part - we need two masks for this - __m128i mask0 = _mm_set_epi32(maska[2],maska[2],maska[3],maska[3]); - __m128 lo = _mm_maskload_ps(reinterpret_cast(a ), (__m128i) mask0); - __m128i mask1 = _mm_set_epi32(maska[0],maska[0],maska[1],maska[1]); - __m128 hi = _mm_maskload_ps(reinterpret_cast(a+2), (__m128i) mask1); - __m128 value_r, value_i; - arrange_from_load(value_r, value_i, lo, hi); - return SIMDVector,simd_abi::sse>(value_r,value_i); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const double * FASTOR_RESTRICT a, const int (&maska)[2]) { - __m128i mask = _mm_set_epi64x(maska[0],maska[1]); - return _mm_maskload_pd(a,(__m128i) mask); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const Int64 * FASTOR_RESTRICT a, const int (&maska)[2]) { - __m128i mask = _mm_set_epi64x(maska[0],maska[1]); - return _mm_maskload_epi64(reinterpret_cast(a),(__m128i) mask); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const float * FASTOR_RESTRICT a, const int (&maska)[4]) { - __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); - return _mm_maskload_ps(a,(__m128i) mask); -} -template<> -FASTOR_INLINE SIMDVector -maskload>(const int * FASTOR_RESTRICT a, const int (&maska)[4]) { - __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); - return _mm_maskload_epi32(a,(__m128i) mask); -} -#endif // FASTOR_AVX2_IMPL - - -template -FASTOR_INLINE -void maskstore(typename V::scalar_value_type * FASTOR_RESTRICT a, const int (&maska)[V::Size], V& v) { - for (FASTOR_INDEX i=0; i -FASTOR_INLINE -void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector,simd_abi::avx> &v) { - // Split the mask in to a higher and lower part - we need two masks for this - __m256i mask0 = _mm256_set_epi64x(maska[2],maska[2],maska[3],maska[3]); - __m256i mask1 = _mm256_set_epi64x(maska[0],maska[0],maska[1],maska[1]); - __m256d lo, hi; - arrange_for_store(lo, hi, v.value_r, v.value_i); - _mm256_maskstore_pd(reinterpret_cast(a ), (__m256i) mask0, lo); - _mm256_maskstore_pd(reinterpret_cast(a+2), (__m256i) mask1, hi); -} -template<> -FASTOR_INLINE -void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[8], SIMDVector,simd_abi::avx> &v) { - // Split the mask in to a higher and lower part - we need two masks for this - __m256i mask0 = _mm256_set_epi32(maska[4],maska[4],maska[5],maska[5],maska[6],maska[6],maska[7],maska[7]); - __m256i mask1 = _mm256_set_epi32(maska[0],maska[0],maska[1],maska[1],maska[2],maska[2],maska[3],maska[3]); - __m256 lo, hi; - arrange_for_store(lo, hi, v.value_r, v.value_i); - _mm256_maskstore_ps(reinterpret_cast(a ), (__m256i) mask0, lo); - _mm256_maskstore_ps(reinterpret_cast(a+4), (__m256i) mask1, hi); -} -template<> -FASTOR_INLINE -void maskstore(double * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { - __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); - _mm256_maskstore_pd(a,(__m256i) mask, v.value); -} -template<> -FASTOR_INLINE -void maskstore(Int64 * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { - __m256i mask = _mm256_set_epi64x(maska[0],maska[1],maska[2],maska[3]); - _mm256_maskstore_epi64(reinterpret_cast(a),(__m256i) mask, v.value); -} -template<> -FASTOR_INLINE -void maskstore(float * FASTOR_RESTRICT a, const int (&maska)[8], SIMDVector &v) { - __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); - _mm256_maskstore_ps(a,(__m256i) mask, v.value); -} -template<> -FASTOR_INLINE -void maskstore(int * FASTOR_RESTRICT a, const int (&maska)[8], SIMDVector &v) { - __m256i mask = _mm256_set_epi32(maska[0],maska[1],maska[2],maska[3],maska[4],maska[5],maska[6],maska[7]); - _mm256_maskstore_epi32(a,(__m256i) mask, v.value); -} -template<> -FASTOR_INLINE -void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[2], SIMDVector,simd_abi::sse> &v) { - // Split the mask in to a higher and lower part - we need two masks for this - __m128i mask0 = _mm_set_epi64x(maska[1],maska[1]); - __m128i mask1 = _mm_set_epi64x(maska[0],maska[0]); - __m128d lo, hi; - arrange_for_store(lo, hi, v.value_r, v.value_i); - _mm_maskstore_pd(reinterpret_cast(a ), (__m128i) mask0, lo); - _mm_maskstore_pd(reinterpret_cast(a+1), (__m128i) mask1, hi); -} -template<> -FASTOR_INLINE -void maskstore(std::complex * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector,simd_abi::sse> &v) { - // Split the mask in to a higher and lower part - we need two masks for this - __m128i mask0 = _mm_set_epi32(maska[2],maska[2],maska[3],maska[3]); - __m128i mask1 = _mm_set_epi32(maska[0],maska[0],maska[1],maska[1]); - __m128 lo, hi; - arrange_for_store(lo, hi, v.value_r, v.value_i); - _mm_maskstore_ps(reinterpret_cast(a ), (__m128i) mask0, lo); - _mm_maskstore_ps(reinterpret_cast(a+2), (__m128i) mask1, hi); -} -template<> -FASTOR_INLINE -void maskstore(double * FASTOR_RESTRICT a, const int (&maska)[2], SIMDVector &v) { - __m128i mask = _mm_set_epi64x(maska[0],maska[1]); - _mm_maskstore_pd(a,(__m128i) mask, v.value); -} -template<> -FASTOR_INLINE -void maskstore(Int64 * FASTOR_RESTRICT a, const int (&maska)[2], SIMDVector &v) { - __m128i mask = _mm_set_epi64x(maska[0],maska[1]); - _mm_maskstore_epi64(reinterpret_cast(a),(__m128i) mask, v.value); -} -template<> -FASTOR_INLINE -void maskstore(float * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { - __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); - _mm_maskstore_ps(a,(__m128i) mask, v.value); -} -template<> -FASTOR_INLINE -void maskstore(int * FASTOR_RESTRICT a, const int (&maska)[4], SIMDVector &v) { - __m128i mask = _mm_set_epi32(maska[0],maska[1],maska[2],maska[3]); - _mm_maskstore_epi32(a,(__m128i) mask, v.value); -} -#endif // FASTOR_AVX2_IMPL -//---------------------------------------------------------------------------------------------------------------- - - - - - - - -// FMAs -//---------------------------------------------------------------------------------------------------------------- -template -FASTOR_INLINE SIMDVector fmadd(const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - return a*b+c; -} -template -FASTOR_INLINE SIMDVector fmsub(const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - return a*b-c; -} -template -FASTOR_INLINE SIMDVector fnmadd(const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - return c-a*b; -} - -#ifdef FASTOR_FMA_IMPL -// fmadd -template<> -FASTOR_INLINE SIMDVector fmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm_fmadd_ps(a.value,b.value,c.value); - return out; -} -template<> -FASTOR_INLINE SIMDVector fmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm256_fmadd_ps(a.value,b.value,c.value); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector fmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm512_fmadd_ps(a.value,b.value,c.value); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector fmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm_fmadd_pd(a.value,b.value,c.value); - return out; -} -template<> -FASTOR_INLINE SIMDVector fmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm256_fmadd_pd(a.value,b.value,c.value); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector fmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm512_fmadd_pd(a.value,b.value,c.value); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> fmadd,simd_abi::sse>( - const SIMDVector,simd_abi::sse> &a, - const SIMDVector,simd_abi::sse> &b, - const SIMDVector,simd_abi::sse> &c) { - SIMDVector,simd_abi::sse> out; - // ar*br - ai*bi + cr - out.value_r = _mm_fnmadd_ps(a.value_i,b.value_i,_mm_fmadd_ps(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm_fmadd_ps (a.value_i,b.value_r,_mm_fmadd_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx> fmadd,simd_abi::avx>( - const SIMDVector,simd_abi::avx> &a, - const SIMDVector,simd_abi::avx> &b, - const SIMDVector,simd_abi::avx> &c) { - SIMDVector,simd_abi::avx> out; - // ar*br - ai*bi + cr - out.value_r = _mm256_fnmadd_ps(a.value_i,b.value_i,_mm256_fmadd_ps(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm256_fmadd_ps (a.value_i,b.value_r,_mm256_fmadd_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx512> fmadd,simd_abi::avx512>( - const SIMDVector,simd_abi::avx512> &a, - const SIMDVector,simd_abi::avx512> &b, - const SIMDVector,simd_abi::avx512> &c) { - SIMDVector,simd_abi::avx512> out; - // ar*br - ai*bi + cr - out.value_r = _mm512_fnmadd_ps(a.value_i,b.value_i,_mm512_fmadd_ps(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm512_fmadd_ps (a.value_i,b.value_r,_mm512_fmadd_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> fmadd,simd_abi::sse>( - const SIMDVector,simd_abi::sse> &a, - const SIMDVector,simd_abi::sse> &b, - const SIMDVector,simd_abi::sse> &c) { - SIMDVector,simd_abi::sse> out; - // ar*br - ai*bi + cr - out.value_r = _mm_fnmadd_pd(a.value_i,b.value_i,_mm_fmadd_pd(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm_fmadd_pd (a.value_i,b.value_r,_mm_fmadd_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx> fmadd,simd_abi::avx>( - const SIMDVector,simd_abi::avx> &a, - const SIMDVector,simd_abi::avx> &b, - const SIMDVector,simd_abi::avx> &c) { - SIMDVector,simd_abi::avx> out; - // ar*br - ai*bi + cr - out.value_r = _mm256_fnmadd_pd(a.value_i,b.value_i,_mm256_fmadd_pd(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm256_fmadd_pd (a.value_i,b.value_r,_mm256_fmadd_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx512> fmadd,simd_abi::avx512>( - const SIMDVector,simd_abi::avx512> &a, - const SIMDVector,simd_abi::avx512> &b, - const SIMDVector,simd_abi::avx512> &c) { - SIMDVector,simd_abi::avx512> out; - // ar*br - ai*bi + cr - out.value_r = _mm512_fnmadd_pd(a.value_i,b.value_i,_mm512_fmadd_pd(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm512_fmadd_pd (a.value_i,b.value_r,_mm512_fmadd_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -#endif - -// fmsub -template<> -FASTOR_INLINE SIMDVector fmsub( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm_fmsub_ps(a.value,b.value,c.value); - return out; -} -template<> -FASTOR_INLINE SIMDVector fmsub( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm256_fmsub_ps(a.value,b.value,c.value); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector fmsub( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm512_fmsub_ps(a.value,b.value,c.value); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector fmsub( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm_fmsub_pd(a.value,b.value,c.value); - return out; -} -template<> -FASTOR_INLINE SIMDVector fmsub( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm256_fmsub_pd(a.value,b.value,c.value); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector fmsub( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm512_fmsub_pd(a.value,b.value,c.value); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> fmsub,simd_abi::sse>( - const SIMDVector,simd_abi::sse> &a, - const SIMDVector,simd_abi::sse> &b, - const SIMDVector,simd_abi::sse> &c) { - SIMDVector,simd_abi::sse> out; - // ar*br - ai*bi - cr - out.value_r = _mm_fnmadd_ps(a.value_i,b.value_i,_mm_fmsub_ps(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br - ci - out.value_i = _mm_fmadd_ps (a.value_i,b.value_r,_mm_fmsub_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx> fmsub,simd_abi::avx>( - const SIMDVector,simd_abi::avx> &a, - const SIMDVector,simd_abi::avx> &b, - const SIMDVector,simd_abi::avx> &c) { - SIMDVector,simd_abi::avx> out; - // ar*br - ai*bi + cr - out.value_r = _mm256_fnmadd_ps(a.value_i,b.value_i,_mm256_fmsub_ps(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm256_fmadd_ps (a.value_i,b.value_r,_mm256_fmsub_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx512> fmsub,simd_abi::avx512>( - const SIMDVector,simd_abi::avx512> &a, - const SIMDVector,simd_abi::avx512> &b, - const SIMDVector,simd_abi::avx512> &c) { - SIMDVector,simd_abi::avx512> out; - // ar*br - ai*bi + cr - out.value_r = _mm512_fnmadd_ps(a.value_i,b.value_i,_mm512_fmsub_ps(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm512_fmadd_ps (a.value_i,b.value_r,_mm512_fmsub_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> fmsub,simd_abi::sse>( - const SIMDVector,simd_abi::sse> &a, - const SIMDVector,simd_abi::sse> &b, - const SIMDVector,simd_abi::sse> &c) { - SIMDVector,simd_abi::sse> out; - // ar*br - ai*bi - cr - out.value_r = _mm_fnmadd_pd(a.value_i,b.value_i,_mm_fmsub_pd(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br - ci - out.value_i = _mm_fmadd_pd (a.value_i,b.value_r,_mm_fmsub_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx> fmsub,simd_abi::avx>( - const SIMDVector,simd_abi::avx> &a, - const SIMDVector,simd_abi::avx> &b, - const SIMDVector,simd_abi::avx> &c) { - SIMDVector,simd_abi::avx> out; - // ar*br - ai*bi + cr - out.value_r = _mm256_fnmadd_pd(a.value_i,b.value_i,_mm256_fmsub_pd(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm256_fmadd_pd (a.value_i,b.value_r,_mm256_fmsub_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx512> fmsub,simd_abi::avx512>( - const SIMDVector,simd_abi::avx512> &a, - const SIMDVector,simd_abi::avx512> &b, - const SIMDVector,simd_abi::avx512> &c) { - SIMDVector,simd_abi::avx512> out; - // ar*br - ai*bi + cr - out.value_r = _mm512_fnmadd_pd(a.value_i,b.value_i,_mm512_fmsub_pd(a.value_r,b.value_r,c.value_r)); - // ar*bi + ai*br + ci - out.value_i = _mm512_fmadd_pd (a.value_i,b.value_r,_mm512_fmsub_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -#endif - -// fnmadd -template<> -FASTOR_INLINE SIMDVector fnmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm_fnmadd_ps(a.value,b.value,c.value); - return out; -} -template<> -FASTOR_INLINE SIMDVector fnmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm256_fnmadd_ps(a.value,b.value,c.value); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector fnmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm512_fnmadd_ps(a.value,b.value,c.value); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector fnmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm_fnmadd_pd(a.value,b.value,c.value); - return out; -} -template<> -FASTOR_INLINE SIMDVector fnmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm256_fnmadd_pd(a.value,b.value,c.value); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector fnmadd( - const SIMDVector &a, const SIMDVector &b, const SIMDVector &c) { - SIMDVector out; - out.value = _mm512_fnmadd_pd(a.value,b.value,c.value); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> fnmadd,simd_abi::sse>( - const SIMDVector,simd_abi::sse> &a, - const SIMDVector,simd_abi::sse> &b, - const SIMDVector,simd_abi::sse> &c) { - SIMDVector,simd_abi::sse> out; - // -ar*br + ai*bi + cr - out.value_r = _mm_fmadd_ps (a.value_i,b.value_i,_mm_fnmadd_ps(a.value_r,b.value_r,c.value_r)); - // -ar*bi - ai*br + ci - out.value_i = _mm_fnmadd_ps(a.value_i,b.value_r,_mm_fnmadd_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx> fnmadd,simd_abi::avx>( - const SIMDVector,simd_abi::avx> &a, - const SIMDVector,simd_abi::avx> &b, - const SIMDVector,simd_abi::avx> &c) { - SIMDVector,simd_abi::avx> out; - // -ar*br + ai*bi + cr - out.value_r = _mm256_fmadd_ps (a.value_i,b.value_i,_mm256_fnmadd_ps(a.value_r,b.value_r,c.value_r)); - // -ar*bi - ai*br + ci - out.value_i = _mm256_fnmadd_ps(a.value_i,b.value_r,_mm256_fnmadd_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx512> fnmadd,simd_abi::avx512>( - const SIMDVector,simd_abi::avx512> &a, - const SIMDVector,simd_abi::avx512> &b, - const SIMDVector,simd_abi::avx512> &c) { - SIMDVector,simd_abi::avx512> out; - // -ar*br + ai*bi + cr - out.value_r = _mm512_fmadd_ps (a.value_i,b.value_i,_mm512_fnmadd_ps(a.value_r,b.value_r,c.value_r)); - // -ar*bi - ai*br + ci - out.value_i = _mm512_fnmadd_ps(a.value_i,b.value_r,_mm512_fnmadd_ps(a.value_r,b.value_i,c.value_i)); - return out; -} -#endif -template<> -FASTOR_INLINE SIMDVector,simd_abi::sse> fnmadd,simd_abi::sse>( - const SIMDVector,simd_abi::sse> &a, - const SIMDVector,simd_abi::sse> &b, - const SIMDVector,simd_abi::sse> &c) { - SIMDVector,simd_abi::sse> out; - // -ar*br + ai*bi + cr - out.value_r = _mm_fmadd_pd (a.value_i,b.value_i,_mm_fnmadd_pd(a.value_r,b.value_r,c.value_r)); - // -ar*bi - ai*br + ci - out.value_i = _mm_fnmadd_pd(a.value_i,b.value_r,_mm_fnmadd_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx> fnmadd,simd_abi::avx>( - const SIMDVector,simd_abi::avx> &a, - const SIMDVector,simd_abi::avx> &b, - const SIMDVector,simd_abi::avx> &c) { - SIMDVector,simd_abi::avx> out; - // -ar*br + ai*bi + cr - out.value_r = _mm256_fmadd_pd (a.value_i,b.value_i,_mm256_fnmadd_pd(a.value_r,b.value_r,c.value_r)); - // -ar*bi - ai*br + ci - out.value_i = _mm256_fnmadd_pd(a.value_i,b.value_r,_mm256_fnmadd_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -#ifdef FASTOR_AVX512F_IMPL -template<> -FASTOR_INLINE SIMDVector,simd_abi::avx512> fnmadd,simd_abi::avx512>( - const SIMDVector,simd_abi::avx512> &a, - const SIMDVector,simd_abi::avx512> &b, - const SIMDVector,simd_abi::avx512> &c) { - SIMDVector,simd_abi::avx512> out; - // -ar*br + ai*bi + cr - out.value_r = _mm512_fmadd_pd (a.value_i,b.value_i,_mm512_fnmadd_pd(a.value_r,b.value_r,c.value_r)); - // -ar*bi - ai*br + ci - out.value_i = _mm512_fnmadd_pd(a.value_i,b.value_r,_mm512_fnmadd_pd(a.value_r,b.value_i,c.value_i)); - return out; -} -#endif - -#endif -//---------------------------------------------------------------------------------------------------------------- - - - - -// Binary comparison ops -//----------------------------------------------------------------------------------------------------------------// -#define FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(OP) \ -template \ -FASTOR_INLINE SIMDVector::Size>> operator OP(const SIMDVector &a, const SIMDVector &b) { \ - constexpr FASTOR_INDEX Size = SIMDVector::Size;\ - FASTOR_ARCH_ALIGN T val_a[Size];\ - a.store(val_a);\ - FASTOR_ARCH_ALIGN T val_b[Size];\ - b.store(val_b);\ - SIMDVector::Size>> out;\ - FASTOR_ARCH_ALIGN bool val_out[Size];\ - out.store(val_out);\ - for (FASTOR_INDEX i=0; i) -FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(<) -FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(>=) -FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(<=) -FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(&&) -FASTOR_MAKE_BINARY_CMP_SIMDVECTORS_OPS_(||) - - -#define FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(OP) \ -template \ -FASTOR_INLINE SIMDVector::Size>> operator OP(const SIMDVector &a, U b) { \ - constexpr FASTOR_INDEX Size = SIMDVector::Size;\ - FASTOR_ARCH_ALIGN T val_a[Size];\ - a.store(val_a);\ - SIMDVector::Size>> out;\ - FASTOR_ARCH_ALIGN bool val_out[Size];\ - out.store(val_out);\ - for (FASTOR_INDEX i=0; i) -FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(<) -FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(>=) -FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(<=) -FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(&&) -FASTOR_MAKE_BINARY_CMP_SIMDVECTOR_SCALAR_OPS_(||) - - -#define FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(OP) \ -template \ -FASTOR_INLINE SIMDVector::Size>> operator OP(U a, const SIMDVector &b) { \ - constexpr FASTOR_INDEX Size = SIMDVector::Size;\ - FASTOR_ARCH_ALIGN T val_b[Size];\ - b.store(val_b);\ - SIMDVector::Size>> out;\ - FASTOR_ARCH_ALIGN bool val_out[Size];\ - out.store(val_out);\ - for (FASTOR_INDEX i=0; i) -FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(<) -FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(>=) -FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(<=) -FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(&&) -FASTOR_MAKE_BINARY_CMP_SCALAR_SIMDVECTOR_OPS_(||) -//----------------------------------------------------------------------------------------------------------------// - - -} // end of namespace Fastor - -#endif // SIMD_VECTOR_COMMON_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_complex_double.h b/noarch/include/Fastor/simd_vector/simd_vector_complex_double.h deleted file mode 100644 index 59e8b88c..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_complex_double.h +++ /dev/null @@ -1,1611 +0,0 @@ -#ifndef SIMD_VECTOR_COMPLEX_DOUBLE_H -#define SIMD_VECTOR_COMPLEX_DOUBLE_H - -#include "Fastor/util/extended_algorithms.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/simd_vector_base.h" -#include "Fastor/simd_vector/simd_vector_double.h" -#include -#include - -namespace Fastor { - - -// AVX512 VERSION -//------------------------------------------------------------------------------------------------------------ - -#ifdef FASTOR_AVX512F_IMPL - -template <> -struct SIMDVector, simd_abi::avx512> { - using vector_type = SIMDVector, simd_abi::avx512>; - using value_type = __m512d; - using scalar_value_type = std::complex; - using abi_type = simd_abi::avx512; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value_r(_mm512_setzero_pd()), value_i(_mm512_setzero_pd()) {} - FASTOR_INLINE SIMDVector(std::complex num) { - value_r = _mm512_set1_pd(num.real()); - value_i = _mm512_set1_pd(num.imag()); - } - FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} - FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - - FASTOR_INLINE SIMDVector,simd_abi::avx512> operator=(std::complex num) { - value_r = _mm512_set1_pd(num.real()); - value_i = _mm512_set1_pd(num.imag()); - return *this; - } - - FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { - if (Aligned) - complex_aligned_store(data); - else - complex_unaligned_store(data); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { - if (!Aligned) - complex_mask_unaligned_load(data,mask); - else - complex_mask_aligned_load(data,mask); - } - FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { - if (!Aligned) - complex_mask_unaligned_store(data,mask); - else - complex_mask_aligned_store(data,mask); - } - - FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { - return scalar_value_type(reinterpret_cast(&value_r)[i],reinterpret_cast(&value_i)[i]); - } - - FASTOR_INLINE SIMDVector real() const { - return value_r; - } - FASTOR_INLINE SIMDVector imag() const { - return value_i; - } - - FASTOR_INLINE void set(std::complex num) { - value_r = _mm512_set1_pd(num.real()); - value_i = _mm512_set1_pd(num.imag()); - } - FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, - scalar_value_type num2, scalar_value_type num3, - scalar_value_type num4, scalar_value_type num5, - scalar_value_type num6, scalar_value_type num7) { - const scalar_value_type tmp[Size] = {num0,num1,num2,num3,num4,num5,num6,num7}; - complex_unaligned_load(tmp); - } - - // In-place operators - template,bool> = false> - FASTOR_INLINE void operator+=(U num) { - value_r = _mm512_add_pd(value_r,_mm512_set1_pd(num)); - } - FASTOR_INLINE void operator+=(scalar_value_type num) { - *this += vector_type(num); - } - FASTOR_INLINE void operator+=(const vector_type &a) { - value_r = _mm512_add_pd(value_r,a.value_r); - value_i = _mm512_add_pd(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - value_r = _mm512_sub_pd(value_r,_mm512_set1_pd(num)); - } - FASTOR_INLINE void operator-=(scalar_value_type num) { - *this -= vector_type(num); - } - FASTOR_INLINE void operator-=(const vector_type &a) { - value_r = _mm512_sub_pd(value_r,a.value_r); - value_i = _mm512_sub_pd(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - __m512d val = _mm512_set1_pd(num); - value_r = _mm512_mul_pd(value_r,val); - value_i = _mm512_mul_pd(value_i,val); - } - FASTOR_INLINE void operator*=(scalar_value_type num) { - *this *= vector_type(num); - } - FASTOR_INLINE void operator*=(const vector_type &a) { -#ifdef FASTOR_FMA_IMPL - __m512d tmp = _mm512_fmsub_pd(value_r,a.value_r,_mm512_mul_pd(value_i,a.value_i)); - value_i = _mm512_fmadd_pd(value_r,a.value_i,_mm512_mul_pd(value_i,a.value_r)); -#else - __m512d tmp = _mm512_sub_pd(_mm512_mul_pd(value_r,a.value_r),_mm512_mul_pd(value_i,a.value_i)); - value_i = _mm512_add_pd(_mm512_mul_pd(value_r,a.value_i),_mm512_mul_pd(value_i,a.value_r)); -#endif - value_r = tmp; - } - - template,bool> = false> - FASTOR_INLINE void operator/=(U num) { - __m512d val = _mm512_set1_pd(num); - value_r = _mm512_div_pd(value_r,val); - value_i = _mm512_div_pd(value_i,val); - } - FASTOR_INLINE void operator/=(scalar_value_type num) { - *this /= vector_type(num); - } - FASTOR_INLINE void operator/=(const vector_type &a) { - __m512d tmp = value_r; -#ifdef FASTOR_FMA_IMPL - value_r = _mm512_fmadd_pd(value_r , a.value_r, _mm512_mul_pd(value_i,a.value_i)); - value_i = _mm512_fmsub_pd(value_i , a.value_r, _mm512_mul_pd(tmp,a.value_i)); - __m512d den = _mm512_fmadd_pd(a.value_r, a.value_r, _mm512_mul_pd(a.value_i,a.value_i)); -#else - value_r = _mm512_add_pd(_mm512_mul_pd(value_r , a.value_r), _mm512_mul_pd(value_i,a.value_i)); - value_i = _mm512_sub_pd(_mm512_mul_pd(value_i , a.value_r), _mm512_mul_pd(tmp,a.value_i)); - __m512d den = _mm512_add_pd(_mm512_mul_pd(a.value_r, a.value_r), _mm512_mul_pd(a.value_i,a.value_i)); -#endif - value_r = _mm512_div_pd(value_r,den); - value_i = _mm512_div_pd(value_i,den); - } - // end of in-place operators - - FASTOR_INLINE scalar_value_type sum() const { -#ifdef FASTOR_HAS_AVX512_REDUCE_ADD - return scalar_value_type(_mm512_reduce_add_pd(value_r),_mm512_reduce_add_pd(value_i)); -#else - __m256d lor = _mm512_castpd512_pd256(value_r); - __m256d hir = _mm512_extractf64x4_pd(value_r,0x1); - __m256d loi = _mm512_castpd512_pd256(value_i); - __m256d hii = _mm512_extractf64x4_pd(value_i,0x1); - return scalar_value_type(_mm256_sum_pd(_mm256_add_pd(lor,hir)), _mm256_sum_pd(_mm256_add_pd(loi,hii))); -#endif - } - FASTOR_INLINE scalar_value_type product() const { - vector_type tmp(*this); - return tmp[0]*tmp[1]*tmp[2]*tmp[3]*tmp[4]*tmp[5]*tmp[6]*tmp[7]; - } - FASTOR_INLINE vector_type reverse() const { - vector_type out; - out.value_r = _mm512_reverse_pd(value_r); - out.value_i = _mm512_reverse_pd(value_i); - return out; - } - /* Actual magnitude - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector magnitude() const { -#ifdef FASTOR_FMA_IMPL - return _mm512_sqrt_pd(_mm512_fmadd_pd(value_r,value_r,_mm512_mul_pd(value_i,value_i))); -#else - return _mm512_sqrt_pd(_mm512_add_pd(_mm512_mul_pd(value_r,value_r),_mm512_mul_pd(value_i,value_i))); -#endif - } - /* STL compliant squared norm - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector norm() const { -#ifdef FASTOR_FMA_IMPL - return _mm512_fmadd_pd(value_r,value_r,_mm512_mul_pd(value_i,value_i)); -#else - return _mm512_add_pd(_mm512_mul_pd(value_r,value_r),_mm512_mul_pd(value_i,value_i)); -#endif - } - // Magnitude based minimum - FASTOR_INLINE scalar_value_type minimum() const; - // Magnitude based maximum - FASTOR_INLINE scalar_value_type maximum() const; - - FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { - vector_type out(*this); - out *= other; - return out.sum(); - } - - value_type value_r; - value_type value_i; - -protected: - FASTOR_INLINE void complex_aligned_load(const std::complex *data) { - __m512d lo = _mm512_load_pd(reinterpret_cast(data )); - __m512d hi = _mm512_load_pd(reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); - } - FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { - __m512d lo = _mm512_loadu_pd(reinterpret_cast(data )); - __m512d hi = _mm512_loadu_pd(reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); - } - - FASTOR_INLINE void complex_aligned_store(std::complex *data) const { - __m512d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm512_store_pd(reinterpret_cast(data ), lo); - _mm512_store_pd(reinterpret_cast(data+4), hi); - } - FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { - __m512d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm512_storeu_pd(reinterpret_cast(data ), lo); - _mm512_storeu_pd(reinterpret_cast(data+4), hi); - } - - FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { -#ifdef FASTOR_HAS_AVX512_MASKS - __m512d lo, hi; - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - lo = _mm512_mask_load_pd(lo, mask0, reinterpret_cast(data )); - hi = _mm512_mask_load_pd(hi, mask1, reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm512_setzero_pd(); - value_i = _mm512_setzero_pd(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - lo = _mm512_mask_loadu_pd(lo, mask0, reinterpret_cast(data )); - hi = _mm512_mask_loadu_pd(hi, mask1, reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm512_setzero_pd(); - value_i = _mm512_setzero_pd(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - _mm512_mask_store_pd(reinterpret_cast(data ), mask0, lo); - _mm512_mask_store_pd(reinterpret_cast(data+4), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { -#ifdef FASTOR_HAS_AVX512_MASKS - __m512d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - _mm512_mask_storeu_pd(reinterpret_cast(data ), mask0, lo); - _mm512_mask_storeu_pd(reinterpret_cast(data+4), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx512> a) { - // ICC crashes without a copy - const __m512d vr = a.value_r; - const __m512d vi = a.value_i; - const double* value_r = reinterpret_cast(&vr); - const double* value_i = reinterpret_cast(&vi); - os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " - << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " - << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " - << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j, " - << value_r[4] << signum_string(value_i[4]) << std::abs(value_i[4]) << "j, " - << value_r[5] << signum_string(value_i[5]) << std::abs(value_i[5]) << "j, " - << value_r[6] << signum_string(value_i[6]) << std::abs(value_i[6]) << "j, " - << value_r[7] << signum_string(value_i[7]) << std::abs(value_i[7]) << "j" << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; - out.value_r = _mm512_add_pd(a.value_r,b.value_r); - out.value_i = _mm512_add_pd(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a + SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) + b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out(a); - out.value_r = _mm512_add_pd(a.value_r,_mm512_set1_pd(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(U a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out(b); - out.value_r = _mm512_add_pd(_mm512_set1_pd(a), b.value_r); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &b) { - return b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; - out.value_r = _mm512_sub_pd(a.value_r,b.value_r); - out.value_i = _mm512_sub_pd(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a - SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) - b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out(a); - out.value_r = _mm512_sub_pd(a.value_r,_mm512_set1_pd(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(U a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(std::complex(a,0)) - b; -} -/* This is negation and not complex conjugate */ -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out; - out.value_r = _mm512_neg_pd(a.value_r); - out.value_i = _mm512_neg_pd(a.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm512_fmsub_pd(a.value_r,b.value_r,_mm512_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm512_fmadd_pd(a.value_r,b.value_i,_mm512_mul_pd(a.value_i,b.value_r)); -#else - out.value_r = _mm512_sub_pd(_mm512_mul_pd(a.value_r,b.value_r),_mm512_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm512_add_pd(_mm512_mul_pd(a.value_r,b.value_i),_mm512_mul_pd(a.value_i,b.value_r)); -#endif - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a * SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) * b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out; - __m512d val = _mm512_set1_pd(b); - out.value_r = _mm512_mul_pd(a.value_r,val); - out.value_i = _mm512_mul_pd(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(U a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; - __m512d val = _mm512_set1_pd(a); - out.value_r = _mm512_mul_pd(val,b.value_r); - out.value_i = _mm512_mul_pd(val,b.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm512_fmadd_pd(a.value_r,b.value_r,_mm512_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm512_fmsub_pd(a.value_i,b.value_r,_mm512_mul_pd(a.value_r,b.value_i)); - __m512d den = _mm512_fmadd_pd(b.value_r,b.value_r,_mm512_mul_pd(b.value_i,b.value_i)); -#else - out.value_r = _mm512_add_pd(_mm512_mul_pd(a.value_r,b.value_r),_mm512_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm512_sub_pd(_mm512_mul_pd(a.value_i,b.value_r),_mm512_mul_pd(a.value_r,b.value_i)); - __m512d den = _mm512_add_pd(_mm512_mul_pd(b.value_r,b.value_r),_mm512_mul_pd(b.value_i,b.value_i)); -#endif - out.value_r = _mm512_div_pd(out.value_r,den); - out.value_i = _mm512_div_pd(out.value_i,den); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a / SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) / b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out; - __m512d val = _mm512_set1_pd(b); - out.value_r = _mm512_div_pd(a.value_r,val); - out.value_i = _mm512_div_pd(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(U a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(std::complex(a,0)) / b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -rcp(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out; -#ifdef FASTOR_FMA_IMPL - __m512d den = _mm512_fmadd_pd(a.value_r,a.value_r,_mm512_mul_pd(a.value_i,a.value_i)); -#else - __m512d den = _mm512_add_pd(_mm512_mul_pd(a.value_r,a.value_r),_mm512_mul_pd(a.value_i,a.value_i)); -#endif - out.value_r = _mm512_div_pd(out.value_r,den); - out.value_i = _mm512_neg_pd(_mm512_div_pd(out.value_i,den)); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -sqrt(const SIMDVector,simd_abi::avx512> &a) = delete; - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -rsqrt(const SIMDVector,simd_abi::avx512> &a) = delete; - -/* This intentionally return a complex vector */ -FASTOR_INLINE SIMDVector,simd_abi::avx512> -abs(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out; - out.value_r = a.magnitude().value; - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -conj(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out(a); - out.value_i = _mm512_neg_pd(out.value_i); - return out; -} - -/* Argument or phase angle */ -FASTOR_INLINE SIMDVector,simd_abi::avx512> -arg(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out(a); - for (FASTOR_INDEX i=0UL; i<8UL; ++i) { - ((double*)&out.value_r)[i] = std::atan2(((double*)&a.value_i)[i],((double*)&a.value_r)[i]); - } - return out; -} -//------------------------------------------------------------------------------------------------------------ - -#endif - - - -// AVX VERSION -//------------------------------------------------------------------------------------------------------------ - -#ifdef FASTOR_AVX_IMPL - -template <> -struct SIMDVector, simd_abi::avx> { - using vector_type = SIMDVector, simd_abi::avx>; - using value_type = __m256d; - using scalar_value_type = std::complex; - using abi_type = simd_abi::avx; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value_r(_mm256_setzero_pd()), value_i(_mm256_setzero_pd()) {} - FASTOR_INLINE SIMDVector(std::complex num) { - value_r = _mm256_set1_pd(num.real()); - value_i = _mm256_set1_pd(num.imag()); - } - FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} - FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - - FASTOR_INLINE SIMDVector,simd_abi::avx> operator=(std::complex num) { - value_r = _mm256_set1_pd(num.real()); - value_i = _mm256_set1_pd(num.imag()); - return *this; - } - - FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { - if (Aligned) - complex_aligned_store(data); - else - complex_unaligned_store(data); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { - if (!Aligned) - complex_mask_unaligned_load(data,mask); - else - complex_mask_aligned_load(data,mask); - } - FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { - if (!Aligned) - complex_mask_unaligned_store(data,mask); - else - complex_mask_aligned_store(data,mask); - } - - FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { - if (i == 0) { return scalar_value_type(_mm256_get0_pd(value_r), _mm256_get0_pd(value_i)); } - else if (i == 1) { return scalar_value_type(_mm256_get1_pd(value_r), _mm256_get1_pd(value_i)); } - else if (i == 2) { return scalar_value_type(_mm256_get2_pd(value_r), _mm256_get2_pd(value_i)); } - else { return scalar_value_type(_mm256_get3_pd(value_r), _mm256_get3_pd(value_i)); } - } - - FASTOR_INLINE SIMDVector real() const { - return value_r; - } - FASTOR_INLINE SIMDVector imag() const { - return value_i; - } - - FASTOR_INLINE void set(std::complex num) { - value_r = _mm256_set1_pd(num.real()); - value_i = _mm256_set1_pd(num.imag()); - } - FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, - scalar_value_type num2, scalar_value_type num3) { - const scalar_value_type tmp[Size] = {num0,num1,num2,num3}; - complex_unaligned_load(tmp); - } - - // In-place operators - template,bool> = false> - FASTOR_INLINE void operator+=(U num) { - value_r = _mm256_add_pd(value_r,_mm256_set1_pd(num)); - } - FASTOR_INLINE void operator+=(scalar_value_type num) { - *this += vector_type(num); - } - FASTOR_INLINE void operator+=(const vector_type &a) { - value_r = _mm256_add_pd(value_r,a.value_r); - value_i = _mm256_add_pd(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - value_r = _mm256_sub_pd(value_r,_mm256_set1_pd(num)); - } - FASTOR_INLINE void operator-=(scalar_value_type num) { - *this -= vector_type(num); - } - FASTOR_INLINE void operator-=(const vector_type &a) { - value_r = _mm256_sub_pd(value_r,a.value_r); - value_i = _mm256_sub_pd(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - __m256d val = _mm256_set1_pd(num); - value_r = _mm256_mul_pd(value_r,val); - value_i = _mm256_mul_pd(value_i,val); - } - FASTOR_INLINE void operator*=(scalar_value_type num) { - *this *= vector_type(num); - } - FASTOR_INLINE void operator*=(const vector_type &a) { -#ifdef FASTOR_FMA_IMPL - __m256d tmp = _mm256_fmsub_pd(value_r,a.value_r,_mm256_mul_pd(value_i,a.value_i)); - value_i = _mm256_fmadd_pd(value_r,a.value_i,_mm256_mul_pd(value_i,a.value_r)); -#else - __m256d tmp = _mm256_sub_pd(_mm256_mul_pd(value_r,a.value_r),_mm256_mul_pd(value_i,a.value_i)); - value_i = _mm256_add_pd(_mm256_mul_pd(value_r,a.value_i),_mm256_mul_pd(value_i,a.value_r)); -#endif - value_r = tmp; - } - - template,bool> = false> - FASTOR_INLINE void operator/=(U num) { - __m256d val = _mm256_set1_pd(num); - value_r = _mm256_div_pd(value_r,val); - value_i = _mm256_div_pd(value_i,val); - } - FASTOR_INLINE void operator/=(scalar_value_type num) { - *this /= vector_type(num); - } - FASTOR_INLINE void operator/=(const vector_type &a) { - __m256d tmp = value_r; -#ifdef FASTOR_FMA_IMPL - value_r = _mm256_fmadd_pd(value_r , a.value_r, _mm256_mul_pd(value_i,a.value_i)); - value_i = _mm256_fmsub_pd(value_i , a.value_r, _mm256_mul_pd(tmp,a.value_i)); - __m256d den = _mm256_fmadd_pd(a.value_r, a.value_r, _mm256_mul_pd(a.value_i,a.value_i)); -#else - value_r = _mm256_add_pd(_mm256_mul_pd(value_r , a.value_r), _mm256_mul_pd(value_i,a.value_i)); - value_i = _mm256_sub_pd(_mm256_mul_pd(value_i , a.value_r), _mm256_mul_pd(tmp,a.value_i)); - __m256d den = _mm256_add_pd(_mm256_mul_pd(a.value_r, a.value_r), _mm256_mul_pd(a.value_i,a.value_i)); -#endif - value_r = _mm256_div_pd(value_r,den); - value_i = _mm256_div_pd(value_i,den); - } - // end of in-place operators - - FASTOR_INLINE scalar_value_type sum() const { - return scalar_value_type(_mm256_sum_pd(value_r),_mm256_sum_pd(value_i)); - } - FASTOR_INLINE scalar_value_type product() const { - vector_type tmp(*this); - return tmp[0]*tmp[1]*tmp[2]*tmp[3]; - } - FASTOR_INLINE vector_type reverse() const { - vector_type out; - out.value_r = _mm256_reverse_pd(value_r); - out.value_i = _mm256_reverse_pd(value_i); - return out; - } - /* Actual magnitude - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector magnitude() const { -#ifdef FASTOR_FMA_IMPL - return _mm256_sqrt_pd(_mm256_fmadd_pd(value_r,value_r,_mm256_mul_pd(value_i,value_i))); -#else - return _mm256_sqrt_pd(_mm256_add_pd(_mm256_mul_pd(value_r,value_r),_mm256_mul_pd(value_i,value_i))); -#endif - } - /* STL compliant squared norm - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector norm() const { -#ifdef FASTOR_FMA_IMPL - return _mm256_fmadd_pd(value_r,value_r,_mm256_mul_pd(value_i,value_i)); -#else - return _mm256_add_pd(_mm256_mul_pd(value_r,value_r),_mm256_mul_pd(value_i,value_i)); -#endif - } - // Magnitude based minimum - FASTOR_INLINE scalar_value_type minimum() const; - // Magnitude based maximum - FASTOR_INLINE scalar_value_type maximum() const; - - FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { - vector_type out(*this); - out *= other; - return out.sum(); - } - - value_type value_r; - value_type value_i; - -protected: - FASTOR_INLINE void complex_aligned_load(const std::complex *data) { - __m256d lo = _mm256_load_pd(reinterpret_cast(data )); - __m256d hi = _mm256_load_pd(reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); - } - FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { - __m256d lo = _mm256_loadu_pd(reinterpret_cast(data )); - __m256d hi = _mm256_loadu_pd(reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); - } - - FASTOR_INLINE void complex_aligned_store(std::complex *data) const { - __m256d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm256_store_pd(reinterpret_cast(data ), lo); - _mm256_store_pd(reinterpret_cast(data+2), hi); - } - FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { - __m256d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm256_storeu_pd(reinterpret_cast(data ), lo); - _mm256_storeu_pd(reinterpret_cast(data+2), hi); - } - - FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { -#ifdef FASTOR_HAS_AVX512_MASKS - __m256d lo, hi; - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - lo = _mm256_mask_load_pd(lo, mask0, reinterpret_cast(data )); - hi = _mm256_mask_load_pd(hi, mask1, reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm256_setzero_pd(); - value_i = _mm256_setzero_pd(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - lo = _mm256_mask_loadu_pd(lo, mask0, reinterpret_cast(data )); - hi = _mm256_mask_loadu_pd(hi, mask1, reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm256_setzero_pd(); - value_i = _mm256_setzero_pd(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - _mm256_mask_store_pd(reinterpret_cast(data ), mask0, lo); - _mm256_mask_store_pd(reinterpret_cast(data+2), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { -#ifdef FASTOR_HAS_AVX512_MASKS - __m256d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - _mm256_mask_storeu_pd(reinterpret_cast(data ), mask0, lo); - _mm256_mask_storeu_pd(reinterpret_cast(data+2), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx> a) { - // ICC crashes without a copy - const __m256d vr = a.value_r; - const __m256d vi = a.value_i; - const double* value_r = reinterpret_cast(&vr); - const double* value_i = reinterpret_cast(&vi); - os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " - << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " - << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " - << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j" << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; - out.value_r = _mm256_add_pd(a.value_r,b.value_r); - out.value_i = _mm256_add_pd(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a + SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) + b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out(a); - out.value_r = _mm256_add_pd(a.value_r,_mm256_set1_pd(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(U a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out(b); - out.value_r = _mm256_add_pd(_mm256_set1_pd(a), b.value_r); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &b) { - return b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; - out.value_r = _mm256_sub_pd(a.value_r,b.value_r); - out.value_i = _mm256_sub_pd(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a - SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) - b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out(a); - out.value_r = _mm256_sub_pd(a.value_r,_mm256_set1_pd(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(U a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(std::complex(a,0)) - b; -} -/* This is negation and not complex conjugate */ -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out; - out.value_r = _mm256_neg_pd(a.value_r); - out.value_i = _mm256_neg_pd(a.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm256_fmsub_pd(a.value_r,b.value_r,_mm256_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm256_fmadd_pd(a.value_r,b.value_i,_mm256_mul_pd(a.value_i,b.value_r)); -#else - out.value_r = _mm256_sub_pd(_mm256_mul_pd(a.value_r,b.value_r),_mm256_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm256_add_pd(_mm256_mul_pd(a.value_r,b.value_i),_mm256_mul_pd(a.value_i,b.value_r)); -#endif - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a * SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) * b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out; - __m256d val = _mm256_set1_pd(b); - out.value_r = _mm256_mul_pd(a.value_r,val); - out.value_i = _mm256_mul_pd(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(U a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; - __m256d val = _mm256_set1_pd(a); - out.value_r = _mm256_mul_pd(val,b.value_r); - out.value_i = _mm256_mul_pd(val,b.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm256_fmadd_pd(a.value_r,b.value_r,_mm256_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm256_fmsub_pd(a.value_i,b.value_r,_mm256_mul_pd(a.value_r,b.value_i)); - __m256d den = _mm256_fmadd_pd(b.value_r,b.value_r,_mm256_mul_pd(b.value_i,b.value_i)); -#else - out.value_r = _mm256_add_pd(_mm256_mul_pd(a.value_r,b.value_r),_mm256_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm256_sub_pd(_mm256_mul_pd(a.value_i,b.value_r),_mm256_mul_pd(a.value_r,b.value_i)); - __m256d den = _mm256_add_pd(_mm256_mul_pd(b.value_r,b.value_r),_mm256_mul_pd(b.value_i,b.value_i)); -#endif - out.value_r = _mm256_div_pd(out.value_r,den); - out.value_i = _mm256_div_pd(out.value_i,den); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a / SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) / b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out; - __m256d val = _mm256_set1_pd(b); - out.value_r = _mm256_div_pd(a.value_r,val); - out.value_i = _mm256_div_pd(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(U a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(std::complex(a,0)) / b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -rcp(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out; -#ifdef FASTOR_FMA_IMPL - __m256d den = _mm256_fmadd_pd(a.value_r,a.value_r,_mm256_mul_pd(a.value_i,a.value_i)); -#else - __m256d den = _mm256_add_pd(_mm256_mul_pd(a.value_r,a.value_r),_mm256_mul_pd(a.value_i,a.value_i)); -#endif - out.value_r = _mm256_div_pd(out.value_r,den); - out.value_i = _mm256_neg_pd(_mm256_div_pd(out.value_i,den)); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -sqrt(const SIMDVector,simd_abi::avx> &a) = delete; - -FASTOR_INLINE SIMDVector,simd_abi::avx> -rsqrt(const SIMDVector,simd_abi::avx> &a) = delete; - -/* This intentionally return a complex vector */ -FASTOR_INLINE SIMDVector,simd_abi::avx> -abs(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out; - out.value_r = a.magnitude().value; - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -conj(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out(a); - out.value_i = _mm256_neg_pd(out.value_i); - return out; -} - -/* Argument or phase angle */ -FASTOR_INLINE SIMDVector,simd_abi::avx> -arg(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out(a); - for (FASTOR_INDEX i=0UL; i<4UL; ++i) { - ((double*)&out.value_r)[i] = std::atan2(((double*)&a.value_i)[i],((double*)&a.value_r)[i]); - } - return out; -} -//------------------------------------------------------------------------------------------------------------ - -#endif - - - - -// SSE VERSION -//------------------------------------------------------------------------------------------------------------ - -#ifdef FASTOR_SSE2_IMPL - -template <> -struct SIMDVector, simd_abi::sse> { - using vector_type = SIMDVector, simd_abi::sse>; - using value_type = __m128d; - using scalar_value_type = std::complex; - using abi_type = simd_abi::sse; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value_r(_mm_setzero_pd()), value_i(_mm_setzero_pd()) {} - FASTOR_INLINE SIMDVector(std::complex num) { - value_r = _mm_set1_pd(num.real()); - value_i = _mm_set1_pd(num.imag()); - } - FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} - FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - - FASTOR_INLINE SIMDVector,simd_abi::sse> operator=(std::complex num) { - value_r = _mm_set1_pd(num.real()); - value_i = _mm_set1_pd(num.imag()); - return *this; - } - - FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { - if (Aligned) - complex_aligned_store(data); - else - complex_unaligned_store(data); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { - if (!Aligned) - complex_mask_unaligned_load(data,mask); - else - complex_mask_aligned_load(data,mask); - } - FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { - if (!Aligned) - complex_mask_unaligned_store(data,mask); - else - complex_mask_aligned_store(data,mask); - } - - FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { - if (i == 0) { return scalar_value_type(_mm_get0_pd(value_r), _mm_get0_pd(value_i)); } - else { return scalar_value_type(_mm_get1_pd(value_r), _mm_get1_pd(value_i)); } - } - - FASTOR_INLINE SIMDVector real() const { - return value_r; - } - FASTOR_INLINE SIMDVector imag() const { - return value_i; - } - - FASTOR_INLINE void set(std::complex num) { - value_r = _mm_set1_pd(num.real()); - value_i = _mm_set1_pd(num.imag()); - } - FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1) { - const scalar_value_type tmp[Size] = {num0,num1}; - complex_unaligned_load(tmp); - } - - // In-place operators - template,bool> = false> - FASTOR_INLINE void operator+=(U num) { - value_r = _mm_add_pd(value_r,_mm_set1_pd(num)); - } - FASTOR_INLINE void operator+=(scalar_value_type num) { - *this += vector_type(num); - } - FASTOR_INLINE void operator+=(const vector_type &a) { - value_r = _mm_add_pd(value_r,a.value_r); - value_i = _mm_add_pd(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - value_r = _mm_sub_pd(value_r,_mm_set1_pd(num)); - } - FASTOR_INLINE void operator-=(scalar_value_type num) { - *this -= vector_type(num); - } - FASTOR_INLINE void operator-=(const vector_type &a) { - value_r = _mm_sub_pd(value_r,a.value_r); - value_i = _mm_sub_pd(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - __m128d val = _mm_set1_pd(num); - value_r = _mm_mul_pd(value_r,val); - value_i = _mm_mul_pd(value_i,val); - } - FASTOR_INLINE void operator*=(scalar_value_type num) { - *this *= vector_type(num); - } - FASTOR_INLINE void operator*=(const vector_type &a) { -#ifdef FASTOR_FMA_IMPL - __m128d tmp = _mm_fmsub_pd(value_r,a.value_r,_mm_mul_pd(value_i,a.value_i)); - value_i = _mm_fmadd_pd(value_r,a.value_i,_mm_mul_pd(value_i,a.value_r)); -#else - __m128d tmp = _mm_sub_pd(_mm_mul_pd(value_r,a.value_r),_mm_mul_pd(value_i,a.value_i)); - value_i = _mm_add_pd(_mm_mul_pd(value_r,a.value_i),_mm_mul_pd(value_i,a.value_r)); -#endif - value_r = tmp; - } - - template,bool> = false> - FASTOR_INLINE void operator/=(U num) { - __m128d val = _mm_set1_pd(num); - value_r = _mm_div_pd(value_r,val); - value_i = _mm_div_pd(value_i,val); - } - FASTOR_INLINE void operator/=(scalar_value_type num) { - *this /= vector_type(num); - } - FASTOR_INLINE void operator/=(const vector_type &a) { - __m128d tmp = value_r; -#ifdef FASTOR_FMA_IMPL - value_r = _mm_fmadd_pd(value_r , a.value_r, _mm_mul_pd(value_i,a.value_i)); - value_i = _mm_fmsub_pd(value_i , a.value_r, _mm_mul_pd(tmp,a.value_i)); - __m128d den = _mm_fmadd_pd(a.value_r, a.value_r, _mm_mul_pd(a.value_i,a.value_i)); -#else - value_r = _mm_add_pd(_mm_mul_pd(value_r , a.value_r), _mm_mul_pd(value_i,a.value_i)); - value_i = _mm_sub_pd(_mm_mul_pd(value_i , a.value_r), _mm_mul_pd(tmp,a.value_i)); - __m128d den = _mm_add_pd(_mm_mul_pd(a.value_r, a.value_r), _mm_mul_pd(a.value_i,a.value_i)); -#endif - value_r = _mm_div_pd(value_r,den); - value_i = _mm_div_pd(value_i,den); - } - // end of in-place operators - - FASTOR_INLINE scalar_value_type sum() const { - return scalar_value_type(_mm_sum_pd(value_r),_mm_sum_pd(value_i)); - } - FASTOR_INLINE scalar_value_type product() const { - vector_type tmp(*this); - // Multiply vector of complex numbers - tmp *= tmp.reverse(); - // Alternatively if relied on the compiler - // return tmp[0]*tmp[1]; - return tmp[0]; - } - FASTOR_INLINE vector_type reverse() const { - vector_type out; - out.value_r = _mm_reverse_pd(value_r); - out.value_i = _mm_reverse_pd(value_i); - return out; - } - /* Actual magnitude - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector magnitude() const { -#ifdef FASTOR_FMA_IMPL - return _mm_sqrt_pd(_mm_fmadd_pd(value_r,value_r,_mm_mul_pd(value_i,value_i))); -#else - return _mm_sqrt_pd(_mm_add_pd(_mm_mul_pd(value_r,value_r),_mm_mul_pd(value_i,value_i))); -#endif - } - /* STL compliant squared norm - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector norm() const { -#ifdef FASTOR_FMA_IMPL - return _mm_fmadd_pd(value_r,value_r,_mm_mul_pd(value_i,value_i)); -#else - return _mm_add_pd(_mm_mul_pd(value_r,value_r),_mm_mul_pd(value_i,value_i)); -#endif - } - // Magnitude based minimum - FASTOR_INLINE scalar_value_type minimum() const { - vector_type out; - SIMDVector tmp(this->norm()); - if ( _mm_get0_pd(tmp.value) < _mm_get1_pd(tmp.value) ) { - return scalar_value_type(_mm_get0_pd(value_r),_mm_get0_pd(value_i)); - } - else { - return scalar_value_type(_mm_get1_pd(value_r),_mm_get1_pd(value_i)); - } - } - // Magnitude based maximum - FASTOR_INLINE scalar_value_type maximum() const { - vector_type out; - SIMDVector tmp(this->norm()); - if ( _mm_get0_pd(tmp.value) > _mm_get1_pd(tmp.value) ) { - return scalar_value_type(_mm_get0_pd(value_r),_mm_get0_pd(value_i)); - } - else { - return scalar_value_type(_mm_get1_pd(value_r),_mm_get1_pd(value_i)); - } - } - - FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { - vector_type out(*this); - out *= other; - return out.sum(); - } - - value_type value_r; - value_type value_i; - -protected: - FASTOR_INLINE void complex_aligned_load(const std::complex *data) { - __m128d lo = _mm_load_pd(reinterpret_cast(data )); - __m128d hi = _mm_load_pd(reinterpret_cast(data+1)); - arrange_from_load(value_r, value_i, lo, hi); - } - FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { - __m128d lo = _mm_loadu_pd(reinterpret_cast(data )); - __m128d hi = _mm_loadu_pd(reinterpret_cast(data+1)); - arrange_from_load(value_r, value_i, lo, hi); - } - - FASTOR_INLINE void complex_aligned_store(std::complex *data) const { - __m128d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm_store_pd(reinterpret_cast(data ),lo); - _mm_store_pd(reinterpret_cast(data+1),hi); - } - FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { - __m128d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm_storeu_pd(reinterpret_cast(data ),lo); - _mm_storeu_pd(reinterpret_cast(data+1),hi); - } - - FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { -#ifdef FASTOR_HAS_AVX512_MASKS - __m128d lo, hi; - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - lo = _mm_mask_load_pd(lo, mask0, reinterpret_cast(data )); - hi = _mm_mask_load_pd(hi, mask1, reinterpret_cast(data+1)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm_setzero_pd(); - value_i = _mm_setzero_pd(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - lo = _mm_mask_loadu_pd(lo, mask0, reinterpret_cast(data )); - hi = _mm_mask_loadu_pd(hi, mask1, reinterpret_cast(data+1)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm_setzero_pd(); - value_i = _mm_setzero_pd(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - _mm_mask_store_pd(reinterpret_cast(data ), mask0, lo); - _mm_mask_store_pd(reinterpret_cast(data+1), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { -#ifdef FASTOR_HAS_AVX512_MASKS - __m128d lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - _mm_mask_storeu_pd(reinterpret_cast(data ), mask0, lo); - _mm_mask_storeu_pd(reinterpret_cast(data+1), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::sse> a) { - // ICC crashes without a copy - const __m128d vr = a.value_r; - const __m128d vi = a.value_i; - const double* value_r = reinterpret_cast(&vr); - const double* value_i = reinterpret_cast(&vi); - os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " - << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j" << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; - out.value_r = _mm_add_pd(a.value_r,b.value_r); - out.value_i = _mm_add_pd(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a + SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) + b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out(a); - out.value_r = _mm_add_pd(a.value_r,_mm_set1_pd(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(U a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out(b); - out.value_r = _mm_add_pd(_mm_set1_pd(a), b.value_r); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &b) { - return b; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; - out.value_r = _mm_sub_pd(a.value_r,b.value_r); - out.value_i = _mm_sub_pd(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a - SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) - b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out(a); - out.value_r = _mm_sub_pd(a.value_r,_mm_set1_pd(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(U a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(std::complex(a,0)) - b; -} -/* This is negation and not complex conjugate */ -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out; - out.value_r = _mm_neg_pd(a.value_r); - out.value_i = _mm_neg_pd(a.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm_fmsub_pd(a.value_r,b.value_r,_mm_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm_fmadd_pd(a.value_r,b.value_i,_mm_mul_pd(a.value_i,b.value_r)); -#else - out.value_r = _mm_sub_pd(_mm_mul_pd(a.value_r,b.value_r),_mm_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm_add_pd(_mm_mul_pd(a.value_r,b.value_i),_mm_mul_pd(a.value_i,b.value_r)); -#endif - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a * SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) * b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out; - __m128d val = _mm_set1_pd(b); - out.value_r = _mm_mul_pd(a.value_r,val); - out.value_i = _mm_mul_pd(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(U a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; - __m128d val = _mm_set1_pd(a); - out.value_r = _mm_mul_pd(val,b.value_r); - out.value_i = _mm_mul_pd(val,b.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm_fmadd_pd(a.value_r,b.value_r,_mm_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm_fmsub_pd(a.value_i,b.value_r,_mm_mul_pd(a.value_r,b.value_i)); - __m128d den = _mm_fmadd_pd(b.value_r,b.value_r,_mm_mul_pd(b.value_i,b.value_i)); -#else - out.value_r = _mm_add_pd(_mm_mul_pd(a.value_r,b.value_r),_mm_mul_pd(a.value_i,b.value_i)); - out.value_i = _mm_sub_pd(_mm_mul_pd(a.value_i,b.value_r),_mm_mul_pd(a.value_r,b.value_i)); - __m128d den = _mm_add_pd(_mm_mul_pd(b.value_r,b.value_r),_mm_mul_pd(b.value_i,b.value_i)); -#endif - out.value_r = _mm_div_pd(out.value_r,den); - out.value_i = _mm_div_pd(out.value_i,den); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a / SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) / b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out; - __m128d val = _mm_set1_pd(b); - out.value_r = _mm_div_pd(a.value_r,val); - out.value_i = _mm_div_pd(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(U a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(std::complex(a,0)) / b; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -rcp(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out; -#ifdef FASTOR_FMA_IMPL - __m128d den = _mm_fmadd_pd(a.value_r,a.value_r,_mm_mul_pd(a.value_i,a.value_i)); -#else - __m128d den = _mm_add_pd(_mm_mul_pd(a.value_r,a.value_r),_mm_mul_pd(a.value_i,a.value_i)); -#endif - out.value_r = _mm_div_pd(out.value_r,den); - out.value_i = _mm_neg_pd(_mm_div_pd(out.value_i,den)); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -sqrt(const SIMDVector,simd_abi::sse> &a) = delete; - -FASTOR_INLINE SIMDVector,simd_abi::sse> -rsqrt(const SIMDVector,simd_abi::sse> &a) = delete; - -/* This intentionally return a complex vector */ -FASTOR_INLINE SIMDVector,simd_abi::sse> -abs(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out; - out.value_r = a.magnitude().value; - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -conj(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out(a); - out.value_i = _mm_neg_pd(out.value_i); - return out; -} - -/* Argument or phase angle */ -FASTOR_INLINE SIMDVector,simd_abi::sse> -arg(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out(a); - for (FASTOR_INDEX i=0UL; i<2UL; ++i) { - ((double*)&out.value_r)[i] = std::atan2(((double*)&a.value_i)[i],((double*)&a.value_r)[i]); - } - return out; -} -//------------------------------------------------------------------------------------------------------------ - -#endif - - -} // end of namespace Fastor - -#endif // SIMD_VECTOR_COMPLEX_DOUBLE_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_complex_float.h b/noarch/include/Fastor/simd_vector/simd_vector_complex_float.h deleted file mode 100644 index cfe0a6c6..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_complex_float.h +++ /dev/null @@ -1,1619 +0,0 @@ -#ifndef SIMD_VECTOR_COMPLEX_FLOAT_H -#define SIMD_VECTOR_COMPLEX_FLOAT_H - -#include "Fastor/util/extended_algorithms.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/simd_vector_base.h" -#include "Fastor/simd_vector/simd_vector_float.h" -#include -#include - -namespace Fastor { - - -// AVX512 VERSION -//------------------------------------------------------------------------------------------------------------ - -#ifdef FASTOR_AVX512F_IMPL - -template <> -struct SIMDVector, simd_abi::avx512> { - using vector_type = SIMDVector, simd_abi::avx512>; - using value_type = __m512; - using scalar_value_type = std::complex; - using abi_type = simd_abi::avx512; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value_r(_mm512_setzero_ps()), value_i(_mm512_setzero_ps()) {} - FASTOR_INLINE SIMDVector(std::complex num) { - value_r = _mm512_set1_ps(num.real()); - value_i = _mm512_set1_ps(num.imag()); - } - FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} - FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - - FASTOR_INLINE SIMDVector,simd_abi::avx512> operator=(std::complex num) { - value_r = _mm512_set1_ps(num.real()); - value_i = _mm512_set1_ps(num.imag()); - return *this; - } - - FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { - if (Aligned) - complex_aligned_store(data); - else - complex_unaligned_store(data); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *data, uint16_t mask, bool Aligned=false) { - if (!Aligned) - complex_mask_unaligned_load(data,mask); - else - complex_mask_aligned_load(data,mask); - } - FASTOR_INLINE void mask_store(scalar_value_type *data, uint16_t mask, bool Aligned=false) const { - if (!Aligned) - complex_mask_unaligned_store(data,mask); - else - complex_mask_aligned_store(data,mask); - } - - FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { - return scalar_value_type(reinterpret_cast(&value_r)[i],reinterpret_cast(&value_i)[i]); - } - - FASTOR_INLINE SIMDVector real() const { - return value_r; - } - FASTOR_INLINE SIMDVector imag() const { - return value_i; - } - - FASTOR_INLINE void set(std::complex num) { - value_r = _mm512_set1_ps(num.real()); - value_i = _mm512_set1_ps(num.imag()); - } - FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, - scalar_value_type num2, scalar_value_type num3, - scalar_value_type num4, scalar_value_type num5, - scalar_value_type num6, scalar_value_type num7, - scalar_value_type num8, scalar_value_type num9, - scalar_value_type num10, scalar_value_type num11, - scalar_value_type num12, scalar_value_type num13, - scalar_value_type num14, scalar_value_type num15) { - const scalar_value_type tmp[Size] = {num0,num1,num2,num3,num4,num5,num6,num7, - num8,num9,num10,num11,num12,num13,num14,num15}; - complex_unaligned_load(tmp); - } - - // In-place operators - template,bool> = false> - FASTOR_INLINE void operator+=(U num) { - value_r = _mm512_add_ps(value_r,_mm512_set1_ps(num)); - } - FASTOR_INLINE void operator+=(scalar_value_type num) { - *this += vector_type(num); - } - FASTOR_INLINE void operator+=(const vector_type &a) { - value_r = _mm512_add_ps(value_r,a.value_r); - value_i = _mm512_add_ps(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - value_r = _mm512_sub_ps(value_r,_mm512_set1_ps(num)); - } - FASTOR_INLINE void operator-=(scalar_value_type num) { - *this -= vector_type(num); - } - FASTOR_INLINE void operator-=(const vector_type &a) { - value_r = _mm512_sub_ps(value_r,a.value_r); - value_i = _mm512_sub_ps(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - __m512 val = _mm512_set1_ps(num); - value_r = _mm512_mul_ps(value_r,val); - value_i = _mm512_mul_ps(value_i,val); - } - FASTOR_INLINE void operator*=(scalar_value_type num) { - *this *= vector_type(num); - } - FASTOR_INLINE void operator*=(const vector_type &a) { -#ifdef FASTOR_FMA_IMPL - __m512 tmp = _mm512_fmsub_ps(value_r,a.value_r,_mm512_mul_ps(value_i,a.value_i)); - value_i = _mm512_fmadd_ps(value_r,a.value_i,_mm512_mul_ps(value_i,a.value_r)); -#else - __m512 tmp = _mm512_sub_ps(_mm512_mul_ps(value_r,a.value_r),_mm512_mul_ps(value_i,a.value_i)); - value_i = _mm512_add_ps(_mm512_mul_ps(value_r,a.value_i),_mm512_mul_ps(value_i,a.value_r)); -#endif - value_r = tmp; - } - - template,bool> = false> - FASTOR_INLINE void operator/=(U num) { - __m512 val = _mm512_set1_ps(num); - value_r = _mm512_div_ps(value_r,val); - value_i = _mm512_div_ps(value_i,val); - } - FASTOR_INLINE void operator/=(scalar_value_type num) { - *this /= vector_type(num); - } - FASTOR_INLINE void operator/=(const vector_type &a) { - __m512 tmp = value_r; -#ifdef FASTOR_FMA_IMPL - value_r = _mm512_fmadd_ps(value_r , a.value_r, _mm512_mul_ps(value_i,a.value_i)); - value_i = _mm512_fmsub_ps(value_i , a.value_r, _mm512_mul_ps(tmp,a.value_i)); - __m512 den = _mm512_fmadd_ps(a.value_r, a.value_r, _mm512_mul_ps(a.value_i,a.value_i)); -#else - value_r = _mm512_add_ps(_mm512_mul_ps(value_r , a.value_r), _mm512_mul_ps(value_i,a.value_i)); - value_i = _mm512_sub_ps(_mm512_mul_ps(value_i , a.value_r), _mm512_mul_ps(tmp,a.value_i)); - __m512 den = _mm512_add_ps(_mm512_mul_ps(a.value_r, a.value_r), _mm512_mul_ps(a.value_i,a.value_i)); -#endif - value_r = _mm512_div_ps(value_r,den); - value_i = _mm512_div_ps(value_i,den); - } - // end of in-place operators - - FASTOR_INLINE scalar_value_type sum() const { -#ifdef FASTOR_HAS_AVX512_REDUCE_ADD - return scalar_value_type(_mm512_reduce_add_ps(value_r),_mm512_reduce_add_ps(value_i)); -#else - __m256 lor = _mm512_castps512_ps256(value_r); - __m256 hir = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value_r),0x1)); - __m256 loi = _mm512_castps512_ps256(value_i); - __m256 hii = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value_i),0x1)); - return scalar_value_type(_mm256_sum_ps(_mm256_add_ps(lor,hir)), _mm256_sum_ps(_mm256_add_ps(loi,hii))); -#endif - } - FASTOR_INLINE scalar_value_type product() const { - vector_type tmp(*this); - scalar_value_type out(tmp[0]); - for (FASTOR_INDEX i=1; i magnitude() const { -#ifdef FASTOR_FMA_IMPL - return _mm512_sqrt_ps(_mm512_fmadd_ps(value_r,value_r,_mm512_mul_ps(value_i,value_i))); -#else - return _mm512_sqrt_ps(_mm512_add_ps(_mm512_mul_ps(value_r,value_r),_mm512_mul_ps(value_i,value_i))); -#endif - } - /* STL compliant squared norm - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector norm() const { -#ifdef FASTOR_FMA_IMPL - return _mm512_fmadd_ps(value_r,value_r,_mm512_mul_ps(value_i,value_i)); -#else - return _mm512_add_ps(_mm512_mul_ps(value_r,value_r),_mm512_mul_ps(value_i,value_i)); -#endif - } - // Magnitude based minimum - FASTOR_INLINE scalar_value_type minimum() const; - // Magnitude based maximum - FASTOR_INLINE scalar_value_type maximum() const; - - FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { - vector_type out(*this); - out *= other; - return out.sum(); - } - - value_type value_r; - value_type value_i; - -protected: - FASTOR_INLINE void complex_aligned_load(const std::complex *data) { - __m512 lo = _mm512_load_ps(reinterpret_cast(data )); - __m512 hi = _mm512_load_ps(reinterpret_cast(data+8)); - arrange_from_load(value_r, value_i, lo, hi); - } - FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { - __m512 lo = _mm512_loadu_ps(reinterpret_cast(data )); - __m512 hi = _mm512_loadu_ps(reinterpret_cast(data+8)); - arrange_from_load(value_r, value_i, lo, hi); - } - - FASTOR_INLINE void complex_aligned_store(std::complex *data) const { - __m512 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm512_store_ps(reinterpret_cast(data ), lo); - _mm512_store_ps(reinterpret_cast(data+8), hi); - } - FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { - __m512 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm512_storeu_ps(reinterpret_cast(data ), lo); - _mm512_storeu_ps(reinterpret_cast(data+8), hi); - } - - FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint16_t mask) { -#ifdef FASTOR_HAS_AVX512_MASKS - __m512 lo, hi; - uint16_t mask0, mask1; - split_mask(mask, mask0, mask1); - lo = _mm512_mask_load_ps(lo, mask0, reinterpret_cast(data )); - hi = _mm512_mask_load_ps(hi, mask1, reinterpret_cast(data+8)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm512_setzero_ps(); - value_i = _mm512_setzero_ps(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - lo = _mm512_mask_loadu_ps(lo, mask0, reinterpret_cast(data )); - hi = _mm512_mask_loadu_ps(hi, mask1, reinterpret_cast(data+8)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm512_setzero_ps(); - value_i = _mm512_setzero_ps(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - _mm512_mask_store_ps(reinterpret_cast(data ), mask0, lo); - _mm512_mask_store_ps(reinterpret_cast(data+8), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint16_t mask) const { -#ifdef FASTOR_HAS_AVX512_MASKS - __m512 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - uint16_t mask0, mask1; - split_mask(mask, mask0, mask1); - _mm512_mask_storeu_ps(reinterpret_cast(data ), mask0, lo); - _mm512_mask_storeu_ps(reinterpret_cast(data+8), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx512> a) { - // ICC crashes without a copy - const __m512 vr = a.value_r; - const __m512 vi = a.value_i; - const float* value_r = reinterpret_cast(&vr); - const float* value_i = reinterpret_cast(&vi); - os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " - << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " - << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " - << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j, " - << value_r[4] << signum_string(value_i[4]) << std::abs(value_i[4]) << "j, " - << value_r[5] << signum_string(value_i[5]) << std::abs(value_i[5]) << "j, " - << value_r[6] << signum_string(value_i[6]) << std::abs(value_i[6]) << "j, " - << value_r[7] << signum_string(value_i[7]) << std::abs(value_i[7]) << "j, " - << value_r[8] << signum_string(value_i[8]) << std::abs(value_i[8]) << "j, " - << value_r[9] << signum_string(value_i[9]) << std::abs(value_i[9]) << "j, " - << value_r[10] << signum_string(value_i[10]) << std::abs(value_i[10]) << "j, " - << value_r[11] << signum_string(value_i[11]) << std::abs(value_i[11]) << "j, " - << value_r[12] << signum_string(value_i[12]) << std::abs(value_i[12]) << "j, " - << value_r[13] << signum_string(value_i[13]) << std::abs(value_i[13]) << "j, " - << value_r[14] << signum_string(value_i[14]) << std::abs(value_i[14]) << "j, " - << value_r[15] << signum_string(value_i[15]) << std::abs(value_i[15]) << "j" << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; - out.value_r = _mm512_add_ps(a.value_r,b.value_r); - out.value_i = _mm512_add_ps(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a + SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) + b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out(a); - out.value_r = _mm512_add_ps(a.value_r,_mm512_set1_ps(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(U a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out(b); - out.value_r = _mm512_add_ps(_mm512_set1_ps(a), b.value_r); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator+(const SIMDVector,simd_abi::avx512> &b) { - return b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; - out.value_r = _mm512_sub_ps(a.value_r,b.value_r); - out.value_i = _mm512_sub_ps(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a - SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) - b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out(a); - out.value_r = _mm512_sub_ps(a.value_r,_mm512_set1_ps(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(U a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(std::complex(a,0)) - b; -} -/* This is negation and not complex conjugate */ -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator-(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out; - out.value_r = _mm512_neg_ps(a.value_r); - out.value_i = _mm512_neg_ps(a.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm512_fmsub_ps(a.value_r,b.value_r,_mm512_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm512_fmadd_ps(a.value_r,b.value_i,_mm512_mul_ps(a.value_i,b.value_r)); -#else - out.value_r = _mm512_sub_ps(_mm512_mul_ps(a.value_r,b.value_r),_mm512_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm512_add_ps(_mm512_mul_ps(a.value_r,b.value_i),_mm512_mul_ps(a.value_i,b.value_r)); -#endif - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a * SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) * b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out; - __m512 val = _mm512_set1_ps(b); - out.value_r = _mm512_mul_ps(a.value_r,val); - out.value_i = _mm512_mul_ps(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator*(U a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; - __m512 val = _mm512_set1_ps(a); - out.value_r = _mm512_mul_ps(val,b.value_r); - out.value_i = _mm512_mul_ps(val,b.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(const SIMDVector,simd_abi::avx512> &a, const SIMDVector,simd_abi::avx512> &b) { - SIMDVector,simd_abi::avx512> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm512_fmadd_ps(a.value_r,b.value_r,_mm512_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm512_fmsub_ps(a.value_i,b.value_r,_mm512_mul_ps(a.value_r,b.value_i)); - __m512 den = _mm512_fmadd_ps(b.value_r,b.value_r,_mm512_mul_ps(b.value_i,b.value_i)); -#else - out.value_r = _mm512_add_ps(_mm512_mul_ps(a.value_r,b.value_r),_mm512_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm512_sub_ps(_mm512_mul_ps(a.value_i,b.value_r),_mm512_mul_ps(a.value_r,b.value_i)); - __m512 den = _mm512_add_ps(_mm512_mul_ps(b.value_r,b.value_r),_mm512_mul_ps(b.value_i,b.value_i)); -#endif - out.value_r = _mm512_div_ps(out.value_r,den); - out.value_i = _mm512_div_ps(out.value_i,den); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(const SIMDVector,simd_abi::avx512> &a, std::complex b) { - return a / SIMDVector,simd_abi::avx512>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(std::complex a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(a) / b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(const SIMDVector,simd_abi::avx512> &a, U b) { - SIMDVector,simd_abi::avx512> out; - __m512 val = _mm512_set1_ps(b); - out.value_r = _mm512_div_ps(a.value_r,val); - out.value_i = _mm512_div_ps(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx512> -operator/(U a, const SIMDVector,simd_abi::avx512> &b) { - return SIMDVector,simd_abi::avx512>(std::complex(a,0)) / b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -rcp(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out; -#ifdef FASTOR_FMA_IMPL - __m512 den = _mm512_fmadd_ps(a.value_r,a.value_r,_mm512_mul_ps(a.value_i,a.value_i)); -#else - __m512 den = _mm512_add_ps(_mm512_mul_ps(a.value_r,a.value_r),_mm512_mul_ps(a.value_i,a.value_i)); -#endif - out.value_r = _mm512_div_ps(out.value_r,den); - out.value_i = _mm512_neg_ps(_mm512_div_ps(out.value_i,den)); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -sqrt(const SIMDVector,simd_abi::avx512> &a) = delete; - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -rsqrt(const SIMDVector,simd_abi::avx512> &a) = delete; - -/* This intentionally return a complex vector */ -FASTOR_INLINE SIMDVector,simd_abi::avx512> -abs(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out; - out.value_r = a.magnitude().value; - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx512> -conj(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out(a); - out.value_i = _mm512_neg_ps(out.value_i); - return out; -} - -/* Argument or phase angle */ -FASTOR_INLINE SIMDVector,simd_abi::avx512> -arg(const SIMDVector,simd_abi::avx512> &a) { - SIMDVector,simd_abi::avx512> out(a); - for (FASTOR_INDEX i=0UL; i<16UL; ++i) { - ((float*)&out.value_r)[i] = std::atan2(((float*)&a.value_i)[i],((float*)&a.value_r)[i]); - } - return out; -} -//------------------------------------------------------------------------------------------------------------ - -#endif - - - -// AVX VERSION -//------------------------------------------------------------------------------------------------------------ - -#ifdef FASTOR_AVX_IMPL - -template <> -struct SIMDVector, simd_abi::avx> { - using vector_type = SIMDVector, simd_abi::avx>; - using value_type = __m256; - using scalar_value_type = std::complex; - using abi_type = simd_abi::avx; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value_r(_mm256_setzero_ps()), value_i(_mm256_setzero_ps()) {} - FASTOR_INLINE SIMDVector(std::complex num) { - value_r = _mm256_set1_ps(num.real()); - value_i = _mm256_set1_ps(num.imag()); - } - FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} - FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - - FASTOR_INLINE SIMDVector,simd_abi::avx> operator=(std::complex num) { - value_r = _mm256_set1_ps(num.real()); - value_i = _mm256_set1_ps(num.imag()); - return *this; - } - - FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { - if (Aligned) - complex_aligned_store(data); - else - complex_unaligned_store(data); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { - if (!Aligned) - complex_mask_unaligned_load(data,mask); - else - complex_mask_aligned_load(data,mask); - } - FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { - if (!Aligned) - complex_mask_unaligned_store(data,mask); - else - complex_mask_aligned_store(data,mask); - } - - FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { - if (i == 0) { return scalar_value_type(_mm256_get0_ps(value_r), _mm256_get0_ps(value_i)); } - else if (i == 1) { return scalar_value_type(_mm256_get1_ps(value_r), _mm256_get1_ps(value_i)); } - else if (i == 2) { return scalar_value_type(_mm256_get2_ps(value_r), _mm256_get2_ps(value_i)); } - else if (i == 3) { return scalar_value_type(_mm256_get3_ps(value_r), _mm256_get3_ps(value_i)); } - else if (i == 4) { return scalar_value_type(_mm256_get4_ps(value_r), _mm256_get4_ps(value_i)); } - else if (i == 5) { return scalar_value_type(_mm256_get5_ps(value_r), _mm256_get5_ps(value_i)); } - else if (i == 6) { return scalar_value_type(_mm256_get6_ps(value_r), _mm256_get6_ps(value_i)); } - else { return scalar_value_type(_mm256_get7_ps(value_r), _mm256_get7_ps(value_i)); } - } - - FASTOR_INLINE SIMDVector real() const { - return value_r; - } - FASTOR_INLINE SIMDVector imag() const { - return value_i; - } - - FASTOR_INLINE void set(std::complex num) { - value_r = _mm256_set1_ps(num.real()); - value_i = _mm256_set1_ps(num.imag()); - } - FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, - scalar_value_type num2, scalar_value_type num3, - scalar_value_type num4, scalar_value_type num5, - scalar_value_type num6, scalar_value_type num7) { - const scalar_value_type tmp[Size] = {num0,num1,num2,num3,num4,num5,num6,num7}; - complex_unaligned_load(tmp); - } - - // In-place operators - template,bool> = false> - FASTOR_INLINE void operator+=(U num) { - value_r = _mm256_add_ps(value_r,_mm256_set1_ps(num)); - } - FASTOR_INLINE void operator+=(scalar_value_type num) { - *this += vector_type(num); - } - FASTOR_INLINE void operator+=(const vector_type &a) { - value_r = _mm256_add_ps(value_r,a.value_r); - value_i = _mm256_add_ps(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - value_r = _mm256_sub_ps(value_r,_mm256_set1_ps(num)); - } - FASTOR_INLINE void operator-=(scalar_value_type num) { - *this -= vector_type(num); - } - FASTOR_INLINE void operator-=(const vector_type &a) { - value_r = _mm256_sub_ps(value_r,a.value_r); - value_i = _mm256_sub_ps(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - __m256 val = _mm256_set1_ps(num); - value_r = _mm256_mul_ps(value_r,val); - value_i = _mm256_mul_ps(value_i,val); - } - FASTOR_INLINE void operator*=(scalar_value_type num) { - *this *= vector_type(num); - } - FASTOR_INLINE void operator*=(const vector_type &a) { -#ifdef FASTOR_FMA_IMPL - __m256 tmp = _mm256_fmsub_ps(value_r,a.value_r,_mm256_mul_ps(value_i,a.value_i)); - value_i = _mm256_fmadd_ps(value_r,a.value_i,_mm256_mul_ps(value_i,a.value_r)); -#else - __m256 tmp = _mm256_sub_ps(_mm256_mul_ps(value_r,a.value_r),_mm256_mul_ps(value_i,a.value_i)); - value_i = _mm256_add_ps(_mm256_mul_ps(value_r,a.value_i),_mm256_mul_ps(value_i,a.value_r)); -#endif - value_r = tmp; - } - - template,bool> = false> - FASTOR_INLINE void operator/=(U num) { - __m256 val = _mm256_set1_ps(num); - value_r = _mm256_div_ps(value_r,val); - value_i = _mm256_div_ps(value_i,val); - } - FASTOR_INLINE void operator/=(scalar_value_type num) { - *this /= vector_type(num); - } - FASTOR_INLINE void operator/=(const vector_type &a) { - __m256 tmp = value_r; -#ifdef FASTOR_FMA_IMPL - value_r = _mm256_fmadd_ps(value_r , a.value_r, _mm256_mul_ps(value_i,a.value_i)); - value_i = _mm256_fmsub_ps(value_i , a.value_r, _mm256_mul_ps(tmp,a.value_i)); - __m256 den = _mm256_fmadd_ps(a.value_r, a.value_r, _mm256_mul_ps(a.value_i,a.value_i)); -#else - value_r = _mm256_add_ps(_mm256_mul_ps(value_r , a.value_r), _mm256_mul_ps(value_i,a.value_i)); - value_i = _mm256_sub_ps(_mm256_mul_ps(value_i , a.value_r), _mm256_mul_ps(tmp,a.value_i)); - __m256 den = _mm256_add_ps(_mm256_mul_ps(a.value_r, a.value_r), _mm256_mul_ps(a.value_i,a.value_i)); -#endif - value_r = _mm256_div_ps(value_r,den); - value_i = _mm256_div_ps(value_i,den); - } - // end of in-place operators - - FASTOR_INLINE scalar_value_type sum() const { - return scalar_value_type(_mm256_sum_ps(value_r),_mm256_sum_ps(value_i)); - } - FASTOR_INLINE scalar_value_type product() const { - vector_type tmp(*this); - return tmp[0]*tmp[1]*tmp[2]*tmp[3]*tmp[4]*tmp[5]*tmp[6]*tmp[7]; - } - FASTOR_INLINE vector_type reverse() const { - vector_type out; - out.value_r = _mm256_reverse_ps(value_r); - out.value_i = _mm256_reverse_ps(value_i); - return out; - } - /* Actual magnitude - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector magnitude() const { -#ifdef FASTOR_FMA_IMPL - return _mm256_sqrt_ps(_mm256_fmadd_ps(value_r,value_r,_mm256_mul_ps(value_i,value_i))); -#else - return _mm256_sqrt_ps(_mm256_add_ps(_mm256_mul_ps(value_r,value_r),_mm256_mul_ps(value_i,value_i))); -#endif - } - /* STL compliant squared norm - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector norm() const { -#ifdef FASTOR_FMA_IMPL - return _mm256_fmadd_ps(value_r,value_r,_mm256_mul_ps(value_i,value_i)); -#else - return _mm256_add_ps(_mm256_mul_ps(value_r,value_r),_mm256_mul_ps(value_i,value_i)); -#endif - } - // Magnitude based minimum - FASTOR_INLINE scalar_value_type minimum() const; - // Magnitude based maximum - FASTOR_INLINE scalar_value_type maximum() const; - - FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { - vector_type out(*this); - out *= other; - return out.sum(); - } - - value_type value_r; - value_type value_i; - -protected: - FASTOR_INLINE void complex_aligned_load(const std::complex *data) { - __m256 lo = _mm256_load_ps(reinterpret_cast(data )); - __m256 hi = _mm256_load_ps(reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); - } - FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { - __m256 lo = _mm256_loadu_ps(reinterpret_cast(data )); - __m256 hi = _mm256_loadu_ps(reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); - } - - FASTOR_INLINE void complex_aligned_store(std::complex *data) const { - __m256 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm256_store_ps(reinterpret_cast(data ), lo); - _mm256_store_ps(reinterpret_cast(data+4), hi); - } - FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { - __m256 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm256_storeu_ps(reinterpret_cast(data ), lo); - _mm256_storeu_ps(reinterpret_cast(data+4), hi); - } - - FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { -#ifdef FASTOR_HAS_AVX512_MASKS - __m256 lo, hi; - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - lo = _mm256_mask_load_ps(lo, mask0, reinterpret_cast(data )); - hi = _mm256_mask_load_ps(hi, mask1, reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm256_setzero_ps(); - value_i = _mm256_setzero_ps(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - lo = _mm256_mask_loadu_ps(lo, mask0, reinterpret_cast(data )); - hi = _mm256_mask_loadu_ps(hi, mask1, reinterpret_cast(data+4)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm256_setzero_ps(); - value_i = _mm256_setzero_ps(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - _mm256_mask_store_ps(reinterpret_cast(data ), mask0, lo); - _mm256_mask_store_ps(reinterpret_cast(data+4), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { -#ifdef FASTOR_HAS_AVX512_MASKS - __m256 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - _mm256_mask_storeu_ps(reinterpret_cast(data ), mask0, lo); - _mm256_mask_storeu_ps(reinterpret_cast(data+4), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::avx> a) { - // ICC crashes without a copy - const __m256 vr = a.value_r; - const __m256 vi = a.value_i; - const float* value_r = reinterpret_cast(&vr); - const float* value_i = reinterpret_cast(&vi); - os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " - << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " - << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " - << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j, " - << value_r[4] << signum_string(value_i[4]) << std::abs(value_i[4]) << "j, " - << value_r[5] << signum_string(value_i[5]) << std::abs(value_i[5]) << "j, " - << value_r[6] << signum_string(value_i[6]) << std::abs(value_i[6]) << "j, " - << value_r[7] << signum_string(value_i[7]) << std::abs(value_i[7]) << "j" << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; - out.value_r = _mm256_add_ps(a.value_r,b.value_r); - out.value_i = _mm256_add_ps(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a + SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) + b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out(a); - out.value_r = _mm256_add_ps(a.value_r,_mm256_set1_ps(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(U a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out(b); - out.value_r = _mm256_add_ps(_mm256_set1_ps(a), b.value_r); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator+(const SIMDVector,simd_abi::avx> &b) { - return b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; - out.value_r = _mm256_sub_ps(a.value_r,b.value_r); - out.value_i = _mm256_sub_ps(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a - SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) - b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out(a); - out.value_r = _mm256_sub_ps(a.value_r,_mm256_set1_ps(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(U a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(std::complex(a,0)) - b; -} -/* This is negation and not complex conjugate */ -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator-(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out; - out.value_r = _mm256_neg_ps(a.value_r); - out.value_i = _mm256_neg_ps(a.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm256_fmsub_ps(a.value_r,b.value_r,_mm256_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm256_fmadd_ps(a.value_r,b.value_i,_mm256_mul_ps(a.value_i,b.value_r)); -#else - out.value_r = _mm256_sub_ps(_mm256_mul_ps(a.value_r,b.value_r),_mm256_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm256_add_ps(_mm256_mul_ps(a.value_r,b.value_i),_mm256_mul_ps(a.value_i,b.value_r)); -#endif - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a * SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) * b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out; - __m256 val = _mm256_set1_ps(b); - out.value_r = _mm256_mul_ps(a.value_r,val); - out.value_i = _mm256_mul_ps(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator*(U a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; - __m256 val = _mm256_set1_ps(a); - out.value_r = _mm256_mul_ps(val,b.value_r); - out.value_i = _mm256_mul_ps(val,b.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(const SIMDVector,simd_abi::avx> &a, const SIMDVector,simd_abi::avx> &b) { - SIMDVector,simd_abi::avx> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm256_fmadd_ps(a.value_r,b.value_r,_mm256_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm256_fmsub_ps(a.value_i,b.value_r,_mm256_mul_ps(a.value_r,b.value_i)); - __m256 den = _mm256_fmadd_ps(b.value_r,b.value_r,_mm256_mul_ps(b.value_i,b.value_i)); -#else - out.value_r = _mm256_add_ps(_mm256_mul_ps(a.value_r,b.value_r),_mm256_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm256_sub_ps(_mm256_mul_ps(a.value_i,b.value_r),_mm256_mul_ps(a.value_r,b.value_i)); - __m256 den = _mm256_add_ps(_mm256_mul_ps(b.value_r,b.value_r),_mm256_mul_ps(b.value_i,b.value_i)); -#endif - out.value_r = _mm256_div_ps(out.value_r,den); - out.value_i = _mm256_div_ps(out.value_i,den); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(const SIMDVector,simd_abi::avx> &a, std::complex b) { - return a / SIMDVector,simd_abi::avx>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(std::complex a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(a) / b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(const SIMDVector,simd_abi::avx> &a, U b) { - SIMDVector,simd_abi::avx> out; - __m256 val = _mm256_set1_ps(b); - out.value_r = _mm256_div_ps(a.value_r,val); - out.value_i = _mm256_div_ps(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::avx> -operator/(U a, const SIMDVector,simd_abi::avx> &b) { - return SIMDVector,simd_abi::avx>(std::complex(a,0)) / b; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -rcp(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out; -#ifdef FASTOR_FMA_IMPL - __m256 den = _mm256_fmadd_ps(a.value_r,a.value_r,_mm256_mul_ps(a.value_i,a.value_i)); -#else - __m256 den = _mm256_add_ps(_mm256_mul_ps(a.value_r,a.value_r),_mm256_mul_ps(a.value_i,a.value_i)); -#endif - out.value_r = _mm256_div_ps(out.value_r,den); - out.value_i = _mm256_neg_ps(_mm256_div_ps(out.value_i,den)); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -sqrt(const SIMDVector,simd_abi::avx> &a) = delete; - -FASTOR_INLINE SIMDVector,simd_abi::avx> -rsqrt(const SIMDVector,simd_abi::avx> &a) = delete; - -/* This intentionally return a complex vector */ -FASTOR_INLINE SIMDVector,simd_abi::avx> -abs(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out; - out.value_r = a.magnitude().value; - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::avx> -conj(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out(a); - out.value_i = _mm256_neg_ps(out.value_i); - return out; -} - -/* Argument or phase angle */ -FASTOR_INLINE SIMDVector,simd_abi::avx> -arg(const SIMDVector,simd_abi::avx> &a) { - SIMDVector,simd_abi::avx> out(a); - for (FASTOR_INDEX i=0UL; i<8UL; ++i) { - ((float*)&out.value_r)[i] = std::atan2(((float*)&a.value_i)[i],((float*)&a.value_r)[i]); - } - return out; -} -//------------------------------------------------------------------------------------------------------------ - -#endif - - - - -// SSE VERSION -//------------------------------------------------------------------------------------------------------------ - -#ifdef FASTOR_SSE2_IMPL - -template <> -struct SIMDVector, simd_abi::sse> { - using vector_type = SIMDVector, simd_abi::sse>; - using value_type = __m128; - using scalar_value_type = std::complex; - using abi_type = simd_abi::sse; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value_r(_mm_setzero_ps()), value_i(_mm_setzero_ps()) {} - FASTOR_INLINE SIMDVector(std::complex num) { - value_r = _mm_set1_ps(num.real()); - value_i = _mm_set1_ps(num.imag()); - } - FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} - FASTOR_INLINE SIMDVector(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - - FASTOR_INLINE SIMDVector,simd_abi::sse> operator=(std::complex num) { - value_r = _mm_set1_ps(num.real()); - value_i = _mm_set1_ps(num.imag()); - return *this; - } - - FASTOR_INLINE void load(const std::complex *data, bool Aligned=true) { - if (Aligned) - complex_aligned_load(data); - else - complex_unaligned_load(data); - } - FASTOR_INLINE void store(std::complex *data, bool Aligned=true) const { - if (Aligned) - complex_aligned_store(data); - else - complex_unaligned_store(data); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool Aligned=false) { - if (!Aligned) - complex_mask_unaligned_load(data,mask); - else - complex_mask_aligned_load(data,mask); - } - FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool Aligned=false) const { - if (!Aligned) - complex_mask_unaligned_store(data,mask); - else - complex_mask_aligned_store(data,mask); - } - - FASTOR_INLINE scalar_value_type operator[](FASTOR_INDEX i) const { - if (i == 0) { return scalar_value_type(_mm_get0_ps(value_r), _mm_get0_ps(value_i)); } - else if (i == 1) { return scalar_value_type(_mm_get1_ps(value_r), _mm_get1_ps(value_i)); } - else if (i == 2) { return scalar_value_type(_mm_get2_ps(value_r), _mm_get2_ps(value_i)); } - else { return scalar_value_type(_mm_get3_ps(value_r), _mm_get3_ps(value_i)); } - } - - FASTOR_INLINE SIMDVector real() const { - return value_r; - } - FASTOR_INLINE SIMDVector imag() const { - return value_i; - } - - FASTOR_INLINE void set(std::complex num) { - value_r = _mm_set1_ps(num.real()); - value_i = _mm_set1_ps(num.imag()); - } - FASTOR_INLINE void set(scalar_value_type num0, scalar_value_type num1, - scalar_value_type num2, scalar_value_type num3) { - const scalar_value_type tmp[Size] = {num0,num1,num2,num3}; - complex_unaligned_load(tmp); - } - - // In-place operators - template,bool> = false> - FASTOR_INLINE void operator+=(U num) { - value_r = _mm_add_ps(value_r,_mm_set1_ps(num)); - } - FASTOR_INLINE void operator+=(scalar_value_type num) { - *this += vector_type(num); - } - FASTOR_INLINE void operator+=(const vector_type &a) { - value_r = _mm_add_ps(value_r,a.value_r); - value_i = _mm_add_ps(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - value_r = _mm_sub_ps(value_r,_mm_set1_ps(num)); - } - FASTOR_INLINE void operator-=(scalar_value_type num) { - *this -= vector_type(num); - } - FASTOR_INLINE void operator-=(const vector_type &a) { - value_r = _mm_sub_ps(value_r,a.value_r); - value_i = _mm_sub_ps(value_i,a.value_i); - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - __m128 val = _mm_set1_ps(num); - value_r = _mm_mul_ps(value_r,val); - value_i = _mm_mul_ps(value_i,val); - } - FASTOR_INLINE void operator*=(scalar_value_type num) { - *this *= vector_type(num); - } - FASTOR_INLINE void operator*=(const vector_type &a) { -#ifdef FASTOR_FMA_IMPL - __m128 tmp = _mm_fmsub_ps(value_r,a.value_r,_mm_mul_ps(value_i,a.value_i)); - value_i = _mm_fmadd_ps(value_r,a.value_i,_mm_mul_ps(value_i,a.value_r)); -#else - __m128 tmp = _mm_sub_ps(_mm_mul_ps(value_r,a.value_r),_mm_mul_ps(value_i,a.value_i)); - value_i = _mm_add_ps(_mm_mul_ps(value_r,a.value_i),_mm_mul_ps(value_i,a.value_r)); -#endif - value_r = tmp; - } - - template,bool> = false> - FASTOR_INLINE void operator/=(U num) { - __m128 val = _mm_set1_ps(num); - value_r = _mm_div_ps(value_r,val); - value_i = _mm_div_ps(value_i,val); - } - FASTOR_INLINE void operator/=(scalar_value_type num) { - *this /= vector_type(num); - } - FASTOR_INLINE void operator/=(const vector_type &a) { - __m128 tmp = value_r; -#ifdef FASTOR_FMA_IMPL - value_r = _mm_fmadd_ps(value_r , a.value_r, _mm_mul_ps(value_i,a.value_i)); - value_i = _mm_fmsub_ps(value_i , a.value_r, _mm_mul_ps(tmp,a.value_i)); - __m128 den = _mm_fmadd_ps(a.value_r, a.value_r, _mm_mul_ps(a.value_i,a.value_i)); -#else - value_r = _mm_add_ps(_mm_mul_ps(value_r , a.value_r), _mm_mul_ps(value_i,a.value_i)); - value_i = _mm_sub_ps(_mm_mul_ps(value_i , a.value_r), _mm_mul_ps(tmp,a.value_i)); - __m128 den = _mm_add_ps(_mm_mul_ps(a.value_r, a.value_r), _mm_mul_ps(a.value_i,a.value_i)); -#endif - value_r = _mm_div_ps(value_r,den); - value_i = _mm_div_ps(value_i,den); - } - // end of in-place operators - - FASTOR_INLINE scalar_value_type sum() const { - return scalar_value_type(_mm_sum_ps(value_r),_mm_sum_ps(value_i)); - } - FASTOR_INLINE scalar_value_type product() const { - vector_type tmp(*this); - return tmp[0]*tmp[1]*tmp[2]*tmp[3]; - } - FASTOR_INLINE vector_type reverse() const { - vector_type out; - out.value_r = _mm_reverse_ps(value_r); - out.value_i = _mm_reverse_ps(value_i); - return out; - } - /* Actual magnitude - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector magnitude() const { -#ifdef FASTOR_FMA_IMPL - return _mm_sqrt_ps(_mm_fmadd_ps(value_r,value_r,_mm_mul_ps(value_i,value_i))); -#else - return _mm_sqrt_ps(_mm_add_ps(_mm_mul_ps(value_r,value_r),_mm_mul_ps(value_i,value_i))); -#endif - } - /* STL compliant squared norm - Note that this is a vertical operation */ - FASTOR_INLINE SIMDVector norm() const { -#ifdef FASTOR_FMA_IMPL - return _mm_fmadd_ps(value_r,value_r,_mm_mul_ps(value_i,value_i)); -#else - return _mm_add_ps(_mm_mul_ps(value_r,value_r),_mm_mul_ps(value_i,value_i)); -#endif - } - // Magnitude based minimum - FASTOR_INLINE scalar_value_type minimum() const; - // Magnitude based maximum - FASTOR_INLINE scalar_value_type maximum() const; - - FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { - vector_type out(*this); - out *= other; - return out.sum(); - } - - value_type value_r; - value_type value_i; - -protected: - FASTOR_INLINE void complex_aligned_load(const std::complex *data) { - __m128 lo = _mm_load_ps(reinterpret_cast(data )); - __m128 hi = _mm_load_ps(reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); - } - FASTOR_INLINE void complex_unaligned_load(const std::complex *data) { - __m128 lo = _mm_loadu_ps(reinterpret_cast(data )); - __m128 hi = _mm_loadu_ps(reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); - } - - FASTOR_INLINE void complex_aligned_store(std::complex *data) const { - __m128 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm_store_ps(reinterpret_cast(data ),lo); - _mm_store_ps(reinterpret_cast(data+2),hi); - } - FASTOR_INLINE void complex_unaligned_store(std::complex *data) const { - __m128 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - _mm_storeu_ps(reinterpret_cast(data ),lo); - _mm_storeu_ps(reinterpret_cast(data+2),hi); - } - - FASTOR_INLINE void complex_mask_aligned_load(const scalar_value_type *data, uint8_t mask) { -#ifdef FASTOR_HAS_AVX512_MASKS - __m128 lo, hi; - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - lo = _mm_mask_load_ps(lo, mask0, reinterpret_cast(data )); - hi = _mm_mask_load_ps(hi, mask1, reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm_setzero_ps(); - value_i = _mm_setzero_ps(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - lo = _mm_mask_loadu_ps(lo, mask0, reinterpret_cast(data )); - hi = _mm_mask_loadu_ps(hi, mask1, reinterpret_cast(data+2)); - arrange_from_load(value_r, value_i, lo, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - value_r = _mm_setzero_ps(); - value_i = _mm_setzero_ps(); - for (FASTOR_INDEX i=0; i(mask, mask0, mask1); - _mm_mask_store_ps(reinterpret_cast(data ), mask0, lo); - _mm_mask_store_ps(reinterpret_cast(data+2), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - FASTOR_INLINE void complex_mask_unaligned_store(scalar_value_type *data, uint8_t mask) const { -#ifdef FASTOR_HAS_AVX512_MASKS - __m128 lo, hi; - arrange_for_store(lo, hi, value_r, value_i); - uint8_t mask0, mask1; - split_mask(mask, mask0, mask1); - _mm_mask_storeu_ps(reinterpret_cast(data ), mask0, lo); - _mm_mask_storeu_ps(reinterpret_cast(data+2), mask1, hi); -#else - int maska[Size]; - mask_to_array(mask,maska); - for (FASTOR_INDEX i=0; i(_real,_imag); - } - else { - data[Size - i - 1] = std::complex(0,0); - } - } -#endif - } - -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::sse> a) { - // ICC crashes without a copy - const __m128 vr = a.value_r; - const __m128 vi = a.value_i; - const float* value_r = reinterpret_cast(&vr); - const float* value_i = reinterpret_cast(&vi); - os << "[" << value_r[0] << signum_string(value_i[0]) << std::abs(value_i[0]) << "j, " - << value_r[1] << signum_string(value_i[1]) << std::abs(value_i[1]) << "j, " - << value_r[2] << signum_string(value_i[2]) << std::abs(value_i[2]) << "j, " - << value_r[3] << signum_string(value_i[3]) << std::abs(value_i[3]) << "j" << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; - out.value_r = _mm_add_ps(a.value_r,b.value_r); - out.value_i = _mm_add_ps(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a + SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) + b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out(a); - out.value_r = _mm_add_ps(a.value_r,_mm_set1_ps(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(U a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out(b); - out.value_r = _mm_add_ps(_mm_set1_ps(a), b.value_r); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator+(const SIMDVector,simd_abi::sse> &b) { - return b; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; - out.value_r = _mm_sub_ps(a.value_r,b.value_r); - out.value_i = _mm_sub_ps(a.value_i,b.value_i); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a - SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) - b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out(a); - out.value_r = _mm_sub_ps(a.value_r,_mm_set1_ps(b)); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(U a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(std::complex(a,0)) - b; -} -/* This is negation and not complex conjugate */ -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator-(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out; - out.value_r = _mm_neg_ps(a.value_r); - out.value_i = _mm_neg_ps(a.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm_fmsub_ps(a.value_r,b.value_r,_mm_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm_fmadd_ps(a.value_r,b.value_i,_mm_mul_ps(a.value_i,b.value_r)); -#else - out.value_r = _mm_sub_ps(_mm_mul_ps(a.value_r,b.value_r),_mm_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm_add_ps(_mm_mul_ps(a.value_r,b.value_i),_mm_mul_ps(a.value_i,b.value_r)); -#endif - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a * SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) * b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out; - __m128 val = _mm_set1_ps(b); - out.value_r = _mm_mul_ps(a.value_r,val); - out.value_i = _mm_mul_ps(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator*(U a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; - __m128 val = _mm_set1_ps(a); - out.value_r = _mm_mul_ps(val,b.value_r); - out.value_i = _mm_mul_ps(val,b.value_i); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(const SIMDVector,simd_abi::sse> &a, const SIMDVector,simd_abi::sse> &b) { - SIMDVector,simd_abi::sse> out; -#ifdef FASTOR_FMA_IMPL - out.value_r = _mm_fmadd_ps(a.value_r,b.value_r,_mm_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm_fmsub_ps(a.value_i,b.value_r,_mm_mul_ps(a.value_r,b.value_i)); - __m128 den = _mm_fmadd_ps(b.value_r,b.value_r,_mm_mul_ps(b.value_i,b.value_i)); -#else - out.value_r = _mm_add_ps(_mm_mul_ps(a.value_r,b.value_r),_mm_mul_ps(a.value_i,b.value_i)); - out.value_i = _mm_sub_ps(_mm_mul_ps(a.value_i,b.value_r),_mm_mul_ps(a.value_r,b.value_i)); - __m128 den = _mm_add_ps(_mm_mul_ps(b.value_r,b.value_r),_mm_mul_ps(b.value_i,b.value_i)); -#endif - out.value_r = _mm_div_ps(out.value_r,den); - out.value_i = _mm_div_ps(out.value_i,den); - return out; -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(const SIMDVector,simd_abi::sse> &a, std::complex b) { - return a / SIMDVector,simd_abi::sse>(b); -} -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(std::complex a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(a) / b; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(const SIMDVector,simd_abi::sse> &a, U b) { - SIMDVector,simd_abi::sse> out; - __m128 val = _mm_set1_ps(b); - out.value_r = _mm_div_ps(a.value_r,val); - out.value_i = _mm_div_ps(a.value_i,val); - return out; -} -template,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::sse> -operator/(U a, const SIMDVector,simd_abi::sse> &b) { - return SIMDVector,simd_abi::sse>(std::complex(a,0)) / b; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -rcp(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out; -#ifdef FASTOR_FMA_IMPL - __m128 den = _mm_fmadd_ps(a.value_r,a.value_r,_mm_mul_ps(a.value_i,a.value_i)); -#else - __m128 den = _mm_add_ps(_mm_mul_ps(a.value_r,a.value_r),_mm_mul_ps(a.value_i,a.value_i)); -#endif - out.value_r = _mm_div_ps(out.value_r,den); - out.value_i = _mm_neg_ps(_mm_div_ps(out.value_i,den)); - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -sqrt(const SIMDVector,simd_abi::sse> &a) = delete; - -FASTOR_INLINE SIMDVector,simd_abi::sse> -rsqrt(const SIMDVector,simd_abi::sse> &a) = delete; - -/* This intentionally return a complex vector */ -FASTOR_INLINE SIMDVector,simd_abi::sse> -abs(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out; - out.value_r = a.magnitude().value; - return out; -} - -FASTOR_INLINE SIMDVector,simd_abi::sse> -conj(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out(a); - out.value_i = _mm_neg_ps(out.value_i); - return out; -} - -/* Argument or phase angle */ -FASTOR_INLINE SIMDVector,simd_abi::sse> -arg(const SIMDVector,simd_abi::sse> &a) { - SIMDVector,simd_abi::sse> out(a); - for (FASTOR_INDEX i=0UL; i<4UL; ++i) { - ((float*)&out.value_r)[i] = std::atan2(((float*)&a.value_i)[i],((float*)&a.value_r)[i]); - } - return out; -} -//------------------------------------------------------------------------------------------------------------ - -#endif - - -} // end of namespace Fastor - -#endif // SIMD_VECTOR_COMPLEX_FLOAT_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h b/noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h deleted file mode 100644 index c6fcc996..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h +++ /dev/null @@ -1,378 +0,0 @@ -#ifndef SIMD_VECTOR_COMPLEX_SCALAR -#define SIMD_VECTOR_COMPLEX_SCALAR - -#include "Fastor/util/extended_algorithms.h" -#include "Fastor/simd_vector/extintrin.h" -#include "Fastor/simd_vector/simd_vector_base.h" -#include "Fastor/simd_vector/simd_vector_double.h" -#include -#include - - -namespace Fastor { - - -// SCALAR IMPLEMENTATION OF SIMDVECTOR FOR COMPLEX -//------------------------------------------------------------------------------------------------------------ - -template -struct SIMDVector, simd_abi::scalar> { - using vector_type = SIMDVector, simd_abi::scalar>; - using value_type = T; - using scalar_value_type = std::complex; - using abi_type = simd_abi::scalar; - static constexpr FASTOR_INDEX Size = 1; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;} - - FASTOR_INLINE SIMDVector() : value_r(0), value_i(0) {} - FASTOR_INLINE SIMDVector(scalar_value_type num) { - value_r = num.real(); - value_i = num.imag(); - } - FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {} - FASTOR_INLINE SIMDVector(const scalar_value_type *data, bool Aligned=true) { - value_r = (*data).real(); - value_i = (*data).imag(); - } - - FASTOR_INLINE SIMDVector operator=(std::complex num) { - value_r = num.real(); - value_i = num.imag(); - return *this; - } - - FASTOR_INLINE void load(const scalar_value_type *data, bool ) { - value_r = (*data).real(); - value_i = (*data).imag(); - } - FASTOR_INLINE void store(scalar_value_type *data, bool ) const { - data[0] = scalar_value_type(value_r,value_i); - } - - FASTOR_INLINE void aligned_load(const T *data) { - value_r = (*data).real(); - value_i = (*data).imag(); - } - FASTOR_INLINE void aligned_store(T *data) const { - data[0] = scalar_value_type(value_r,value_i); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool ) { - if (mask != 0x0) { - value_r = (*data).real(); - value_i = (*data).imag(); - } - } - FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool) const { - if (mask != 0x0) { - data[0] = scalar_value_type(value_r,value_i); - } - } - - FASTOR_INLINE T operator[](FASTOR_INDEX) const {return scalar_value_type(value_r,value_i);} - - FASTOR_INLINE SIMDVector real() const { - return value_r; - } - FASTOR_INLINE SIMDVector imag() const { - return value_i; - } - - FASTOR_INLINE void set(scalar_value_type num) { - value_r = num.real(); - value_i = num.imag(); - } - - FASTOR_INLINE void set_sequential(scalar_value_type num) { - value_r = num.real(); - value_i = num.imag(); - } - - // In-place operators - template,bool> = false> - FASTOR_INLINE void operator+=(U num) { - value_r += num; - } - FASTOR_INLINE void operator+=(scalar_value_type num) { - value_r += num.real(); - value_i += num.imag(); - } - FASTOR_INLINE void operator+=(const vector_type &a) { - value_r += a.value_r; - value_i += a.value_i; - } - - template,bool> = false> - FASTOR_INLINE void operator-=(U num) { - value_r -= num; - } - FASTOR_INLINE void operator-=(scalar_value_type num) { - value_r -= num.real(); - value_i -= num.imag(); - } - FASTOR_INLINE void operator-=(const vector_type &a) { - value_r -= a.value_r; - value_i -= a.value_i; - } - - template,bool> = false> - FASTOR_INLINE void operator*=(U num) { - value_r *= num; - value_i *= num; - } - FASTOR_INLINE void operator*=(scalar_value_type num) { - scalar_value_type tmp(value_r, value_i); - tmp *= num; - value_r = tmp.real(); - value_i = tmp.imag(); - } - FASTOR_INLINE void operator*=(const vector_type &a) { - scalar_value_type tmp0(value_r, value_i); - scalar_value_type tmp1(a.value_r, a.value_i); - tmp0 *= tmp1; - value_r = tmp0.real(); - value_i = tmp0.imag(); - } - - template,bool> = false> - FASTOR_INLINE void operator/=(U num) { - value_r /= num; - value_i /= num; - } - FASTOR_INLINE void operator/=(scalar_value_type num) { - scalar_value_type tmp(value_r, value_i); - tmp /= num; - value_r = tmp.real(); - value_i = tmp.imag(); - } - FASTOR_INLINE void operator/=(const vector_type &a) { - scalar_value_type tmp0(value_r, value_i); - scalar_value_type tmp1(a.value_r, a.value_i); - tmp0 /= tmp1; - value_r = tmp0.real(); - value_i = tmp0.imag(); - } - // end of in-place operators - - FASTOR_INLINE scalar_value_type sum() const {return scalar_value_type(value_r, value_i);} - FASTOR_INLINE scalar_value_type product() const {return scalar_value_type(value_r, value_i);} - FASTOR_INLINE vector_type reverse() const { return scalar_value_type(value_r, value_i); } - FASTOR_INLINE SIMDVector magnitude() const { return std::abs(scalar_value_type(value_r, value_i));} - FASTOR_INLINE SIMDVector norm() const { return std::norm(scalar_value_type(value_r, value_i));} - FASTOR_INLINE scalar_value_type minimum() const {return scalar_value_type(value_r, value_i);} - FASTOR_INLINE scalar_value_type maximum() const {return scalar_value_type(value_r, value_i);} - FASTOR_INLINE scalar_value_type dot(const vector_type &other) const { - return vector_type(value_r, value_i)*vector_type(other.value_r, other.value_i); - } - - value_type value_r; - value_type value_i; -}; - -template -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector,simd_abi::scalar> a) { - os << "[" << a.value_r << signum_string(a.value_i) << std::abs(a.value_i) << "j]\n"; - return os; -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator+(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out += b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator+(const SIMDVector,simd_abi::scalar> &a, std::complex b) { - SIMDVector,simd_abi::scalar> out(a); - out += b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator+(std::complex a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out += b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator+(const SIMDVector,simd_abi::scalar> &a, U b) { - SIMDVector,simd_abi::scalar> out(a); - out += b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator+(U a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(std::complex(a,0)); - out += b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator+(const SIMDVector,simd_abi::scalar> &b) { - return b; -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator-(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out -= b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator-(const SIMDVector,simd_abi::scalar> &a, std::complex b) { - SIMDVector,simd_abi::scalar> out(a); - out -= b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator-(std::complex a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out -= b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator-(const SIMDVector,simd_abi::scalar> &a, U b) { - SIMDVector,simd_abi::scalar> out(a); - out -= b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator-(U a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(std::complex(a,0)); - out -= b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator-(const SIMDVector,simd_abi::scalar> &b) { - return SIMDVector,simd_abi::scalar>(0,0) - b; -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator*(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out *= b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator*(const SIMDVector,simd_abi::scalar> &a, std::complex b) { - SIMDVector,simd_abi::scalar> out(a); - out *= b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator*(std::complex a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out *= b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator*(const SIMDVector,simd_abi::scalar> &a, U b) { - SIMDVector,simd_abi::scalar> out(a); - out *= b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator*(U a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(std::complex(a,0)); - out *= b; - return out; -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator/(const SIMDVector,simd_abi::scalar> &a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out /= b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator/(const SIMDVector,simd_abi::scalar> &a, std::complex b) { - SIMDVector,simd_abi::scalar> out(a); - out /= b; - return out; -} -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator/(std::complex a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(a); - out /= b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator/(const SIMDVector,simd_abi::scalar> &a, U b) { - SIMDVector,simd_abi::scalar> out(a); - out /= b; - return out; -} -template ,bool> = false> -FASTOR_INLINE SIMDVector,simd_abi::scalar> -operator/(U a, const SIMDVector,simd_abi::scalar> &b) { - SIMDVector,simd_abi::scalar> out(std::complex(a,0)); - out /= b; - return out; -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -rcp(const SIMDVector,simd_abi::scalar> &a) { - SIMDVector,simd_abi::scalar> out(std::complex(1,0)); - out /= a; - return out; -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -sqrt(const SIMDVector,simd_abi::scalar> &a) { - std::complex out(a.value_r,a.value_i); - return std::sqrt(out); -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -rsqrt(const SIMDVector,simd_abi::scalar> &a) { - return rcp(sqrt(a)); -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -abs(const SIMDVector,simd_abi::scalar> &a) { - SIMDVector,simd_abi::scalar> out; - out.value_r = a.magnitude().value; - return out; -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -conj(const SIMDVector,simd_abi::scalar> &a) { - std::complex out(a.value_r,a.value_i); - return std::conj(out); -} - -template -FASTOR_INLINE SIMDVector,simd_abi::scalar> -arg(const SIMDVector,simd_abi::scalar> &a) { - std::complex out(a.value_r,a.value_i); - return SIMDVector,simd_abi::scalar>(std::arg(out),0); -} -//------------------------------------------------------------------------------------------------------------ - -} // end of namespace Fastor - -#endif // SIMD_VECTOR_COMPLEX_SCALAR diff --git a/noarch/include/Fastor/simd_vector/simd_vector_double.h b/noarch/include/Fastor/simd_vector/simd_vector_double.h deleted file mode 100644 index 55543d9a..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_double.h +++ /dev/null @@ -1,914 +0,0 @@ -#ifndef SIMD_VECTOR_DOUBLE_H -#define SIMD_VECTOR_DOUBLE_H - -#include "Fastor/simd_vector/simd_vector_base.h" - -namespace Fastor { - -// AVX512 VERSION -//-------------------------------------------------------------------------------------------------- -#ifdef FASTOR_AVX512_IMPL - -template <> -struct SIMDVector { - using value_type = __m512d; - using scalar_value_type = double; - using abi_type = simd_abi::avx512; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm512_setzero_pd()) {} - FASTOR_INLINE SIMDVector(double num) : value(_mm512_set1_pd(num)) {} - FASTOR_INLINE SIMDVector(__m512d regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_pd(data); - else - value = _mm512_loadu_pd(data); - } - - FASTOR_INLINE SIMDVector operator=(double num) { - value = _mm512_set1_pd(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m512d regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const double *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_pd(data); - else - value = _mm512_loadu_pd(data); - } - FASTOR_INLINE void store(double *data, bool Aligned=true) const { - if (Aligned) - _mm512_store_pd(data,value); - else - _mm512_storeu_pd(data,value); - } - - FASTOR_INLINE void aligned_load(const double *data) { - value =_mm512_load_pd(data); - } - FASTOR_INLINE void aligned_store(double *data) const { - _mm512_store_pd(data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm512_mask_loadu_pd(value, mask, a); - else - value = _mm512_mask_load_pd(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm512_setzero_pd(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(double num) { - value = _mm512_set1_pd(num); - } - FASTOR_INLINE void set(double num0, double num1, double num2, double num3, double num4, double num5, double num6, double num7) { - value = _mm512_set_pd(num0,num1,num2,num3,num4,num5,num6,num7); - } - FASTOR_INLINE void set_sequential(double num0) { - value = _mm512_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0,num0+4.0,num0+5.0,num0+6.0,num0+7.0); - } - FASTOR_INLINE void broadcast(const double *data) { - // value = _mm512_broadcast_sd(data); - } - - // In-place operators - FASTOR_INLINE void operator+=(double num) { - value = _mm512_add_pd(value,_mm512_set1_pd(num)); - } - FASTOR_INLINE void operator+=(__m512d regi) { - value = _mm512_add_pd(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm512_add_pd(value,a.value); - } - - FASTOR_INLINE void operator-=(double num) { - value = _mm512_sub_pd(value,_mm512_set1_pd(num)); - } - FASTOR_INLINE void operator-=(__m512d regi) { - value = _mm512_sub_pd(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm512_sub_pd(value,a.value); - } - - FASTOR_INLINE void operator*=(double num) { - value = _mm512_mul_pd(value,_mm512_set1_pd(num)); - } - FASTOR_INLINE void operator*=(__m512d regi) { - value = _mm512_mul_pd(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm512_mul_pd(value,a.value); - } - - FASTOR_INLINE void operator/=(double num) { - value = _mm512_div_pd(value,_mm512_set1_pd(num)); - } - FASTOR_INLINE void operator/=(__m512d regi) { - value = _mm512_div_pd(value,regi); - } - FASTOR_INLINE void operator/=(const SIMDVector &a) { - value = _mm512_div_pd(value,a.value); - } - // end of in-place operators - - // FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { - // SIMDVector out; - // if (i==1) - // out.value = _mm512_shift1_pd(value); - // else if (i==2) - // out.value = _mm512_shift2_pd(value); - // else if (i==3) - // out.value = _mm512_shift3_pd(value); - // return out; - // } - FASTOR_INLINE double sum() { -#ifdef FASTOR_HAS_AVX512_REDUCE_ADD - return _mm512_reduce_add_pd(value); -#else - __m256d low = _mm512_castpd512_pd256(value); - __m256d high = _mm512_extractf64x4_pd(value,0x1); - return _mm256_sum_pd(_mm256_add_pd(low,high)); -#endif - } - FASTOR_INLINE double product() { - __m256d low = _mm512_castpd512_pd256(value); - __m256d high = _mm512_extractf64x4_pd(value,1); - return _mm256_prod_pd(_mm256_mul_pd(low,high)); - } - FASTOR_INLINE SIMDVector reverse() { - return _mm512_reverse_pd(value); - } - // FASTOR_INLINE double minimum() {return _mm512_hmin_pd(value);} - // FASTOR_INLINE double maximum() {return _mm512_hmax_pd(value);} - - FASTOR_INLINE double dot(const SIMDVector &other) { - __m512d res = _mm512_mul_pd(value,other.value); - __m256d low = _mm512_castpd512_pd256(res); - __m256d high = _mm512_extractf64x4_pd(res,1); - return _mm256_sum_pd(_mm256_add_pd(low,high)); - } - - __m512d value; -}; - - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - // ICC crashes without a copy - const __m512d v = a.value; - const double* value = reinterpret_cast(&v); - os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] - << " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm512_add_pd(a.value,_mm512_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_pd(_mm512_set1_pd(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm512_sub_pd(a.value,_mm512_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_pd(_mm512_set1_pd(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - return _mm512_neg_pd(b.value); -} - -FASTOR_INLINE SIMDVector operator*( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_mul_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm512_mul_pd(a.value,_mm512_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_mul_pd(_mm512_set1_pd(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_div_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm512_div_pd(a.value,_mm512_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator/(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_div_pd(_mm512_set1_pd(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - SIMDVector out; - out.value = _mm512_rcp14_pd(a.value); - return out; -} - -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm512_sqrt_pd(a.value); - return out; -} - -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm512_rsqrt14_pd(a.value); - return out; -} - -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - SIMDVector out; -#ifdef FASTOR_HAS_AVX512_ABS - out.value = _mm512_abs_pd(a.value); -#else - for (FASTOR_INDEX i=0UL; i<8UL; ++i) { - ((double*)&out.value)[i] = std::abs(((double*)&a.value)[i]); - } -#endif - return out; -} - -#endif - - - -// AVX VERSION -//-------------------------------------------------------------------------------------------------- -#ifdef FASTOR_AVX_IMPL - -template <> -struct SIMDVector { - using value_type = __m256d; - using scalar_value_type = double; - using abi_type = simd_abi::avx; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm256_setzero_pd()) {} - FASTOR_INLINE SIMDVector(double num) : value(_mm256_set1_pd(num)) {} - FASTOR_INLINE SIMDVector(__m256d regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_pd(data); - else - value = _mm256_loadu_pd(data); - } - - FASTOR_INLINE SIMDVector operator=(double num) { - value = _mm256_set1_pd(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m256d regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const double *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_pd(data); - else - value = _mm256_loadu_pd(data); - } - FASTOR_INLINE void store(double *data, bool Aligned=true) const { - if (Aligned) - _mm256_store_pd(data,value); - else - _mm256_storeu_pd(data,value); - } - - FASTOR_INLINE void aligned_load(const double *data) { - value =_mm256_load_pd(data); - } - FASTOR_INLINE void aligned_store(double *data) const { - _mm256_store_pd(data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm256_mask_loadu_pd(value, mask, a); - else - value = _mm256_mask_load_pd(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm256_setzero_pd(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(double num) { - value = _mm256_set1_pd(num); - } - FASTOR_INLINE void set(double num0, double num1, double num2, double num3) { - value = _mm256_set_pd(num0,num1,num2,num3); - } - FASTOR_INLINE void set_sequential(double num0) { - value = _mm256_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0); - } - FASTOR_INLINE void broadcast(const double *data) { - value = _mm256_broadcast_sd(data); - } - - // In-place operators - FASTOR_INLINE void operator+=(double num) { - value = _mm256_add_pd(value,_mm256_set1_pd(num)); - } - FASTOR_INLINE void operator+=(__m256d regi) { - value = _mm256_add_pd(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm256_add_pd(value,a.value); - } - - FASTOR_INLINE void operator-=(double num) { - value = _mm256_sub_pd(value,_mm256_set1_pd(num)); - } - FASTOR_INLINE void operator-=(__m256d regi) { - value = _mm256_sub_pd(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm256_sub_pd(value,a.value); - } - - FASTOR_INLINE void operator*=(double num) { - value = _mm256_mul_pd(value,_mm256_set1_pd(num)); - } - FASTOR_INLINE void operator*=(__m256d regi) { - value = _mm256_mul_pd(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm256_mul_pd(value,a.value); - } - - FASTOR_INLINE void operator/=(double num) { - value = _mm256_div_pd(value,_mm256_set1_pd(num)); - } - FASTOR_INLINE void operator/=(__m256d regi) { - value = _mm256_div_pd(value,regi); - } - FASTOR_INLINE void operator/=(const SIMDVector &a) { - value = _mm256_div_pd(value,a.value); - } - // end of in-place operators - - FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { - SIMDVector out; - if (i==1) - out.value = _mm256_shift1_pd(value); - else if (i==2) - out.value = _mm256_shift2_pd(value); - else if (i==3) - out.value = _mm256_shift3_pd(value); - return out; - } - FASTOR_INLINE double sum() {return _mm256_sum_pd(value);} - FASTOR_INLINE double product() {return _mm256_prod_pd(value);} - FASTOR_INLINE SIMDVector reverse() { - SIMDVector out; - out.value = _mm256_reverse_pd(value); - return out; - } - FASTOR_INLINE double minimum() {return _mm256_hmin_pd(value);} - FASTOR_INLINE double maximum() {return _mm256_hmax_pd(value);} - - FASTOR_INLINE double dot(const SIMDVector &other) { - return _mm_cvtsd_f64(_mm256_dp_pd(value,other.value)); - } - - __m256d value; -}; - - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - // ICC crashes without a copy - const __m256d v = a.value; - const double* value = reinterpret_cast(&v); - os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm256_add_pd(a.value,_mm256_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_pd(_mm256_set1_pd(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm256_sub_pd(a.value,_mm256_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_pd(_mm256_set1_pd(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_neg_pd(b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm256_mul_pd(a.value,_mm256_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_pd(_mm256_set1_pd(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_div_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm256_div_pd(a.value,_mm256_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator/(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_div_pd(_mm256_set1_pd(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - SIMDVector out; - // This is very inaccurate for double precision - out.value = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value))); - return out; - - // // For making it more accurate using Newton Raphson use this - // __m128d xmm0 = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value))); - // xmm0 = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0))); - // out.value = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0))); - // return out; - -} - -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm256_sqrt_pd(a.value); - return out; -} - -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - SIMDVector out; - // This is very inaccurate for double precision - out.value = _mm256_cvtps_pd(_mm_rsqrt_ps(_mm256_cvtpd_ps(a.value))); - return out; -} - -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - SIMDVector out; - out.value = _mm256_abs_pd(a.value); - return out; -} - -#endif - - - - - - - -// SSE VERSION -//------------------------------------------------------------------------------------------------------------ - -#ifdef FASTOR_SSE2_IMPL - -template <> -struct SIMDVector { - using value_type = __m128d; - using scalar_value_type = double; - using abi_type = simd_abi::sse; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm_setzero_pd()) {} - FASTOR_INLINE SIMDVector(double num) : value(_mm_set1_pd(num)) {} - FASTOR_INLINE SIMDVector(__m128d regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_pd(data); - else - value = _mm_loadu_pd(data); - } - - FASTOR_INLINE SIMDVector operator=(double num) { - value = _mm_set1_pd(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m128d regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const double *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_pd(data); - else - value = _mm_loadu_pd(data); - } - FASTOR_INLINE void store(double *data, bool Aligned=true) const { - if (Aligned) - _mm_store_pd(data,value); - else - _mm_storeu_pd(data,value); - } - - FASTOR_INLINE void aligned_load(const double *data) { - value =_mm_load_pd(data); - } - FASTOR_INLINE void aligned_store(double *data) const { - _mm_store_pd(data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm_mask_loadu_pd(value, mask, a); - else - value = _mm_mask_load_pd(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm_setzero_pd(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(double num) { - value = _mm_set1_pd(num); - } - FASTOR_INLINE void set(double num0, double num1) { - value = _mm_set_pd(num0,num1); - } - FASTOR_INLINE void set_sequential(double num0) { - value = _mm_setr_pd(num0,num0+1.0); - } - FASTOR_INLINE void broadcast(const double *data) { - value = _mm_load1_pd(data); - } - - // In-place operators - FASTOR_INLINE void operator+=(double num) { - value = _mm_add_pd(value,_mm_set1_pd(num)); - } - FASTOR_INLINE void operator+=(__m128d regi) { - value = _mm_add_pd(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm_add_pd(value,a.value); - } - - FASTOR_INLINE void operator-=(double num) { - value = _mm_sub_pd(value,_mm_set1_pd(num)); - } - FASTOR_INLINE void operator-=(__m128d regi) { - value = _mm_sub_pd(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm_sub_pd(value,a.value); - } - - FASTOR_INLINE void operator*=(double num) { - value = _mm_mul_pd(value,_mm_set1_pd(num)); - } - FASTOR_INLINE void operator*=(__m128d regi) { - value = _mm_mul_pd(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm_mul_pd(value,a.value); - } - - FASTOR_INLINE void operator/=(double num) { - value = _mm_div_pd(value,_mm_set1_pd(num)); - } - FASTOR_INLINE void operator/=(__m128d regi) { - value = _mm_div_pd(value,regi); - } - FASTOR_INLINE void operator/=(const SIMDVector &a) { - value = _mm_div_pd(value,a.value); - } - // end of in-place operators - - FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { - SIMDVector out; - FASTOR_ASSERT(i==1,"INCORRECT SHIFT INDEX"); - out.value = _mm_shift1_pd(value); - return out; - } - FASTOR_INLINE double sum() {return _mm_sum_pd(value);} - FASTOR_INLINE double product() {return _mm_prod_pd(value);} - FASTOR_INLINE SIMDVector reverse() { - SIMDVector out; - out.value = _mm_reverse_pd(value); - return out; - } - FASTOR_INLINE double minimum() {return _mm_hmin_pd(value);} - FASTOR_INLINE double maximum() {return _mm_hmax_pd(value);} - - FASTOR_INLINE double dot(const SIMDVector &other) { -#ifdef FASTOR_SSE4_1_IMPL - return _mm_cvtsd_f64(_mm_dp_pd(value,other.value,0xff)); -#else - return _mm_sum_pd(_mm_mul_pd(value,other.value)); -#endif - } - - __m128d value; -}; - - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - // ICC crashes without a copy - const __m128d v = a.value; - const double* value = reinterpret_cast(&v); - os << "[" << value[0] << " " << value[1] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_add_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm_add_pd(a.value,_mm_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_add_pd(_mm_set1_pd(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_sub_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm_sub_pd(a.value,_mm_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_sub_pd(_mm_set1_pd(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - out.value = _mm_neg_pd(b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_mul_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm_mul_pd(a.value,_mm_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_mul_pd(_mm_set1_pd(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_div_pd(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, double b) { - SIMDVector out; - out.value = _mm_div_pd(a.value,_mm_set1_pd(b)); - return out; -} -FASTOR_INLINE SIMDVector operator/(double a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_div_pd(_mm_set1_pd(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - SIMDVector out; - // This is very inaccurate for double precision - out.value = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value))); - return out; - - /* - // For making it more accurate using Newton Raphson use this - __m128d xmm0 = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value))); - xmm0 = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0))); - out.value = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0))); - return out; - */ -} - -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm_sqrt_pd(a.value); - return out; -} - -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - SIMDVector out; - // This is very inaccurate for double precision - out.value = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(a.value))); - return out; -} - -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - SIMDVector out; - out.value = _mm_abs_pd(a.value); - return out; -} - - -#endif - - -} // end of namespace Fastor - -#endif // // SIMD_VECTOR_DOUBLE_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_float.h b/noarch/include/Fastor/simd_vector/simd_vector_float.h deleted file mode 100644 index ce78cfb8..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_float.h +++ /dev/null @@ -1,918 +0,0 @@ -#ifndef SIMD_VECTOR_FLOAT_H -#define SIMD_VECTOR_FLOAT_H - -#include "Fastor/simd_vector/simd_vector_base.h" - -namespace Fastor { - - -// AVX512 VERSION -//-------------------------------------------------------------------------------------------------- - -#ifdef FASTOR_AVX512_IMPL - -template <> -struct SIMDVector { - using value_type = __m512; - using scalar_value_type = float; - using abi_type = simd_abi::avx512; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm512_setzero_ps()) {} - FASTOR_INLINE SIMDVector(float num) : value(_mm512_set1_ps(num)) {} - FASTOR_INLINE SIMDVector(__m512 regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_ps(data); - else - value = _mm512_loadu_ps(data); - } - - FASTOR_INLINE SIMDVector operator=(float num) { - value = _mm512_set1_ps(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m512 regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const float *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_ps(data); - else - value = _mm512_loadu_ps(data); - } - FASTOR_INLINE void store(float *data, bool Aligned=true) const { - if (Aligned) - _mm512_store_ps(data,value); - else - _mm512_storeu_ps(data,value); - } - - FASTOR_INLINE void aligned_load(const float *data) { - value =_mm512_load_ps(data); - } - FASTOR_INLINE void aligned_store(float *data) const { - _mm512_store_ps(data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint16_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm512_mask_loadu_ps(value, mask, a); - else - value = _mm512_mask_load_ps(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm512_setzero_ps(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(float num) { - value = _mm512_set1_ps(num); - } - FASTOR_INLINE void set(float num0, float num1, float num2, float num3, - float num4, float num5, float num6, float num7, - float num8, float num9, float num10, float num11, - float num12, float num13, float num14, float num15) { - value = _mm512_set_ps(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15); - } - FASTOR_INLINE void set_sequential(float num0) { - value = _mm512_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f, - num0+8.f,num0+9.f,num0+10.f,num0+11.f,num0+12.f,num0+13.f,num0+14.f,num0+15.f); - } - FASTOR_INLINE void broadcast(const float *data) { - // value = _mm512_broadcast_ss(data); - } - - // In-place operators - FASTOR_INLINE void operator+=(float num) { - value = _mm512_add_ps(value,_mm512_set1_ps(num)); - } - FASTOR_INLINE void operator+=(__m512 regi) { - value = _mm512_add_ps(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm512_add_ps(value,a.value); - } - - FASTOR_INLINE void operator-=(float num) { - value = _mm512_sub_ps(value,_mm512_set1_ps(num)); - } - FASTOR_INLINE void operator-=(__m512 regi) { - value = _mm512_sub_ps(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm512_sub_ps(value,a.value); - } - - FASTOR_INLINE void operator*=(float num) { - value = _mm512_mul_ps(value,_mm512_set1_ps(num)); - } - FASTOR_INLINE void operator*=(__m512 regi) { - value = _mm512_mul_ps(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm512_mul_ps(value,a.value); - } - - FASTOR_INLINE void operator/=(float num) { - value = _mm512_div_ps(value,_mm512_set1_ps(num)); - } - FASTOR_INLINE void operator/=(__m512 regi) { - value = _mm512_div_ps(value,regi); - } - FASTOR_INLINE void operator/=(const SIMDVector &a) { - value = _mm512_div_ps(value,a.value); - } - // end of in-place operators - - FASTOR_INLINE float sum() { -#ifdef FASTOR_HAS_AVX512_REDUCE_ADD - return _mm512_reduce_add_ps(value); -#else - __m256 low = _mm512_castps512_ps256(value); - __m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1)); - return _mm256_sum_ps(_mm256_add_ps(low,high)); -#endif - } - FASTOR_INLINE float product() { - __m256 low = _mm512_castps512_ps256(value); - __m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1)); - return _mm256_prod_ps(_mm256_mul_ps(low,high)); - } - FASTOR_INLINE SIMDVector reverse() { - return _mm512_reverse_ps(value); - } - // FASTOR_INLINE float minimum() {return _mm512_hmin_ps(value);} - // FASTOR_INLINE float maximum() {return _mm512_hmax_ps(value);} - - FASTOR_INLINE float dot(const SIMDVector &other) { - __m512 res = _mm512_mul_ps(value,other.value); - __m256 low = _mm512_castps512_ps256(res); - __m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(res),1)); - return _mm256_sum_ps(_mm256_add_ps(low,high)); - } - - __m512 value; -}; - - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - // ICC crashes without a copy - const __m512 v = a.value; - const float* value = reinterpret_cast(&v); - os << "[" - << value[0] << " " << value[1] << " " - << value[2] << " " << value[3] << " " - << value[4] << " " << value[5] << " " - << value[6] << " " << value[7] << " " - << value[8] << " " << value[9] << " " - << value[10] << " " << value[11] << " " - << value[12] << " " << value[13] << " " - << value[14] << " " << value[15] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm512_add_ps(a.value,_mm512_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_ps(_mm512_set1_ps(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm512_sub_ps(a.value,_mm512_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_ps(_mm512_set1_ps(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - return _mm512_neg_ps(b.value); -} - -FASTOR_INLINE SIMDVector operator*( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_mul_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm512_mul_ps(a.value,_mm512_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_mul_ps(_mm512_set1_ps(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/( - const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_div_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm512_div_ps(a.value,_mm512_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator/(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_div_ps(_mm512_set1_ps(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - SIMDVector out; - out.value = _mm512_rcp14_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm512_sqrt_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm512_rsqrt14_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - SIMDVector out; -#ifdef FASTOR_HAS_AVX512_ABS - out.value = _mm512_abs_ps(a.value); -#else - for (FASTOR_INDEX i=0UL; i<16UL; ++i) { - ((float*)&out.value)[i] = std::abs(((float*)&a.value)[i]); - } -#endif - return out; -} - - -#endif - - - - -// AVX VERSION -//-------------------------------------------------------------------------------------------------- - -#ifdef FASTOR_AVX_IMPL - -template <> -struct SIMDVector { - using value_type = __m256; - using scalar_value_type = float; - using abi_type = simd_abi::avx; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm256_setzero_ps()) {} - FASTOR_INLINE SIMDVector(float num) : value(_mm256_set1_ps(num)) {} - FASTOR_INLINE SIMDVector(__m256 regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_ps(data); - else - value = _mm256_loadu_ps(data); - } - - FASTOR_INLINE SIMDVector operator=(float num) { - value = _mm256_set1_ps(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m256 regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const float *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_ps(data); - else - value = _mm256_loadu_ps(data); - } - FASTOR_INLINE void store(float *data, bool Aligned=true) const { - if (Aligned) - _mm256_store_ps(data,value); - else - _mm256_storeu_ps(data,value); - } - - FASTOR_INLINE void aligned_load(const float *data) { - value =_mm256_load_ps(data); - } - FASTOR_INLINE void aligned_store(float *data) const { - _mm256_store_ps(data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm256_mask_loadu_ps(value, mask, a); - else - value = _mm256_mask_load_ps(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm256_setzero_ps(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(float num) { - value = _mm256_set1_ps(num); - } - FASTOR_INLINE void set(float num0, float num1, float num2, float num3, - float num4, float num5, float num6, float num7) { - value = _mm256_set_ps(num0,num1,num2,num3,num4,num5,num6,num7); - } - FASTOR_INLINE void set_sequential(float num0) { - value = _mm256_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f); - } - FASTOR_INLINE void broadcast(const float *data) { - value = _mm256_broadcast_ss(data); - } - - // In-place operators - FASTOR_INLINE void operator+=(float num) { - value = _mm256_add_ps(value,_mm256_set1_ps(num)); - } - FASTOR_INLINE void operator+=(__m256 regi) { - value = _mm256_add_ps(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm256_add_ps(value,a.value); - } - - FASTOR_INLINE void operator-=(float num) { - value = _mm256_sub_ps(value,_mm256_set1_ps(num)); - } - FASTOR_INLINE void operator-=(__m256 regi) { - value = _mm256_sub_ps(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm256_sub_ps(value,a.value); - } - - FASTOR_INLINE void operator*=(float num) { - value = _mm256_mul_ps(value,_mm256_set1_ps(num)); - } - FASTOR_INLINE void operator*=(__m256 regi) { - value = _mm256_mul_ps(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm256_mul_ps(value,a.value); - } - - FASTOR_INLINE void operator/=(float num) { - value = _mm256_div_ps(value,_mm256_set1_ps(num)); - } - FASTOR_INLINE void operator/=(__m256 regi) { - value = _mm256_div_ps(value,regi); - } - FASTOR_INLINE void operator/=(const SIMDVector &a) { - value = _mm256_div_ps(value,a.value); - } - // end of in-place operators - - FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { - SIMDVector out; - if (i==1) - out.value = _mm256_shift1_ps(value); - else if (i==2) - out.value = _mm256_shift2_ps(value); - else if (i==3) - out.value = _mm256_shift3_ps(value); - else if (i==4) - out.value = _mm256_shift4_ps(value); - else if (i==5) - out.value = _mm256_shift5_ps(value); - else if (i==6) - out.value = _mm256_shift6_ps(value); - else if (i==7) - out.value = _mm256_shift7_ps(value); - return out; - } - FASTOR_INLINE float sum() {return _mm256_sum_ps(value);} - FASTOR_INLINE float product() {return _mm256_prod_ps(value);} - FASTOR_INLINE SIMDVector reverse() { - SIMDVector out; - out.value = _mm256_reverse_ps(value); - return out; - } - FASTOR_INLINE float minimum() {return _mm256_hmin_ps(value);} - FASTOR_INLINE float maximum() {return _mm256_hmax_ps(value);} - - FASTOR_INLINE float dot(const SIMDVector &other) { - __m256 tmp = _mm256_dp_ps(value,other.value,0xff); - return _mm256_get0_ps(tmp)+_mm256_get4_ps(tmp); - } - - __m256 value; -}; - - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - // ICC crashes without a copy - const __m256 v = a.value; - const float* value = reinterpret_cast(&v); - os << "[" << value[0] << " " << value[1] << " " - << value[2] << " " << value[3] << " " - << value[4] << " " << value[5] << " " - << value[6] << " " << value[7] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm256_add_ps(a.value,_mm256_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_ps(_mm256_set1_ps(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm256_sub_ps(a.value,_mm256_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_ps(_mm256_set1_ps(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_neg_ps(b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm256_mul_ps(a.value,_mm256_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_ps(_mm256_set1_ps(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_div_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm256_div_ps(a.value,_mm256_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator/(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_div_ps(_mm256_set1_ps(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - SIMDVector out; - out.value = _mm256_rcp_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm256_sqrt_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm256_rsqrt_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - SIMDVector out; - out.value = _mm256_abs_ps(a.value); - return out; -} - - -#endif - - - - - - -// SSE VERSION -//-------------------------------------------------------------------------------------------------- - -#ifdef FASTOR_SSE2_IMPL - -template <> -struct SIMDVector { - using value_type = __m128; - using scalar_value_type = float; - using abi_type = simd_abi::sse; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm_setzero_ps()) {} - FASTOR_INLINE SIMDVector(float num) : value(_mm_set1_ps(num)) {} - FASTOR_INLINE SIMDVector(__m128 regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_ps(data); - else - value = _mm_loadu_ps(data); - } - - FASTOR_INLINE SIMDVector operator=(float num) { - value = _mm_set1_ps(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m128 regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const float *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_ps(data); - else - value = _mm_loadu_ps(data); - } - FASTOR_INLINE void store(float *data, bool Aligned=true) const { - if (Aligned) - _mm_store_ps(data,value); - else - _mm_storeu_ps(data,value); - } - - FASTOR_INLINE void aligned_load(const float *data) { - value =_mm_load_ps(data); - } - FASTOR_INLINE void aligned_store(float *data) const { - _mm_store_ps(data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm_mask_loadu_ps(value, mask, a); - else - value = _mm_mask_load_ps(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm_setzero_ps(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(float num) { - value = _mm_set1_ps(num); - } - FASTOR_INLINE void set(float num0, float num1, float num2, float num3) { - value = _mm_set_ps(num0,num1,num2,num3); - } - FASTOR_INLINE void set_sequential(float num0) { - value = _mm_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f); - } - FASTOR_INLINE void broadcast(const float *data) { - value = _mm_load1_ps(data); - } - - // In-place operators - FASTOR_INLINE void operator+=(float num) { - value = _mm_add_ps(value,_mm_set1_ps(num)); - } - FASTOR_INLINE void operator+=(__m128 regi) { - value = _mm_add_ps(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm_add_ps(value,a.value); - } - - FASTOR_INLINE void operator-=(float num) { - value = _mm_sub_ps(value,_mm_set1_ps(num)); - } - FASTOR_INLINE void operator-=(__m128 regi) { - value = _mm_sub_ps(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm_sub_ps(value,a.value); - } - - FASTOR_INLINE void operator*=(float num) { - value = _mm_mul_ps(value,_mm_set1_ps(num)); - } - FASTOR_INLINE void operator*=(__m128 regi) { - value = _mm_mul_ps(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm_mul_ps(value,a.value); - } - - FASTOR_INLINE void operator/=(float num) { - value = _mm_div_ps(value,_mm_set1_ps(num)); - } - FASTOR_INLINE void operator/=(__m128 regi) { - value = _mm_div_ps(value,regi); - } - FASTOR_INLINE void operator/=(const SIMDVector &a) { - value = _mm_div_ps(value,a.value); - } - // end of in-place operators - - FASTOR_INLINE SIMDVector shift(FASTOR_INDEX i) { - SIMDVector out; - if (i==1) - out.value = _mm_shift1_ps(value); - else if (i==2) - out.value = _mm_shift2_ps(value); - else if (i==3) - out.value = _mm_shift3_ps(value); - return out; - } - FASTOR_INLINE float sum() {return _mm_sum_ps(value);} - FASTOR_INLINE float product() {return _mm_prod_ps(value);} - FASTOR_INLINE SIMDVector reverse() { - SIMDVector out; - out.value = _mm_reverse_ps(value); - return out; - } - FASTOR_INLINE float minimum() {return _mm_hmin_ps(value);} - FASTOR_INLINE float maximum() {return _mm_hmax_ps(value);} - - FASTOR_INLINE float dot(const SIMDVector &other) { -#ifdef FASTOR_SSE4_1_IMPL - return _mm_cvtss_f32(_mm_dp_ps(value,other.value,0xff)); -#else - return _mm_sum_ps(_mm_mul_ps(value,other.value)); -#endif - } - - __m128 value; -}; - - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - // ICC crashes without a copy - const __m128 v = a.value; - const float* value = reinterpret_cast(&v); - os << "[" << value[0] << " " << value[1] << " " - << value[2] << " " << value[3] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_add_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm_add_ps(a.value,_mm_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_add_ps(_mm_set1_ps(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_sub_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm_sub_ps(a.value,_mm_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_sub_ps(_mm_set1_ps(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - out.value = _mm_neg_ps(b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_mul_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm_mul_ps(a.value,_mm_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_mul_ps(_mm_set1_ps(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_div_ps(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, float b) { - SIMDVector out; - out.value = _mm_div_ps(a.value,_mm_set1_ps(b)); - return out; -} -FASTOR_INLINE SIMDVector operator/(float a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_div_ps(_mm_set1_ps(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - SIMDVector out; - out.value = _mm_rcp_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm_sqrt_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - SIMDVector out; - out.value = _mm_rsqrt_ps(a.value); - return out; -} - -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - SIMDVector out; - out.value = _mm_abs_ps(a.value); - return out; -} - - -#endif - - -} // end of namespace Fastor - - -#endif // SIMD_VECTOR_FLOAT_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_int32.h b/noarch/include/Fastor/simd_vector/simd_vector_int32.h deleted file mode 100644 index 0521defa..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_int32.h +++ /dev/null @@ -1,988 +0,0 @@ -#ifndef SIMD_VECTOR_INT_H -#define SIMD_VECTOR_INT_H - -#include "Fastor/simd_vector/simd_vector_base.h" -#include - -namespace Fastor { - - -// AVX512 VERSION -//----------------------------------------------------------------------------------------------- - -#ifdef FASTOR_AVX512F_IMPL - -template<> -struct SIMDVector { - using value_type = __m512i; - using scalar_value_type = int32_t; - using abi_type = simd_abi::avx512; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm512_setzero_si512()) {} - FASTOR_INLINE SIMDVector(int32_t num) : value(_mm512_set1_epi32(num)) {} - FASTOR_INLINE SIMDVector(__m512i regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_si512((__m512i*)data); - else - value = _mm512_loadu_si512((__m512i*)data); - } - - FASTOR_INLINE SIMDVector operator=(int32_t num) { - value = _mm512_set1_epi32(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m512i regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_si512((__m512i*)data); - else - value = _mm512_loadu_si512((__m512i*)data); - } - FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const { - if (Aligned) - _mm512_store_si512((__m512i*)data,value); - else - _mm512_storeu_si512((__m512i*)data,value); - } - - FASTOR_INLINE void aligned_load(const int32_t *data) { - value =_mm512_load_si512((__m512i*)data); - } - FASTOR_INLINE void aligned_store(int32_t *data) const { - _mm512_store_si512((__m512i*)data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm512_mask_loadu_epi32(value, mask, a); - else - value = _mm512_mask_load_epi32(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm512_setzero_si512(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(int32_t num) { - value = _mm512_set1_epi32(num); - } - FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7, - int32_t num8, int32_t num9, int32_t num10, int32_t num11, int32_t num12, int32_t num13, int32_t num14, int32_t num15) { - value = _mm512_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15); - } - FASTOR_INLINE void set_sequential(int32_t num0) { - value = _mm512_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7, - num0+8,num0+9,num0+10,num0+11,num0+12,num0+13,num0+14,num0+15); - } - - // In-place operators - FASTOR_INLINE void operator+=(int32_t num) { - value = _mm512_add_epi32(value,_mm512_set1_epi32(num)); - - } - FASTOR_INLINE void operator+=(__m512i regi) { - value = _mm512_add_epi32(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm512_add_epi32(value,a.value); - } - - FASTOR_INLINE void operator-=(int32_t num) { - value = _mm512_sub_epi32(value,_mm512_set1_epi32(num)); - } - FASTOR_INLINE void operator-=(__m512i regi) { - value = _mm512_sub_epi32(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm512_sub_epi32(value,a.value); - } - - FASTOR_INLINE void operator*=(int32_t num) { - value = _mm512_mullo_epi32(value,_mm512_set1_epi32(num)); - } - FASTOR_INLINE void operator*=(__m512i regi) { - value = _mm512_mullo_epi32(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm512_mullo_epi32(value,a.value); - } - - FASTOR_INLINE void operator/=(int32_t num) { -#ifdef FASTOR_INTEL - value = _mm512_div_epi32(value,_mm512_set1_epi32(num)); -#else - int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value); - for (FASTOR_INDEX i=0; i &a) { -#ifdef FASTOR_INTEL - value = _mm512_div_epi32(value,a.value); -#else - int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value); - int32_t val_a[Size]; _mm512_storeu_si512((__m512i*)val_a, a.value); - for (FASTOR_INDEX i=0; iquan) - quan = vals[i]; - return quan; - } - FASTOR_INLINE SIMDVector reverse() { - return _mm512_reverse_epi32(value); - } - - FASTOR_INLINE int32_t sum() { -#ifdef FASTOR_HAS_AVX512_REDUCE_ADD - return _mm512_reduce_add_epi32(value); -#else - int32_t vals[Size]; _mm512_storeu_si512((__m512i*)vals, value); - int32_t quan = 0; - for (FASTOR_INDEX i=0; i &other) { -#ifdef FASTOR_HAS_AVX512_REDUCE_ADD - return _mm512_reduce_add_epi32(_mm512_mullo_epi32(value,other.value)); -#else - return SIMDVector(_mm512_mullo_epi32(value,other.value)).sum(); -#endif - } - - __m512i value; -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - const int32_t *value = (int32_t*) &a.value; - os << "[" - << value[0] << " " << value[1] << " " - << value[2] << " " << value[3] << " " - << value[4] << " " << value[5] << " " - << value[6] << " " << value[7] << " " - << value[8] << " " << value[9] << " " - << value[10] << " " << value[11] << " " - << value[12] << " " << value[13] << " " - << value[14] << " " << value[15] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_epi32(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm512_add_epi32(a.value,_mm512_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_epi32(_mm512_set1_epi32(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_epi32(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm512_sub_epi32(a.value,_mm512_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_epi32(_mm512_set1_epi32(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - return _mm512_castps_si512(_mm512_neg_ps(_mm512_castsi512_ps(b.value))); -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_mullo_epi32(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm512_mullo_epi32(a.value,_mm512_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_mullo_epi32(_mm512_set1_epi32(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; -#ifdef FASTOR_INTEL - out.value = _mm512_div_epi32(a.value,b.value); -#else - int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); - int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); - int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); - for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int32_t b) { - SIMDVector out; -#ifdef FASTOR_INTEL - out.value = _mm512_div_epi32(a.value,_mm512_set1_epi32(b)); -#else - int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); - int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); - for (FASTOR_INDEX i=0; i operator/(int32_t a, const SIMDVector &b) { - SIMDVector out; -#ifdef FASTOR_INTEL - out.value = _mm512_div_epi32(_mm512_set1_epi32(a),b.value); -#else - int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); - int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); - for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { - SIMDVector out; -#ifdef FASTOR_HAS_AVX512_ABS - out.value = _mm512_abs_epi32(a.value); -#else - for (FASTOR_INDEX i=0UL; i<16UL; ++i) { - ((int32_t*)&out.value)[i] = std::abs(((int32_t*)&a.value)[i]); - } -#endif - return out; -} - - -#endif - - -// AVX VERSION -//----------------------------------------------------------------------------------------------- - -#ifdef FASTOR_AVX2_IMPL - -template<> -struct SIMDVector { - using value_type = __m256i; - using scalar_value_type = int32_t; - using abi_type = simd_abi::avx; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm256_setzero_si256()) {} - FASTOR_INLINE SIMDVector(int32_t num) : value(_mm256_set1_epi32(num)) {} - FASTOR_INLINE SIMDVector(__m256i regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_si256((__m256i*)data); - else - value = _mm256_loadu_si256((__m256i*)data); - } - - FASTOR_INLINE SIMDVector operator=(int32_t num) { - value = _mm256_set1_epi32(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m256i regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_si256((__m256i*)data); - else - value = _mm256_loadu_si256((__m256i*)data); - } - FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const { - if (Aligned) - _mm256_store_si256((__m256i*)data,value); - else - _mm256_storeu_si256((__m256i*)data,value); - } - - FASTOR_INLINE void aligned_load(const int32_t *data) { - value =_mm256_load_si256((__m256i*)data); - } - FASTOR_INLINE void aligned_store(int32_t *data) const { - _mm256_store_si256((__m256i*)data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm256_mask_loadu_epi32(value, mask, a); - else - value = _mm256_mask_load_epi32(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm256_setzero_si256(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(int32_t num) { - value = _mm256_set1_epi32(num); - } - FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7) { - value = _mm256_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7); - } - FASTOR_INLINE void set_sequential(int32_t num0) { - value = _mm256_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7); - } - - // In-place operators - FASTOR_INLINE void operator+=(int32_t num) { - value = _mm256_add_epi32x(value,_mm256_set1_epi32(num)); - - } - FASTOR_INLINE void operator+=(__m256i regi) { - value = _mm256_add_epi32x(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm256_add_epi32x(value,a.value); - } - - FASTOR_INLINE void operator-=(int32_t num) { - value = _mm256_sub_epi32x(value,_mm256_set1_epi32(num)); - } - FASTOR_INLINE void operator-=(__m256i regi) { - value = _mm256_sub_epi32x(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm256_sub_epi32x(value,a.value); - } - - FASTOR_INLINE void operator*=(int32_t num) { - value = _mm256_mul_epi32x(value,_mm256_set1_epi32(num)); - } - FASTOR_INLINE void operator*=(__m256i regi) { - value = _mm256_mul_epi32x(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm256_mul_epi32x(value,a.value); - } - - FASTOR_INLINE void operator/=(int32_t num) { - int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); - for (FASTOR_INDEX i=0; i &a) { - int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); - int32_t val_a[Size]; _mm256_storeu_si256((__m256i*)val_a, a.value); - for (FASTOR_INDEX i=0; iquan) - quan = vals[i]; - return quan; - } - FASTOR_INLINE SIMDVector reverse() { - SIMDVector out; - out.value = _mm256_reverse_epi32(value); - return out; - } - - FASTOR_INLINE int32_t sum() { - int32_t vals[Size]; _mm256_storeu_si256((__m256i*)vals, value); - int32_t quan = 0; - for (FASTOR_INDEX i=0; i &other) { - int32_t vals0[Size]; _mm256_storeu_si256((__m256i*)vals0, value); - int32_t vals1[Size]; _mm256_storeu_si256((__m256i*)vals1, other.value); - int32_t quan = 0; - for (FASTOR_INDEX i=0; i a) { - const int32_t *value = (int32_t*) &a.value; - os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] - << " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_epi32x(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm256_add_epi32x(a.value,_mm256_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_epi32x(_mm256_set1_epi32(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_epi32x(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm256_sub_epi32x(a.value,_mm256_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_epi32x(_mm256_set1_epi32(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - return _mm256_castps_si256(_mm256_neg_ps(_mm256_castsi256_ps(b.value))); -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_epi32x(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm256_mul_epi32x(a.value,_mm256_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_epi32x(_mm256_set1_epi32(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); - int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); - int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); - for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int32_t b) { - SIMDVector out; - int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); - int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); - for (FASTOR_INDEX i=0; i operator/(int32_t a, const SIMDVector &b) { - SIMDVector out; - int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); - int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); - for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { - SIMDVector out; -#ifdef __AVX2__ - out.value = _mm256_abs_epi32(a.value); -#else - // THIS IS ALSO AVX2 VERSION! - // __m128i lo = _mm_abs_epi32(_mm256_castsi256_si128(a.value)); - // __m128i hi = _mm_abs_epi32(_mm256_extracti128_si256(a.value,0x1)); - // out.value = _mm256_castsi128_si256(lo); - // out.value = _mm256_insertf128_si256(out.value,hi,0x1); - - int32_t *value = (int32_t*) &a.value; - for (int32_t i=0; i<8; ++i) { - value[i] = std::abs(value[i]); - } -#endif - return out; -} - - -#endif - - -// SSE VERSION -//----------------------------------------------------------------------------------------------- - -#ifdef FASTOR_SSE2_IMPL - -template<> -struct SIMDVector { - using value_type = __m128i; - using scalar_value_type = int32_t; - using abi_type = simd_abi::sse; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm_setzero_si128()) {} - FASTOR_INLINE SIMDVector(int32_t num) : value(_mm_set1_epi32(num)) {} - FASTOR_INLINE SIMDVector(__m128i regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_si128((__m128i*)data); - else - value = _mm_loadu_si128((__m128i*)data); - } - - FASTOR_INLINE SIMDVector operator=(int32_t num) { - value = _mm_set1_epi32(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m128i regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_si128((__m128i*)data); - else - value = _mm_loadu_si128((__m128i*)data); - } - FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const { - if (Aligned) - _mm_store_si128((__m128i*)data,value); - else - _mm_storeu_si128((__m128i*)data,value); - } - - FASTOR_INLINE void aligned_load(const int32_t *data) { - value =_mm_load_si128((__m128i*)data); - } - FASTOR_INLINE void aligned_store(int32_t *data) const { - _mm_store_si128((__m128i*)data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm_mask_loadu_epi32(value, mask, a); - else - value = _mm_mask_load_epi32(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm_setzero_si128(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(int32_t num) { - value = _mm_set1_epi32(num); - } - FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3) { - value = _mm_set_epi32(num0,num1,num2,num3); - } - FASTOR_INLINE void set_sequential(int32_t num0) { - value = _mm_setr_epi32(num0,num0+1,num0+2,num0+3); - } - - // In-place operators - FASTOR_INLINE void operator+=(int32_t num) { - value = _mm_add_epi32(value,_mm_set1_epi32(num)); - - } - FASTOR_INLINE void operator+=(__m128i regi) { - value = _mm_add_epi32(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm_add_epi32(value,a.value); - } - - FASTOR_INLINE void operator-=(int32_t num) { - value = _mm_sub_epi32(value,_mm_set1_epi32(num)); - } - FASTOR_INLINE void operator-=(__m128i regi) { - value = _mm_sub_epi32(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm_sub_epi32(value,a.value); - } - - FASTOR_INLINE void operator*=(int32_t num) { - value = _mm_mul_epi32x(value,_mm_set1_epi32(num)); - } - FASTOR_INLINE void operator*=(__m128i regi) { - value = _mm_mul_epi32x(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm_mul_epi32x(value,a.value); - } - - FASTOR_INLINE void operator/=(int32_t num) { - int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value); - for (FASTOR_INDEX i=0; i &a) { - int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value); - int32_t val_a[Size]; _mm_storeu_si128((__m128i*)val_a, a.value); - for (FASTOR_INDEX i=0; iquan) - quan = vals[i]; - return quan; - } - FASTOR_INLINE SIMDVector reverse() { - return _mm_reverse_epi32(value); - } - - FASTOR_INLINE int32_t sum() {return _mm_sum_epi32(value);} - FASTOR_INLINE int32_t product() {return _mm_prod_epi32(value);} - - FASTOR_INLINE int32_t dot(const SIMDVector &other) { - return _mm_sum_epi32(_mm_mul_epi32x(value,other.value)); - } - - __m128i value; -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - const int32_t *value = (int32_t*) &a.value; - os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_add_epi32(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm_add_epi32(a.value,_mm_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_add_epi32(_mm_set1_epi32(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_sub_epi32(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm_sub_epi32(a.value,_mm_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_sub_epi32(_mm_set1_epi32(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - return _mm_castps_si128(_mm_neg_ps(_mm_castsi128_ps(b.value))); -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_mul_epi32x(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int32_t b) { - SIMDVector out; - out.value = _mm_mul_epi32x(a.value,_mm_set1_epi32(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(int32_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_mul_epi32x(_mm_set1_epi32(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); - int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); - int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); - for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int32_t b) { - SIMDVector out; - int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); - int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); - for (FASTOR_INDEX i=0; i operator/(int32_t a, const SIMDVector &b) { - SIMDVector out; - int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); - int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); - for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { - SIMDVector out; -#ifdef FASTOR_SSSE3_IMPL - out.value = _mm_abs_epi32(a.value); -#else // SSE2 - __m128i sign = _mm_srai_epi32(a.value, 31); - __m128i inv = _mm_xor_si128(a.value, sign); - out.value = _mm_sub_epi32(inv, sign); -#endif - return out; -} - - -#endif - - -} // end of namespace Fastor - - -#endif // SIMD_VECTOR_INT_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_int64.h b/noarch/include/Fastor/simd_vector/simd_vector_int64.h deleted file mode 100644 index f76e9468..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_int64.h +++ /dev/null @@ -1,1040 +0,0 @@ -#ifndef SIMD_VECTOR_INT64_H -#define SIMD_VECTOR_INT64_H - -#include "Fastor/simd_vector/simd_vector_base.h" -#include - -namespace Fastor { - - -// AVX512 VERSION -//----------------------------------------------------------------------------------------------- - -#ifdef FASTOR_AVX512F_IMPL - -template<> -struct SIMDVector { - using value_type = __m512i; - using scalar_value_type = int64_t; - using abi_type = simd_abi::avx512; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm512_setzero_si512()) {} - FASTOR_INLINE SIMDVector(int64_t num) : value(_mm512_set1_epi64(num)) {} - FASTOR_INLINE SIMDVector(__m512i regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const int64_t *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_si512((__m512i*)data); - else - value = _mm512_loadu_si512((__m512i*)data); - } - - FASTOR_INLINE SIMDVector operator=(int64_t num) { - value = _mm512_set1_epi64(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m512i regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const int64_t *data, bool Aligned=true) { - if (Aligned) - value =_mm512_load_si512((__m512i*)data); - else - value = _mm512_loadu_si512((__m512i*)data); - } - FASTOR_INLINE void store(int64_t *data, bool Aligned=true) const { - if (Aligned) - _mm512_store_si512((__m512i*)data,value); - else - _mm512_storeu_si512((__m512i*)data,value); - } - - FASTOR_INLINE void aligned_load(const int64_t *data) { - value =_mm512_load_si512((__m512i*)data); - } - FASTOR_INLINE void aligned_store(int64_t *data) const { - _mm512_store_si512((__m512i*)data,value); - } - - FASTOR_INLINE int64_t operator[](FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - FASTOR_INLINE int64_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm512_mask_loadu_epi64(value, mask, a); - else - value = _mm512_mask_load_epi64(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm512_setzero_si512(); - for (FASTOR_INDEX i=0; i &a) { - value = _mm512_add_epi64(value,a.value); - } - - FASTOR_INLINE void operator-=(int64_t num) { - value = _mm512_sub_epi64(value,_mm512_set1_epi64(num)); - } - FASTOR_INLINE void operator-=(__m512i regi) { - value = _mm512_sub_epi64(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm512_sub_epi64(value,a.value); - } - - FASTOR_INLINE void operator*=(int64_t num) { -#ifdef FASTOR_AVX512DQ_IMPL - value = _mm512_mullo_epi64(value,_mm512_set1_epi64(num)); -#else - for (FASTOR_INDEX i=0; i &a) { -#ifdef FASTOR_AVX512DQ_IMPL - value = _mm512_mullo_epi64(value,a.value); -#else - for (FASTOR_INDEX i=0; i &a) { -#ifdef FASTOR_INTEL - value = _mm512_div_epi64(value,a.value); -#else - int64_t val[Size]; _mm512_storeu_si512((__m512i*)val, value); - int64_t val_a[Size]; _mm512_storeu_si512((__m512i*)val_a, a.value); - for (FASTOR_INDEX i=0; i(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; iquan) - quan = vals[i]; - return quan; - } - FASTOR_INLINE SIMDVector reverse() { - return _mm512_reverse_epi64(value); - } - - FASTOR_INLINE int64_t sum() { -#ifdef FASTOR_HAS_AVX512_REDUCE_ADD - return _mm512_reduce_add_epi64(value); -#else - const int64_t *vals = reinterpret_cast(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i(&value); - int64_t quan = 1; - for (FASTOR_INDEX i=0; i &other) { - return (*this * other).sum(); - } - - __m512i value; -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - const int64_t *value = reinterpret_cast(&a.value); - os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] - << " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_epi64(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int64_t b) { - SIMDVector out; - out.value = _mm512_add_epi64(a.value,_mm512_set1_epi64(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(int64_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_add_epi64(_mm512_set1_epi64(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_epi64(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int64_t b) { - SIMDVector out; - out.value = _mm512_sub_epi64(a.value,_mm512_set1_epi64(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(int64_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm512_sub_epi64(_mm512_set1_epi64(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - return _mm512_castps_si512(_mm512_neg_ps(_mm512_castsi512_ps(b.value))); -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; -#ifdef FASTOR_AVX512DQ_IMPL - out.value = _mm512_mullo_epi64(a.value,b.value); -#else - for (FASTOR_INDEX i=0; i operator*(const SIMDVector &a, int64_t b) { - SIMDVector out; -#ifdef FASTOR_AVX512DQ_IMPL - out.value = _mm512_mullo_epi64(a.value,_mm512_set1_epi64(b)); -#else - for (FASTOR_INDEX i=0; i operator*(int64_t a, const SIMDVector &b) { - SIMDVector out; -#ifdef FASTOR_AVX512DQ_IMPL - out.value = _mm512_mullo_epi64(_mm512_set1_epi64(a),b.value); -#else - for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; -#ifdef FASTOR_INTEL - out.value = _mm512_div_epi64(a.value,b.value); -#else - int64_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); - int64_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); - int64_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); - for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int64_t b) { - SIMDVector out; -#ifdef FASTOR_INTEL - out.value = _mm512_div_epi64(a.value,_mm512_set1_epi64(b)); -#else - int64_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); - int64_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value); - for (FASTOR_INDEX i=0; i operator/(int64_t a, const SIMDVector &b) { - SIMDVector out; -#ifdef FASTOR_INTEL - out.value = _mm512_div_epi64(_mm512_set1_epi64(a),b.value); -#else - int64_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value); - int64_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value); - for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { - SIMDVector out; -#ifdef FASTOR_HAS_AVX512_ABS - out.value = _mm512_abs_epi64(a.value); -#else - for (FASTOR_INDEX i=0UL; i<8UL; ++i) { - ((int64_t*)&out.value)[i] = std::abs(((int64_t*)&a.value)[i]); - } -#endif - return out; -} - -#endif - - - - -// AVX VERSION -//----------------------------------------------------------------------------------------------- - -#ifdef FASTOR_AVX2_IMPL - -template<> -struct SIMDVector { - using value_type = __m256i; - using scalar_value_type = int64_t; - using abi_type = simd_abi::avx; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm256_setzero_si256()) {} - FASTOR_INLINE SIMDVector(int64_t num) : value(_mm256_set1_epi64x(num)) {} - FASTOR_INLINE SIMDVector(__m256i regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const int64_t *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_si256((__m256i*)data); - else - value = _mm256_loadu_si256((__m256i*)data); - } - - FASTOR_INLINE SIMDVector operator=(int64_t num) { - value = _mm256_set1_epi64x(num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m256i regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const int64_t *data, bool Aligned=true) { - if (Aligned) - value =_mm256_load_si256((__m256i*)data); - else - value = _mm256_loadu_si256((__m256i*)data); - } - FASTOR_INLINE void store(int64_t *data, bool Aligned=true) const { - if (Aligned) - _mm256_store_si256((__m256i*)data,value); - else - _mm256_storeu_si256((__m256i*)data,value); - } - - FASTOR_INLINE void aligned_load(const int64_t *data) { - value =_mm256_load_si256((__m256i*)data); - } - FASTOR_INLINE void aligned_store(int64_t *data) const { - _mm256_store_si256((__m256i*)data,value); - } - - FASTOR_INLINE int64_t operator[](FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - FASTOR_INLINE int64_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm256_mask_loadu_epi64(value, mask, a); - else - value = _mm256_mask_load_epi64(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm256_setzero_si256(); - for (FASTOR_INDEX i=0; i &a) { - value = _mm256_add_epi64x(value,a.value); - } - - FASTOR_INLINE void operator-=(int64_t num) { - value = _mm256_sub_epi64x(value,_mm256_set1_epi64x(num)); - } - FASTOR_INLINE void operator-=(__m256i regi) { - value = _mm256_sub_epi64x(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm256_sub_epi64x(value,a.value); - } - - FASTOR_INLINE void operator*=(int64_t num) { - value = _mm256_mul_epi64x(value,_mm256_set1_epi64x(num)); - } - FASTOR_INLINE void operator*=(__m256i regi) { - value = _mm256_mul_epi64x(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm256_mul_epi64x(value,a.value); - } - - FASTOR_INLINE void operator/=(int64_t num) { - int64_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); - for (FASTOR_INDEX i=0; i &a) { - int64_t val[Size]; _mm256_storeu_si256((__m256i*)val, value); - int64_t val_a[Size]; _mm256_storeu_si256((__m256i*)val_a, a.value); - for (FASTOR_INDEX i=0; i(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; iquan) - quan = vals[i]; - return quan; - } - FASTOR_INLINE SIMDVector reverse() { - // Reversing a 64 bit vector seems really expensive regardless - // of which of the following methods being used - // SIMDVector out(_mm256_set_epi64x(value[0],value[1],value[2],value[3])); - // SIMDVector out; out.set(value[3],value[2],value[1],value[0]); - // return out; - SIMDVector out; - out.value = _mm256_reverse_epi64(value); - return out; - } - - FASTOR_INLINE int64_t sum() { - const int64_t *vals = reinterpret_cast(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i(&value); - int64_t quan = 1; - for (FASTOR_INDEX i=0; i &other) { - const int64_t *vals0 = reinterpret_cast(&value); - const int64_t *vals1 = reinterpret_cast(&other.value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i a) { - const int64_t *value = reinterpret_cast(&a.value); - os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_epi64x(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int64_t b) { - SIMDVector out; - out.value = _mm256_add_epi64x(a.value,_mm256_set1_epi64x(b)); - return out; -} -FASTOR_INLINE SIMDVector operator+(int64_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_add_epi64x(_mm256_set1_epi64x(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_epi64x(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int64_t b) { - SIMDVector out; - out.value = _mm256_sub_epi64x(a.value,_mm256_set1_epi64x(b)); - return out; -} -FASTOR_INLINE SIMDVector operator-(int64_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_sub_epi64x(_mm256_set1_epi64x(a),b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_castpd_si256(_mm256_neg_pd(_mm256_castsi256_pd(b.value))); - return out; -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_epi64x(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int64_t b) { - SIMDVector out; - out.value = _mm256_mul_epi64x(a.value,_mm256_set1_epi64x(b)); - return out; -} -FASTOR_INLINE SIMDVector operator*(int64_t a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm256_mul_epi64x(_mm256_set1_epi64x(a),b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - int64_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); - int64_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); - int64_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); - for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int64_t b) { - SIMDVector out; - int64_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); - int64_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value); - for (FASTOR_INDEX i=0; i operator/(int64_t a, const SIMDVector &b) { - SIMDVector out; - int64_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value); - int64_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value); - for (FASTOR_INDEX i=0; i -struct SIMDVector { - using value_type = __m128i; - using scalar_value_type = int64_t; - using abi_type = simd_abi::sse; - static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size>::value; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size>::value;} - - FASTOR_INLINE SIMDVector() : value(_mm_setzero_si128()) {} - FASTOR_INLINE SIMDVector(int64_t num) { - value = _mm_set_epi64x(num,num); - } - FASTOR_INLINE SIMDVector(__m128i regi) : value(regi) {} - FASTOR_INLINE SIMDVector(const int64_t *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_si128((__m128i*)data); - else - value = _mm_loadu_si128((__m128i*)data); - } - - FASTOR_INLINE SIMDVector operator=(int64_t num) { - value = _mm_set_epi64x(num,num); - return *this; - } - FASTOR_INLINE SIMDVector operator=(__m128i regi) { - value = regi; - return *this; - } - - FASTOR_INLINE void load(const int64_t *data, bool Aligned=true) { - if (Aligned) - value =_mm_load_si128((__m128i*)data); - else - value = _mm_loadu_si128((__m128i*)data); - } - FASTOR_INLINE void store(int64_t *data, bool Aligned=true) const { - if (Aligned) - _mm_store_si128((__m128i*)data,value); - else - _mm_storeu_si128((__m128i*)data,value); - } - - FASTOR_INLINE void aligned_load(const int64_t *data) { - value =_mm_load_si128((__m128i*)data); - } - FASTOR_INLINE void aligned_store(int64_t *data) const { - _mm_store_si128((__m128i*)data,value); - } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) { -#ifdef FASTOR_HAS_AVX512_MASKS - if (!Aligned) - value = _mm_mask_loadu_epi64(value, mask, a); - else - value = _mm_mask_load_epi64(value, mask, a); -#else - // perhaps very inefficient but they never get used - int maska[Size]; - mask_to_array(mask,maska); - value = _mm_setzero_si128(); - for (FASTOR_INDEX i=0; i(&value)[i];} - FASTOR_INLINE int64_t operator()(FASTOR_INDEX i) const {return reinterpret_cast(&value)[i];} - - FASTOR_INLINE void set(int64_t num) { - value = _mm_set_epi64x(num,num); - } - FASTOR_INLINE void set(int64_t num0, int64_t num1) { - value = _mm_set_epi64x(num0,num1); - } - FASTOR_INLINE void set_sequential(int64_t num0) { - value = _mm_set_epi64x(num0+1,num0); - } - - // In-place operators - FASTOR_INLINE void operator+=(int64_t num) { - auto numb = _mm_set_epi64x(num,num); - value = _mm_add_epi64(value,numb); - } - FASTOR_INLINE void operator+=(__m128i regi) { - value = _mm_add_epi64(value,regi); - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value = _mm_add_epi64(value,a.value); - } - - FASTOR_INLINE void operator-=(int64_t num) { - auto numb = _mm_set_epi64x(num,num); - value = _mm_sub_epi64(value,numb); - } - FASTOR_INLINE void operator-=(__m128i regi) { - value = _mm_sub_epi64(value,regi); - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value = _mm_sub_epi64(value,a.value); - } - - FASTOR_INLINE void operator*=(int64_t num) { - auto numb = _mm_set_epi64x(num,num); - value = _mm_mul_epi64(value,numb); - } - FASTOR_INLINE void operator*=(__m128i regi) { - value = _mm_mul_epi64(value,regi); - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value = _mm_mul_epi64(value,a.value); - } - - FASTOR_INLINE void operator/=(int64_t num) { - int64_t val[Size]; _mm_storeu_si128((__m128i*)val, value); - for (FASTOR_INDEX i=0; i &a) { - int64_t val[Size]; _mm_storeu_si128((__m128i*)val, value); - int64_t val_a[Size]; _mm_storeu_si128((__m128i*)val_a, a.value); - for (FASTOR_INDEX i=0; i(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i(quan); - } - FASTOR_INLINE int64_t maximum() { - const int64_t *vals = reinterpret_cast(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; iquan) - quan = vals[i]; - return static_cast(quan); - } - FASTOR_INLINE SIMDVector reverse() { - return _mm_reverse_epi64(value); - } - - FASTOR_INLINE int64_t sum() { - const int64_t *vals = reinterpret_cast(&value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i<2; ++i) - quan += vals[i]; - return static_cast(quan); - } - FASTOR_INLINE int64_t product() { - const int64_t *vals = reinterpret_cast(&value); - int64_t quan = 1; - for (FASTOR_INDEX i=0; i &other) { - const int64_t *vals0 = reinterpret_cast(&value); - const int64_t *vals1 = reinterpret_cast(&other.value); - int64_t quan = 0; - for (FASTOR_INDEX i=0; i<2; ++i) - quan += vals0[i]*vals1[i]; - return static_cast(quan); - } - - __m128i value; -}; - -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - const int64_t *value = reinterpret_cast(&a.value); - os << "[" << value[0] << " " << value[1] << "]\n"; - return os; -} - -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_add_epi64(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, int64_t b) { - SIMDVector out; - auto numb = _mm_set_epi64x(b,b); - out.value = _mm_add_epi64(a.value,numb); - return out; -} -FASTOR_INLINE SIMDVector operator+(int64_t a, const SIMDVector &b) { - SIMDVector out; - auto numb = _mm_set_epi64x(a,a); - out.value = _mm_add_epi64(numb,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_sub_epi64(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, int64_t b) { - SIMDVector out; - auto numb = _mm_set_epi64x(b,b); - out.value = _mm_sub_epi64(a.value,numb); - return out; -} -FASTOR_INLINE SIMDVector operator-(int64_t a, const SIMDVector &b) { - SIMDVector out; - auto numb = _mm_set_epi64x(a,a); - out.value = _mm_sub_epi64(numb,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - out.value = _mm_castpd_si128(_mm_neg_pd(_mm_castsi128_pd(b.value))); - return out; -} - -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = _mm_mul_epi64(a.value,b.value); - return out; -} -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, int64_t b) { - SIMDVector out; - auto numb = _mm_set_epi64x(b,b); - out.value = _mm_mul_epi64(a.value,numb); - return out; -} -FASTOR_INLINE SIMDVector operator*(int64_t a, const SIMDVector &b) { - SIMDVector out; - auto numb = _mm_set_epi64x(a,a); - out.value = _mm_mul_epi64(numb,b.value); - return out; -} - -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - int64_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); - int64_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); - int64_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); - for (FASTOR_INDEX i=0; i operator/(const SIMDVector &a, int64_t b) { - SIMDVector out; - int64_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); - int64_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value); - for (FASTOR_INDEX i=0; i operator/(int64_t a, const SIMDVector &b) { - SIMDVector out; - int64_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value); - int64_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value); - for (FASTOR_INDEX i=0; i abs(const SIMDVector &a) { - SIMDVector out; -#ifdef FASTOR_AVX512VL_IMPL - return _mm_abs_epi64(a.value); -#elif defined (FASTOR_SSE4_2_IMPL) - __m128i sign = _mm_cmpgt_epi64(_mm_setzero_si128(), a.value); // 0 > a - __m128i inv = _mm_xor_si128(a.value, sign); // invert bits if negative - return _mm_sub_epi64(inv, sign); // add 1 -#else // SSE2 - __m128i signh = _mm_srai_epi32(a.value, 31); // sign in high dword - __m128i sign = _mm_shuffle_epi32(signh, 0xF5); // copy sign to low dword - __m128i inv = _mm_xor_si128(a.value, sign); // invert bits if negative - return _mm_sub_epi64(inv, sign); // add 1 -#endif -} - - -#endif - - -} // end of namespace Fastor - - -#endif // SIMD_VECTOR_INT64_H diff --git a/noarch/include/Fastor/simd_vector/simd_vector_scalar.h b/noarch/include/Fastor/simd_vector/simd_vector_scalar.h deleted file mode 100644 index d3861072..00000000 --- a/noarch/include/Fastor/simd_vector/simd_vector_scalar.h +++ /dev/null @@ -1,216 +0,0 @@ -#ifndef SIMD_VECTOR_T_SCALAR_H -#define SIMD_VECTOR_T_SCALAR_H - -#include "Fastor/simd_vector/simd_vector_base.h" - -namespace Fastor { - -template -struct SIMDVector { - using value_type = T; - using scalar_value_type = T; - using abi_type = simd_abi::scalar; - static constexpr FASTOR_INDEX Size = 1; - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;} - - FASTOR_INLINE SIMDVector() : value(0) {} - FASTOR_INLINE SIMDVector(T num) : value(num) {} - FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) : value(*data) {} - - FASTOR_INLINE SIMDVector operator=(T num) { - value = num; - return *this; - } - - FASTOR_INLINE void load(const T *data, bool Aligned=true) { value = *data; unused(Aligned); } - FASTOR_INLINE void store(T *data, bool Aligned=true) const { data[0] = value; unused(Aligned); } - - FASTOR_INLINE void aligned_load(const T *data) { value = *data; } - FASTOR_INLINE void aligned_store(T *data) const { data[0] = value; } - - FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool ) { - if (mask != 0x0) value = *a; - } - FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool) const { - if (mask != 0x0) a[0] = value; - } - - FASTOR_INLINE T operator[](FASTOR_INDEX) const {return value;} - FASTOR_INLINE T operator()(FASTOR_INDEX) const {return value;} - - FASTOR_INLINE void set(T num) { - value = num; - } - - FASTOR_INLINE void set_sequential(T num) { - value = num; - } - - FASTOR_INLINE void broadcast(const T *data) { - value = *data; - } - - // In-place operators - FASTOR_INLINE void operator+=(T num) { - value += num; - } - FASTOR_INLINE void operator+=(const SIMDVector &a) { - value += a.value; - } - - FASTOR_INLINE void operator-=(T num) { - value -= num; - } - FASTOR_INLINE void operator-=(const SIMDVector &a) { - value -= a.value; - } - - FASTOR_INLINE void operator*=(T num) { - value *= num; - } - FASTOR_INLINE void operator*=(const SIMDVector &a) { - value *= a.value; - } - - FASTOR_INLINE void operator/=(T num) { - value /= num; - } - FASTOR_INLINE void operator/=(const SIMDVector &a) { - value /= a.value; - } - // end of in-place operators - - FASTOR_INLINE SIMDVector shift(FASTOR_INDEX) { - return *this; - } - FASTOR_INLINE T sum() {return value;} - FASTOR_INLINE T product() {return value;} - FASTOR_INLINE SIMDVector reverse() { - return *this; - } - FASTOR_INLINE T minimum() {return value;} - FASTOR_INLINE T maximum() {return value;} - - FASTOR_INLINE T dot(const SIMDVector &other) { - return value*other.value; - } - - T value; -}; - -template -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector a) { - os << "[" << a.value << "]\n"; - return os; -} - -template -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = a.value+b.value; - return out; -} -template -FASTOR_INLINE SIMDVector operator+(const SIMDVector &a, T b) { - SIMDVector out; - out.value = a.value+b; - return out; -} -template -FASTOR_INLINE SIMDVector operator+(T a, const SIMDVector &b) { - SIMDVector out; - out.value = a+b.value; - return out; -} -template -FASTOR_INLINE SIMDVector operator+(const SIMDVector &b) { - return b; -} - -template -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = a.value-b.value; - return out; -} -template -FASTOR_INLINE SIMDVector operator-(const SIMDVector &a, T b) { - SIMDVector out; - out.value = a.value-b; - return out; -} -template -FASTOR_INLINE SIMDVector operator-(T a, const SIMDVector &b) { - SIMDVector out; - out.value = a-b.value; - return out; -} -template -FASTOR_INLINE SIMDVector operator-(const SIMDVector &b) { - SIMDVector out; - out.value = -b.value; - return out; -} - -template -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = a.value*b.value; - return out; -} -template -FASTOR_INLINE SIMDVector operator*(const SIMDVector &a, T b) { - SIMDVector out; - out.value = a.value*b; - return out; -} -template -FASTOR_INLINE SIMDVector operator*(T a, const SIMDVector &b) { - SIMDVector out; - out.value = a*b.value; - return out; -} - -template -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, const SIMDVector &b) { - SIMDVector out; - out.value = a.value/b.value; - return out; -} -template -FASTOR_INLINE SIMDVector operator/(const SIMDVector &a, T b) { - SIMDVector out; - out.value = a.value/b; - return out; -} -template -FASTOR_INLINE SIMDVector operator/(T a, const SIMDVector &b) { - SIMDVector out; - out.value = a/b.value; - return out; -} - -template -FASTOR_INLINE SIMDVector rcp(const SIMDVector &a) { - return T(1) / a.value; -} - -template -FASTOR_INLINE SIMDVector sqrt(const SIMDVector &a) { - return std::sqrt(a.value); -} - -template -FASTOR_INLINE SIMDVector rsqrt(const SIMDVector &a) { - return T(1) / std::sqrt(a.value); -} - -template -FASTOR_INLINE SIMDVector abs(const SIMDVector &a) { - return std::abs(a.value); -} - -} // end of namespace Fastor - - -#endif // SIMD_VECTOR_T_SCALAR_H diff --git a/noarch/include/Fastor/tensor/AbstractTensor.h b/noarch/include/Fastor/tensor/AbstractTensor.h deleted file mode 100644 index 92100812..00000000 --- a/noarch/include/Fastor/tensor/AbstractTensor.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef TENSORBASE_H -#define TENSORBASE_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/tensor_meta.h" - - -namespace Fastor { - -template -class Tensor; -template -class TensorMap; - - -template -class AbstractTensor { -public: - constexpr FASTOR_INLINE AbstractTensor() = default; - constexpr FASTOR_INLINE const Derived& self() const {return *static_cast(this);} - FASTOR_INLINE Derived& self() {return *static_cast(this);} - - static constexpr FASTOR_INDEX Dimension = Rank; -#ifndef FASTOR_DYNAMIC_MODE - static constexpr FASTOR_INDEX size() {return Derived::Size;} -#else - FASTOR_INDEX size() const {return (*static_cast(this)).size();} -#endif -}; - -} - -#endif // TENSORBASE_H diff --git a/noarch/include/Fastor/tensor/AbstractTensorFunctions.h b/noarch/include/Fastor/tensor/AbstractTensorFunctions.h deleted file mode 100644 index d928cc41..00000000 --- a/noarch/include/Fastor/tensor/AbstractTensorFunctions.h +++ /dev/null @@ -1,393 +0,0 @@ -#ifndef ABSTRACT_TENSOR_FUNCTIONS_H -#define ABSTRACT_TENSOR_FUNCTIONS_H - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/TensorTraits.h" - -#include - -namespace Fastor { - -/* The implementation of the evaluate function that evaluates any expression in to a tensor -*/ -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE const Tensor& evaluate(const Tensor &src) { - return src; -} -template -FASTOR_INLINE typename Derived::result_type evaluate(const AbstractTensor &src) { - typename Derived::result_type out(src); - return out; -} -//----------------------------------------------------------------------------------------------------------// - - -/* IO for tensor expressions */ -//----------------------------------------------------------------------------------------------------------// -template -inline std::ostream& operator<<(std::ostream &os, const AbstractTensor &src) { - using result_type = typename Expr::result_type; - result_type tmp(src); - print(tmp); - return os; -} - -template -inline void print(const AbstractTensor &src) { - using result_type = typename Expr::result_type; - result_type tmp(src); - print(tmp); -} -//----------------------------------------------------------------------------------------------------------// - - -/* These are the set of functions that work on any expression that evaluate immediately -*/ - -/* Add all the elements of the tensor in a flattened sense -*/ -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type sum(const AbstractTensor &_src) { - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out(src); - return out.sum(); -} -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type sum(const AbstractTensor &_src) { - - const Derived &src = _src.self(); - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - FASTOR_INDEX i; - T _scal=0; V _vec(_scal); - for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { - _vec += src.template eval(i); - } - for (; i < src.size(); ++i) { - _scal += src.template eval_s(i); - } - return _vec.sum() + _scal; -} - -/* Multiply all the elements of the tensor in a flattened sense -*/ -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type product(const AbstractTensor &_src) { - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out(src); - return out.product(); -} -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type product(const AbstractTensor &_src) { - - const Derived &src = _src.self(); - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - FASTOR_INDEX i; - T _scal=1; V _vec(_scal); - for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { - _vec *= src.template eval(i); - } - for (; i < src.size(); ++i) { - _scal *= src.template eval_s(i); - } - return _vec.product() * _scal; -} - -/* Get minimum element of a tensor -*/ -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type min(const AbstractTensor &_src) { - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out(src); - return min(out); -} -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type min(const AbstractTensor &_src) { - - const Derived &src = _src.self(); - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - FASTOR_INDEX i; - T _scal=std::numeric_limits::max(); V _vec(_scal); - for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { - _vec = min(src.template eval(i),_vec); - } - for (; i < src.size(); ++i) { - _scal = std::min(src.template eval_s(i),_scal); - } - return std::min(_vec.minimum(), _scal); -} - -/* Get maximum element of a tensor -*/ -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type max(const AbstractTensor &_src) { - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out(src); - return max(out); -} -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type max(const AbstractTensor &_src) { - - const Derived &src = _src.self(); - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - FASTOR_INDEX i; - T _scal=std::numeric_limits::min(); V _vec(_scal); - for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) { - _vec = max(src.template eval(i),_vec); - } - for (; i < src.size(); ++i) { - _scal = std::max(src.template eval_s(i),_scal); - } - return std::max(_vec.maximum(), _scal); -} - -/* Get the lower triangular matrix from a 2D expression -*/ -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type tril(const AbstractTensor &_src, int k = 0) { - static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIL"); - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out(src); - return tril(out,k); -} -template,bool> = false> -FASTOR_INLINE typename Derived::result_type tril(const AbstractTensor &_src, int k = 0) { - static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIL"); - const Derived &src = _src.self(); - using T = typename Derived::scalar_type; - typename Derived::result_type out(0); - - int M = int(src.dimension(0)); - int N = int(src.dimension(1)); - for (int i = 0; i < M; ++i) { - int jcount = k + i < N ? k + i : N - 1; - for (int j = 0; j <= jcount; ++j) { - out(i,j) = src.template eval_s(i,j); - } - } - return out; -} - -/* Get the upper triangular matrix from a 2D expression -*/ -template,bool> = false> -FASTOR_INLINE typename Derived::scalar_type triu(const AbstractTensor &_src, int k = 0) { - static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIU"); - const Derived &src = _src.self(); - using result_type = typename Derived::result_type; - const result_type out(src); - return triu(out,k); -} -template,bool> = false> -FASTOR_INLINE typename Derived::result_type triu(const AbstractTensor &_src, int k = 0) { - static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIU"); - const Derived &src = _src.self(); - using T = typename Derived::scalar_type; - typename Derived::result_type out(0); - - int M = int(src.dimension(0)); - int N = int(src.dimension(1)); - for (int i = 0; i < M; ++i) { - int jcount = k + i < 0 ? 0 : k + i; - for (int j = jcount; j < N; ++j) { - out(i,j) = src.template eval_s(i,j); - } - } - return out; -} -//----------------------------------------------------------------------------------------------------------// - - -// Boolean functions -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// -template && requires_evaluation_v,bool> = false> -FASTOR_INLINE bool all_of(const AbstractTensor &_src) { - using result_type = typename Derived::result_type; - const result_type tmp(_src.self()); - return all_of(tmp); -} -template && !requires_evaluation_v,bool> = false> -FASTOR_INLINE bool all_of(const AbstractTensor &_src) { - const Derived &src = _src.self(); - bool val = true; - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - if (src.template eval_s(i) == false) { - val = false; - break; - } - } - return val; -} - -template && requires_evaluation_v,bool> = false> -FASTOR_INLINE bool any_of(const AbstractTensor &_src) { - using result_type = typename Derived::result_type; - const result_type tmp(_src.self()); - return any_of(tmp); -} -template && !requires_evaluation_v,bool> = false> -FASTOR_INLINE bool any_of(const AbstractTensor &_src) { - const Derived &src = _src.self(); - bool val = false; - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - if (src.template eval_s(i) == true) { - val = true; - break; - } - } - return val; -} - -template && requires_evaluation_v,bool> = false> -FASTOR_INLINE bool none_of(const AbstractTensor &_src) { - using result_type = typename Derived::result_type; - const result_type tmp(_src.self()); - return none_of(tmp); -} -template && !requires_evaluation_v,bool> = false> -FASTOR_INLINE bool none_of(const AbstractTensor &_src) { - const Derived &src = _src.self(); - bool val = false; - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - if (src.template eval_s(i) == true) { - val = true; - break; - } - } - return val; -} - - -/* Is a second order tensor expression a uniform - A tensor expression is uniform if it spans equally in all dimensions, - i.e. generalisation of square matrix to N-dimensions -*/ -template -constexpr FASTOR_INLINE bool isuniform(const AbstractTensor &_src) { - return is_tensor_uniform_v; -} - -/* Is a second order tensor expression a square matrix -*/ -template = false> -constexpr FASTOR_INLINE bool issquare(const AbstractTensor &_src) { - return is_tensor_uniform_v; -} - -/* Is a second order tensor expression orthogonal -*/ -template = false> -FASTOR_INLINE bool isorthogonal(const AbstractTensor &_src, const double Tol=PRECI_TOL) { - typename Derived::result_type tmp1(_src.self()); - typename Derived::result_type tmp2(matmul(transpose(tmp1),tmp1)); - typename Derived::result_type I; I.eye2(); - return isequal(tmp2,I,Tol); -} - -/* Is a tensor expression symmetric - for higher order tensor two axes can defining a plane - provided to determine if a tensor expression is symmertric in that plane -*/ -template,bool> = false> -FASTOR_INLINE bool issymmetric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { - if (!issquare(_src.self())) return false; - return all_of( abs(evaluate(trans(_src.self()) - _src.self())) < Tol); -} -template,bool> = false> -FASTOR_INLINE bool issymmetric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { - if (!issquare(_src.self())) return false; - // Avoid copies - const Derived& src = _src.self(); - using T = typename Derived::scalar_type; - using result_type = typename Derived::result_type; - constexpr size_t M = get_tensor_dimension_v<0,result_type>; - constexpr size_t N = get_tensor_dimension_v<1,result_type>; - - bool _issym = true; - for (size_t i=0; i(i*N+j) - src.template eval_s(j*N+i) ) > Tol ) { - _issym = false; - break; - } - } - } - return _issym; -} - -/* Is a second order tensor expression deviatoric - a 2D tensor expression is deviatoric if it is - trace free -*/ -template = false> -constexpr FASTOR_INLINE bool isdeviatoric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { - return std::abs( trace(_src.self()) ) < Tol ? true : false; -} - -/* Is a second order tensor expression volumetric - a 2D tensor expression is volumetric if 1/3*[A:I]*I = A -*/ -template = false> -constexpr FASTOR_INLINE bool isvolumetric(const AbstractTensor &_src, const double Tol=PRECI_TOL) { - typename Derived::result_type I; I.eye2(); - typename Derived::result_type tmp = trace(_src.self()) * I; - return all_of( abs(tmp - _src.self()) < Tol); -} - -/* A second order tensor expression belongs to the special linear group if -it's determinant is +1 -*/ -template = false> -FASTOR_INLINE bool doesbelongtoSL3(const AbstractTensor &_src, const double Tol=PRECI_TOL) { - // Expression must be square - if ( !issquare(_src.self()) ) return false; - return std::abs(determinant(_src.self()) - 1) < Tol ? true : false; -} - -/* A second order tensor/expression belongs to the special orthogonal group if -it is orthogonal and it's determinant is +1 -*/ -template = false> -FASTOR_INLINE bool doesbelongtoSO3(const AbstractTensor &_src, const double Tol=PRECI_TOL) { - // Expression must be square and orthogonal - return issquare(_src.self()) && isorthogonal(_src.self()) ? true : false; -} - -/* Are two tensor expressions approximately equal -*/ -template && !requires_evaluation_v,bool> = false> -FASTOR_INLINE bool isequal( - const AbstractTensor &_src0, - const AbstractTensor &_src1, - const double Tol=PRECI_TOL) { - if ( DIMS0 != DIMS1) return false; - if ( _src0.self().size() != _src1.self().size()) return false; - return all_of( abs(_src0.self() - _src1.self()) < Tol); -} -template || requires_evaluation_v,bool> = false> -FASTOR_INLINE bool isequal( - const AbstractTensor &_src0, - const AbstractTensor &_src1, - const double Tol=PRECI_TOL) { - if ( DIMS0 != DIMS1) return false; - if ( _src0.self().size() != _src1.self().size()) return false; - return all_of( abs(evaluate(_src0.self() - _src1.self())) < Tol); -} -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// - -} // end of namespace Fastor - - -#endif // #ifndef TENSOR_FUNCTIONS_H diff --git a/noarch/include/Fastor/tensor/Aliasing.h b/noarch/include/Fastor/tensor/Aliasing.h deleted file mode 100644 index 8b81887a..00000000 --- a/noarch/include/Fastor/tensor/Aliasing.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef ALIASING_H_ -#define ALIASING_H_ - -#include "Fastor/tensor/ForwardDeclare.h" -#include "Fastor/tensor/Tensor.h" - -namespace Fastor { - -// template -// FASTOR_INLINE bool does_alias(const Tensor &dst, const Tensor &src) { -// return dst.data() == src.data() ? true : false; -// } -template -FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const Tensor &src) { - return dst.self().data() == src.data() ? true : false; -} -// template -// FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const AbstractTensor &src) { -// return does_alias(dst.self(),src.self()); -// } - - -#define FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(NAME)\ -template\ -FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const Unary ##NAME ## Op &src) {\ - return does_alias(dst.self(),src.expr().self());\ -}\ - -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Add ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sub ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Abs ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sqrt) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Exp ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Log ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sin ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Cos ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Tan ) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Asin) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Acos) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Atan) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sinh) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Cosh) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Tanh) - - -// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Det ) -// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Norm) -// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Trace) -FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Trans) - - -#define FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(NAME)\ -template\ -FASTOR_INLINE bool does_alias(const AbstractTensor &dst, const Binary ##NAME ## Op &src) {\ - return does_alias(dst.self(),src.lhs().self()) || does_alias(dst.self(),src.rhs().self());\ -}\ - -FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Add) -FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Sub) -FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Mul) -FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Div) -FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(MatMul) - -} - - -#endif // ALIASING_H_ diff --git a/noarch/include/Fastor/tensor/BlockIndexing.h b/noarch/include/Fastor/tensor/BlockIndexing.h deleted file mode 100644 index 5513858c..00000000 --- a/noarch/include/Fastor/tensor/BlockIndexing.h +++ /dev/null @@ -1,502 +0,0 @@ -#ifndef BLOCK_INDEXING_H -#define BLOCK_INDEXING_H - - -//----------------------------------------------------------------------------------------------------------// -// Block indexing -//----------------------------------------------------------------------------------------------------------// -// Calls scalar indexing so they are fully bounds checked. -template -FASTOR_INLINE Tensor::value> operator()(const iseq& idx) { - - static_assert(1==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor::value> out; - FASTOR_INDEX counter = 0; - for (FASTOR_INDEX i=F; ioperator()(i); - counter++; - } - return out; -} - -template -FASTOR_INLINE Tensor::value,range_detector::value> - operator()(iseq, iseq) { - - static_assert(2==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - - Tensor::value,range_detector::value> out; - FASTOR_INDEX counter_i = 0; - for (FASTOR_INDEX i=F0; ioperator()(i,j); - counter_j++; - } - counter_i++; - } - - return out; -} - -template -FASTOR_INLINE Tensor::value,range_detector::value,range_detector::value> - operator()(iseq, iseq, iseq) const { - static_assert(3==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor::value, - range_detector::value, - range_detector::value> out; - FASTOR_INDEX counter_i = 0; - for (FASTOR_INDEX i=F0; ioperator()(i,j,k); - counter_k++; - } - counter_j++; - } - counter_i++; - } - return out; -} - -template -FASTOR_INLINE Tensor::value, - range_detector::value, - range_detector::value, - range_detector::value> - operator ()(iseq, iseq, - iseq, iseq) { - - static_assert(4==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor::value, - range_detector::value, - range_detector::value, - range_detector::value> out; - FASTOR_INDEX counter_i = 0; - for (FASTOR_INDEX i=F0; ioperator()(i,j,k,l); - counter_l++; - } - counter_k++; - } - counter_j++; - } - counter_i++; - } - return out; -} - -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// -FASTOR_INLINE TensorViewExpr,1> operator()(seq _s) { - static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,1>(*this,_s); -} - -FASTOR_INLINE TensorViewExpr,2> operator()(seq _s0, seq _s1) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,2>(*this,_s0,_s1); -} -template -FASTOR_INLINE TensorViewExpr,2> -operator()(fseq _s0, seq _s1) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,2>(*this,_s0,_s1); -} -template -FASTOR_INLINE TensorViewExpr,2> -operator()(seq _s0, fseq _s1) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,2>(*this,_s0,_s1); -} -template::value,bool>::type=0> -FASTOR_INLINE TensorViewExpr,2> operator()(seq _s0, Int num) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,2>(*this,_s0,seq(num)); -} -template::value,bool>::type=0> -FASTOR_INLINE TensorViewExpr,2> operator()(Int num, seq _s1) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,2>(*this,seq(num),_s1); -} - -template && !is_fixed_sequence_pack_v,bool> = false> -FASTOR_INLINE TensorViewExpr,sizeof...(Seq)> operator()(Seq ... _seqs) { - static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,sizeof...(Seq)>(*this, {_seqs...}); -} - -template,bool> = false> -FASTOR_INLINE TensorFixedViewExprnD,Fseq...> operator()(Fseq... ) { - static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorFixedViewExprnD,Fseq...>(*this); -} - -// if fseq == fall - then just return a reference to the tensor -template, - get_value<1,Rest...>::value>::type>::value == get_value<1,Rest...>::value, - bool>::type =0> -FASTOR_INLINE Tensor& -operator()(fseq) { - static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return (*this); -} -// if fseq != fall - return a view -template, - get_value<1,Rest...>::value>::type>::value != get_value<1,Rest...>::value, - bool>::type =0> -FASTOR_INLINE TensorFixedViewExpr1D, - typename to_positive,pack_prod::value>::type,1> -operator()(fseq) { - static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorFixedViewExpr1D, - typename to_positive,pack_prod::value>::type,1>(*this); -} - -// if fseq == fall - then just return a reference to the tensor -template,get_value<1,Rest...>::value>::type>::value == get_value<1,Rest...>::value && - internal::fseq_range_detector,get_value<2,Rest...>::value>::type>::value == get_value<2,Rest...>::value, - bool>::type =0> -FASTOR_INLINE Tensor& -operator()(fseq, fseq) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return (*this); -} -// if fseq != fall - return a view -template,get_value<1,Rest...>::value>::type>::value != get_value<1,Rest...>::value || - internal::fseq_range_detector,get_value<2,Rest...>::value>::type>::value != get_value<2,Rest...>::value, - bool>::type =0> -FASTOR_INLINE TensorFixedViewExpr2D, - typename to_positive,get_value<1,Rest...>::value>::type, - typename to_positive,get_value<2,Rest...>::value>::type,2> -operator()(fseq, fseq) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorFixedViewExpr2D, - typename to_positive,get_value<1,Rest...>::value>::type, - typename to_positive,get_value<2,Rest...>::value>::type,2>(*this); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorViewExpr,2> operator()(fseq _s, Int num) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,2>(*this,seq(_s),seq(num)); -} -template::value,bool>::type=0> -FASTOR_INLINE TensorViewExpr,2> operator()(Int num, fseq _s) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,2>(*this,seq(num),seq(_s)); -} - -// Selecting a row and returning a TensorMap - does not seem to speed up the code -//----------------------------------------------------------------------------------------------------------// -// template::value && -// internal::fseq_range_detector, -// get_value<2,Rest...>::value>::type>::value != get_value<2,Rest...>::value,bool>::type=0> -// FASTOR_INLINE TensorViewExpr,2> operator()(Int num, fseq _s) { -// static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); -// return TensorViewExpr,2>(*this,seq(num),seq(_s)); -// } -// // Selecting a row from a 2D tensor returns a TensorMap -// template::value && -// internal::fseq_range_detector, -// get_value<2,Rest...>::value>::type>::value == get_value<2,Rest...>::value,bool>::type=0> -// FASTOR_INLINE TensorMap::value> operator()(Int num, fseq _s) { -// static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); -// constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; -// return TensorMap(&_data[num*N]); -// } -//----------------------------------------------------------------------------------------------------------// - -template::value,bool>::type=0> -FASTOR_INLINE TensorRandomViewExpr,Tensor,1> operator()(const Tensor &_it) { - static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorRandomViewExpr,Tensor,1>(*this,_it); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorRandomViewExpr,Tensor,sizeof...(Rest)> -operator()(const Tensor &_it) { - static_assert(dimension_t::value==sizeof...(IterSizes),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorRandomViewExpr,Tensor,sizeof...(Rest)>(*this,_it); -} - -template::value && std::is_integral::value,bool>::type=0> -FASTOR_INLINE TensorRandomViewExpr,Tensor,2> -operator()(const Tensor &_it0, const Tensor &_it1) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor tmp_it; - constexpr int NCols = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); -} - -template::value && std::is_integral::value,bool>::type=0> -FASTOR_INLINE TensorRandomViewExpr,Tensor,2> -operator()(const Tensor &_it0, Int1 num) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor tmp_it; - constexpr int NCols = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); -} - -template::value && std::is_integral::value,bool>::type=0> -FASTOR_INLINE TensorRandomViewExpr,Tensor,2> -operator()(Int1 num, const Tensor &_it0) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor tmp_it; - constexpr int NCols = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorRandomViewExpr,Tensor,get_value<2,Rest...>::value>::type::Size>,2> -operator()(const Tensor &_it0, fseq) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - constexpr int NCols = get_value<2,Rest...>::value; - using _seq = typename to_positive,NCols>::type; - constexpr int ColSize = _seq::Size; - Tensor tmp_it; - for (FASTOR_INDEX i = 0; i,Tensor,get_value<2,Rest...>::value>::type::Size>,2> (*this,tmp_it); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorRandomViewExpr,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> -operator()(fseq, const Tensor &_it0) { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - constexpr int NRows = get_value<1,Rest...>::value; - constexpr int NCols = get_value<2,Rest...>::value; - using _seq = typename to_positive,NRows>::type; - constexpr int RowSize = _seq::Size; - Tensor tmp_it; - for (FASTOR_INDEX i = 0; i,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> (*this,tmp_it); -} - - - -//----------------------------------------------------------------------------------------------------------// -// Filter views -FASTOR_INLINE TensorFilterViewExpr,Tensor,sizeof...(Rest)> -operator()(const Tensor &_fl) { - return TensorFilterViewExpr,Tensor,sizeof...(Rest)>(*this,_fl); -} -FASTOR_INLINE TensorFilterViewExpr,TensorMap,sizeof...(Rest)> -operator()(const TensorMap &_fl) { - return TensorFilterViewExpr,TensorMap,sizeof...(Rest)>(*this,_fl); -} -//----------------------------------------------------------------------------------------------------------// - -FASTOR_INLINE TensorConstViewExpr,1> operator()(seq _s) const { - static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstViewExpr,1>(*this,_s); -} - -FASTOR_INLINE TensorConstViewExpr,2> operator()(seq _s0, seq _s1) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstViewExpr,2>(*this,_s0,_s1); -} -template::value,bool>::type=0> -FASTOR_INLINE TensorConstViewExpr,2> operator()(seq _s0, Int num) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstViewExpr,2>(*this,_s0,seq(num)); -} -template::value,bool>::type=0> -FASTOR_INLINE TensorConstViewExpr,2> operator()(Int num, seq _s1) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstViewExpr,2>(*this,seq(num),_s1); -} - -template && !is_fixed_sequence_pack_v,bool> = false> -FASTOR_INLINE TensorConstViewExpr,sizeof...(Seq)> operator()(Seq ... _seqs) const { - static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstViewExpr,sizeof...(Seq)>(*this, {_seqs...}); -} - -template,bool> = false> -FASTOR_INLINE TensorConstFixedViewExprnD,Fseq...> operator()(Fseq... ) const { - static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstFixedViewExprnD,Fseq...>(*this); -} - -template -FASTOR_INLINE TensorConstFixedViewExpr1D, - typename to_positive,pack_prod::value>::type,1> operator()(fseq) const { - static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstFixedViewExpr1D, - typename to_positive,pack_prod::value>::type,1>(*this); -} - -template -FASTOR_INLINE TensorConstFixedViewExpr2D, - typename to_positive,get_value<1,Rest...>::value>::type, - typename to_positive,get_value<2,Rest...>::value>::type,2> -operator()(fseq, fseq) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstFixedViewExpr2D, - typename to_positive,get_value<1,Rest...>::value>::type, - typename to_positive,get_value<2,Rest...>::value>::type,2>(*this); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorConstViewExpr,2> operator()(fseq _s, Int num) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstViewExpr,2>(*this,seq(_s),seq(num)); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorConstViewExpr,2> operator()(Int num, fseq _s) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstViewExpr,2>(*this,seq(num),seq(_s)); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorConstRandomViewExpr,Tensor,1> -operator()(const Tensor &_it) const { - static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstRandomViewExpr,Tensor,1>(*this,_it); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorConstRandomViewExpr,Tensor,sizeof...(Rest)> -operator()(const Tensor &_it) const { - static_assert(dimension_t::value==sizeof...(IterSizes),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorConstRandomViewExpr,Tensor,sizeof...(Rest)>(*this,_it); -} - -template::value && std::is_integral::value,bool>::type=0> -FASTOR_INLINE TensorConstRandomViewExpr,Tensor,2> -operator()(const Tensor &_it0, const Tensor &_it1) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor tmp_it; - constexpr int NCols = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); -} - -template::value && std::is_integral::value,bool>::type=0> -FASTOR_INLINE TensorConstRandomViewExpr,Tensor,2> -operator()(const Tensor &_it0, Int1 num) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor tmp_it; - constexpr int NCols = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); -} - -template::value && std::is_integral::value,bool>::type=0> -FASTOR_INLINE TensorConstRandomViewExpr,Tensor,2> -operator()(Int1 num, const Tensor &_it0) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - Tensor tmp_it; - constexpr int NCols = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i,Tensor,2>(*this,tmp_it); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorConstRandomViewExpr,Tensor,get_value<2,Rest...>::value>::type::Size>,2> -operator()(const Tensor &_it0, fseq) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - constexpr int NCols = get_value<2,Rest...>::value; - using _seq = typename to_positive,NCols>::type; - constexpr int ColSize = _seq::Size; - Tensor tmp_it; - for (FASTOR_INDEX i = 0; i,Tensor,get_value<2,Rest...>::value>::type::Size>,2> (*this,tmp_it); -} - -template::value,bool>::type=0> -FASTOR_INLINE TensorConstRandomViewExpr,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> -operator()(fseq, const Tensor &_it0) const { - static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - constexpr int NRows = get_value<1,Rest...>::value; - constexpr int NCols = get_value<2,Rest...>::value; - using _seq = typename to_positive,NRows>::type; - constexpr int RowSize = _seq::Size; - Tensor tmp_it; - for (FASTOR_INDEX i = 0; i,Tensor,get_value<1,Rest...>::value>::type::Size,N>,2> (*this,tmp_it); -} -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// - - - - -#endif // BLOCK_INDEXING_H diff --git a/noarch/include/Fastor/tensor/ForwardDeclare.h b/noarch/include/Fastor/tensor/ForwardDeclare.h deleted file mode 100644 index 5293fbb9..00000000 --- a/noarch/include/Fastor/tensor/ForwardDeclare.h +++ /dev/null @@ -1,154 +0,0 @@ -#ifndef FORWARD_DECLARE_H -#define FORWARD_DECLARE_H - - -namespace Fastor { - -// FORWARD DECLARATIONS -//---------------------------------------------------------------- -template -struct BinaryAddOp; - -template -struct BinarySubOp; - -template -struct BinaryMulOp; - -template -struct BinaryDivOp; - -template -struct BinaryMatMulOp; - - -template -struct UnaryAddOp; - -template -struct UnarySubOp; - -template -struct UnaryAbsOp; - -template -struct UnarySqrtOp; - -template -struct UnaryExpOp; - -template -struct UnaryLogOp; - -template -struct UnarySinOp; - -template -struct UnaryCosOp; - -template -struct UnaryTanOp; - -template -struct UnaryAsinOp; - -template -struct UnaryAcosOp; - -template -struct UnaryAtanOp; - -template -struct UnarySinhOp; - -template -struct UnaryCoshOp; - -template -struct UnaryTanhOp; - - -template -struct UnaryTransOp; - - -template -struct TensorViewExpr; - -template -struct TensorConstViewExpr; - -template -struct TensorRandomViewExpr; - -template -struct TensorFilterViewExpr; - -template -struct TensorConstRandomViewExpr; - -template -struct TensorFixedViewExpr1D; - -template -struct TensorConstFixedViewExpr1D; - -template -struct TensorFixedViewExpr2D; - -template -struct TensorConstFixedViewExpr2D; - -template -struct TensorConstFixedViewExprnD; - -template -struct TensorFixedViewExprnD; - -template -struct TensorDiagViewExpr; - - -template -struct Index; - -template -struct nprods; - -template -struct nprods_views; - -#define FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(NAME)\ -template\ -struct Unary ##NAME ## Op;\ - -FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Not) -FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isinf) -FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isnan) -FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isfinite) - -template -struct is_unary_bool_op; - - -#define FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(NAME)\ -template\ -struct BinaryCmpOp##NAME ;\ - -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(EQ) -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(NEQ) -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(LT) -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(GT) -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(LE) -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(GE) -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(AND) -FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(OR) - -template -struct is_binary_cmp_op; -//---------------------------------------------------------------- - -} - - -#endif // FORWARD_DECLARE_H diff --git a/noarch/include/Fastor/tensor/IndexRetriever.h b/noarch/include/Fastor/tensor/IndexRetriever.h deleted file mode 100644 index 61a3386f..00000000 --- a/noarch/include/Fastor/tensor/IndexRetriever.h +++ /dev/null @@ -1,120 +0,0 @@ -#ifndef INDEX_RETRIEVER_H -#define INDEX_RETRIEVER_H - -// Retrieving index -// Given a flat index get the flat index in to the tensor - note that for tensor class the incoming index is -// the out-going index but it may not be the same for special tensors for instance for SingleValueTensor and -// views and such the index may be different -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE U get_mem_index(U index) const { -#if FASTOR_BOUNDS_CHECK - FASTOR_ASSERT((index>=0 && index < size()), "INDEX OUT OF BOUNDS"); -#endif - return index; -} - -// Retrieving index for nD tensors -// Given a multi-dimensional index get the flat index in to the tensor -//----------------------------------------------------------------------------------------------------------// -template::value,bool>::type =0> -FASTOR_INLINE size_t get_flat_index(Args ... args) const { - constexpr size_t M = get_value<1,Rest...>::value; - const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); -#if FASTOR_BOUNDS_CHECK - FASTOR_ASSERT( ( (i>=0 && i::value,bool>::type =0> -FASTOR_INLINE size_t get_flat_index(Args ... args) const { - constexpr size_t M = get_value<1,Rest...>::value; - constexpr size_t N = get_value<2,Rest...>::value; - const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); - const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...); -#if FASTOR_BOUNDS_CHECK - FASTOR_ASSERT( ( (i>=0 && i=0 && j::value,bool>::type =0> -FASTOR_INLINE size_t get_flat_index(Args ... args) const { - constexpr size_t M = get_value<1,Rest...>::value; - constexpr size_t N = get_value<2,Rest...>::value; - constexpr size_t P = get_value<3,Rest...>::value; - const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); - const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...); - const size_t k = get_index<2>(args...) < 0 ? P + get_index<2>(args...) : get_index<2>(args...); -#if FASTOR_BOUNDS_CHECK - FASTOR_ASSERT( ( (i>=0 && i=0 && j=0 && k::value,bool>::type =0> -FASTOR_INLINE size_t get_flat_index(Args ... args) const { - constexpr size_t M = get_value<1,Rest...>::value; - constexpr size_t N = get_value<2,Rest...>::value; - constexpr size_t P = get_value<3,Rest...>::value; - constexpr size_t Q = get_value<4,Rest...>::value; - const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...); - const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...); - const size_t k = get_index<2>(args...) < 0 ? P + get_index<2>(args...) : get_index<2>(args...); - const size_t l = get_index<3>(args...) < 0 ? Q + get_index<3>(args...) : get_index<3>(args...); -#if FASTOR_BOUNDS_CHECK - FASTOR_ASSERT( ( (i>=0 && i=0 && j=0 && k=0 && l=5 - && sizeof...(Args)==dimension_t::value && is_arithmetic_pack::value,bool>::type =0> -FASTOR_INLINE size_t get_flat_index(Args ... args) const { - /* The type of largs needs to be the type of incoming pack i.e. - whatever the tensor is indexed with - */ - get_nth_type<0,Args...> largs[sizeof...(Args)] = {args...}; - - constexpr std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; - constexpr size_t DimensionHolder[dimension_t::value] = {Rest...}; - - for (size_t i=0; i=0 && largs[i] &as) const { - constexpr std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; - size_t index = 0; - for (size_t i=0; i=0 && index,bool> = false > - constexpr - FASTOR_INLINE Tensor(const std::initializer_list &lst) { - static_assert(sizeof...(Rest)==1,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); -#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) - FASTOR_ASSERT(pack_prod::value==lst.size(), "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); -#endif - auto counter = 0; - for (const auto &i: lst) {_data[counter] = i; counter++;} - } - - template,bool> = false > - constexpr - FASTOR_INLINE Tensor(const std::initializer_list> &lst2d) { - static_assert(sizeof...(Rest)==2,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); -#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - auto size_ = 0; - FASTOR_ASSERT(M==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - for (auto &lst: lst2d) { - auto curr_size = lst.size(); - FASTOR_ASSERT(N==lst.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - size_ += curr_size; - } - FASTOR_ASSERT(pack_prod::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); -#endif - auto counter = 0; - for (const auto &lst1d: lst2d) { - for (const auto &i: lst1d) { - _data[counter] = T(i); - counter++; - } - } - } - - template,bool> = false > - constexpr - FASTOR_INLINE Tensor(const std::initializer_list>> &lst3d) { - static_assert(sizeof...(Rest)==3,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); -#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - constexpr FASTOR_INDEX P = get_value<3,Rest...>::value; - auto size_ = 0; - FASTOR_ASSERT(M==lst3d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - for (const auto &lst2d: lst3d) { - FASTOR_ASSERT(N==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - for (const auto &lst: lst2d) { - const auto curr_size = lst.size(); - FASTOR_ASSERT(P==lst.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - size_ += curr_size; - } - } - FASTOR_ASSERT(pack_prod::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); -#endif - auto counter = 0; - for (const auto &lst2d: lst3d) { - for (const auto &lst1d: lst2d) { - for (const auto &i: lst1d) { - _data[counter] = i; - counter++; - } - } - } - } - - template,bool> = false > - constexpr - FASTOR_INLINE Tensor(const std::initializer_list>>> &lst4d) { - static_assert(sizeof...(Rest)==4,"TENSOR RANK MISMATCH WITH LIST-INITIALISER"); -#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE)) - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - constexpr FASTOR_INDEX P = get_value<3,Rest...>::value; - constexpr FASTOR_INDEX Q = get_value<4,Rest...>::value; - auto size_ = 0; - FASTOR_ASSERT(M==lst4d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - for (const auto &lst3d: lst4d) { - FASTOR_ASSERT(N==lst3d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - for (const auto &lst2d: lst3d) { - FASTOR_ASSERT(P==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - for (const auto &lst: lst2d) { - const auto curr_size = lst.size(); - FASTOR_ASSERT(Q==curr_size,"TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); - size_ += curr_size; - } - } - } - FASTOR_ASSERT(pack_prod::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER"); -#endif - auto counter = 0; - for (const auto &lst3d: lst4d) { - for (const auto &lst2d: lst3d) { - for (const auto &lst1d: lst2d) { - for (const auto &i: lst1d) { - _data[counter] = i; - counter++; - } - } - } - } - } -//----------------------------------------------------------------------------------------------------------// - -#endif // INITIALIZER_LIST_CONSTRUCTORS_H diff --git a/noarch/include/Fastor/tensor/PODConverters.h b/noarch/include/Fastor/tensor/PODConverters.h deleted file mode 100644 index dee58222..00000000 --- a/noarch/include/Fastor/tensor/PODConverters.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef PODCONVERTERS_H -#define PODCONVERTERS_H - -FASTOR_INLINE T toscalar() const { - //! Returns a scalar - static_assert(size()==1,"ONLY TENSORS OF SIZE 1 CAN BE CONVERTED TO A SCALAR"); - return (*_data); -} - -FASTOR_INLINE std::array toarray() const { - //! Returns std::array - std::array out; - std::copy(_data,_data+size(),out.begin()); - return out; -} - -FASTOR_INLINE std::vector tovector() const { - //! Returns std::vector - std::vector out(size()); - std::copy(_data,_data+size(),out.begin()); - return out; -} - -#endif //PODCONVERTERS_H diff --git a/noarch/include/Fastor/tensor/Ranges.h b/noarch/include/Fastor/tensor/Ranges.h deleted file mode 100644 index 7b1e3d71..00000000 --- a/noarch/include/Fastor/tensor/Ranges.h +++ /dev/null @@ -1,210 +0,0 @@ -#ifndef RANGES_H -#define RANGES_H - -#include "Fastor/config/config.h" -#include "Fastor/meta/meta.h" -#include - -namespace Fastor { - -// range detector for fseq -//----------------------------------------------------------------------------------------------------------// -template -struct range_detector { - static constexpr int range = last - first; - static constexpr int value = range % step==0 ? range/step : range/step+1; -}; - -namespace internal { -template -struct fseq_range_detector; - -template class Seq, int first, int last, int step> -struct fseq_range_detector> { - static constexpr int range = last - first; - static constexpr int value = range % step==0 ? range/step : range/step+1; -}; -} // internal -//----------------------------------------------------------------------------------------------------------// - - - - -// Immediate sequence -//----------------------------------------------------------------------------------------------------------// -template -struct iseq { - static constexpr size_t _first = F; - static constexpr size_t _last= L; - static constexpr size_t _step = S; -}; -//----------------------------------------------------------------------------------------------------------// - - -// Fixed sequence -//----------------------------------------------------------------------------------------------------------// -template -struct fseq { - static constexpr int _first = F; - static constexpr int _last= L; - static constexpr int _step = S; - - static constexpr int Size = range_detector::value; - constexpr FASTOR_INLINE int size() const {return range_detector::value;} -}; - -static constexpr fseq<0,-1,1> fall; - -template -static constexpr fseq fix{}; - -static constexpr fseq<0 ,1 ,1> ffirst; -static constexpr fseq<-1,-1,1> flast; -//----------------------------------------------------------------------------------------------------------// - - -// Dynamic sequence -//----------------------------------------------------------------------------------------------------------// -struct seq { - - int _first; - int _last; - int _step = 1; - - constexpr FASTOR_INLINE seq(int _f, int _l, int _s=1) : _first(_f), _last(_l), _step(_s) {} - constexpr FASTOR_INLINE seq(int num) : _first(num), _last(num+1), _step(1) {} - - template - constexpr FASTOR_INLINE seq(fseq) : _first(F), _last(L), _step(S) {} - - // Do not allow construction of seq using std::initializer_list, as it happens - // implicitly. Overloading operator() with std::initializer_list should imply - // TensorRandomView, not TensorView - template - constexpr FASTOR_INLINE seq(std::initializer_list _s1) = delete; - - // Do not provide this overload as it is meaningless [iseq stands for immediate evaluation] - // template - // constexpr FASTOR_INLINE seq(iseq) : _first(F), _last(L), _step(S) {} - - FASTOR_INLINE int size() const { - int range = _last - _first; - return range % _step==0 ? range/_step : range/_step+1; - } - - constexpr FASTOR_INLINE bool operator==(seq other) const { - return (_first==other._first && _last==other._last && _step==other._step) ? true : false; - } - constexpr FASTOR_INLINE bool operator!=(seq other) const { - return (_first!=other._first || _last!=other._last || _step!=other._step) ? true : false; - } - -}; - -static constexpr int first = 0; -static constexpr int last = -1; - -// static constexpr seq all = seq(0,-1,1); -// why not this? -static constexpr fseq<0,-1,1> all; -//----------------------------------------------------------------------------------------------------------// - - -// traits -//----------------------------------------------------------------------------------------------------------// -template -struct is_fixed_sequence { - static constexpr bool value = false; -}; -template -struct is_fixed_sequence> { - static constexpr bool value = true; -}; -template -struct is_fixed_sequence> { - static constexpr bool value = true; -}; - -template -static constexpr bool is_fixed_sequence_v = is_fixed_sequence::value; - -template -struct is_fixed_sequence_pack; -template -struct is_fixed_sequence_pack { - static constexpr bool value = is_fixed_sequence::value && is_fixed_sequence_pack::value; -}; -template -struct is_fixed_sequence_pack { - static constexpr bool value = is_fixed_sequence::value; -}; - -template -static constexpr bool is_fixed_sequence_pack_v = is_fixed_sequence_pack::value; -//----------------------------------------------------------------------------------------------------------// - - - -// Transform sequence with negative indices to positive indices; -//----------------------------------------------------------------------------------------------------------// -template -struct to_positive; - -template -struct to_positive,N> { - // Same logic as seq used in the constructor of dynamic tensor views - static constexpr int _first = (L==0 && F==-1) ? N-1 : ( L < 0 && F < 0 ? F + N + 1 : F); - static constexpr int _last = (L < 0 && F >=0) ? L + N + 1 : ( (L==0 && F==-1) ? N : ( L < 0 && F < 0 ? L + N + 1 : L) ); - using type = fseq<_first,_last,S>; -}; - -template -struct to_positive,N> { - // Same logic as seq used in the constructor of dynamic tensor views - static constexpr int _first = (L==0 && F==-1) ? N-1 : ( L < 0 && F < 0 ? F + N + 1 : F); - static constexpr int _last = (L < 0 && F >=0) ? L + N + 1 : ( (L==0 && F==-1) ? N : ( L < 0 && F < 0 ? L + N + 1 : L) ); - using type = iseq<_first,_last,S>; -}; - -template -using to_positive_t = typename to_positive::type; -//----------------------------------------------------------------------------------------------------------// - - -//----------------------------------------------------------------------------------------------------------// -template -struct get_fixed_sequence_pack_dimensions; - -template class Derived, typename T, size_t ...Rest, size_t... ss, typename ... Fseqs> -struct get_fixed_sequence_pack_dimensions, std_ext::index_sequence, Fseqs...>{ - static constexpr std::array dims = { internal::fseq_range_detector>::value... }; - // using type = Derived::value...>; - using type = Derived>::value...>; - static constexpr size_t Size = pack_prod::value; -}; - -template class Derived, typename T, size_t ...Rest, size_t... ss, typename ... Fseqs> -constexpr std::array get_fixed_sequence_pack_dimensions, std_ext::index_sequence, Fseqs...>::dims; -//----------------------------------------------------------------------------------------------------------// - - - -//----------------------------------------------------------------------------------------------------------// -template -struct ravel_2d_indices; - -template -struct ravel_2d_indices> { - static constexpr size_t size_1 = range_detector::value; - static constexpr std::array idx = {(S0*(ss/size_1)*Ncol + S1*(ss%size_1) + F0*Ncol + F1)...}; -}; -template -constexpr std::array -ravel_2d_indices>::idx; -//----------------------------------------------------------------------------------------------------------// - - -} - -#endif // RANGES_H - diff --git a/noarch/include/Fastor/tensor/ScalarIndexing.h b/noarch/include/Fastor/tensor/ScalarIndexing.h deleted file mode 100644 index 93404bc7..00000000 --- a/noarch/include/Fastor/tensor/ScalarIndexing.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef SCALAR_INDEXING_NONCONST_H -#define SCALAR_INDEXING_NONCONST_H - - -// Scalar indexing non-const -//----------------------------------------------------------------------------------------------------------// -template::value,bool>::type =0> -FASTOR_INLINE T& operator()(Args ... args) { - return _data[get_flat_index(args...)]; -} -template::value,bool>::type =0> -FASTOR_INLINE T& operator[](Arg arg) { - return _data[get_flat_index(arg)]; -} -//----------------------------------------------------------------------------------------------------------// - -#endif // SCALAR_INDEXING_NONCONST_H - - -#ifndef SCALAR_INDEXING_CONST_H -#define SCALAR_INDEXING_CONST_H - -// Scalar indexing const -//----------------------------------------------------------------------------------------------------------// -template::value,bool>::type =0> -constexpr FASTOR_INLINE const T& operator()(Args ... args) const { - return _data[get_flat_index(args...)]; -} -template::value,bool>::type =0> -constexpr FASTOR_INLINE const T& operator[](Arg arg) const { - return _data[get_flat_index(arg)]; -} -//----------------------------------------------------------------------------------------------------------// - -#endif // SCALAR_INDEXING_CONST_H diff --git a/noarch/include/Fastor/tensor/SpecialisedConstructors.h b/noarch/include/Fastor/tensor/SpecialisedConstructors.h deleted file mode 100644 index 44c28e3c..00000000 --- a/noarch/include/Fastor/tensor/SpecialisedConstructors.h +++ /dev/null @@ -1,374 +0,0 @@ -#ifndef SPECIALISED_CONSTRUCTORS_H -#define SPECIALISED_CONSTRUCTORS_H - - -//----------------------------------------------------------------------------------------------------------// -template::type=0> -FASTOR_INLINE Tensor(const TensorFixedViewExpr2D,Seq0,Seq1,2>& src) { - using scalar_type_ = T; - constexpr FASTOR_INDEX Stride_ = simd_size_v; -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); - } -#endif - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); - } - for (; j < N; ++j) { - _data[i*N+j] = src.template eval_s(i,j); - } - } -} - - -template = false> -FASTOR_INLINE Tensor(const TensorFixedViewExprnD,Fseqs...>& src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); -#endif - constexpr int DimensionHolder[dimension_t::value] = {Rest...}; - std::array as = {}; - int jt, counter=0; - - if (src.is_vectorisable() || src.is_strided_vectorisable()) - { - using V = SIMDVector; - V _vec; - while(counter < size()) - { - _vec = src.template teval(as); - _vec.store(&_data[counter],false); - - counter+=V::Size; - for(jt = dimension_t::value-1; jt>=0; jt--) - { - if (jt == dimension_t::value-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt](as); - - counter++; - for(jt = dimension_t::value-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt] && !has_tensor_fixed_view_nd_v && has_tensor_fixed_view_2d_v - && DIMS==sizeof...(Rest) - && requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - // const typename Derived::result_type& tmp = evaluate(src_.self()); - FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - assign(*this,src_.self()); -} -template && !has_tensor_fixed_view_nd_v && has_tensor_fixed_view_2d_v - && DIMS==sizeof...(Rest) - && !requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - using scalar_type_ = typename scalar_type_finder::type; - constexpr FASTOR_INDEX Stride_ = simd_size_v; - const Derived &src = src_.self(); -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); - } -#endif - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); - } - for (; j (i,j); - } - } - } - else { - for (FASTOR_INDEX i = 0; i (i,j); - } - } - } -} - - -template && has_tensor_fixed_view_nd_v - && DIMS!=2 && DIMS==sizeof...(Rest) && - requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - assign(*this,src_.self()); -} -template && has_tensor_fixed_view_nd_v - && DIMS!=2 && DIMS==sizeof...(Rest) && - !requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - const Derived &src = src_.self(); -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); - } -#endif - - constexpr int DimensionHolder[dimension_t::value] = {Rest...}; - std::array as = {}; - int jt, counter=0; - - while(counter < size()) - { - _data[counter] = src.template teval_s(as); - - counter++; - for(jt = dimension_t::value-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt] = false> -FASTOR_INLINE Tensor(const TensorViewExpr,2>& src) { - using scalar_type_ = T; - constexpr FASTOR_INDEX Stride_ = simd_size_v; -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); - } -#endif - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); - } - for (; j < N; ++j) { - _data[i*N+j] = src.template eval_s(i,j); - } - } -} - - -template::value,bool> = false> -FASTOR_INLINE Tensor(const TensorViewExpr,sizeof...(Rest)>& src) { -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); - } -#endif - constexpr int DimensionHolder[dimension_t::value] = {Rest...}; - std::array as = {}; - int jt, counter=0; - - if (src.is_vectorisable() || src.is_strided_vectorisable()) - { - using V = SIMDVector; - V _vec; - while(counter < size()) - { - _vec = src.template teval(as); - _vec.store(&_data[counter],false); - - counter+=V::Size; - for(jt = dimension_t::value-1; jt>=0; jt--) - { - if (jt == dimension_t::value-1) as[jt]+=V::Size; - else as[jt] +=1; - if(as[jt](as); - - counter++; - for(jt = dimension_t::value-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt] && DIMS==2 && DIMS==sizeof...(Rest) && - requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - assign(*this,src_.self()); -} -template && DIMS==2 && DIMS==sizeof...(Rest) && - !requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - using scalar_type_ = typename scalar_type_finder::type; - constexpr FASTOR_INDEX Stride_ = simd_size_v; - const Derived &src = src_.self(); -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); - } -#endif - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { - for (FASTOR_INDEX i = 0; i (i,j).store(&_data[i*N+j], false); - } - for (; j < N; ++j) { - _data[i*N+j] = src.template eval_s(i,j); - } - } - } - else { - for (FASTOR_INDEX i = 0; i (i,j); - } - } - } -} - -template && DIMS!=2 && DIMS==sizeof...(Rest) && - requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - FASTOR_ASSERT(src_.self().size()==this->size(), "TENSOR SIZE MISMATCH"); - assign(*this,src_.self()); -} -template && DIMS!=2 && DIMS==sizeof...(Rest) && - !requires_evaluation_v,bool> = false> -FASTOR_INLINE Tensor(const AbstractTensor& src_) { - // using scalar_type_ = typename scalar_type_finder::type; - // constexpr FASTOR_INDEX Stride_ = simd_size_v; - const Derived &src = src_.self(); -#ifndef NDEBUG - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - for (FASTOR_INDEX i = 0; idimension(i), "TENSOR SHAPE MISMATCH"); - } -#endif - - constexpr int DimensionHolder[dimension_t::value] = {Rest...}; - std::array as = {}; - int jt, counter=0; - - while(counter < size()) - { - _data[counter] = src.template teval_s(as); - - counter++; - for(jt = dimension_t::value-1; jt>=0; jt--) - { - as[jt] +=1; - if(as[jt]; - // while(counter < size()) - // { - // const FASTOR_INDEX remainder = DimensionHolder[dimension_t::value-1] - as[dimension_t::value-1]; - // if (remainder > V::Size) { - // // V _vec = src.template eval(counter); - // V _vec = src.template teval(as); - // _vec.store(&_data[counter],false); - // counter+=V::Size; - // } - // else { - // // _data[counter] = src.template eval_s(counter); - // _data[counter] = src.template teval_s(as); - // counter++; - // } - - // for(jt = dimension_t::value-1; jt>=0; jt--) - // { - // if (jt == dimension_t::value-1 && remainder > V::Size) as[jt]+=V::Size; - // else as[jt] +=1; - // if(as[jt] -#include - -namespace Fastor { - -template -class Tensor: public AbstractTensor,sizeof...(Rest)> { -public: - using scalar_type = T; - using simd_vector_type = choose_best_simd_vector_t; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor; - using dimension_t = std::integral_constant; - static constexpr FASTOR_INLINE FASTOR_INDEX rank() {return sizeof...(Rest);} - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return pack_prod::value;} - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX dim) const { -#if FASTOR_SHAPE_CHECK - FASTOR_ASSERT(dim>=0 && dim < sizeof...(Rest), "TENSOR SHAPE MISMATCH"); -#endif - constexpr FASTOR_INDEX DimensionHolder[sizeof...(Rest)] = {Rest...}; - return DimensionHolder[dim]; - } - FASTOR_INLINE Tensor& noalias() {return *this;} - - // Classic constructors - //----------------------------------------------------------------------------------------------------------// - // Default constructor - constexpr FASTOR_INLINE Tensor() = default; - - // Copy constructor - FASTOR_INLINE Tensor(const Tensor &other) { - // This constructor cannot be default - if (_data == other.data()) return; - // fast memcopy - std::copy(other.data(),other.data()+size(),_data); - }; - - // Constructor from a scalar - template,bool> = false> - constexpr FASTOR_INLINE Tensor(U num) { -#ifdef FASTOR_ZERO_INITIALISE - // This is for compile time initialisation, so it is fine - scalar_type cnum = (scalar_type)num; - FASTOR_INDEX i = 0; - for (; i::value> &arr, int layout=RowMajor) { - std::copy(arr.data(),arr.data()+pack_prod::value,_data); - if (layout == RowMajor) - return; - else - *this = tocolumnmajor(*this); - } - FASTOR_INLINE Tensor(const std::vector &arr, int layout=RowMajor) { - std::copy(arr.data(),arr.data()+pack_prod::value,_data); - if (layout == RowMajor) - return; - else - *this = tocolumnmajor(*this); - } - //----------------------------------------------------------------------------------------------------------// - - // CRTP constructors - //----------------------------------------------------------------------------------------------------------// - //----------------------------------------------------------------------------------------------------------// - // Generic AbstractTensors -#ifndef FASTOR_DISABLE_SPECIALISED_CTR - template && !has_tensor_fixed_view_2d_v && - !has_tensor_fixed_view_nd_v) || DIMS!=sizeof...(Rest),bool> = false> -#else - template -#endif - FASTOR_INLINE Tensor(const AbstractTensor& src) { - FASTOR_ASSERT(src.self().size()==size(), "TENSOR SIZE MISMATCH"); - assign(*this, src.self()); - } - //----------------------------------------------------------------------------------------------------------// - - // Specialised constructors - //----------------------------------------------------------------------------------------------------------// - #include "Fastor/tensor/SpecialisedConstructors.h" - //----------------------------------------------------------------------------------------------------------// - - // AbstractTensor and scalar in-place operators - //----------------------------------------------------------------------------------------------------------// - #include "Fastor/tensor/TensorInplaceOperators.h" - //----------------------------------------------------------------------------------------------------------// - - // Raw pointer providers - //----------------------------------------------------------------------------------------------------------// -#ifdef FASTOR_ZERO_INITIALISE - constexpr FASTOR_INLINE T* data() const { return const_cast(this->_data);} -#else - FASTOR_INLINE T* data() const { return const_cast(this->_data);} -#endif - - FASTOR_INLINE T* data() {return this->_data;} - //----------------------------------------------------------------------------------------------------------// - - // Scalar & block indexing - //----------------------------------------------------------------------------------------------------------// - #include "Fastor/tensor/IndexRetriever.h" - #include "Fastor/tensor/ScalarIndexing.h" - #include "Fastor/tensor/BlockIndexing.h" - //----------------------------------------------------------------------------------------------------------// - - // Expression templates evaluators - //----------------------------------------------------------------------------------------------------------// - #include "Fastor/tensor/TensorEvaluator.h" - //----------------------------------------------------------------------------------------------------------// - - // Tensor methods - //----------------------------------------------------------------------------------------------------------// - #include "Fastor/tensor/TensorMethods.h" - //----------------------------------------------------------------------------------------------------------// - - // Converters - //----------------------------------------------------------------------------------------------------------// - #include "Fastor/tensor/PODConverters.h" - //----------------------------------------------------------------------------------------------------------// - - // Cast method - //----------------------------------------------------------------------------------------------------------// - template - FASTOR_INLINE Tensor cast() const { - Tensor out; - U *out_data = out.data(); - for (FASTOR_INDEX i=0; i(_data[i]); - } - return out; - } - //----------------------------------------------------------------------------------------------------------// - - - //----------------------------------------------------------------------------------------------------------// -protected: - template - FASTOR_INLINE void verify_dimensions(const AbstractTensor& src_) const { - static_assert(DIMS==dimension_t::value, "TENSOR RANK MISMATCH"); -#ifndef NDEBUG - const Derived &src = src_.self(); - FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH"); - // Check if shape of tensors match - for (FASTOR_INDEX i=0; i::value] = {}; -#else - FASTOR_ALIGN T _data[pack_prod::value]; -#endif - //----------------------------------------------------------------------------------------------------------// -}; - - -} // end of namespace Fastor - - -#include "Fastor/tensor/TensorAssignment.h" - - -#endif // TENSOR_H - diff --git a/noarch/include/Fastor/tensor/TensorAssignment.h b/noarch/include/Fastor/tensor/TensorAssignment.h deleted file mode 100644 index a21db81b..00000000 --- a/noarch/include/Fastor/tensor/TensorAssignment.h +++ /dev/null @@ -1,299 +0,0 @@ -#ifndef TENSOR_ASSIGNMENT_H -#define TENSOR_ASSIGNMENT_H - -namespace Fastor { - -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE void trivial_assign(AbstractTensor &dst, const AbstractTensor &src_) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - const OtherDerived &src = src_.self(); - FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); - T* _data = dst.self().data(); - - FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { - FASTOR_INDEX i = 0; - for (; i (i).store(&_data[i], FASTOR_ALIGNED); - } - for (; i < src.size(); ++i) { - _data[i] = src.template eval_s(i); - } - } - else { - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - _data[i] = src.template eval_s(i); - } - } -} - -template -FASTOR_INLINE void trivial_assign_add(AbstractTensor &dst, const AbstractTensor &src_) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - const OtherDerived &src = src_.self(); - FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); - T* _data = dst.self().data(); - - FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { - FASTOR_INDEX i = 0; - for (; i (i); - _vec.store(&_data[i], FASTOR_ALIGNED); - } - for (; i < src.size(); ++i) { - _data[i] += src.template eval_s(i); - } - } - else { - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - _data[i] += src.template eval_s(i); - } - } -} - -template -FASTOR_INLINE void trivial_assign_sub(AbstractTensor &dst, const AbstractTensor &src_) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - const OtherDerived &src = src_.self(); - FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); - T* _data = dst.self().data(); - - FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { - FASTOR_INDEX i = 0; - for (; i (i); - _vec.store(&_data[i], FASTOR_ALIGNED); - } - for (; i < src.size(); ++i) { - _data[i] -= src.template eval_s(i); - } - } - else { - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - _data[i] -= src.template eval_s(i); - } - } -} - -template -FASTOR_INLINE void trivial_assign_mul(AbstractTensor &dst, const AbstractTensor &src_) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - const OtherDerived &src = src_.self(); - FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); - T* _data = dst.self().data(); - - FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { - FASTOR_INDEX i = 0; - for (; i (i); - _vec.store(&_data[i], FASTOR_ALIGNED); - } - for (; i < src.size(); ++i) { - _data[i] *= src.template eval_s(i); - } - } - else { - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - _data[i] *= src.template eval_s(i); - } - } -} - -template -FASTOR_INLINE void trivial_assign_div(AbstractTensor &dst, const AbstractTensor &src_) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - const OtherDerived &src = src_.self(); - FASTOR_ASSERT(src.size()==dst.self().size(), "TENSOR SIZE MISMATCH"); - T* _data = dst.self().data(); - - FASTOR_IF_CONSTEXPR(!is_boolean_expression_v) { - FASTOR_INDEX i = 0; - for (; i (i); - _vec.store(&_data[i], FASTOR_ALIGNED); - } - for (; i < src.size(); ++i) { - _data[i] /= src.template eval_s(i); - } - } - else { - for (FASTOR_INDEX i = 0; i < src.size(); ++i) { - _data[i] /= src.template eval_s(i); - } - } -} -//----------------------------------------------------------------------------------------------------------// - -//----------------------------------------------------------------------------------------------------------// -template, bool> = false> -FASTOR_INLINE void trivial_assign(AbstractTensor &dst, U num) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - T* _data = dst.self().data(); - T cnum = (T)num; - V _vec(cnum); - FASTOR_INDEX i = 0; - for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { - _vec.store(&_data[i], FASTOR_ALIGNED); - } - for (; i, bool> = false> -FASTOR_INLINE void trivial_assign_add(AbstractTensor &dst, U num) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - T* _data = dst.self().data(); - T cnum = (T)num; - V _vec(cnum); - FASTOR_INDEX i = 0; - for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { - V _vec_out(&_data[i], FASTOR_ALIGNED); - _vec_out += _vec; - _vec_out.store(&_data[i], FASTOR_ALIGNED); - } - for (; i, bool> = false> -FASTOR_INLINE void trivial_assign_sub(AbstractTensor &dst, U num) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - T* _data = dst.self().data(); - T cnum = (T)num; - V _vec(cnum); - FASTOR_INDEX i = 0; - for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { - V _vec_out(&_data[i], FASTOR_ALIGNED); - _vec_out -= _vec; - _vec_out.store(&_data[i], FASTOR_ALIGNED); - } - for (; i, bool> = false> -FASTOR_INLINE void trivial_assign_mul(AbstractTensor &dst, U num) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - T* _data = dst.self().data(); - T cnum = (T)num; - V _vec(cnum); - FASTOR_INDEX i = 0; - for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { - V _vec_out(&_data[i], FASTOR_ALIGNED); - _vec_out *= _vec; - _vec_out.store(&_data[i], FASTOR_ALIGNED); - } - for (; i && !is_integral_v_, bool> = false> -FASTOR_INLINE void trivial_assign_div(AbstractTensor &dst, U num) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - T* _data = dst.self().data(); - T cnum = T(1) / (T)num; - V _vec(cnum); - FASTOR_INDEX i = 0; - for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { - V _vec_out(&_data[i], FASTOR_ALIGNED); - _vec_out *= _vec; - _vec_out.store(&_data[i], FASTOR_ALIGNED); - } - for (; i && is_integral_v_, bool> = false> -FASTOR_INLINE void trivial_assign_div(AbstractTensor &dst, U num) { - using T = typename Derived::scalar_type; - using V = typename Derived::simd_vector_type; - T* _data = dst.self().data(); - T cnum = (T)num; - V _vec(cnum); - FASTOR_INDEX i = 0; - for (; i< ROUND_DOWN(dst.self().size(),V::Size); i+=V::Size) { - V _vec_out(&_data[i], FASTOR_ALIGNED); - _vec_out /= _vec; - _vec_out.store(&_data[i], FASTOR_ALIGNED); - } - for (; i -constexpr FASTOR_INLINE void assign(AbstractTensor &dst, const Tensor &src) { - if (dst.self().data()==src.data()) return; - trivial_assign(dst.self(),src); -} -template -constexpr FASTOR_INLINE void assign_add(AbstractTensor &dst, const Tensor &src) { - trivial_assign_add(dst.self(),src); -} -template -constexpr FASTOR_INLINE void assign_sub(AbstractTensor &dst, const Tensor &src) { - trivial_assign_sub(dst.self(),src); -} -template -constexpr FASTOR_INLINE void assign_mul(AbstractTensor &dst, const Tensor &src) { - trivial_assign_mul(dst.self(),src); -} -template -constexpr FASTOR_INLINE void assign_div(AbstractTensor &dst, const Tensor &src) { - trivial_assign_div(dst.self(),src); -} - -template,bool> = false> -constexpr FASTOR_INLINE void assign(AbstractTensor &dst, U num) { - trivial_assign(dst.self(),num); -} -template,bool> = false> -constexpr FASTOR_INLINE void assign_add(AbstractTensor &dst, U num) { - trivial_assign_add(dst.self(),num); -} -template,bool> = false> -constexpr FASTOR_INLINE void assign_sub(AbstractTensor &dst, U num) { - trivial_assign_sub(dst.self(),num); -} -template,bool> = false> -constexpr FASTOR_INLINE void assign_mul(AbstractTensor &dst, U num) { - trivial_assign_mul(dst.self(),num); -} -template,bool> = false> -constexpr FASTOR_INLINE void assign_div(AbstractTensor &dst, U num) { - trivial_assign_div(dst.self(),num); -} -//----------------------------------------------------------------------------------------------------------// -//----------------------------------------------------------------------------------------------------------// - -} // end of namespace Fastor - -#endif // TENSOR_ASSIGNMENT_H diff --git a/noarch/include/Fastor/tensor/TensorEvaluator.h b/noarch/include/Fastor/tensor/TensorEvaluator.h deleted file mode 100644 index 27bc8d1e..00000000 --- a/noarch/include/Fastor/tensor/TensorEvaluator.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef TENSOR_EVALUATOR_H -#define TENSOR_EVALUATOR_H - -// Expression templates evaluators -//----------------------------------------------------------------------------------------------------------// -template -FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i) const { - SIMDVector _vec; - _vec.load(&_data[get_mem_index(i)],false); - return _vec; -} -template -FASTOR_INLINE T eval_s(FASTOR_INDEX i) const { - return _data[get_mem_index(i)]; -} -template -FASTOR_INLINE SIMDVector eval(FASTOR_INDEX i, FASTOR_INDEX j) const { - SIMDVector _vec; - _vec.load(&_data[get_flat_index(i,j)],false); - return _vec; -} -template -FASTOR_INLINE T eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const { - return _data[get_flat_index(i,j)]; -} - -template -FASTOR_INLINE SIMDVector teval(const std::array &as) const { - SIMDVector _vec; - _vec.load(&_data[get_flat_index(as)],false); - return _vec; -} -template -FASTOR_INLINE T teval_s(const std::array &as) const { - return _data[get_flat_index(as)]; -} -//----------------------------------------------------------------------------------------------------------// - -#endif // end of TENSOR_EVALUATOR_H diff --git a/noarch/include/Fastor/tensor/TensorFunctions.h b/noarch/include/Fastor/tensor/TensorFunctions.h deleted file mode 100644 index 235d5622..00000000 --- a/noarch/include/Fastor/tensor/TensorFunctions.h +++ /dev/null @@ -1,215 +0,0 @@ -#ifndef TENSOR_FUNCTIONS_H -#define TENSOR_FUNCTIONS_H - -#include "Fastor/meta/meta.h" -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor/TensorMap.h" -#include "Fastor/tensor/TensorTraits.h" - -namespace Fastor { - -/* Turns a row-major tensor to column-major */ -template class TensorType, typename T, size_t ... Rest> -FASTOR_INLINE Tensor tocolumnmajor(const TensorType &a) { - constexpr int Dimension = sizeof...(Rest); - if (Dimension < 2) { - return a; - } - else { - Tensor out; - T *arr_out = out.data(); - const T *a_data = a.data(); - - if (Dimension == 2) { - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - for (FASTOR_INDEX i=0; i::value; - std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; - FASTOR_INDEX DimensionHolder[Dimension] = {Rest...}; - std::reverse(DimensionHolder,DimensionHolder+Dimension); - std::reverse(products_.begin(),products_.end()); - std::array as = {}; - - int jt; - FASTOR_INDEX counter=0; - while(counter < Size) - { - FASTOR_INDEX index = 0; - for (int ii=0; ii=0; jt--) - { - as[jt] +=1; - if(as[jt] class TensorType, typename T, size_t ... Rest> -FASTOR_INLINE Tensor torowmajor(const TensorType &a) { - constexpr int Dimension = sizeof...(Rest); - if (Dimension < 2) { - return a; - } - else { - Tensor out; - T *arr_out = out.data(); - const T *a_data = a.data(); - - if (Dimension == 2) { - constexpr FASTOR_INDEX M = get_value<1,Rest...>::value; - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - for (FASTOR_INDEX i=0; i::value; - std::array products_ = nprods_views, - typename std_ext::make_index_sequence::type>::values; - FASTOR_INDEX DimensionHolder[Dimension] = {Rest...}; - std::reverse(DimensionHolder,DimensionHolder+Dimension); - std::reverse(products_.begin(),products_.end()); - std::array as = {}; - - int jt; - FASTOR_INDEX counter=0; - while(counter < Size) - { - FASTOR_INDEX index = 0; - for (int ii=0; ii=0; jt--) - { - as[jt] +=1; - if(as[jt] class TensorType, typename T, size_t ... Rest> -FASTOR_INLINE -index_to_tensor_map_t> -squeeze(const TensorType &a) { - return index_to_tensor_map_t>(a.data()); -} - -/* reshape - reshapes a tensor to a tensor of different shape and returns a TensorMap - A TensorMap is a view in to an existing tensor so modifying the reshaped - tensor will modify the original tensor and vice versa - Note that you cannot call this function on tensor expressions as this is - a view in to a concrete tensor type holding storage - - example: - auto b = reshape(a); -*/ -template -FASTOR_INLINE TensorMap reshape(const Tensor &a) { - static_assert(pack_prod::value==pack_prod::value, "SIZE OF TENSOR SHOULD REMAIN THE SAME DURING RESHAPE"); - return TensorMap(a.data()); -} - -/* flatten - creates a flattened 1D view of a tensor and returns a TensorMap - A TensorMap is a view in to an existing tensor so modifying the reshaped - tensor will modify the original tensor and vice versa - Note that you cannot call this function on tensor expressions as this is - a view in to a concrete tensor type holding storage - - example: - auto b = flatten(a); -*/ -template -FASTOR_INLINE TensorMap::value> flatten(const Tensor &a) { - return TensorMap::value>(a.data()); -} - - - - - - - - -#if FASTOR_NIL -// Constant tensors -static FASTOR_INLINE -Tensor levi_civita_ps() { - Tensor LeCi_ps; - LeCi_ps(0,1,2) = 1.f; - LeCi_ps(1,2,0) = 1.f; - LeCi_ps(2,0,1) = 1.f; - LeCi_ps(1,0,2) = -1.f; - LeCi_ps(2,1,0) = -1.f; - LeCi_ps(0,2,1) = -1.f; - - return LeCi_ps; -} - -static FASTOR_INLINE -Tensor levi_civita_pd() { - Tensor LeCi_pd; - LeCi_pd(0,1,2) = 1.; - LeCi_pd(1,2,0) = 1.; - LeCi_pd(2,0,1) = 1.; - LeCi_pd(1,0,2) = -1.; - LeCi_pd(2,1,0) = -1.; - LeCi_pd(0,2,1) = -1.; - - return LeCi_pd; -} - -template -static FASTOR_INLINE -Tensor kronecker_delta() { - Tensor out; out.eye(); - return out; -} -#endif - -} - -#endif // TENSOR_FUNCTIONS_H - diff --git a/noarch/include/Fastor/tensor/TensorIO.h b/noarch/include/Fastor/tensor/TensorIO.h deleted file mode 100644 index e58fb73b..00000000 --- a/noarch/include/Fastor/tensor/TensorIO.h +++ /dev/null @@ -1,180 +0,0 @@ -#ifndef TENSOR_PRINT_H -#define TENSOR_PRINT_H - -#include "Fastor/tensor/Tensor.h" - -namespace Fastor { - - -namespace internal { - -template -using std_matrix = typename std::vector>::type; - -// Generate combinations -template -FASTOR_INLINE std::vector> index_generator() { - // Do NOT change int to size_t, comparison overflows - std::vector> idx; idx.resize(pack_prod::value); - std::array maxes = {M,N,Rest...}; - std::array a; - int i,j; - std::fill(a.begin(),a.end(),0); - - auto counter=0; - while(1) - { - std::vector current_idx; //current_idx.reserve(sizeof...(Rest)+2); - for(i = 0; i< sizeof...(Rest)+2; i++) { - current_idx.push_back(a[i]); - } - idx[counter] = current_idx; - counter++; - for(j = sizeof...(Rest)+2-1 ; j>=0 ; j--) - { - if(++a[j] -int get_row_width(const std::ostream &os, const T *a_data, size_t size) { - // compute the largest width - int width = 0; - for(size_t j = 0; j < size; ++j) - { - std::stringstream sstr; - sstr.copyfmt(os); - sstr << a_data[j]; - width = std::max(width, int(sstr.str().length())); - } - return width; -} - - -template class t_type, typename T, size_t ...Rest> -int get_row_width(const std::ostream &os, const t_type& a) { - int width = 0; - for(size_t j = 0; j < pack_prod::value; ++j) - { - std::stringstream sstr; - sstr.copyfmt(os); - sstr << a.eval_s(j); - width = std::max(width, int(sstr.str().length())); - } - return width; -} - -} // end of namespace internal - - -#define FASTOR_MAKE_OS_STREAM_TENSOR0(t_type) \ -template\ -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) {\ - IOFormat fmt = FASTOR_DEFINE_IO_FORMAT;\ - os.precision(fmt._precision);\ - os << *a.data();\ - return os;\ -}\ - -#define FASTOR_MAKE_OS_STREAM_TENSOR1(t_type) \ -template \ -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) {\ - IOFormat fmt = FASTOR_DEFINE_IO_FORMAT;\ - os.precision(fmt._precision);\ - int width = internal::get_row_width(os, a);\ - for(size_t i = 0; i < M; ++i)\ - {\ - os << fmt._rowprefix;\ - if(width) os.width(width);\ - os << a(i);\ - os << fmt._rowsuffix;\ - os << fmt._rowsep;\ - }\ - return os;\ -}\ - -#define FASTOR_MAKE_OS_STREAM_TENSOR2(t_type) \ -template \ -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) { \ - IOFormat fmt = FASTOR_DEFINE_IO_FORMAT; \ - os.precision(fmt._precision); \ - int width = internal::get_row_width(os, a); \ - for(size_t i = 0; i < M; ++i) \ - {\ - os << fmt._rowprefix;\ - if(width) os.width(width);\ - os << a(i, 0);\ - for(size_t j = 1; j < N; ++j)\ - {\ - os << fmt._colsep;\ - if(width) os.width(width);\ - os << a(i, j);\ - }\ - os << fmt._rowsuffix;\ - if( i < M - 1)\ - os << fmt._rowsep;\ - }\ - return os;\ -}\ - - -#define FASTOR_MAKE_OS_STREAM_TENSORn(t_type) \ -template=3,bool>::type=0> \ -FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, const t_type &a) {\ - IOFormat fmt = FASTOR_DEFINE_IO_FORMAT;\ - constexpr std::array DimensionHolder = {Rest...};\ - constexpr int M = get_value::value;\ - constexpr int N = get_value::value;\ - constexpr int lastrowcol = M*N;\ - constexpr size_t prods = pack_prod::value;\ - std::vector> combs = internal::index_generator();\ - os.precision(fmt._precision);\ - int width = internal::get_row_width(os, a);\ - for (size_t dims=0; dims -FASTOR_INLINE auto& operator +=(const AbstractTensor& src_) { - assign_add(*this, src_.self()); - return *this; -} - -template -FASTOR_INLINE auto& operator -=(const AbstractTensor& src_) { - assign_sub(*this, src_.self()); - return *this; -} - -template -FASTOR_INLINE auto& operator *=(const AbstractTensor& src_) { - assign_mul(*this, src_.self()); - return *this; -} - -template -FASTOR_INLINE auto& operator /=(const AbstractTensor& src_) { - assign_div(*this, src_.self()); - return *this; -} -//---------------------------------------------------------------------------------------------// - -// Scalar overloads for in-place operators -//---------------------------------------------------------------------------------------------// -template,bool> = 0 > -FASTOR_INLINE auto& operator +=(U num) { - trivial_assign_add(*this, num); - return *this; -} - -template,bool> = 0 > -FASTOR_INLINE auto& operator -=(U num) { - trivial_assign_sub(*this, num); - return *this; -} - -template,bool> = 0 > -FASTOR_INLINE auto& operator *=(U num) { - trivial_assign_mul(*this, num); - return *this; -} - -template,bool> = 0 > -FASTOR_INLINE auto& operator /=(U num) { - trivial_assign_div(*this, num); - return *this; -} -//---------------------------------------------------------------------------------------------// - -#endif // TENSOR_INPLACE_OPERATORS_H diff --git a/noarch/include/Fastor/tensor/TensorMap.h b/noarch/include/Fastor/tensor/TensorMap.h deleted file mode 100644 index 5b2a452d..00000000 --- a/noarch/include/Fastor/tensor/TensorMap.h +++ /dev/null @@ -1,170 +0,0 @@ -#ifndef TENSOR_MAP_H -#define TENSOR_MAP_H - -#include "Fastor/config/config.h" -#include "Fastor/backend/backend.h" -#include "Fastor/simd_vector/SIMDVector.h" -#include "Fastor/tensor/AbstractTensor.h" -#include "Fastor/tensor/Ranges.h" -#include "Fastor/tensor/ForwardDeclare.h" -#include "Fastor/expressions/linalg_ops/linalg_ops.h" -#include "Fastor/tensor/TensorIO.h" - - -namespace Fastor { - -template -class TensorMap: public AbstractTensor,sizeof...(Rest)> { -public: - using scalar_type = T; - using simd_vector_type = choose_best_simd_vector_t; - using simd_abi_type = typename simd_vector_type::abi_type; - using result_type = Tensor,Rest...>; - using dimension_t = std::integral_constant; - static constexpr FASTOR_INLINE FASTOR_INDEX rank() {return sizeof...(Rest);} - static constexpr FASTOR_INLINE FASTOR_INDEX size() {return pack_prod::value;} - FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX dim) const { -#if FASTOR_SHAPE_CHECK - FASTOR_ASSERT(dim>=0 && dim < sizeof...(Rest), "TENSOR SHAPE MISMATCH"); -#endif - const FASTOR_INDEX DimensionHolder[sizeof...(Rest)] = {Rest...}; - return DimensionHolder[dim]; - } - FASTOR_INLINE Tensor& noalias() {return *this;} - - // Constructors - //----------------------------------------------------------------------------------------------------------// - constexpr TensorMap(scalar_type* data) : _data(data) {} - template constexpr TensorMap(Tensor &a) : _data(a.data()) {} - //----------------------------------------------------------------------------------------------------------// - - // Raw pointer providers - //----------------------------------------------------------------------------------------------------------// - FASTOR_INLINE T* data() const { return const_cast(this->_data);} - FASTOR_INLINE T* data() {return this->_data;} - //----------------------------------------------------------------------------------------------------------// - - // Scalar indexing - //----------------------------------------------------------------------------------------------------------// -#undef SCALAR_INDEXING_NONCONST_H -#undef SCALAR_INDEXING_CONST_H -#undef INDEX_RETRIEVER_H - #include "Fastor/tensor/IndexRetriever.h" - #include "Fastor/tensor/ScalarIndexing.h" -#define INDEX_RETRIEVER_H -#define SCALAR_INDEXING_NONCONST_H -#define SCALAR_INDEXING_CONST_H - - // Block indexing (all variants excluding iseq) - //----------------------------------------------------------------------------------------------------------// - template && !is_fixed_sequence_pack_v,bool> = false> - FASTOR_INLINE TensorViewExpr,sizeof...(Seq)> operator()(Seq ... _seqs) { - static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorViewExpr,sizeof...(Seq)>(*this, {_seqs...}); - } - - template,bool> = false> - FASTOR_INLINE TensorFixedViewExprnD,Fseq...> operator()(Fseq... ) { - static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS"); - return TensorFixedViewExprnD,Fseq...>(*this); - } - - FASTOR_INLINE TensorFilterViewExpr,Tensor,sizeof...(Rest)> - operator()(const Tensor &_fl) { - return TensorFilterViewExpr,Tensor,sizeof...(Rest)>(*this,_fl); - } - FASTOR_INLINE TensorFilterViewExpr,TensorMap,sizeof...(Rest)> - operator()(const TensorMap &_fl) { - return TensorFilterViewExpr,TensorMap,sizeof...(Rest)>(*this,_fl); - } - - //----------------------------------------------------------------------------------------------------------// - - // Expression templates evaluators - //----------------------------------------------------------------------------------------------------------// -#undef TENSOR_EVALUATOR_H - #include "Fastor/tensor/TensorEvaluator.h" -#define TENSOR_EVALUATOR_H - //----------------------------------------------------------------------------------------------------------// - - // No constructor should be added - // Provide generic AbstractTensors copy constructor though - //----------------------------------------------------------------------------------------------------------// - template - FASTOR_INLINE void operator=(const AbstractTensor& src) { - FASTOR_ASSERT(src.self().size()==size(), "TENSOR SIZE MISMATCH"); - assign(*this, src.self()); - } - - // AbstractTensor and scalar in-place operators - //----------------------------------------------------------------------------------------------------------// -#undef TENSOR_INPLACE_OPERATORS_H - #include "Fastor/tensor/TensorInplaceOperators.h" -#define TENSOR_INPLACE_OPERATORS_H - //----------------------------------------------------------------------------------------------------------// - - //----------------------------------------------------------------------------------------------------------// -#undef TENSOR_METHODS_CONST_H -#undef TENSOR_METHODS_NONCONST_H - #include "Fastor/tensor/TensorMethods.h" -#define TENSOR_METHODS_CONST_H -#define TENSOR_METHODS_NONCONST_H - //----------------------------------------------------------------------------------------------------------// - - // Converters - //----------------------------------------------------------------------------------------------------------// -#undef PODCONVERTERS_H - #include "Fastor/tensor/PODConverters.h" -#define PODCONVERTERS_H - //----------------------------------------------------------------------------------------------------------// - - // Cast method - //----------------------------------------------------------------------------------------------------------// - template - FASTOR_INLINE Tensor cast() const { - Tensor out; - U *out_data = out.data(); - for (FASTOR_INDEX i=0; i(_data[i]); - } - return out; - } - //----------------------------------------------------------------------------------------------------------// - - -private: - scalar_type* _data; -}; - -FASTOR_MAKE_OS_STREAM_TENSOR0(TensorMap) -FASTOR_MAKE_OS_STREAM_TENSOR1(TensorMap) -FASTOR_MAKE_OS_STREAM_TENSOR2(TensorMap) -FASTOR_MAKE_OS_STREAM_TENSORn(TensorMap) - - -template -FASTOR_INLINE void assign(AbstractTensor &dst, const TensorMap &src) { - if (dst.self().data()==src.data()) return; - trivial_assign(dst.self(),src); -} -template -FASTOR_INLINE void assign_add(AbstractTensor &dst, const TensorMap &src) { - trivial_assign_add(dst.self(),src); -} -template -FASTOR_INLINE void assign_sub(AbstractTensor &dst, const TensorMap &src) { - trivial_assign_sub(dst.self(),src); -} -template -FASTOR_INLINE void assign_mul(AbstractTensor &dst, const TensorMap &src) { - trivial_assign_mul(dst.self(),src); -} -template -FASTOR_INLINE void assign_div(AbstractTensor &dst, const TensorMap &src) { - trivial_assign_div(dst.self(),src); -} - - -} - -#endif // TENSOR_MAP_H diff --git a/noarch/include/Fastor/tensor/TensorMethods.h b/noarch/include/Fastor/tensor/TensorMethods.h deleted file mode 100644 index 99ef9e8e..00000000 --- a/noarch/include/Fastor/tensor/TensorMethods.h +++ /dev/null @@ -1,167 +0,0 @@ -#ifndef TENSOR_METHODS_NONCONST_H -#define TENSOR_METHODS_NONCONST_H - -FASTOR_INLINE void fill(T num0) { - FASTOR_INDEX i = 0UL; - using V = simd_vector_type; - V _vec(num0); - for (; i(num0); - // using V = SIMDVector; - // V _vec; - // FASTOR_INDEX i=0; - // for (; ifill(static_cast(1)); -} - -FASTOR_INLINE void eye2() { - // Second order identity tensor (identity matrices) - static_assert(sizeof...(Rest)==2, "CANNOT BUILD AN IDENTITY TENSOR"); - static_assert(no_of_unique::value==1, "TENSOR MUST BE UNIFORM"); - constexpr FASTOR_INDEX N = get_value<2,Rest...>::value; - zeros(); - for (FASTOR_INDEX i=0; i=2, "CANNOT BUILD AN IDENTITY TENSOR"); - static_assert(no_of_unique::value==1, "TENSOR MUST BE UNIFORM"); - zeros(); - - constexpr int ndim = sizeof...(Rest); - constexpr std::array maxes_a = {Rest...}; - std::array products; - std::fill(products.begin(),products.end(),0); - - for (int j=ndim-1; j>0; --j) { - int num = maxes_a[ndim-1]; - for (int k=0; k(1); - } -} - -FASTOR_INLINE void random() { - //! Populate tensor with random FP numbers - for (FASTOR_INDEX i=0; i; - V vec; - FASTOR_INDEX i = 0; - for (; i< ROUND_DOWN(size(),V::Size); i+=V::Size) { - vec.load(&tmp[size() - i - V::Size],false); - vec.reverse().store(&_data[i],false); - } - for (; i< size(); ++i) { - _data[i] = tmp[size()-i-1]; - } -} - -#endif // TENSOR_METHODS_NONCONST_H - - -#ifndef TENSOR_METHODS_CONST_H -#define TENSOR_METHODS_CONST_H - -FASTOR_INLINE T sum() const { - - if ((size()==0) || (size()==1)) return _data[0]; - using V = SIMDVector; - V vec = static_cast(0); - V _vec_in; - FASTOR_INDEX i = 0; - for (; i(0); - for (; i< size(); ++i) { - scalar += _data[i]; - } - return vec.sum() + scalar; -} - -FASTOR_INLINE T product() const { - - if ((size()==0) || (size()==1)) return _data[0]; - - using V = SIMDVector; - FASTOR_INDEX i = 0; - - V vec = static_cast(1); - for (; i< ROUND_DOWN(size(),V::Size); i+=V::Size) { - vec *= V(&_data[i],false); - } - T scalar = static_cast(1); - for (; i< size(); ++i) { - scalar *= _data[i]; - } - - return vec.product()*scalar; -} - -#endif // TENSOR_METHODS_CONST_H diff --git a/noarch/include/Fastor/tensor/TensorTraits.h b/noarch/include/Fastor/tensor/TensorTraits.h deleted file mode 100644 index e98aaa8d..00000000 --- a/noarch/include/Fastor/tensor/TensorTraits.h +++ /dev/null @@ -1,303 +0,0 @@ -#ifndef TENSOR_POST_META_H -#define TENSOR_POST_META_H - -#include "Fastor/tensor/Tensor.h" -#include "Fastor/tensor_algebra/indicial.h" - -namespace Fastor { - - -/* Classify/specialise Tensor as primitive if needed. - This specialisation hurts the performance of some specialised - kernels like matmul/norm/LU/inv that unroll aggressively unless - Tensor is specialised to wrap T only -*/ -//--------------------------------------------------------------------------------------------------------------------// -// template -// struct is_primitive> { -// static constexpr bool value = is_primitive_v_ ? true : false; -// }; -//--------------------------------------------------------------------------------------------------------------------// - - -/* Find the underlying scalar type of an expression */ -//--------------------------------------------------------------------------------------------------------------------// -template -struct scalar_type_finder { - using type = T; -}; - -template -struct scalar_type_finder> { - using type = T; -}; -// This specific specialisation is needed to avoid ambiguity for vectors -template -struct scalar_type_finder> { - using type = T; -}; -template -struct scalar_type_finder> { - using type = T; -}; -// This specific specialisation is needed to avoid ambiguity for vectors -template -struct scalar_type_finder> { - using type = T; -}; - -template