bgirgis/cpp-thirdparty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Fastor

Browse Source
This commit is contained in:
Bassem Girgis
2025-03-22 12:00:35 -05:00
parent 4dd5939693
commit 6d6438268a
132 changed files with 0 additions and 55086 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
noarch/include/Fastor/Fastor.h
View File

@@ -1,39 +0,0 @@
#ifndef FASTOR_ALL_INCLUDE_H
#define FASTOR_ALL_INCLUDE_H
#if defined(_MSC_VER)
// Disable MSVC warnings for macros
#pragma warning (disable: 4003)
// Disable MSVC warnings for conversion
#pragma warning (disable: 4244)
// Disable MSVC unreferenced parameters
#pragma warning (disable: 4100)
// Disable MSVC narrowing conversion
#pragma warning (disable: 4267)
#endif
#include "config/config.h"
#include "util/util.h"
#include "simd_vector/SIMDVector.h"
#include "simd_math/simd_math.h"
#include "tensor/Tensor.h"
#include "tensor/TensorMap.h"
#include "tensor/TensorIO.h"
#include "tensor/TensorFunctions.h"
#include "tensor/AbstractTensorFunctions.h"
#include "tensor_algebra/einsum.h"
#include "tensor_algebra/network_einsum.h"
#include "tensor_algebra/einsum_explicit.h"
#include "tensor_algebra/abstract_contraction.h"
#include "expressions/expressions.h"
#include "backend/voigt.h"
#if defined(_MSC_VER)
#pragma warning (default: 4003)
#pragma warning (default: 4244)
#pragma warning (default: 4100)
#pragma warning (default: 4267)
#endif
#endif // FASTOR_ALL_INCLUDE_H

105
noarch/include/Fastor/backend/adjoint.h
View File

@@ -1,105 +0,0 @@
#ifndef ADJOINT_H
#define ADJOINT_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = *src;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
/* Compute adjoint: */
dst[0] = + src3;
dst[1] = - src1;
dst[2] = - src2;
dst[3] = + src0;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
T src4 = src[4];
T src5 = src[5];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute adjoint: */
dst[0] = + src4 * src8 - src5 * src7;
dst[1] = - src1 * src8 + src2 * src7;
dst[2] = + src1 * src5 - src2 * src4;
dst[3] = - src3 * src8 + src5 * src6;
dst[4] = + src0 * src8 - src2 * src6;
dst[5] = - src0 * src5 + src2 * src3;
dst[6] = + src3 * src7 - src4 * src6;
dst[7] = - src0 * src7 + src1 * src6;
dst[8] = + src0 * src4 - src1 * src3;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
/* Compute adjoint: */
T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2];
T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1];
dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3;
dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3;
T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0];
dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5;
dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5;
t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0];
dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1;
dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1;
dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1;
dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1;
t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2];
t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1];
t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1];
dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3;
dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3;
t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0];
t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0];
dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5;
dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5;
t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0];
dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1;
dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1;
dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1;
dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1;
}
} // end of namespace Fastor
#endif // ADJOINT_H

24
noarch/include/Fastor/backend/backend.h
View File

@@ -1,24 +0,0 @@
#ifndef BACKEND_H
#define BACKEND_H
#include "Fastor/backend/adjoint.h"
#include "Fastor/backend/cofactor.h"
#include "Fastor/backend/cyclic_0.h"
#include "Fastor/backend/determinant.h"
#include "Fastor/backend/doublecontract.h"
#include "Fastor/backend/dyadic.h"
#include "Fastor/backend/inner.h"
#include "Fastor/backend/inverse.h"
#include "Fastor/backend/lufact.h"
#include "Fastor/backend/lut_inverse.h"
#include "Fastor/backend/matmul/matmul.h"
#include "Fastor/backend/matmul/tmatmul.h"
#include "Fastor/backend/norm.h"
#include "Fastor/backend/outer.h"
#include "Fastor/backend/tensor_cross.h"
#include "Fastor/backend/trace.h"
#include "Fastor/backend/transpose/transpose.h"
#endif // BACKEND_H

105
noarch/include/Fastor/backend/cofactor.h
View File

@@ -1,105 +0,0 @@
#ifndef COFACTOR_H
#define COFACTOR_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = *src;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
/* Compute cofactor: */
dst[0] = + src3;
dst[1] = - src2;
dst[2] = - src1;
dst[3] = + src0;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
T src4 = src[4];
T src5 = src[5];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute cofactor: */
dst[0] = + src4 * src8 - src5 * src7;
dst[1] = - src3 * src8 + src5 * src6;
dst[2] = + src3 * src7 - src4 * src6;
dst[3] = - src1 * src8 + src2 * src7;
dst[4] = + src0 * src8 - src2 * src6;
dst[5] = - src0 * src7 + src1 * src6;
dst[6] = + src1 * src5 - src2 * src4;
dst[7] = - src0 * src5 + src2 * src3;
dst[8] = + src0 * src4 - src1 * src3;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
/* Compute cofactor: */
T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2];
T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1];
dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3;
dst[4] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3;
T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0];
dst[1] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5;
dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5;
t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0];
dst[2] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1;
dst[6] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1;
dst[3] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1;
dst[7] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1;
t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2];
t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1];
t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1];
dst[8] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3;
dst[12] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3;
t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0];
t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0];
dst[9] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5;
dst[13] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5;
t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0];
dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1;
dst[14] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1;
dst[11] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1;
dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1;
}
}
#endif // COFACTOR_H

172
noarch/include/Fastor/backend/cyclic_0.h
View File

@@ -1,172 +0,0 @@
#ifndef CYCLIC_0_H
#define CYCLIC_0_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
//! Version 0 of cyclic product of two second order tensors i.e. C_ijkl = A_ik * B_jl
template<typename T, size_t M0, size_t N0, size_t M1, size_t N1>
FASTOR_HINT_INLINE void _cyclic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
for (size_t i=0; i<M0; ++i) {
for (size_t j=0; j<N0; ++j) {
for (size_t k=0; k<M1; ++k) {
for (size_t l=0; l<N1; ++l) {
out[i*N1*M1*N0+j*M1*N0+k*N0+l] += a[i*N0+k]*b[j*N1+l];
}
}
}
}
}
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_HINT_INLINE void _cyclic<double,2,2,2,2>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
__m256d as = _mm256_load_pd(a);
__m256d bs = _mm256_load_pd(b);
// First 2x2 block
__m256d c0 = _mm256_mul_pd(as,bs);
_mm_store_sd(out,_mm256_castpd256_pd128(c0));
_mm_store_sd(out+1,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1));
__m128d c0_high = _mm256_extractf128_pd(c0,0x1);
_mm_store_sd(out+3,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1));
_mm_store_sd(out+4,_mm_shuffle_pd(c0_high,c0_high,0x1));
// c_02
__m128d a0 = _mm256_castpd256_pd128(as);
__m128d b0 = _mm256_castpd256_pd128(bs);
b0 = _mm_shuffle_pd(b0,b0,0x1);
__m128d c1 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0)));
_mm_store_sd(out+2,c1);
_mm_store_sd(out+6,c1);
// c_12
__m128d a1 = _mm256_extractf128_pd(as,0x1);
__m128d b1 = _mm256_extractf128_pd(bs,0x1);
b1 = _mm_shuffle_pd(b1,b1,0x1);
__m128d c2 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1)));
_mm_store_sd(out+5,c2);
_mm_store_sd(out+7,c2);
// c_22
__m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b1)));
_mm_store_sd(out+8,c3);
}
template<>
FASTOR_HINT_INLINE void _cyclic<double,3,3,3,3>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// 34+ OPS
__m256d a_low = _mm256_load_pd(a);
__m256d a_high = _mm256_load_pd(a+4);
__m128d a_end = _mm_load_sd(a+8);
__m256d b_low = _mm256_load_pd(b);
__m256d b_high = _mm256_load_pd(b+4);
__m128d b_end = _mm_load_sd(b+8);
// The first 3x3 block
__m256d c0 = _mm256_mul_pd(a_low,b_low);
__m256d c1 = _mm256_mul_pd(a_high,b_high);
__m128d c2 = _mm_mul_sd(a_end,b_end);
_mm256_store_pd(out,c0);
_mm_store_sd(out+6,_mm_set_sd(_mm256_get1_pd(c0)));
_mm256_storeu_pd(out+7,c1);
// _mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0)));
_mm_store_sd(out+12,_mm256_extractf128_pd(c0,0x1));
_mm_store_sd(out+13,_mm_set_sd(_mm256_get1_pd(c1)));
_mm_store_sd(out+14,c2);
// rest
// c_03
__m128d a0 = _mm256_castpd256_pd128(a_low);
__m128d b0 = _mm256_castpd256_pd128(b_low);
b0 = _mm_shuffle_pd(b0,b0,0x1);
__m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0)));
_mm_store_sd(out+3,c3);
_mm_store_sd(out+18,c3);
// c_04
__m128d a1 = _mm_set_pd(_mm256_get0_pd(a_low),_mm256_get2_pd(a_low));
__m128d b1 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get0_pd(b_low));
__m128d c4 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1)));
_mm_store_sd(out+4,c4);
_mm_store_sd(out+24,c4);
// c_05
__m128d a2 = _mm_set_pd(_mm256_get1_pd(a_low),_mm256_get2_pd(a_low));
__m128d b2 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get1_pd(b_low));
__m128d c5 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b2)));
_mm_store_sd(out+5,c5);
_mm_store_sd(out+30,c5);
// c_13
__m128d a3 = _mm_setr_pd(_mm256_get3_pd(a_low),_mm256_get0_pd(a_high));
__m128d b3 = _mm_setr_pd(_mm256_get0_pd(b_high),_mm256_get3_pd(b_low));
__m128d c6 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b3)));
_mm_store_sd(out+9,c6);
_mm_store_sd(out+19,c6);
// c_14
__m128d a4 = _mm_set_pd(_mm256_get3_pd(a_low),_mm256_get1_pd(a_high));
__m128d b4 = _mm_set_pd(_mm256_get1_pd(b_high),_mm256_get3_pd(b_low));
__m128d c7 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b4)));
_mm_store_sd(out+10,c7);
_mm_store_sd(out+25,c7);
// c_15
__m128d a5 = _mm256_castpd256_pd128(a_high);
__m128d b5 = _mm256_castpd256_pd128(b_high);
b5 = _mm_shuffle_pd(b5,b5,0x1);
__m128d c8 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,b5)));
_mm_store_sd(out+11,c8);
_mm_store_sd(out+31,c8);
// c_23
__m128d a6 = _mm256_extractf128_pd(a_high,0x1);
__m128d b6 = _mm256_extractf128_pd(b_high,0x1);
b6 = _mm_shuffle_pd(b6,b6,0x1);
__m128d c9 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a6,b6)));
_mm_store_sd(out+15,c9);
_mm_store_sd(out+20,c9);
// c_24
__m128d a7 = _mm_set_pd(_mm256_get2_pd(a_high),_mm_get0_pd(a_end));
__m128d b7 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get2_pd(b_high));
__m128d c10 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a7,b7)));
_mm_store_sd(out+16,c10);
_mm_store_sd(out+26,c10);
// c_25
__m128d a8 = _mm_set_pd(_mm256_get3_pd(a_high),_mm_get0_pd(a_end));
__m128d b8 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get3_pd(b_high));
__m128d c11 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a8,b8)));
_mm_store_sd(out+17,c11);
_mm_store_sd(out+32,c11);
// c_33
__m128d c12 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b3)));
_mm_store_sd(out+21,c12);
// c_34
__m128d c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b4)));
_mm_store_sd(out+22,c13);
// c_43
c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b6)));
_mm_store_sd(out+27,c13);
// c_35
__m128d c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,_mm_shuffle_pd(b5,b5,0x1))));
_mm_store_sd(out+23,c14);
// c_53
c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b6)));
_mm_store_sd(out+33,c14);
// c_44
__m128d c15 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b7)));
_mm_store_sd(out+28,c15);
// c_45
__m128d c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b8)));
_mm_store_sd(out+29,c16);
c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b7)));
_mm_store_sd(out+34,c16);
// c_55
__m128d c17 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,_mm_shuffle_pd(b8,b8,0x1))));
_mm_store_sd(out+35,c17);
}
#endif
}
#endif // CYCLIC_0_H

116
noarch/include/Fastor/backend/determinant.h
View File

@@ -1,116 +0,0 @@
#ifndef DETERMINANT_H
#define DETERMINANT_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/meta/tensor_meta.h"
namespace Fastor {
#ifndef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N, typename std::enable_if<M==2 && N==2, bool>::type=0>
#else
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==2 && N==2, bool>::type=0>
#endif
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
return a[0] * a[3] - a[1] * a[2];
}
#ifndef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N, typename std::enable_if<M==3 && N==3, bool>::type=0>
#else
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==3 && N==3, bool>::type=0>
#endif
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
return a[0]*a[4]*a[8] + a[1]*a[5]*a[6] + a[2]*a[3]*a[7] - a[2]*a[4]*a[6] - a[1]*a[3]*a[8] - a[0]*a[5]*a[7];
}
template<typename T, size_t M, size_t N, typename std::enable_if<M==4 && N==4, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT m) {
return m[12] * m[9] * m[6] * m[3] - m[8] * m[13] * m[6] * m[3] -
m[12] * m[5] * m[10] * m[3] + m[4] * m[13] * m[10] * m[3] +
m[8] * m[5] * m[14] * m[3] - m[4] * m[9] * m[14] * m[3] -
m[12] * m[9] * m[2] * m[7] + m[8] * m[13] * m[2] * m[7] +
m[12] * m[1] * m[10] * m[7] - m[0] * m[13] * m[10] * m[7] -
m[8] * m[1] * m[14] * m[7] + m[0] * m[9] * m[14] * m[7] +
m[12] * m[5] * m[2] * m[11] - m[4] * m[13] * m[2] * m[11] -
m[12] * m[1] * m[6] * m[11] + m[0] * m[13] * m[6] * m[11] +
m[4] * m[1] * m[14] * m[11] - m[0] * m[5] * m[14] * m[11] -
m[8] * m[5] * m[2] * m[15] + m[4] * m[9] * m[2] * m[15] +
m[8] * m[1] * m[6] * m[15] - m[0] * m[9] * m[6] * m[15] -
m[4] * m[1] * m[10] * m[15] + m[0] * m[5] * m[10] * m[15];
}
template<typename T, size_t M, size_t N, typename std::enable_if<is_greater<M,4>::value || is_greater<N,4>::value, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
static_assert(M==N, "2D TENSOR MUST BE SQUARE");
assert(false && "2D TENSOR MUST BE SQUARE");
}
#ifdef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
std::is_same<T,float>::value && M==2 && N==2, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// 10 OPS
__m128 a1 = _mm_load_ps(a);
__m128 a2 = _mm_shuffle_ps(a1,a1,_MM_SHUFFLE(0,1,2,3));
__m128 a3 = _mm_mul_ps(a1,a2);
return _mm_cvtss_f32(_mm_sub_ss(a3,_mm_shuffle_ps(a3,a3,_MM_SHUFFLE(0,0,0,1))));
}
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
std::is_same<T,float>::value && M==3 && N==3, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// ?? OPS
__m128 r0 = {a[2],a[1],a[0],0.};
__m128 r1 = {a[3],a[5],a[4],0.};
__m128 r2 = {a[7],a[6],a[8],0.};
__m128 r3 = {a[6],a[7],a[8],0.};
__m128 r4 = {a[4],a[5],a[3],0.};
__m128 r5 = {a[2],a[0],a[1],0.};
__m128 out0 = _mm_mul_ps(r2,_mm_mul_ps(r0,r1));
__m128 out1 = _mm_mul_ps(r3,_mm_mul_ps(r4,r5));
return _mm_cvtss_f32(_mm_sub_ss(_add_ps(out0),_add_ps(out1)));
}
template<typename T, size_t M, size_t N, typename std::enable_if<std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==2 && N==2, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// 10 OPS
__m128d a1 = _mm_load_pd(a);
__m128d a2 = _mm_load_pd(a+2);
__m128d a3 = _mm_mul_pd(a1,_mm_shuffle_pd(a2,a2,1));
return _mm_cvtsd_f64(_mm_sub_pd(a3,_mm_shuffle_pd(a3,a3,0x1)));
}
template<typename T, size_t M, size_t N, typename std::enable_if<std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==3 && N==3, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// ?? OPS
__m256d r0 = {a[2],a[1],a[0],0.};
__m256d r1 = {a[3],a[5],a[4],0.};
__m256d r2 = {a[7],a[6],a[8],0.};
__m256d r3 = {a[6],a[7],a[8],0.};
__m256d r4 = {a[4],a[5],a[3],0.};
__m256d r5 = {a[2],a[0],a[1],0.};
__m256d out0 = _mm256_mul_pd(r2,_mm256_mul_pd(r0,r1));
__m256d out1 = _mm256_mul_pd(r3,_mm256_mul_pd(r4,r5));
return _mm_cvtsd_f64(_mm_sub_sd(_add_pd(out0),_add_pd(out1)));
}
#endif
}
#endif // DETERMINANT_H

128
noarch/include/Fastor/backend/doublecontract.h
View File

@@ -1,128 +0,0 @@
#ifndef DOUBLECONTRACT_H
#define DOUBLECONTRACT_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
template<typename T, size_t M, size_t N,
enable_if_t_<is_greater_equal_v_<4*choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,M*N>::Size, M*N >, bool> = false>
FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
constexpr size_t Size = M*N;
using V = choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,Size>;
V omm0;
size_t i = 0;
for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) {
const V amm0(&a[i],false);
const V bmm0(&b[i],false);
omm0 = fmadd(amm0,bmm0,omm0);
}
T scalar = static_cast<T>(0);
for (; i < Size; ++i) {
scalar += a[i]*b[i];
}
return omm0.sum() + scalar;
}
template<typename T, size_t M, size_t N,
enable_if_t_<is_less_v_<4*choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,M*N>::Size, M*N >, bool> = false>
FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
constexpr size_t Size = M*N;
using V = choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,Size>;
V omm0, omm1, omm2, omm3;
size_t i = 0;
for (; i< ROUND_DOWN(Size,4*V::Size); i+=4*V::Size) {
const V amm0(&a[i],false);
const V amm1(&a[i+V::Size],false);
const V amm2(&a[i+2*V::Size],false);
const V amm3(&a[i+3*V::Size],false);
const V bmm0(&b[i],false);
const V bmm1(&b[i+V::Size],false);
const V bmm2(&b[i+2*V::Size],false);
const V bmm3(&b[i+3*V::Size],false);
omm0 = fmadd(amm0,bmm0,omm0);
omm1 = fmadd(amm1,bmm1,omm1);
omm2 = fmadd(amm2,bmm2,omm2);
omm3 = fmadd(amm3,bmm3,omm3);
}
for (; i< ROUND_DOWN(Size,2*V::Size); i+=2*V::Size) {
const V amm0(&a[i],false);
const V amm1(&a[i+V::Size],false);
const V bmm0(&b[i],false);
const V bmm1(&b[i+V::Size],false);
omm0 = fmadd(amm0,bmm0,omm0);
omm1 = fmadd(amm1,bmm1,omm1);
}
for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) {
const V amm0(&a[i],false);
const V bmm0(&b[i],false);
omm0 = fmadd(amm0,bmm0,omm0);
}
T scalar = static_cast<T>(0);
for (; i < Size; ++i) {
scalar += a[i]*b[i];
}
return (omm0 + omm1 + omm2 + omm3).sum() + scalar;
}
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE float _doublecontract<float,2,2>(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) {
return _mm_sum_ps(_mm_mul_ps(_mm_loadu_ps(a),_mm_loadu_ps(b)));
}
template<>
FASTOR_INLINE float _doublecontract<float,3,3>(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) {
float r1 = _mm256_sum_ps(_mm256_mul_ps(_mm256_loadu_ps(a),_mm256_loadu_ps(b)));
float r2 = _mm_sum_ps(_mm_mul_ss(_mm_load_ss(a+8),_mm_load_ss(b+8)));
return r1+r2;
}
template<>
FASTOR_INLINE double _doublecontract<double,2,2>(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) {
return _mm256_sum_pd(_mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b)));
}
template<>
FASTOR_INLINE double _doublecontract<double,3,3>(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) {
__m256d r1 = _mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b));
__m256d r2 = _mm256_mul_pd(_mm256_loadu_pd(a+4),_mm256_loadu_pd(b+4));
__m128d r3 = _mm_mul_sd(_mm_load_sd(a+8),_mm_load_sd(b+8));
__m128d r4 = _add_pd(_mm256_add_pd(r1,r2));
__m128d summ = _mm_add_pd(_add_pd(r3),r4);
return _mm_cvtsd_f64(summ);
}
#endif
// doublecontract and transpose
template<typename T, size_t M, size_t N>
FASTOR_INLINE T _doublecontract_transpose(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
T dc = static_cast<T>(0);
for (FASTOR_INDEX i=0; i<M; ++i)
for (FASTOR_INDEX j=0; j<N; ++j)
dc += a[i*N+j]*b[j*M+i];
return dc;
}
}
#endif // DOUBLECONTRACT_H

212
noarch/include/Fastor/backend/dyadic.h
View File

@@ -1,212 +0,0 @@
#ifndef DYADIC_H
#define DYADIC_H
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
// The non-voigt version of outer product
//---------------------------------------------------------------------------------------------------
// dyadic template parameters are based on size
// of the two tensors and not the dimensions
//---------------------------------------------------------------------------------------------------
template<typename T, size_t SizeA, size_t SizeB>
FASTOR_INLINE
void _dyadic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,SizeB>::type;
// constexpr size_t unrollOuterloop = 4UL;
// constexpr size_t M0 = SizeA / unrollOuterloop * unrollOuterloop;
// Unrolling the inner loop beyond 4 does not give any benefit neither on AVX
// nor on AVX512
size_t i = 0;
for (; i<SizeA; ++i) {
const V amm0(a[i ]);
size_t j=0;
for (; j<ROUND_DOWN(SizeB,4*V::Size); j+=4*V::Size) {
const V bmm0(&b[j],false);
const V bmm1(&b[j+V::Size],false);
const V bmm2(&b[j+2*V::Size],false);
const V bmm3(&b[j+3*V::Size],false);
V omm0(amm0*bmm0);
V omm1(amm0*bmm1);
V omm2(amm0*bmm2);
V omm3(amm0*bmm3);
omm0.store(&out[(i )*SizeB+j],false);
omm1.store(&out[(i )*SizeB+j+V::Size],false);
omm2.store(&out[(i )*SizeB+j+2*V::Size],false);
omm3.store(&out[(i )*SizeB+j+3*V::Size],false);
}
for (; j<ROUND_DOWN(SizeB,2*V::Size); j+=2*V::Size) {
const V bmm0(&b[j],false);
const V bmm1(&b[j+V::Size],false);
V omm0(amm0*bmm0);
V omm1(amm0*bmm1);
omm0.store(&out[(i )*SizeB+j],false);
omm1.store(&out[(i )*SizeB+j+V::Size],false);
}
for (; j<ROUND_DOWN(SizeB,V::Size); j+=V::Size) {
const V bmm0(&b[j],false);
V omm0(amm0*bmm0);
omm0.store(&out[(i )*SizeB+j],false);
}
for (; j<SizeB; ++j) {
const T bmm0(b[j]);
out[(i )*SizeB+j] = a[i ]*bmm0;
}
}
}
//---------------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX_IMPL
// Outer product (2x2) x (2x2)
template<>
FASTOR_INLINE
void _dyadic<float,4,4>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
__m128 vec_a = _mm_load_ps(a);
__m128 vec_b = _mm_load_ps(b);
__m128 a00 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0));
__m128 a01 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1));
__m128 a10 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2));
__m128 a11 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(3,3,3,3));
__m256 a0001 = _mm256_castps128_ps256(a00);
a0001 = _mm256_insertf128_ps(a0001,a01,0x1);
__m256 a1011 = _mm256_castps128_ps256(a10);
a1011 = _mm256_insertf128_ps(a1011,a11,0x1);
__m256 vec_b2 = _mm256_castps128_ps256(vec_b);
vec_b2 = _mm256_insertf128_ps(vec_b2,vec_b,0x1);
_mm256_store_ps(out,_mm256_mul_ps(a0001,vec_b2));
_mm256_store_ps(out+8,_mm256_mul_ps(a1011,vec_b2));
}
// Outer product (2x2) x (2x2)
template<>
FASTOR_INLINE
void _dyadic<double,4,4>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
__m256d vec_b = _mm256_loadu_pd(b);
__m256d a00 = _mm256_set1_pd(a[0]);
__m256d a01 = _mm256_set1_pd(a[1]);
__m256d a10 = _mm256_set1_pd(a[2]);
__m256d a11 = _mm256_set1_pd(a[3]);
_mm256_storeu_pd(out,_mm256_mul_pd(a00,vec_b));
_mm256_storeu_pd(out+4,_mm256_mul_pd(a01,vec_b));
_mm256_storeu_pd(out+8,_mm256_mul_pd(a10,vec_b));
_mm256_storeu_pd(out+12,_mm256_mul_pd(a11,vec_b));
}
// Outer product (1x2) x (1x2) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<float,2,2>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 7 OPS
__m128 vec_a = _mm_loadu_ps(a);
__m128 vec_b = _mm_loadu_ps(b);
vec_a = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,0,0));
vec_b = _mm_shuffle_ps(vec_b,vec_b,_MM_SHUFFLE(1,0,1,0));
_mm_storeu_ps(out,_mm_mul_ps(vec_a,vec_b));
}
// Outer product (1x2) x (1x2) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<double,2,2>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// IVY 9 OPS / HW 13 OPS
__m128d vec_a = _mm_loadu_pd(a);
__m128d vec_b = _mm_loadu_pd(b);
__m128d a0 = _mm_shuffle_pd(vec_a,vec_a,0x0);
__m128d a1 = _mm_shuffle_pd(vec_a,vec_a,0x3);
__m256d as = _mm256_castpd128_pd256(a0);
as = _mm256_insertf128_pd(as,a1,0x1);
__m256d bs = _mm256_castpd128_pd256(vec_b);
bs = _mm256_insertf128_pd(bs,vec_b,0x1);
_mm256_storeu_pd(out,_mm256_mul_pd(as,bs));
}
// Outer product (1x3) x (1x3) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<float,3,3>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 18 OPS
__m128 vec_a = _mm_loadu_ps(a);
__m128 vec_b = _mm_loadu_ps(b);
__m128 a0 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0));
__m128 a1 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1));
__m128 a2 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2));
_mm_storeu_ps(out,_mm_mul_ps(a0,vec_b));
_mm_storeu_ps(out+3,_mm_mul_ps(a1,vec_b));
_mm_storeu_ps(out+6,_mm_mul_ps(a2,vec_b));
}
// Outer product (1x3) x (1x3) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<double,3,3>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// 15 OPS + set OPS
__m256d vec_b = _mm256_loadu_pd(b);
__m256d a0 = _mm256_set1_pd(a[0]);
__m256d a1 = _mm256_set1_pd(a[1]);
__m256d a2 = _mm256_set1_pd(a[2]);
_mm256_storeu_pd(out,_mm256_mul_pd(a0,vec_b));
_mm256_storeu_pd(out+3,_mm256_mul_pd(a1,vec_b));
_mm256_storeu_pd(out+6,_mm256_mul_pd(a2,vec_b));
}
#endif
//---------------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------------
// Outer product of scalars
template<>
FASTOR_INLINE
void _dyadic<double,1,1>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
out[0] = a[0]*b[0];
}
template<>
FASTOR_INLINE
void _dyadic<float,1,1>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
out[0] = a[0]*b[0];
}
//---------------------------------------------------------------------------------------------------
}
#endif // DYADIC_H

27
noarch/include/Fastor/backend/inner.h
View File

@@ -1,27 +0,0 @@
#ifndef INNER_H_
#define INNER_H_
#include "Fastor/meta/meta.h"
#include "Fastor/backend/doublecontract.h"
namespace Fastor {
/* The dependency on doublecontract here is on purpose
as it creates a necessary layer of indirection to avoid
the case where M == 0
*/
template<typename T, size_t M,
enable_if_t_<is_greater_v_<M,0>, bool> = false>
FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
return _doublecontract<T,M,1>(a,b);
}
template<typename T, size_t M,
enable_if_t_<M==0, bool> = false>
FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
return (*a)*(*b);
}
} // end of namespace Fastor
#endif // INNER_H_

450
noarch/include/Fastor/backend/inverse.h
View File

@@ -1,450 +0,0 @@
#ifndef INVERSE_H
#define INVERSE_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = T(1) / (*src);
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2> && !is_same_v_<T,float> && !is_same_v_<T,double>, bool> = false>
#else
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
#endif
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T det;
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
/* Compute adjoint: */
dst[0] = + src3;
dst[1] = - src1;
dst[2] = - src2;
dst[3] = + src0;
/* Compute determinant: */
det = src0 * dst[0] + src1 * dst[2];
/* Multiply adjoint with reciprocal of determinant: */
det = T(1.0) / det;
dst[0] *= det;
dst[1] *= det;
dst[2] *= det;
dst[3] *= det;
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2> && is_same_v_<T,float>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// This is much superior to the scalar code as
// gcc/clang can't auto-vectorise the scalar code
// 6 shuffles + 1 add + 1 mul + 1 div
// Sky 6 + 4 + 4 + 11 = 25
__m128 mat = _mm_loadu_ps(src);
// xor to swap off-diagonals sings
__m128 nmat = _mm_neg_ps(mat);
// two shuffles to get adjoint
__m128 adj = _mm_shuffle_ps(mat, nmat, 0x009C );
adj = _mm_shuffle_ps(adj, adj , 0x39 );
// compute determinat
__m128 tmp0 = _mm_shuffle_ps(mat , mat , 0x00D8);
tmp0 = _mm_mul_ps (adj , tmp0 );
__m128 tmp1 = _mm_shuffle_ps(tmp0, tmp0, 0x1 );
__m128 det = _mm_div_ss (ONEPS, _mm_add_ss(tmp0,tmp1));
// broadcast det to all elements of __m128
det = _mm_shuffle_ps(det, det, 0x0 );
// divide adjoint by determinant
__m128 inv = _mm_mul_ps (adj, det);
_mm_storeu_ps(dst, inv);
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2> && is_same_v_<T,double>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// This is much superior to the scalar code as
// gcc/clang can't auto-vectorise the scalar code
// 8 shuffles + 1 add + 3 mul + 1 div
// Sky 8 + 4 + 12 + 14 = 38
__m128d row0 = _mm_loadu_pd(src);
__m128d row1 = _mm_loadu_pd(src+2);
__m128d tmp = row0;
row0 = _mm_shuffle_pd(row0,_mm_neg_pd(row0),0x2);
row1 = _mm_shuffle_pd(_mm_neg_pd(row1),row1,0x2);
// these two registers hold the adjoint
__m128d irow0 = _mm_shuffle_pd(row1,row0,0x3);
__m128d irow1 = _mm_shuffle_pd(row1,row0,0x0);
// dot product to compute determinant
__m128d det = _mm_mul_pd(tmp,_mm_reverse_pd(row1));
det = _mm_add_pd(det,_mm_reverse_pd(det));
// one by determinant
__m128d invdet = _mm_div_pd(_mm_set1_pd(1.0),det);
// scale
irow0 = _mm_mul_pd(irow0,invdet);
irow1 = _mm_mul_pd(irow1,invdet);
_mm_storeu_pd(dst ,irow0);
_mm_storeu_pd(dst+2,irow1);
}
#endif
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T det;
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
T src4 = src[4];
T src5 = src[5];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute adjoint: */
dst[0] = + src4 * src8 - src5 * src7;
dst[1] = - src1 * src8 + src2 * src7;
dst[2] = + src1 * src5 - src2 * src4;
dst[3] = - src3 * src8 + src5 * src6;
dst[4] = + src0 * src8 - src2 * src6;
dst[5] = - src0 * src5 + src2 * src3;
dst[6] = + src3 * src7 - src4 * src6;
dst[7] = - src0 * src7 + src1 * src6;
dst[8] = + src0 * src4 - src1 * src3;
/* Compute determinant: */
det = src0 * dst[0] + src1 * dst[3] + src2 * dst[6];
/* Multiply adjoint with reciprocal of determinant: */
det = T(1.0) / det;
dst[0] *= det;
dst[1] *= det;
dst[2] *= det;
dst[3] *= det;
dst[4] *= det;
dst[5] *= det;
dst[6] *= det;
dst[7] *= det;
dst[8] *= det;
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4> && !is_same_v_<T,float> && !is_same_v_<T,double>, bool> = false>
#else
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
#endif
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2];
T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1];
dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3;
dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3;
T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0];
dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5;
dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5;
t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0];
dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1;
dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1;
dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1;
dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1;
t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2];
t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1];
t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1];
dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3;
dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3;
t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0];
t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0];
dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5;
dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5;
t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0];
dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1;
dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1;
dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1;
dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1;
const T __det = src[0]*dst[0] + src[1]*dst[4] + src[2]*dst[8] + src[3]*dst[12];
const T __invdet = T(1)/__det;
for (int i=0; i<16; ++i)
dst[i] *= __invdet;
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4> && is_same_v_<T,float>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// From Intel's SSE matrix library
// The inverse is calculated using "Divide and Conquer" technique. The
// original matrix is divide into four 2x2 sub-matrices. Since each
// register of the matrix holds two elements, the smaller matrices are
// consisted of two registers. Hence we get a better locality of the
// calculations.
const __m128 p4f_sign_PNNP = _mm_castsi128_ps(_mm_set_epi32(0x00000000, 0x80000000, 0x80000000, 0x00000000));
// Load the full matrix into registers
__m128 _L1 = _mm_loadu_ps(src + 0);
__m128 _L2 = _mm_loadu_ps(src + 4);
__m128 _L3 = _mm_loadu_ps(src + 8);
__m128 _L4 = _mm_loadu_ps(src + 12);
__m128 A, B, C, D; // the four sub-matrices
A = _mm_movelh_ps(_L1, _L2);
B = _mm_movehl_ps(_L2, _L1);
C = _mm_movelh_ps(_L3, _L4);
D = _mm_movehl_ps(_L4, _L3);
// partial inverse of the sub-matrices
__m128 iA, iB, iC, iD, DC, AB;
__m128 dA, dB, dC, dD; // determinant of the sub-matrices
__m128 det, d, d1, d2;
__m128 rd; // reciprocal of the determinant
// AB = A# * B
AB = _mm_mul_ps(_mm_shuffle_ps(A,A,0x0F), B);
AB = _mm_sub_ps(AB,_mm_mul_ps(_mm_shuffle_ps(A,A,0xA5), _mm_shuffle_ps(B,B,0x4E)));
// DC = D# * C
DC = _mm_mul_ps(_mm_shuffle_ps(D,D,0x0F), C);
DC = _mm_sub_ps(DC,_mm_mul_ps(_mm_shuffle_ps(D,D,0xA5), _mm_shuffle_ps(C,C,0x4E)));
// dA = |A|
dA = _mm_mul_ps(_mm_shuffle_ps(A, A, 0x5F),A);
dA = _mm_sub_ss(dA, _mm_movehl_ps(dA,dA));
// dB = |B|
dB = _mm_mul_ps(_mm_shuffle_ps(B, B, 0x5F),B);
dB = _mm_sub_ss(dB, _mm_movehl_ps(dB,dB));
// dC = |C|
dC = _mm_mul_ps(_mm_shuffle_ps(C, C, 0x5F),C);
dC = _mm_sub_ss(dC, _mm_movehl_ps(dC,dC));
// dD = |D|
dD = _mm_mul_ps(_mm_shuffle_ps(D, D, 0x5F),D);
dD = _mm_sub_ss(dD, _mm_movehl_ps(dD,dD));
// d = trace(AB*DC) = trace(A#*B*D#*C)
d = _mm_mul_ps(_mm_shuffle_ps(DC,DC,0xD8),AB);
// iD = C*A#*B
iD = _mm_mul_ps(_mm_shuffle_ps(C,C,0xA0), _mm_movelh_ps(AB,AB));
iD = _mm_add_ps(iD,_mm_mul_ps(_mm_shuffle_ps(C,C,0xF5), _mm_movehl_ps(AB,AB)));
// iA = B*D#*C
iA = _mm_mul_ps(_mm_shuffle_ps(B,B,0xA0), _mm_movelh_ps(DC,DC));
iA = _mm_add_ps(iA,_mm_mul_ps(_mm_shuffle_ps(B,B,0xF5), _mm_movehl_ps(DC,DC)));
// d = trace(AB*DC) = trace(A#*B*D#*C) [continue]
d = _mm_add_ps(d, _mm_movehl_ps(d, d));
d = _mm_add_ss(d, _mm_shuffle_ps(d, d, 1));
d1 = _mm_mul_ss(dA,dD);
d2 = _mm_mul_ss(dB,dC);
// iD = D*|A| - C*A#*B
iD = _mm_sub_ps(_mm_mul_ps(D,_mm_shuffle_ps(dA,dA,0)), iD);
// iA = A*|D| - B*D#*C;
iA = _mm_sub_ps(_mm_mul_ps(A,_mm_shuffle_ps(dD,dD,0)), iA);
// det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C)
det = _mm_sub_ss(_mm_add_ss(d1,d2),d);
rd = _mm_div_ss(_mm_set_ss(1.0f), det);
// iB = D * (A#B)# = D*B#*A
iB = _mm_mul_ps(D, _mm_shuffle_ps(AB,AB,0x33));
iB = _mm_sub_ps(iB, _mm_mul_ps(_mm_shuffle_ps(D,D,0xB1), _mm_shuffle_ps(AB,AB,0x66)));
// iC = A * (D#C)# = A*C#*D
iC = _mm_mul_ps(A, _mm_shuffle_ps(DC,DC,0x33));
iC = _mm_sub_ps(iC, _mm_mul_ps(_mm_shuffle_ps(A,A,0xB1), _mm_shuffle_ps(DC,DC,0x66)));
rd = _mm_shuffle_ps(rd,rd,0);
rd = _mm_xor_ps(rd, p4f_sign_PNNP);
// iB = C*|B| - D*B#*A
iB = _mm_sub_ps(_mm_mul_ps(C,_mm_shuffle_ps(dB,dB,0)), iB);
// iC = B*|C| - A*C#*D;
iC = _mm_sub_ps(_mm_mul_ps(B,_mm_shuffle_ps(dC,dC,0)), iC);
// iX = iX / det
iA = _mm_mul_ps(rd,iA);
iB = _mm_mul_ps(rd,iB);
iC = _mm_mul_ps(rd,iC);
iD = _mm_mul_ps(rd,iD);
_mm_storeu_ps(dst+0, _mm_shuffle_ps(iA,iB,0x77));
_mm_storeu_ps(dst+4, _mm_shuffle_ps(iA,iB,0x22));
_mm_storeu_ps(dst+8, _mm_shuffle_ps(iC,iD,0x77));
_mm_storeu_ps(dst+12, _mm_shuffle_ps(iC,iD,0x22));
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4> && is_same_v_<T,double>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// From Intel's SSE matrix library
// The inverse is calculated using "Divide and Conquer" technique. The
// original matrix is divide into four 2x2 sub-matrices. Since each
// register of the matrix holds two elements, the smaller matrices are
// consisted of two registers. Hence we get a better locality of the
// calculations.
const __m128d _Sign_NP = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0));
const __m128d _Sign_PN = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0));
// the four sub-matrices
__m128d A1, A2, B1, B2, C1, C2, D1, D2;
A1 = _mm_loadu_pd(src + 0); B1 = _mm_loadu_pd(src + 2);
A2 = _mm_loadu_pd(src + 4); B2 = _mm_loadu_pd(src + 6);
C1 = _mm_loadu_pd(src + 8); D1 = _mm_loadu_pd(src +10);
C2 = _mm_loadu_pd(src +12); D2 = _mm_loadu_pd(src +14);
// partial inverse of the sub-matrices
__m128d iA1, iA2, iB1, iB2, iC1, iC2, iD1, iD2, DC1, DC2, AB1, AB2;
__m128d dA, dB, dC, dD; // determinant of the sub-matrices
__m128d det, d1, d2, rd;
// dA = |A|
dA = _mm_shuffle_pd(A2, A2, 1);
dA = _mm_mul_pd(A1, dA);
dA = _mm_sub_sd(dA, _mm_shuffle_pd(dA,dA,3));
// dB = |B|
dB = _mm_shuffle_pd(B2, B2, 1);
dB = _mm_mul_pd(B1, dB);
dB = _mm_sub_sd(dB, _mm_shuffle_pd(dB,dB,3));
// AB = A# * B
AB1 = _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,3));
AB2 = _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,0));
AB1 = _mm_sub_pd(AB1, _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,3)));
AB2 = _mm_sub_pd(AB2, _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,0)));
// dC = |C|
dC = _mm_shuffle_pd(C2, C2, 1);
dC = _mm_mul_pd(C1, dC);
dC = _mm_sub_sd(dC, _mm_shuffle_pd(dC,dC,3));
// dD = |D|
dD = _mm_shuffle_pd(D2, D2, 1);
dD = _mm_mul_pd(D1, dD);
dD = _mm_sub_sd(dD, _mm_shuffle_pd(dD,dD,3));
// DC = D# * C
DC1 = _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,3));
DC2 = _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,0));
DC1 = _mm_sub_pd(DC1, _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,3)));
DC2 = _mm_sub_pd(DC2, _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,0)));
// rd = trace(AB*DC) = trace(A#*B*D#*C)
d1 = _mm_mul_pd(AB1, _mm_shuffle_pd(DC1, DC2, 0));
d2 = _mm_mul_pd(AB2, _mm_shuffle_pd(DC1, DC2, 3));
rd = _mm_add_pd(d1, d2);
rd = _mm_add_sd(rd, _mm_shuffle_pd(rd, rd,3));
// iD = C*A#*B
iD1 = _mm_mul_pd(AB1, _mm_shuffle_pd(C1,C1,0));
iD2 = _mm_mul_pd(AB1, _mm_shuffle_pd(C2,C2,0));
iD1 = _mm_add_pd(iD1, _mm_mul_pd(AB2, _mm_shuffle_pd(C1,C1,3)));
iD2 = _mm_add_pd(iD2, _mm_mul_pd(AB2, _mm_shuffle_pd(C2,C2,3)));
// iA = B*D#*C
iA1 = _mm_mul_pd(DC1, _mm_shuffle_pd(B1,B1,0));
iA2 = _mm_mul_pd(DC1, _mm_shuffle_pd(B2,B2,0));
iA1 = _mm_add_pd(iA1, _mm_mul_pd(DC2, _mm_shuffle_pd(B1,B1,3)));
iA2 = _mm_add_pd(iA2, _mm_mul_pd(DC2, _mm_shuffle_pd(B2,B2,3)));
// iD = D*|A| - C*A#*B
dA = _mm_shuffle_pd(dA,dA,0);
iD1 = _mm_sub_pd(_mm_mul_pd(D1, dA), iD1);
iD2 = _mm_sub_pd(_mm_mul_pd(D2, dA), iD2);
// iA = A*|D| - B*D#*C;
dD = _mm_shuffle_pd(dD,dD,0);
iA1 = _mm_sub_pd(_mm_mul_pd(A1, dD), iA1);
iA2 = _mm_sub_pd(_mm_mul_pd(A2, dD), iA2);
d1 = _mm_mul_sd(dA, dD);
d2 = _mm_mul_sd(dB, dC);
// iB = D * (A#B)# = D*B#*A
iB1 = _mm_mul_pd(D1, _mm_shuffle_pd(AB2,AB1,1));
iB2 = _mm_mul_pd(D2, _mm_shuffle_pd(AB2,AB1,1));
iB1 = _mm_sub_pd(iB1, _mm_mul_pd(_mm_shuffle_pd(D1,D1,1), _mm_shuffle_pd(AB2,AB1,2)));
iB2 = _mm_sub_pd(iB2, _mm_mul_pd(_mm_shuffle_pd(D2,D2,1), _mm_shuffle_pd(AB2,AB1,2)));
// det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C)
det = _mm_add_sd(d1, d2);
det = _mm_sub_sd(det, rd);
// iC = A * (D#C)# = A*C#*D
iC1 = _mm_mul_pd(A1, _mm_shuffle_pd(DC2,DC1,1));
iC2 = _mm_mul_pd(A2, _mm_shuffle_pd(DC2,DC1,1));
iC1 = _mm_sub_pd(iC1, _mm_mul_pd(_mm_shuffle_pd(A1,A1,1), _mm_shuffle_pd(DC2,DC1,2)));
iC2 = _mm_sub_pd(iC2, _mm_mul_pd(_mm_shuffle_pd(A2,A2,1), _mm_shuffle_pd(DC2,DC1,2)));
rd = _mm_div_sd(_mm_set_sd(1.0), det);
rd = _mm_shuffle_pd(rd,rd,0);
// iB = C*|B| - D*B#*A
dB = _mm_shuffle_pd(dB,dB,0);
iB1 = _mm_sub_pd(_mm_mul_pd(C1, dB), iB1);
iB2 = _mm_sub_pd(_mm_mul_pd(C2, dB), iB2);
d1 = _mm_xor_pd(rd, _Sign_PN);
d2 = _mm_xor_pd(rd, _Sign_NP);
// iC = B*|C| - A*C#*D;
dC = _mm_shuffle_pd(dC,dC,0);
iC1 = _mm_sub_pd(_mm_mul_pd(B1, dC), iC1);
iC2 = _mm_sub_pd(_mm_mul_pd(B2, dC), iC2);
_mm_storeu_pd(dst+0, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 3), d1));
_mm_storeu_pd(dst+4, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 0), d2));
_mm_storeu_pd(dst+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 3), d1));
_mm_storeu_pd(dst+4+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 0), d2));
_mm_storeu_pd(dst+2*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 3), d1));
_mm_storeu_pd(dst+3*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 0), d2));
_mm_storeu_pd(dst+2*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 3), d1));
_mm_storeu_pd(dst+3*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 0), d2));
}
#endif
} // end of namespace Fastor
#endif // INVERSE_H

753
noarch/include/Fastor/backend/lufact.h
View File

@@ -1,753 +0,0 @@
#ifndef LUFACT_H
#define LUFACT_H
#include "Fastor/meta/meta.h"
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,8>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u) {
*l = 1;
*u = *a;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
// [a11 a12] [1 0] [u11 u12]
// [a21 a22] [l21 1] [0 u22]
const T L21 = A[2]/A[0];
L[0] = 1;
L[1] = 0;
L[2] = L21;
L[3] = 1;
U[0] = A[0];
U[1] = A[1];
U[2] = 0;
U[3] = A[3] - L21 * A[1];
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0];
const T L21 = A[3]/A00;
const T L31 = A[6]/A00;
const T U22 = A[4] - L21 * A[1];
const T U23 = A[5] - L21 * A[2];
const T L32 = (A[7] - L31 * A[1]) / U22;
const T U33 = A[8] - L31 * A[2] - L32 * U23;
L[0] = 1;
L[1] = 0;
L[2] = 0;
L[3] = L21;
L[4] = 1;
L[5] = 0;
L[6] = L31;
L[7] = L32;
L[8] = 1;
U[0] = A00;
U[1] = A[1];
U[2] = A[2];
U[3] = 0;
U[4] = U22;
U[5] = U23;
U[6] = 0;
U[7] = 0;
U[8] = U33;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0];
const T L21 = A[4] /A00;
const T L31 = A[8] /A00;
const T L41 = A[12]/A00;
const T U22 = A[5] - L21 * A[1];
const T U23 = A[6] - L21 * A[2];
const T U24 = A[7] - L21 * A[3];
const T L32 = (A[9 ] - L31 * A[1]) / U22;
const T L42 = (A[13] - L41 * A[1]) / U22;
const T U33 = A[10] - L31 * A[2] - L32 * U23;
const T U34 = A[11] - L31 * A[3] - L32 * U24;
const T L43 = (A[14] - L41 * A[2] - L42 * U23) / U33;
const T U44 = A[15] - L41 * A[3] - L42 * U24 - L43 * U34;
L[0] = 1;
L[1] = 0;
L[2] = 0;
L[3] = 0;
L[4] = L21;
L[5] = 1;
L[6] = 0;
L[7] = 0;
L[8] = L31;
L[9] = L32;
L[10] = 1;
L[11] = 0;
L[12] = L41;
L[13] = L42;
L[14] = L43;
L[15] = 1;
U[0] = A00;
U[1] = A[1];
U[2] = A[2];
U[3] = A[3];
U[4] = 0;
U[5] = U22;
U[6] = U23;
U[7] = U24;
U[8] = 0;
U[9] = 0;
U[10] = U33;
U[11] = U34;
U[12] = 0;
U[13] = 0;
U[14] = 0;
U[15] = U44;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,5>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,6>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T L61 = A[5*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T U26 = A[1*N+5] - L21 * A[5];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T L62 = (A[5*N+1] - L61 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46;
const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55;
const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[0*N+5] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[1*N+5] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[2*N+5] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[3*N+5] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
L[4*N+5] = 0;
L[5*N+0] = L61;
L[5*N+1] = L62;
L[5*N+2] = L63;
L[5*N+3] = L64;
L[5*N+4] = L65;
L[5*N+5] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[0*N+5] = A[5];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[1*N+5] = U26;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[2*N+5] = U36;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[3*N+5] = U46;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
U[4*N+5] = U56;
U[5*N+0] = 0;
U[5*N+1] = 0;
U[5*N+2] = 0;
U[5*N+3] = 0;
U[5*N+4] = 0;
U[5*N+5] = U66;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,7>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T L61 = A[5*N] /A00;
const T L71 = A[6*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T U26 = A[1*N+5] - L21 * A[5];
const T U27 = A[1*N+6] - L21 * A[6];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T L62 = (A[5*N+1] - L61 * A[1]) / U22;
const T L72 = (A[6*N+1] - L71 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26;
const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33;
const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36;
const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44;
const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46;
const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47;
const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55;
const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55;
const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56;
const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57;
const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66;
const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[0*N+5] = 0;
L[0*N+6] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[1*N+5] = 0;
L[1*N+6] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[2*N+5] = 0;
L[2*N+6] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[3*N+5] = 0;
L[3*N+6] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
L[4*N+5] = 0;
L[4*N+6] = 0;
L[5*N+0] = L61;
L[5*N+1] = L62;
L[5*N+2] = L63;
L[5*N+3] = L64;
L[5*N+4] = L65;
L[5*N+5] = 1;
L[5*N+6] = 0;
L[6*N+0] = L71;
L[6*N+1] = L72;
L[6*N+2] = L73;
L[6*N+3] = L74;
L[6*N+4] = L75;
L[6*N+5] = L76;
L[6*N+6] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[0*N+5] = A[5];
U[0*N+6] = A[6];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[1*N+5] = U26;
U[1*N+6] = U27;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[2*N+5] = U36;
U[2*N+6] = U37;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[3*N+5] = U46;
U[3*N+6] = U47;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
U[4*N+5] = U56;
U[4*N+6] = U57;
U[5*N+0] = 0;
U[5*N+1] = 0;
U[5*N+2] = 0;
U[5*N+3] = 0;
U[5*N+4] = 0;
U[5*N+5] = U66;
U[5*N+6] = U67;
U[6*N+0] = 0;
U[6*N+1] = 0;
U[6*N+2] = 0;
U[6*N+3] = 0;
U[6*N+4] = 0;
U[6*N+5] = 0;
U[6*N+6] = U77;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,8>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T L61 = A[5*N] /A00;
const T L71 = A[6*N] /A00;
const T L81 = A[7*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T U26 = A[1*N+5] - L21 * A[5];
const T U27 = A[1*N+6] - L21 * A[6];
const T U28 = A[1*N+7] - L21 * A[7];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T L62 = (A[5*N+1] - L61 * A[1]) / U22;
const T L72 = (A[6*N+1] - L71 * A[1]) / U22;
const T L82 = (A[7*N+1] - L81 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26;
const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27;
const T U38 = A[2*N+7] - L31 * A[7] - L32 * U28;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33;
const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33;
const T L83 = (A[7*N+2] - L81 * A[2] - L82 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36;
const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37;
const T U48 = A[3*N+7] - L41 * A[7] - L42 * U28 - L43 * U38;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44;
const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44;
const T L84 = (A[7*N+3] - L81 * A[3] - L82 * U24 - L83 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46;
const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47;
const T U58 = A[4*N+7] - L51 * A[7] - L52 * U28 - L53 * U38 - L54 * U48;
const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55;
const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55;
const T L85 = (A[7*N+4] - L81 * A[4] - L82 * U25 - L83 * U35 - L84 * U45) / U55;
const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56;
const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57;
const T U68 = A[5*N+7] - L61 * A[7] - L62 * U28 - L63 * U38 - L64 * U48 - L65 * U58;
const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66;
const T L86 = (A[7*N+5] - L81 * A[5] - L82 * U26 - L83 * U36 - L84 * U46 - L85 * U56) / U66;
const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67;
const T U78 = A[6*N+7] - L71 * A[7] - L72 * U28 - L73 * U38 - L74 * U48 - L75 * U58 - L76 * U68;
const T L87 = (A[7*N+6] - L81 * A[6] - L82 * U27 - L83 * U37 - L84 * U47 - L85 * U57 - L86 * U67) / U77;
const T U88 = A[7*N+7] - L81 * A[7] - L82 * U28 - L83 * U38 - L84 * U48 - L85 * U58 - L86 * U68 - L87 * U78;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[0*N+5] = 0;
L[0*N+6] = 0;
L[0*N+7] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[1*N+5] = 0;
L[1*N+6] = 0;
L[1*N+7] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[2*N+5] = 0;
L[2*N+6] = 0;
L[2*N+7] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[3*N+5] = 0;
L[3*N+6] = 0;
L[3*N+7] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
L[4*N+5] = 0;
L[4*N+6] = 0;
L[4*N+7] = 0;
L[5*N+0] = L61;
L[5*N+1] = L62;
L[5*N+2] = L63;
L[5*N+3] = L64;
L[5*N+4] = L65;
L[5*N+5] = 1;
L[5*N+6] = 0;
L[5*N+7] = 0;
L[6*N+0] = L71;
L[6*N+1] = L72;
L[6*N+2] = L73;
L[6*N+3] = L74;
L[6*N+4] = L75;
L[6*N+5] = L76;
L[6*N+6] = 1;
L[6*N+7] = 0;
L[7*N+0] = L81;
L[7*N+1] = L82;
L[7*N+2] = L83;
L[7*N+3] = L84;
L[7*N+4] = L85;
L[7*N+5] = L86;
L[7*N+6] = L87;
L[7*N+7] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[0*N+5] = A[5];
U[0*N+6] = A[6];
U[0*N+7] = A[7];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[1*N+5] = U26;
U[1*N+6] = U27;
U[1*N+7] = U28;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[2*N+5] = U36;
U[2*N+6] = U37;
U[2*N+7] = U38;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[3*N+5] = U46;
U[3*N+6] = U47;
U[3*N+7] = U48;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
U[4*N+5] = U56;
U[4*N+6] = U57;
U[4*N+7] = U58;
U[5*N+0] = 0;
U[5*N+1] = 0;
U[5*N+2] = 0;
U[5*N+3] = 0;
U[5*N+4] = 0;
U[5*N+5] = U66;
U[5*N+6] = U67;
U[5*N+7] = U68;
U[6*N+0] = 0;
U[6*N+1] = 0;
U[6*N+2] = 0;
U[6*N+3] = 0;
U[6*N+4] = 0;
U[6*N+5] = 0;
U[6*N+6] = U77;
U[6*N+7] = U78;
U[7*N+0] = 0;
U[7*N+1] = 0;
U[7*N+2] = 0;
U[7*N+3] = 0;
U[7*N+4] = 0;
U[7*N+5] = 0;
U[7*N+6] = 0;
U[7*N+7] = U88;
}
} // end of namespace Fastor
#endif // LUFACT_H

80
noarch/include/Fastor/backend/lut_inverse.h
View File

@@ -1,80 +0,0 @@
#ifndef LOWUNITRI_INVERSE_H
#define LOWUNITRI_INVERSE_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = T(1);
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
/* Compute adjoint: */
dst[0] = 1;
dst[1] = 0;
dst[2] = - src[2];
dst[3] = 1;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src3 = src[3];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute adjoint: */
dst[0] = 1;
dst[1] = 0;
dst[2] = 0;
dst[3] = - src3 * src8;
dst[4] = 1;
dst[5] = 0;
dst[6] = + src3 * src7 - src6;
dst[7] = - src7;
dst[8] = 1;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
dst[0] = 1;
dst[1] = 0;
dst[2] = 0;
dst[3] = 0;
dst[4] = - src[1*4+0];
dst[5] = 1;
dst[6] = 0;
dst[7] = 0;
T t2 = src[2*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[3*4+1];
T t4 = src[2*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[3*4+0];
dst[8] = src[1*4+0]*t2 - t4;
dst[9] = - t2;
dst[10] = 1;
dst[11] = 0;
dst[12] = t5 - src[1*4+0]*t3;
dst[13] = t3;
dst[14] = - src[3*4+2];
dst[15] = 1;
}
} // end of namespace Fastor
#endif // LOWUNITRI_INVERSE_H

209
noarch/include/Fastor/backend/matmul/libxsmm_backend.h
View File

@@ -1,209 +0,0 @@
#ifndef LIBXSMM_BACKEND_H
#define LIBXSMM_BACKEND_H
#include <Fastor/tensor/Tensor.h>
#ifdef FASTOR_USE_LIBXSMM
#include <libxsmm.h>
namespace Fastor {
namespace blas {
// single
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulNN_libxsmm(
const float * FASTOR_RESTRICT a_data,
const float * FASTOR_RESTRICT b_data,
float * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr float alpha = 1.0;
constexpr float beta = 0.0;
constexpr char transa = 'N';
constexpr char transb = 'N';
libxsmm_sgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&NN /*lda*/,
a_data /*required*/,
&KK /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
// double
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulNN_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'N';
constexpr char transb = 'N';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&NN /*lda*/,
a_data /*required*/,
&KK /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulTN_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'N';
constexpr char transb = 'Y';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&NN /*lda*/,
a_data /*required*/,
&MM /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulNT_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'Y';
constexpr char transb = 'N';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&KK /*lda*/,
a_data /*required*/,
&KK /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulTT_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'Y';
constexpr char transb = 'Y';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&KK /*lda*/,
a_data /*required*/,
&MM /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<typename T, size_t M, size_t K, size_t N,
typename std::enable_if<std::is_same<T,double>::value,bool>::type=0>
FASTOR_INLINE
void matmul_libxsmm(
const T * FASTOR_RESTRICT a_data,
const T * FASTOR_RESTRICT b_data,
T * FASTOR_RESTRICT out_data) {
matmulNN_libxsmm<M,K,N>(a_data,b_data,out_data);
}
template<typename T, size_t M, size_t K, size_t N,
typename std::enable_if<std::is_same<T,float>::value,bool>::type=0>
FASTOR_INLINE
void matmul_libxsmm(
const T * FASTOR_RESTRICT a_data,
const T * FASTOR_RESTRICT b_data,
T * FASTOR_RESTRICT out_data) {
matmulNN_libxsmm<M,K,N>(a_data,b_data,out_data);
}
} // end of namespace blas
} // end of namespace Fastor
#endif // FASTOR_USE_LIBXSMM
#endif // LIBXSMM_BACKEND_H

150
noarch/include/Fastor/backend/matmul/matmul.h
View File

@@ -1,150 +0,0 @@
#ifndef MATMUL_H
#define MATMUL_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/matmul/matmul_kernels.h"
#ifdef FASTOR_USE_LIBXSMM
#include "Fastor/backend/matmul/libxsmm_backend.h"
#endif
#ifdef FASTOR_USE_MKL
#include "Fastor/backend/matmul/mkl_backend.h"
#endif
namespace Fastor {
// Forward declare
//-----------------------------------------------------------------------------------------------------------
namespace internal {
template<typename T, size_t M, size_t N>
FASTOR_INLINE
void _matvecmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out);
} // internal
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#if !defined(FASTOR_USE_LIBXSMM) && !defined(FASTOR_USE_MKL)
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) ),bool> = 0>
#else
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<
!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) )
&& is_less_equal<M*N*K/internal::meta_cube<FASTOR_BLAS_SWITCH_MATRIX_SIZE>::value,1>::value,
bool> = 0>
#endif
FASTOR_INLINE
void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
// Non-primitive types
FASTOR_IF_CONSTEXPR (!is_primitive_v_<T>) {
internal::_matmul_base_non_primitive<T,M,K,N>(a,b,out);
return;
}
// Matrix-vector specialisation
FASTOR_IF_CONSTEXPR (N==1UL) {
internal::_matvecmul<T,M,K>(a,b,out);
return;
}
using nativeV = SIMDVector<T,DEFAULT_ABI>;
using V = choose_best_simd_t<nativeV,N>;
// Use specialised kernels
FASTOR_IF_CONSTEXPR((N==V::Size || N==2*V::Size || N==3*V::Size || N==4*V::Size || N==5*V::Size) && V::Size!=1UL) {
internal::_matmul_mk_smalln<T,M,K,N>(a,b,out);
return;
}
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
FASTOR_IF_CONSTEXPR((N<5*V::Size && N!=1UL)) {
internal::_matmul_mk_smalln<T,M,K,N>(a,b,out);
return;
}
#endif
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size <= 1UL) {
internal::_matmul_base<T,M,K,N>(a,b,out);
return;
}
else FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size > 1UL) {
internal::_matmul_base_masked<T,M,K,N>(a,b,out);
return;
}
#else
FASTOR_IF_CONSTEXPR( M*N*K > 27UL ) {
internal::_matmul_base<T,M,K,N>(a,b,out);
return;
}
#endif
else
{
// For all other cases where M,N,K is too small
// this simple version is sufficient
constexpr int ROUND_ = ROUND_DOWN(N,V::Size);
for (size_t j=0; j<M; ++j) {
size_t k=0;
for (; k<ROUND_; k+=V::Size) {
V out_row;
for (size_t i=0; i<K; ++i) {
const V brow(&b[i*N+k],false);
const V vec_a(a[j*K+i]);
out_row = fmadd(vec_a,brow,out_row);
}
out_row.store(&out[k+N*j],false);
}
for (; k<N; k++) {
T out_row = 0.;
for (size_t i=0; i<K; ++i) {
out_row += a[j*K+i]*b[i*N+k];
}
out[N*j+k] = out_row;
}
}
}
}
#if defined(FASTOR_USE_LIBXSMM) && !defined(FASTOR_USE_MKL)
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<
!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) )
&& is_greater<M*N*K/internal::meta_cube<FASTOR_BLAS_SWITCH_MATRIX_SIZE>::value,1>::value,
bool> = 0>
FASTOR_INLINE
void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
blas::matmul_libxsmm<T,M,K,N>(a,b,c);
}
#endif
#if !defined(FASTOR_USE_LIBXSMM) && defined(FASTOR_USE_MKL)
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<
!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) )
&& is_greater<M*N*K/internal::meta_cube<FASTOR_BLAS_SWITCH_MATRIX_SIZE>::value,1>::value,
bool> = 0>
FASTOR_INLINE
void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
blas::matmul_mkl<T,M,K,N>(a,b,c);
}
#endif
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#include "Fastor/backend/matmul/matmul_specialisations_kernels.h"
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace
#endif // MATMUL_H

823
noarch/include/Fastor/backend/matmul/matmul_kernels.h
View File

@@ -1,823 +0,0 @@
#ifndef MATMUL_KERNELS_H
#define MATMUL_KERNELS_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/meta/tensor_meta.h"
namespace Fastor {
namespace internal {
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel)
// unroll the inner-most loop (on unrollOuterloop)
//-----------------------------------------------------------------------------------------------------------
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==2,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==3,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==4,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==5,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
const V bmm4(&b[k*N+j+4*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_scalar_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
T c_ij[unrollOuterloop*numSIMDCols] = {};
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const T bmm0(b[k*N+j]);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] += amm0*bmm0;
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n];
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const int (&maska)[V::Size]) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(maskload<V>(&b[k*N+j],maska));
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]);
}
}
}
template<typename T, typename MaskT, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const MaskT mask) {
V bmm0;
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
bmm0.mask_load(&b[k*N+j],mask,false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false);
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the base implementation of matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as gemm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in a scalar fashion
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; ++j) {
interior_block_matmul_scalar_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[unrollOuterloop];
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
T c_ij[unrollOuterloop] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * b[k*N+j];
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i + n)*N+j] = c_ij[n];
}
}
}
// Now treat the remaining M-M1 rows
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_matmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
T c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] += a[n*K+k] * b[k*N+j];
c[n*N+j] = c_ij[n-M1];
}
}
for (size_t n = M1; n < M; ++n) {
c[n*N+j] = c_ij[n-M1];
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the base implementation of matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as gemm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in vector mode with masked and conditional load/stores.
// Note that conditional load/store requires at least AVX intrinsics
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
int maska[V::Size];
std::fill(maska,&maska[V::Size], -1);
for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0;
#ifdef FASTOR_HAS_AVX512_MASKS
const auto mask = array_to_mask(maska);
#endif
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; j+= N-N1) {
#ifdef FASTOR_HAS_AVX512_MASKS
interior_block_matmul_mask_impl<T,decltype(mask),V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,mask);
#else
interior_block_matmul_mask_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,maska);
#endif
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[unrollOuterloop];
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
V c_ij[unrollOuterloop];
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[(i + n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n].mask_store(&c[(i+n)*N+j],mask,false);
#else
maskstore(&c[(i+n)*N+j],maska,c_ij[n]);
#endif
}
}
}
// Now treat the remaining M-M1 rows
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_matmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
V c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[n*K+k];
c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]);
}
}
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n-M1].mask_store(&c[n*N+j],mask,false);
#else
maskstore(&c[n*N+j],maska,c_ij[n-M1]);
#endif
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// matmul kernel for non-fundamental types
// The assumption here is that non-fundamental types are not SIMD vectorisable for instance
// Tensor<std::vector<T>,3,3> or Tensor<Tensor<...>,...> plus they cannot fuse [do fused-add-multiply]
// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in
// the inner most loops of matmul
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
// There is no SIMD here as V::Size == 1 anyway
// No outer loop unrolling otherwise the innermost loop
// will create unnecessary temporaries
for (size_t i=0; i<M; ++i) {
// V::Size == 1 so this loop can't be unrolled
for (size_t j=0; j<N; ++j) {
// This could potentially cost as opposed to directly writing in to c
T tmp {};
for (size_t k=0; k<K; ++k) {
tmp += a[i*K+k]*b[k*N+j];
}
c[i*N+j] = tmp;
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// Other variants and slightly older implementations
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the same implementation as the above case but does not unroll on block sizes and does not require
// the registers to be zeroed out but K must be !=1
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_mkn_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// Get 10 parallel independent chains of accumulators for bigger matrices
constexpr size_t unrollOuterloop = M >= 64 ? 10UL : (M % 8 == 0 ? 8UL : V::Size);
// The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore
// the last rows may need special treatment if M is not a multiple of unrollOuterloop.
// M0 is the number of rows that can safely be iterated with a stride of
// unrollOuterloop.
constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop;
for (size_t i = 0; i < M0; i += unrollOuterloop) {
// The iteration over the column index of b and c uses a stride of V::Size. This
// enables row-vector loads (from b) and stores (to c). The matrix storage is
// padded accordingly, ensuring correct bounds and alignment.
for (size_t j = 0; j < N; j += V::Size) {
// This temporary variables are used to accumulate the results of the products
// producing the new values for the c matrix. This variable is necessary
// because we need a V object for data-parallel accumulation. Storing to c
// directly stores to scalar objects and thus would drop the ability for
// data-parallel (SIMD) addition.
V c_ij[unrollOuterloop];
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = a[(i + n)*K]*V(&b[j]);
}
for (size_t k = 1; k < K - 1; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j]);
c_ij[n].store(&c[(i + n)*N+j]);
}
}
}
}
// This is the same implementation as the above case but does not unroll on block sizes and does not require
// the registers to be zeroed out but K must be !=1
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_mkn_non_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
// This variant strictly cannot deal outer-product i.e. with K==1
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// Get 10 parallel independent chains of accumulators for bigger matrices
constexpr size_t unrollOuterloop = M < V::Size ? 1UL :
(( M >= 64 && K > 10 && N > V::Size ) ? 10UL : (M % 8 == 0 && N > V::Size ? 8UL : V::Size));
constexpr bool isPadded = N % V::Size == 0;
// The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore
// the last rows may need special treatment if M is not a multiple of unrollOuterloop.
// M0 is the number of rows that can safely be iterated with a stride of
// unrollOuterloop.
constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop;
constexpr size_t N0 = N / V::Size * V::Size;
for (size_t i = 0; i < M0; i += unrollOuterloop) {
// The iteration over the column index of b and c uses a stride of V::size(). This
// enables row-vector loads (from b) and stores (to c). The matrix storage is
// padded accordingly, ensuring correct bounds and alignment.
size_t j = 0;
for (; j < N0; j += V::Size) {
// This temporary variables are used to accumulate the results of the products
// producing the new values for the c matrix. This variable is necessary
// because we need a V object for data-parallel accumulation. Storing to c
// directly stores to scalar objects and thus would drop the ability for
// data-parallel (SIMD) addition.
V c_ij[unrollOuterloop];
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] = a[(i + n)*K]*V(&b[j], isPadded);
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j], false);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j], false);
c_ij[n].store(&c[(i + n)*N+j], isPadded);
}
}
// Remainder N - N0 columns
for (; j < N; ++j) {
T c_ij[unrollOuterloop];
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] = a[(i + n)*K]*b[j];
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * b[k*N+j];
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] += a[(i + n)*K+(K - 1)] * b[(K - 1)*N+j];
c[(i + n)*N+j] = c_ij[n];
}
}
}
// This final loop treats the remaining M - M0 rows.
size_t j = 0;
for (; j < N0; j += V::Size) {
V c_ij[M-M0];
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] = a[n*K] * V(&b[j], isPadded);
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+k] * V(&b[k*N+j], false);
}
}
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+(K - 1)] * V(&b[(K - 1)*N+j], false);
c_ij[n - M0].store(&c[n*N+j], isPadded);
}
}
for (; j < N; ++j) {
T c_ij[M-M0];
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] = a[n*K] * b[j];
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+k] * b[k*N+j];
}
}
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+(K - 1)] * b[(K - 1)*N+j];
c[n*N+j] = c_ij[n - M0];
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace internal
} // end of namespace Fastor
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#include "Fastor/backend/matmul/matmul_mk_smalln.h"
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#endif // MATMUL_KERNELS_H

3271
noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h
View File

File diff suppressed because it is too large Load Diff

1187
noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h
View File

File diff suppressed because it is too large Load Diff

67
noarch/include/Fastor/backend/matmul/mkl_backend.h
View File

@@ -1,67 +0,0 @@
#ifndef MKL_BACKEND_H
#define MKL_BACKEND_H
#include <Fastor/tensor/Tensor.h>
#ifdef FASTOR_USE_MKL
// Explicitly activate the jit
#ifndef MKL_DIRECT_CALL_SEQ_JIT
#define MKL_DIRECT_CALL_SEQ_JIT
#endif
#include <mkl.h>
namespace Fastor {
namespace blas {
// single
template<typename T, size_t M, size_t K, size_t N, enable_if_t_<is_same_v_<T,float>,bool> = false>
void matmul_mkl(
const float * FASTOR_RESTRICT a_data,
const float * FASTOR_RESTRICT b_data,
float * FASTOR_RESTRICT out_data) {
cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N);
}
// double
template<typename T, size_t M, size_t K, size_t N, enable_if_t_<is_same_v_<T,double>,bool> = false>
void matmul_mkl(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N);
}
#if 0
// dedicated jit api, but the jit kernel has to be created before hand
template<typename T, size_t M, size_t K, size_t N>
Tensor<T,M,N> matmul_mkl_jit_api(const Tensor<T,M,K> &a, const Tensor<T,K,N> &b) {
Tensor<T,M,N> out;
void* jitter;
mkl_jit_status_t status = mkl_jit_create_dgemm(&jitter, MKL_ROW_MAJOR, MKL_NOTRANS, MKL_NOTRANS, M, N, K, 1.0, K, N, 0.0, N);
dgemm_jit_kernel_t _dgemm_kernel = mkl_jit_get_dgemm_ptr(jitter);
_dgemm_kernel(jitter, a.data(), b.data(), out.data());
mkl_jit_destroy(jitter);
return out;
}
#endif
} // end of namespace blas
} // end of namespace Fastor
#endif // FASTOR_USE_MKL
#endif // MKL_BACKEND_H

803
noarch/include/Fastor/backend/matmul/tmatmul.h
View File

@@ -1,803 +0,0 @@
#ifndef MATMUL_KERNELS2_H
#define MATMUL_KERNELS2_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/meta/meta.h"
#include "Fastor/meta/tensor_meta.h"
namespace Fastor {
namespace internal {
// TRMM implementation of Fastor - matrix-matrix multiplication when either or both operands are
// lower or upper triangular. The matrices do not need to be square and trapezoidal cases are also
// covered. For big matrices the speed-up is 2X or even better over matmul for when one operand is
// is triangular and nearly 4X for when both operands are triangular.
// For small matrices due to aggressive unrolling for SIMD the matrices cannot be exactly traversed
// within their triangular part(s) and a bit of the non-triangular part(s) need(s) to be loaded as well
// hence, the performance may not be exactly 2X/4X over the general matmul case
// The functions here are exact replica of those in matmul_kernels.h and will be eventually
// merged together as these variants have no associated overhead for the general case of matmul
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
/*
For triangular matmul only the iteration span of K is modified
using the following logic [lt = Lower, ut=Upper]
// if lhs == lt
const size_t kfirst = 0;
const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K);
// if lhs == ut
const size_t kfirst = i;
const size_t klast = K;
// if rhs == lt
const size_t kfirst = j;
const size_t klast = K;
// if rhs == ut
const size_t kfirst = 0;
const size_t klast = min(j+1,K); // or min(j+unrollOuterloop,K);
// both lower
const size_t kfirst = j;
const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K);
// if lhs == lt && rhs == ut
const size_t kfirst = 0;
const size_t klast = min(min(i+1,j+1),K); // or min(min(i+unrollOuterloop,j+unrollInnerloop),K);
// if lhs == ut && rhs == lt
const size_t kfirst = max(i,j);
const size_t klast = K;
// if both upper
const size_t kfirst = i;
const size_t klast = min(j+1,K); // or min(j+unrollInnerloop,K);
*/
template<typename T, T K, T unrollOuterloop=1,T unrollInnerloop=1, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
constexpr FASTOR_INLINE T find_kfirst(const T i, const T j) {
return is_same_v_<LhsType,UpLoType::Lower> || is_same_v_<LhsType,UpLoType::General> ?
( is_same_v_<RhsType,UpLoType::Lower> ? j : 0UL ) :
(is_same_v_<LhsType,UpLoType::Upper> ? ( is_same_v_<RhsType,UpLoType::Lower> ? internal::max_(i,j) : i ) : 0UL );
}
template<typename T, T K, T unrollOuterloop=1,T unrollInnerloop=1, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
constexpr FASTOR_INLINE T find_klast(const T i, const T j) {
return is_same_v_<LhsType,UpLoType::Lower> ?
( is_same_v_<RhsType,UpLoType::Upper> ? internal::min_(internal::min_(i+unrollOuterloop,j+unrollInnerloop),K) : internal::min_(i+unrollOuterloop,K) ) :
(is_same_v_<LhsType,UpLoType::Upper> || (is_same_v_<LhsType,UpLoType::General>) ?
( is_same_v_<RhsType,UpLoType::Upper> ? internal::min_(j+unrollInnerloop,K) : K ) : K );
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel)
// unroll the inner-most loop (on unrollOuterloop)
//-----------------------------------------------------------------------------------------------------------
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==2,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==3,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==4,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==5,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
const V bmm4(&b[k*N+j+4*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_scalar_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
T c_ij[unrollOuterloop*numSIMDCols] = {};
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const T bmm0(b[k*N+j]);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] += amm0*bmm0;
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n];
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const int (&maska)[V::Size]) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(maskload<V>(&b[k*N+j],maska));
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]);
}
}
}
template<typename T, typename MaskT, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const MaskT mask) {
V bmm0;
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
bmm0.mask_load(&b[k*N+j],mask,false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false);
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the base implementation of triangular matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as trmm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in a scalar fashion
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,N>;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols,LhsType,RhsType>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1,LhsType,RhsType>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; ++j) {
interior_block_tmatmul_scalar_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1,LhsType,RhsType>(a,b,c,i,j);
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols,LhsType,RhsType>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
V c_ij[unrollOuterloop];
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,1,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,1,LhsType,RhsType>(i,j);
T c_ij[unrollOuterloop] = {};
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * b[k*N+j];
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i + n)*N+j] = c_ij[n];
}
}
}
// Now treat the remaining M-M1 rows - here the klast - kfirst range is not used
// so the implementation is exactly the same as matmul_base
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_tmatmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
T c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] += a[n*K+k] * b[k*N+j];
c[n*N+j] = c_ij[n-M1];
}
}
for (size_t n = M1; n < M; ++n) {
c[n*N+j] = c_ij[n-M1];
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This the base implementation of triangular matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as trmm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in vector mode with masked and conditional load/stores.
// Note that conditional load/store requires at least AVX intrinsics
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
int maska[V::Size];
std::fill(maska,&maska[V::Size], -1);
for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0;
#ifdef FASTOR_HAS_AVX512_MASKS
const auto mask = array_to_mask(maska);
#endif
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; j+= N-N1) {
#ifdef FASTOR_HAS_AVX512_MASKS
interior_block_matmul_mask_impl<T,decltype(mask),V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,mask);
#else
interior_block_matmul_mask_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,maska);
#endif
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
V c_ij[unrollOuterloop];
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
V c_ij[unrollOuterloop];
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[(i + n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n].mask_store(&c[(i+n)*N+j],mask,false);
#else
maskstore(&c[(i+n)*N+j],maska,c_ij[n]);
#endif
}
}
}
// Now treat the remaining M-M1 rows - here the klast - kfirst range is not used
// so the implementation is exactly the same as matmul_base
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_tmatmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
V c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[n*K+k];
c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]);
}
}
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n-M1].mask_store(&c[n*N+j],mask,false);
#else
maskstore(&c[n*N+j],maska,c_ij[n-M1]);
#endif
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// tmatmul kernel for non-fundamental types
// The assumption here is that non-fundamental types are not SIMD vectorisable for instance
// Tensor<std::vector<T>,3,3> or Tensor<Tensor<...>,...> plus they cannot fuse [do fused-add-multiply]
// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in
// the inner most loops of matmul
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
// There is no SIMD here as V::Size == 1 anyway
// No outer loop unrolling otherwise the innermost loop
// will create unnecessary temporaries
for (size_t i=0; i<M; ++i) {
// V::Size == 1 so this loop can't be unrolled
for (size_t j=0; j<N; ++j) {
const size_t kfirst = find_kfirst<size_t,K,1,1,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,1,1,LhsType,RhsType>(i,j);
// This could potentially cost as opposed to directly writing in to c
T tmp {};
for (size_t k=kfirst; k<klast; ++k) {
tmp += a[i*K+k]*b[k*N+j];
}
c[i*N+j] = tmp;
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace internal
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// Backend tmatmul function
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
// Non-primitive types
FASTOR_IF_CONSTEXPR (!is_primitive_v_<T>) {
internal::_tmatmul_base_non_primitive<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
}
// Use specialised kernels
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
using nativeV = SIMDVector<T,DEFAULT_ABI>;
using V = choose_best_simd_t<nativeV,N>;
FASTOR_IF_CONSTEXPR(N % V::Size <= 1UL) {
internal::_tmatmul_base<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
}
else {
internal::_tmatmul_base_masked<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
}
#else
internal::_tmatmul_base<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
#endif
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace Fastor
#endif // MATMUL_KERNELS_H

149
noarch/include/Fastor/backend/norm.h
View File

@@ -1,149 +0,0 @@
#ifndef NORM_H
#define NORM_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
template<typename T, size_t N,
#ifdef FASTOR_AVX512_IMPL
enable_if_t_<is_less_v_<8*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#else
enable_if_t_<is_less_v_<4*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#endif
FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
T _scal = 0;
#ifdef FASTOR_AVX512_IMPL
V omm0, omm1, omm2, omm3, omm4, omm5, omm6, omm7;
#else
V omm0, omm1, omm2, omm3;
#endif
FASTOR_INDEX i = 0;
// With AVX utilises all the 16 registers but hurts the performance
// due to spill if eval has created temporary registers so only
// activated for AVX512
#ifdef FASTOR_AVX512_IMPL
for (; i < ROUND_DOWN(N,8*V::Size); i+=8*V::Size) {
const V smm0(&a[i] , false);
const V smm1(&a[i+V::Size] , false);
const V smm2(&a[i+2*V::Size] , false);
const V smm3(&a[i+3*V::Size] , false);
const V smm4(&a[i+4*V::Size] , false);
const V smm5(&a[i+5*V::Size] , false);
const V smm6(&a[i+6*V::Size] , false);
const V smm7(&a[i+7*V::Size] , false);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
omm2 = fmadd(smm2,smm2,omm2);
omm3 = fmadd(smm3,smm3,omm3);
omm4 = fmadd(smm4,smm4,omm4);
omm5 = fmadd(smm5,smm5,omm5);
omm6 = fmadd(smm6,smm6,omm6);
omm7 = fmadd(smm7,smm7,omm7);
}
#endif
for (; i < ROUND_DOWN(N,4*V::Size); i+=4*V::Size) {
const V smm0(&a[i] , false);
const V smm1(&a[i+V::Size] , false);
const V smm2(&a[i+2*V::Size] , false);
const V smm3(&a[i+3*V::Size] , false);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
omm2 = fmadd(smm2,smm2,omm2);
omm3 = fmadd(smm3,smm3,omm3);
}
for (; i < ROUND_DOWN(N,2*V::Size); i+=2*V::Size) {
const V smm0(&a[i] , false);
const V smm1(&a[i+V::Size] , false);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
}
for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) {
const V smm0(&a[i] , false);
omm0 = fmadd(smm0,smm0,omm0);
}
for (; i < N; ++i) {
const auto smm0(a[i]);
_scal += smm0*smm0;
}
#ifdef FASTOR_AVX512_IMPL
return sqrts( (omm0 + omm1 + omm2 + omm3 + omm4 + omm5 + omm6 + omm7).sum() + _scal);
#else
return sqrts( (omm0 + omm1 + omm2 + omm3).sum() + _scal);
#endif
}
template<typename T, size_t N,
#ifdef FASTOR_AVX512_IMPL
enable_if_t_<is_greater_equal_v_<8*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#else
enable_if_t_<is_greater_equal_v_<4*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#endif
FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
T _scal = 0;
V omm0;
FASTOR_INDEX i = 0;
for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) {
const V smm0(&a[i] , false);
omm0 = fmadd(smm0,smm0,omm0);
}
for (; i < N; ++i) {
const auto smm0(a[i]);
_scal += smm0*smm0;
}
return sqrts( omm0.sum() + _scal);
}
#ifdef FASTOR_SSE4_2_IMPL
template<>
FASTOR_INLINE float _norm<float,4>(const float * FASTOR_RESTRICT a) {
// IVY 33 OPS / HW 31 OPS
__m128 a_reg = _mm_load_ps(a);
return _mm_cvtss_f32(_mm_sqrt_ps(_add_ps(_mm_mul_ps(a_reg,a_reg))));
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE float _norm<float,9>(const float * FASTOR_RESTRICT a) {
// IVY & HW 61 OPS
__m256 a_reg = _mm256_loadu_ps(a);
__m128 a_end = _mm_load_ss(a+8);
__m128 a0 = _add_ps(_mm256_mul_ps(a_reg,a_reg));
__m128 a1 = _add_ps(_mm_mul_ps(a_end,a_end));
return _mm_cvtss_f32(_mm_sqrt_ps(_mm_add_ss(a0,a1)));
}
template<>
FASTOR_INLINE double _norm<double,4>(const double * FASTOR_RESTRICT a) {
// IVY 34 OPS / HW 36 OPS
__m256d a_reg = _mm256_loadu_pd(a);
return _mm_cvtsd_f64(_mm_sqrt_pd(_add_pd(_mm256_mul_pd(a_reg,a_reg))));
}
template<>
FASTOR_INLINE double _norm<double,9>(const double * FASTOR_RESTRICT a) {
// IVY 63 OPS / HW 67 OPS
__m256d a_low = _mm256_loadu_pd(a);
__m256d a_high = _mm256_loadu_pd(a+4);
__m128d a_end = _mm_load_sd(a+8);
__m128d a0 = _add_pd(_mm256_mul_pd(a_low,a_low));
__m128d a1 = _add_pd(_mm256_mul_pd(a_high,a_high));
__m128d a2 = _add_pd(_mm_mul_pd(a_end,a_end));
return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_add_sd(a2,(_mm_add_sd(a0,a1)))));
}
#endif
}
#endif // NORM_H

292
noarch/include/Fastor/backend/outer.h
View File

@@ -1,292 +0,0 @@
#ifndef OUTER_H
#define OUTER_H
#include "Fastor/config/config.h"
namespace Fastor {
template<typename T, size_t M0, size_t N0, size_t M1, size_t N1>
FASTOR_HINT_INLINE void _outer(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
for (size_t i=0; i<M0; ++i) {
for (size_t j=0; j<N0; ++j) {
for (size_t k=0; k<M1; ++k) {
for (size_t l=0; l<N1; ++l) {
out[i*N1*M1*N0+j*M1*N0+k*N0+l] += a[i*N0+j]*b[k*N1+l];
}
}
}
}
}
#ifdef FASTOR_SSE4_2_IMPL
// The followings are Voigt overloads
template<>
FASTOR_HINT_INLINE void _outer<float,2,2,2,2>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 31 OPS
// Fetch a to L1-cache
// _mm_prefetch(a,_MM_HINT_T0);
__m128 a0 = _mm_set1_ps(a[0]);
__m128 a1 = _mm_set1_ps(a[1]);
// __m128 a2 = _mm_set1_ps(a[2]);
__m128 a3 = _mm_set1_ps(a[3]);
__m128 bs = _mm_load_ps(b);
__m128 r0 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(1,2,3,0));
__m128 r1 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(3,1,2,0));
__m128 r2 = _mm_mul_ps(HALFPS,_mm_add_ps(r1,bs));
r2 = _mm_shuffle_ps(r2,r2,_MM_SHUFFLE(3,0,2,1));
// __m128 r3 = _mm_shift1_ps(_mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0)));
__m128 r3 = _mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0));
// row0
__m128 row0 = _mm_mul_ps(a0,r3);
_mm_store_ps(out, row0);
// row1
__m128 row1 = _mm_mul_ps(a3,r3);
row1 = _mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+3,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,1)));
// _mm_storeu_ps(out+4,row1);
_mm_store_ps(out+4,row1);
// row2
__m128 row2 = _mm_mul_ps(a1,r3);
row2 = _mm_shuffle_ps(row2,row2,_MM_SHUFFLE(1,0,3,2));
_mm_store_ss(out+6,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+7,_mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+8, row2);
}
template<>
FASTOR_HINT_INLINE void _outer<float,3,3,3,3>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 81 OPS
// Fetch a to L1-cache
// _mm_prefetch(a,_MM_HINT_T0);
__m128 a0 = _mm_set1_ps(a[0]);
__m128 a1 = _mm_set1_ps(a[1]);
__m128 a2 = _mm_set1_ps(a[2]);
// __m128 a3 = _mm_set1_ps(a[3]);
__m128 a4 = _mm_set1_ps(a[4]);
__m128 a5 = _mm_set1_ps(a[5]);
// __m128 a6 = _mm_set1_ps(a[6]);
// __m128 a7 = _mm_set1_ps(a[7]);
__m128 a8 = _mm_set1_ps(a[8]);
__m128 b_low = _mm_load_ps(b);
__m128 b_high =_mm_load_ps(b+4);
__m128 b_end = _mm_load_ss(b+8);
__m128 b_diag = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(1,0,1,0));
b_diag = _mm_shuffle_ps(b_diag,b_end,_MM_SHUFFLE(1,0,2,0));
__m128 ofdiag_str = _mm_shift1_ps(_mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,1,2,1)));
__m128 ofdiag_rev = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,2,2,3));
ofdiag_rev = _mm_shift1_ps(_mm_shuffle_ps(ofdiag_rev,ofdiag_rev,_MM_SHUFFLE(3,3,2,0)));
// Compute this only once
__m128 half_add_diag = _mm_mul_ps(HALFPS,_mm_add_ps(ofdiag_str,ofdiag_rev));
// row0
__m128 c0_d = _mm_mul_ps(b_diag,a0);
__m128 c0_off = _mm_mul_ps(a0,half_add_diag);
c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ps(out,c0_d);
_mm_storeu_ps(out+3,c0_off);
// row1
__m128 c1_d = _mm_mul_ps(b_diag,a4);
__m128 c1_off = _mm_mul_ps(a4,half_add_diag);
c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1));
c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1)));
_mm_storeu_ps(out+7,c1_d);
_mm_storeu_ps(out+9,c1_off);
// row2
__m128 c2_d = _mm_mul_ps(b_diag,a8);
__m128 c2_off = _mm_mul_ps(a8,half_add_diag);
c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2)));
_mm_storeu_ps(out+15,c2_off);
// row3
__m128 c3_off = _mm_mul_ps(a1,half_add_diag);
c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+18,c0_off);
_mm_store_ss(out+19,c1_off);
_mm_store_ss(out+20,c2_off);
_mm_storeu_ps(out+21,c3_off);
// row4
__m128 c4_off = _mm_mul_ps(a2,half_add_diag);
c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1)));
_mm_storeu_ps(out+27,c4_off);
// row5
__m128 c5_off = _mm_mul_ps(a5,half_add_diag);
c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2)));
_mm_storeu_ps(out+33,c5_off);
// row0
// __m128 c0_d = _mm_mul_ps(b_diag,a0);
// __m128 c_os = _mm_mul_ps(ofdiag_str,a0);
// __m128 c_or = _mm_mul_ps(ofdiag_rev,a0);
// __m128 c0_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ps(out,c0_d);
// _mm_storeu_ps(out+3,c0_off);
// // row1
// __m128 c1_d = _mm_mul_ps(b_diag,a4);
// c_os = _mm_mul_ps(ofdiag_str,a4);
// c_or = _mm_mul_ps(ofdiag_rev,a4);
// __m128 c1_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1));
// c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1)));
// _mm_storeu_ps(out+7,c1_d);
// _mm_storeu_ps(out+9,c1_off);
// // row2
// __m128 c2_d = _mm_mul_ps(b_diag,a8);
// c_os = _mm_mul_ps(ofdiag_str,a8);
// c_or = _mm_mul_ps(ofdiag_rev,a8);
// __m128 c2_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2)));
// _mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1)));
// _mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2)));
// _mm_storeu_ps(out+15,c2_off);
// // row3
// c_os = _mm_mul_ps(ofdiag_str,a1);
// c_or = _mm_mul_ps(ofdiag_rev,a1);
// __m128 c3_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+18,c0_off);
// _mm_store_ss(out+19,c1_off);
// _mm_store_ss(out+20,c2_off);
// _mm_storeu_ps(out+21,c3_off);
// // row4
// c_os = _mm_mul_ps(ofdiag_str,a2);
// c_or = _mm_mul_ps(ofdiag_rev,a2);
// __m128 c4_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1)));
// _mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1)));
// _mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1)));
// _mm_storeu_ps(out+27,c4_off);
// // row5
// c_os = _mm_mul_ps(ofdiag_str,a5);
// c_or = _mm_mul_ps(ofdiag_rev,a5);
// __m128 c5_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2)));
// _mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2)));
// _mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2)));
// _mm_storeu_ps(out+33,c5_off);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_HINT_INLINE void _outer<double,2,2,2,2>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// Fetch a to L1-cache
// _mm_prefetch(a,_MM_HINT_T0);
__m256d a0 = _mm256_set1_pd(a[0]);
__m256d a1 = _mm256_set1_pd(a[1]);
__m256d a3 = _mm256_set1_pd(a[3]);
__m256d bs = _mm256_load_pd(b);
__m128d r0 = _mm_setr_pd(_mm256_get2_pd(bs),_mm256_get1_pd(bs));
__m128d r1 = _mm_setr_pd(_mm256_get1_pd(bs),_mm256_get2_pd(bs));
__m128d r2 = _mm_mul_pd(HALFPD,_mm_add_pd(r0,r1));
__m256d r3 = _mm256_setr_pd(_mm256_get0_pd(bs),_mm256_get3_pd(bs),_mm_get0_pd(r2),0.0);
// row0
__m256d row0 = _mm256_mul_pd(a0,r3);
_mm256_store_pd(out, row0);
// row1
__m256d row1 = _mm256_mul_pd(a3,r3);
_mm_store_sd(out+3,_mm_set_sd(_mm256_get1_pd(row0)));
_mm_store_pd(out+4,_mm_setr_pd(_mm256_get1_pd(row1),_mm256_get2_pd(row1)));
// row2
__m256d row2 = _mm256_mul_pd(a1,r3);
_mm_store_sd(out+6,_mm256_extractf128_pd(row0,0x1));
_mm_store_sd(out+7,_mm256_extractf128_pd(row1,0x1));
_mm_store_sd(out+8,_mm256_extractf128_pd(row2,0x1));
}
template<>
FASTOR_HINT_INLINE void _outer<double,3,3,3,3>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// OPS
// _mm_prefetch(a,_MM_HINT_T0);
__m256d a0 = _mm256_set1_pd(a[0]);
__m256d a1 = _mm256_set1_pd(a[1]);
__m256d a2 = _mm256_set1_pd(a[2]);
__m256d a4 = _mm256_set1_pd(a[4]);
__m256d a5 = _mm256_set1_pd(a[5]);
__m256d a8 = _mm256_set1_pd(a[8]);
__m256d b_low = _mm256_load_pd(b);
__m256d b_high = _mm256_load_pd(b+4);
__m128d b_end = _mm_load_sd(b+8);
__m256d b_diag = _mm256_setr_pd(_mm_cvtsd_f64(_mm256_castpd256_pd128(b_low)),
_mm_cvtsd_f64(_mm256_castpd256_pd128(b_high)),
_mm_cvtsd_f64(b_end),0.0);
__m256d ofdiag_str = _mm256_setr_pd(b[1],b[2],b[5],0.0);
__m256d ofdiag_rev = _mm256_setr_pd(b[3],b[6],b[7],0.0);
// Compute this only once
__m256d half_add_diag = _mm256_mul_pd(VHALFPD,_mm256_add_pd(ofdiag_str,ofdiag_rev));
// row0
__m256d c0_d = _mm256_mul_pd(b_diag,a0);
__m256d c0_off = _mm256_mul_pd(a0,half_add_diag);
_mm256_store_pd(out,c0_d);
_mm256_storeu_pd(out+3,c0_off);
// row1
__m256d c1_d = _mm256_mul_pd(b_diag,a4);
__m256d c1_off = _mm256_mul_pd(a4,half_add_diag);
_mm256_storeu_pd(out+6,c1_d);
_mm_store_sd(out+6,_mm256_castpd256_pd128(_mm256_shuffle_pd(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1))));
_mm256_storeu_pd(out+9,c1_off);
// row2
__m256d c2_d = _mm256_mul_pd(b_diag,a8);
__m256d c2_off = _mm256_mul_pd(a8,half_add_diag);
_mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0_d)));
_mm_store_sd(out+13,_mm_set_sd(_mm256_get2_pd(c1_d)));
_mm_store_sd(out+14,_mm_set_sd(_mm256_get2_pd(c2_d)));
_mm256_storeu_pd(out+15,c2_off);
// row3
__m256d c3_off = _mm256_mul_pd(a1,half_add_diag);
_mm_store_sd(out+18,_mm256_castpd256_pd128(c0_off));
_mm_store_sd(out+19,_mm256_castpd256_pd128(c1_off));
_mm_store_sd(out+20,_mm256_castpd256_pd128(c2_off));
_mm256_storeu_pd(out+21,c3_off);
// row4
__m256d c4_off = _mm256_mul_pd(a2,half_add_diag);
_mm_store_sd(out+24,_mm_set_sd(_mm256_get1_pd(c0_off)));
_mm_store_sd(out+25,_mm_set_sd(_mm256_get1_pd(c1_off)));
_mm_store_sd(out+26,_mm_set_sd(_mm256_get1_pd(c2_off)));
_mm256_storeu_pd(out+27,c4_off);
// row4
__m256d c5_off = _mm256_mul_pd(a5,half_add_diag);
_mm_store_sd(out+30,_mm_set_sd(_mm256_get2_pd(c0_off)));
_mm_store_sd(out+31,_mm_set_sd(_mm256_get2_pd(c1_off)));
_mm_store_sd(out+32,_mm_set_sd(_mm256_get2_pd(c2_off)));
_mm256_storeu_pd(out+33,c5_off);
}
#endif
}
#endif // OUTER_H

1843
noarch/include/Fastor/backend/tensor_cross.h
View File

File diff suppressed because it is too large Load Diff

59
noarch/include/Fastor/backend/trace.h
View File

@@ -1,59 +0,0 @@
#ifndef TRACE_H
#define TRACE_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t M, size_t N, typename std::enable_if<M==N,bool>::type=0>
FASTOR_INLINE T _trace(const T * FASTOR_RESTRICT a) {
T sum = static_cast<T>(0);
for (FASTOR_INDEX i=0; i<M; ++i)
sum +=a[i*N+i];
return sum;
}
#ifdef FASTOR_SSE4_2_IMPL
template<>
FASTOR_INLINE double _trace<double,2,2>(const double * FASTOR_RESTRICT a) {
// AVX VERSION
// IVY 5 OPS / HW 7 OPS
// __m256d a_reg = _mm256_load_pd(a);
// __m128d a_high = _mm256_extractf128_pd(a_reg,0x1);
// return _mm_cvtsd_f64(_mm_add_sd(_mm256_castpd256_pd128(a_reg),_mm_shuffle_pd(a_high,a_high,0x1)));
// SSE VERSION
// 3 OPS
__m128d a0 = _mm_load_sd(a);
__m128d a1 = _mm_load_sd(a+3);
return _mm_cvtsd_f64(_mm_add_pd(a0,a1));
}
template<>
FASTOR_INLINE double _trace<double,3,3>(const double * FASTOR_RESTRICT a) {
// No benefit in AVX
return _mm_cvtsd_f64(_mm_add_sd(_mm_load_sd(a),_mm_add_sd(_mm_load_sd(a+4),_mm_load_sd(a+8))));
}
template<>
FASTOR_INLINE float _trace<float,2,2>(const float * FASTOR_RESTRICT a) {
__m128 a_reg = _mm_load_ps(a);
return _mm_cvtss_f32(_mm_add_ss(a_reg,_mm_reverse_ps(a_reg)));
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE float _trace<float,3,3>(const float * FASTOR_RESTRICT a) {
__m256 a_reg = _mm256_load_ps(a);
__m128 sum_two = _mm_add_ps(_mm256_castps256_ps128(a_reg),_mm256_extractf128_ps(a_reg,0x1));
return _mm_cvtss_f32(_mm_add_ss(sum_two,_mm_load_ss(a+8)));
}
#endif
}
#endif // TRACE_H

448
noarch/include/Fastor/backend/transpose/transpose.h
View File

@@ -1,448 +0,0 @@
#ifndef TRANSPOSE_H
#define TRANSPOSE_H
#include "Fastor/config/config.h"
#include "Fastor/backend/transpose/transpose_kernels.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
// Forward declare
namespace internal {
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out);
} // internal
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) {
using V = SIMDVector<T,DEFAULT_ABI>;
// Block sizes of 8x8 i.e. numSIMDRows=1
// numSIMDCols=1 and innerBlock=outerBlock=1
// give a much greater speed up, but causes
// significant slow-down for issue #42
#ifndef FASTOR_TRANS_OUTER_BLOCK_SIZE
constexpr size_t numSIMDRows = 1UL;
#else
constexpr size_t numSIMDRows = FASTOR_TRANS_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_TRANS_INNER_BLOCK_SIZE
constexpr size_t numSIMDCols = 1UL;
#else
constexpr size_t numSIMDCols = FASTOR_TRANS_INNER_BLOCK_SIZE;
#endif
constexpr size_t innerBlock = V::Size * numSIMDCols;
constexpr size_t outerBlock = V::Size * numSIMDRows;
FASTOR_ARCH_ALIGN T pack_a[outerBlock*innerBlock];
FASTOR_ARCH_ALIGN T pack_out[outerBlock*innerBlock];
constexpr size_t M0 = M / innerBlock * innerBlock;
constexpr size_t N0 = N / outerBlock * outerBlock;
V _vec;
// For row-major matrices we go over N
// and then M to get contiguous writes
size_t j=0;
for (; j<N0; j+=outerBlock) {
size_t i=0;
for (; i< M0; i+=innerBlock) {
// Pack A
for (size_t ii=0; ii<innerBlock; ++ii) {
_vec.load(&a[(i+ii)*N+(j)],false);
_vec.store(&pack_a[ii*outerBlock]);
}
// Perform transpose on pack_a and get the result
// on pack_out
internal::_transpose_dispatch<T,innerBlock,outerBlock>(pack_a,pack_out);
// Unpack pack_out to out
for (size_t jj=0; jj<outerBlock; ++jj) {
_vec.load(&pack_out[jj*innerBlock]);
_vec.store(&out[(j+jj)*M+(i)],false);
}
}
// Remainer M - M0 columns (of c)
for (; i< M; ++i) {
for (size_t jj=0; jj<outerBlock; ++jj) {
out[(j+jj)*M+(i)] = a[i*N+j+jj];
}
}
}
// Remainder N - N0 rows (of c)
for (; j<N; ++j) {
for (size_t i=0; i< M; ++i) {
out[(j)*M+(i)] = a[i*N+j];
}
}
}
#else
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) {
for (size_t j=0; j<N; ++j)
for (size_t i=0; i< M; ++i)
out[j*M+i] = a[i*N+j];
}
#endif
//----------------------------------------------------------------------------------------------------------//
// Specialisations - float
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE void _transpose<float,2,2>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
__m128 a_reg = _mm_loadu_ps(a);
_mm_storeu_ps(out,_mm_shuffle_ps(a_reg,a_reg,_MM_SHUFFLE(3,1,2,0)));
}
template<>
FASTOR_INLINE void _transpose<float,3,3>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
#ifndef FASTOR_AVX2_IMPL
// 5 OPS
__m128 row0 = _mm_loadu_ps(a);
__m128 row1 = _mm_loadu_ps(a+3);
__m128 row2 = _mm_loadu_ps(a+6);
__m128 T0 = _mm_unpacklo_ps(row0,row1);
__m128 T1 = _mm_unpackhi_ps(row0,row1);
row0 = _mm_movelh_ps ( T0,row2 );
row1 = _mm_shuffle_ps( T0,row2, _MM_SHUFFLE(3,1,3,2) );
row2 = _mm_shuffle_ps( T1,row2, _MM_SHUFFLE(3,2,1,0) );
_mm_storeu_ps(out,row0);
_mm_storeu_ps(out+3,row1);
_mm_storeu_ps(out+6,row2); // out of range for out[9]
#else
// 3 OPS
// gcc/clang emit vpermsps tht operate on (%rsp)
// less pressure on shuffle port perhaps
__m256 trans07 = _mm256_loadu_ps(a);
const __m256i trans_mask = _mm256_setr_epi32(
0,3,6,
1,4,7,
2,5);
// does not shuffle across 256 lanes, only 128 lanes
// __m256 _res = _mm256_permutevar_ps(trans07, trans_mask);
// this one shuffles correctly
__m256 _res = _mm256_permutevar8x32_ps(trans07, trans_mask);
_mm256_storeu_ps(out,_res);
// out[8] = a[8];
_mm_store_ss(out+8,_mm_load_ss(a+8));
#endif
}
template<>
FASTOR_INLINE void _transpose<float,4,4>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
#ifdef FASTOR_AVX512F_IMPL
__m512 amm = _mm512_loadu_ps(a);
__m512i idx = _mm512_setr_epi32( 0, 4, 8, 12,
1, 5, 9, 13,
2, 6, 10, 14,
3, 7, 11, 15);
__m512 omm = _mm512_permutexvar_ps(idx, amm);
_mm512_storeu_ps(out, omm);
#else
__m128 row1 = _mm_loadu_ps(a);
__m128 row2 = _mm_loadu_ps(a+4);
__m128 row3 = _mm_loadu_ps(a+8);
__m128 row4 = _mm_loadu_ps(a+12);
_MM_TRANSPOSE4_PS(row1, row2, row3, row4);
_mm_storeu_ps(out , row1);
_mm_storeu_ps(out+4 , row2);
_mm_storeu_ps(out+8 , row3);
_mm_storeu_ps(out+12, row4);
#endif
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE void _transpose<float,8,8>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
__m256 row1 = _mm256_loadu_ps(a);
__m256 row2 = _mm256_loadu_ps(a+8);
__m256 row3 = _mm256_loadu_ps(a+16);
__m256 row4 = _mm256_loadu_ps(a+24);
__m256 row5 = _mm256_loadu_ps(a+32);
__m256 row6 = _mm256_loadu_ps(a+40);
__m256 row7 = _mm256_loadu_ps(a+48);
__m256 row8 = _mm256_loadu_ps(a+56);
internal::_MM_TRANSPOSE8_PS(row1, row2, row3, row4, row5, row6, row7, row8);
_mm256_storeu_ps(out, row1);
_mm256_storeu_ps(out+8, row2);
_mm256_storeu_ps(out+16, row3);
_mm256_storeu_ps(out+24, row4);
_mm256_storeu_ps(out+32, row5);
_mm256_storeu_ps(out+40, row6);
_mm256_storeu_ps(out+48, row7);
_mm256_storeu_ps(out+56, row8);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL)
template<>
FASTOR_INLINE void _transpose<float,16,16>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
internal::_MM_TRANSPOSE16_PS(a,out);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// Specialisations - double
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE void _transpose<double,2,2>(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) {
/*-------------------------------------------------------*/
// 2 OPS
__m128d row0 = _mm_loadu_pd(a);
__m128d row1 = _mm_loadu_pd(a+2);
__m128d tmp = row0;
row0 = _mm_shuffle_pd(row0,row1,0x0);
row1 = _mm_shuffle_pd(tmp ,row1,0x3);
_mm_storeu_pd(out ,row0);
_mm_storeu_pd(out+2,row1);
/*-------------------------------------------------------*/
/*-------------------------------------------------------*/
// // AVX VERSION
// // IVY 4 OPS / HW 8 OPS
// __m256d a1 = _mm256_loadu_pd(a);
// __m128d a2 = _mm256_castpd256_pd128(a1);
// __m128d a3 = _mm256_extractf128_pd(a1,0x1);
// __m128d a4 = _mm_shuffle_pd(a2,a3,0x0);
// a3 = _mm_shuffle_pd(a2,a3,0x3);
// a1 = _mm256_castpd128_pd256(a4);
// a1 = _mm256_insertf128_pd(a1,a3,0x1);
// _mm256_storeu_pd(out,a1);
/*-------------------------------------------------------*/
}
#endif
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE void _transpose<double,3,3>(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) {
// AVX512 is the fastest & AVX version despite more instructions is faster than SSE
#if defined(FASTOR_AVX512F_IMPL)
// AVX512
/*-------------------------------------------------------*/
__m512d a07 = _mm512_loadu_pd(a);
const __m512i trans_mask = _mm512_setr_epi64(
0,3,6,
1,4,7,
2,5);
__m512d trans07 = _mm512_permutexvar_pd(trans_mask, a07);
_mm512_storeu_pd(out,trans07);
_mm_store_sd(out+8,_mm_load_sd(a+8));
/*-------------------------------------------------------*/
#elif defined(FASTOR_AVX_IMPL)
// AVX
/*-------------------------------------------------------*/
__m256d row1 = _mm256_loadu_pd(a);
__m256d row2 = _mm256_loadu_pd(a+4);
__m128d a11 = _mm256_castpd256_pd128(row1);
__m128d a12 = _mm256_extractf128_pd(row1,0x1);
__m128d a21 = _mm256_castpd256_pd128(row2);
__m128d a22 = _mm256_extractf128_pd(row2,0x1);
row1 = _mm256_castpd128_pd256(_mm_shuffle_pd(a11,a12,0x2));
row1 = _mm256_insertf128_pd(row1,_mm_shuffle_pd(a22,a11,0x2),0x1);
row2 = _mm256_castpd128_pd256(_mm_shuffle_pd(a21,a22,0x2));
row2 = _mm256_insertf128_pd(row2,_mm_shuffle_pd(a12,a21,0x2),0x1);
_mm256_storeu_pd(out,row1);
_mm256_storeu_pd(out+4,row2);
_mm_store_sd(out+8,_mm_load_sd(a+8));
/*-------------------------------------------------------*/
#else
// SSE
/*-------------------------------------------------------*/
__m128d a11 = _mm_loadu_pd(a);
__m128d a12 = _mm_loadu_pd(a+2);
__m128d a21 = _mm_loadu_pd(a+4);
__m128d a22 = _mm_loadu_pd(a+6);
_mm_storeu_pd(out ,_mm_shuffle_pd(a11,a12,0x2));
_mm_storeu_pd(out+2,_mm_shuffle_pd(a22,a11,0x2));
_mm_storeu_pd(out+4,_mm_shuffle_pd(a21,a22,0x2));
_mm_storeu_pd(out+6,_mm_shuffle_pd(a12,a21,0x2));
_mm_store_sd (out+8,_mm_load_sd(a+8));
/*-------------------------------------------------------*/
#endif
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE void _transpose<double,4,4>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
#ifdef FASTOR_AVX512F_IMPL
__m512d amm0 = _mm512_loadu_pd(a);
__m512d amm1 = _mm512_loadu_pd(a+8);
__m512i idx0 = _mm512_setr_epi64(0, 4, 8, 12, 1, 5, 9, 13);
__m512i idx1 = _mm512_setr_epi64(2, 6, 10, 14, 3, 7, 11, 15);
__m512d omm0 = _mm512_permutex2var_pd(amm0, idx0, amm1);
__m512d omm1 = _mm512_permutex2var_pd(amm0, idx1, amm1);
_mm512_storeu_pd(out , omm0);
_mm512_storeu_pd(out+8, omm1);
#else
__m256d row1 = _mm256_loadu_pd(a);
__m256d row2 = _mm256_loadu_pd(a+4);
__m256d row3 = _mm256_loadu_pd(a+8);
__m256d row4 = _mm256_loadu_pd(a+12);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(out, row1);
_mm256_storeu_pd(out+4, row2);
_mm256_storeu_pd(out+8, row3);
_mm256_storeu_pd(out+12, row4);
#endif
}
#endif
#if defined(FASTOR_AVX512F_IMPL)
template<>
FASTOR_INLINE void _transpose<double,8,8>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
__m512d row0 = _mm512_loadu_pd(a);
__m512d row1 = _mm512_loadu_pd(a+8);
__m512d row2 = _mm512_loadu_pd(a+16);
__m512d row3 = _mm512_loadu_pd(a+24);
__m512d row4 = _mm512_loadu_pd(a+32);
__m512d row5 = _mm512_loadu_pd(a+40);
__m512d row6 = _mm512_loadu_pd(a+48);
__m512d row7 = _mm512_loadu_pd(a+56);
internal::_MM_TRANSPOSE8_PD(row0,row1,row2,row3,row4,row5,row6,row7);
_mm512_storeu_pd(out , row0);
_mm512_storeu_pd(out+8 , row1);
_mm512_storeu_pd(out+16, row2);
_mm512_storeu_pd(out+24, row3);
_mm512_storeu_pd(out+32, row4);
_mm512_storeu_pd(out+40, row5);
_mm512_storeu_pd(out+48, row6);
_mm512_storeu_pd(out+56, row7);
}
#elif defined(FASTOR_AVX_IMPL)
template<>
FASTOR_INLINE void _transpose<double,8,8>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
{
__m256d row1 = _mm256_loadu_pd(&a[0]);
__m256d row2 = _mm256_loadu_pd(&a[8]);
__m256d row3 = _mm256_loadu_pd(&a[16]);
__m256d row4 = _mm256_loadu_pd(&a[24]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[0], row1);
_mm256_storeu_pd(&out[8], row2);
_mm256_storeu_pd(&out[16], row3);
_mm256_storeu_pd(&out[24], row4);
}
{
__m256d row1 = _mm256_loadu_pd(&a[32]);
__m256d row2 = _mm256_loadu_pd(&a[40]);
__m256d row3 = _mm256_loadu_pd(&a[48]);
__m256d row4 = _mm256_loadu_pd(&a[56]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[4], row1);
_mm256_storeu_pd(&out[12], row2);
_mm256_storeu_pd(&out[20], row3);
_mm256_storeu_pd(&out[28], row4);
}
{
__m256d row1 = _mm256_loadu_pd(&a[4]);
__m256d row2 = _mm256_loadu_pd(&a[12]);
__m256d row3 = _mm256_loadu_pd(&a[20]);
__m256d row4 = _mm256_loadu_pd(&a[28]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[32], row1);
_mm256_storeu_pd(&out[40], row2);
_mm256_storeu_pd(&out[48], row3);
_mm256_storeu_pd(&out[56], row4);
}
{
__m256d row1 = _mm256_loadu_pd(&a[36]);
__m256d row2 = _mm256_loadu_pd(&a[44]);
__m256d row3 = _mm256_loadu_pd(&a[52]);
__m256d row4 = _mm256_loadu_pd(&a[60]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[36], row1);
_mm256_storeu_pd(&out[44], row2);
_mm256_storeu_pd(&out[52], row3);
_mm256_storeu_pd(&out[60], row4);
}
}
#endif
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
namespace internal {
// To get around compilers recusive inlining depth issue
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) {
for (size_t j=0; j<N; ++j)
for (size_t i=0; i< M; ++i)
out[j*M+i] = a[i*N+j];
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,2,2>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,2,2>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,3,3>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,3,3>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,4,4>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,4,4>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,8,8>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,8,8>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,16,16>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,16,16>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,2,2>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,2,2>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,3,3>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,3,3>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,4,4>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,4,4>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,8,8>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,8,8>(a,out);
}
} // internal
//----------------------------------------------------------------------------------------------------------//
}
#endif // TRANSPOSE_H

233
noarch/include/Fastor/backend/transpose/transpose_kernels.h
View File

@@ -1,233 +0,0 @@
#ifndef TRANSPOSE_KERNELS_H
#define TRANSPOSE_KERNELS_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
namespace internal {
#ifdef FASTOR_SSE_IMPL
// 4x4 PS - defined
// _MM_TRANSPOSE4_PS
#endif
#ifdef FASTOR_AVX_IMPL
// 4x4 PD
FASTOR_INLINE void _MM_TRANSPOSE4_PD(__m256d &row0, __m256d &row1, __m256d &row2, __m256d &row3)
{
__m256d tmp3, tmp2, tmp1, tmp0;
tmp0 = _mm256_shuffle_pd((row0),(row1), 0x0);
tmp2 = _mm256_shuffle_pd((row0),(row1), 0xF);
tmp1 = _mm256_shuffle_pd((row2),(row3), 0x0);
tmp3 = _mm256_shuffle_pd((row2),(row3), 0xF);
row0 = _mm256_permute2f128_pd(tmp0, tmp1, 0x20);
row1 = _mm256_permute2f128_pd(tmp2, tmp3, 0x20);
row2 = _mm256_permute2f128_pd(tmp0, tmp1, 0x31);
row3 = _mm256_permute2f128_pd(tmp2, tmp3, 0x31);
}
// 8x8 PS
FASTOR_INLINE void _MM_TRANSPOSE8_PS(__m256 &row0, __m256 &row1, __m256 &row2, __m256 &row3,
__m256 &row4, __m256 &row5, __m256 &row6, __m256 &row7)
{
__m256 __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7;
__m256 __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7;
__t0 = _mm256_unpacklo_ps(row0, row1);
__t1 = _mm256_unpackhi_ps(row0, row1);
__t2 = _mm256_unpacklo_ps(row2, row3);
__t3 = _mm256_unpackhi_ps(row2, row3);
__t4 = _mm256_unpacklo_ps(row4, row5);
__t5 = _mm256_unpackhi_ps(row4, row5);
__t6 = _mm256_unpacklo_ps(row6, row7);
__t7 = _mm256_unpackhi_ps(row6, row7);
__tt0 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(1,0,1,0));
__tt1 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(3,2,3,2));
__tt2 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(1,0,1,0));
__tt3 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(3,2,3,2));
__tt4 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(1,0,1,0));
__tt5 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(3,2,3,2));
__tt6 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(1,0,1,0));
__tt7 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(3,2,3,2));
row0 = _mm256_permute2f128_ps(__tt0, __tt4, 0x20);
row1 = _mm256_permute2f128_ps(__tt1, __tt5, 0x20);
row2 = _mm256_permute2f128_ps(__tt2, __tt6, 0x20);
row3 = _mm256_permute2f128_ps(__tt3, __tt7, 0x20);
row4 = _mm256_permute2f128_ps(__tt0, __tt4, 0x31);
row5 = _mm256_permute2f128_ps(__tt1, __tt5, 0x31);
row6 = _mm256_permute2f128_ps(__tt2, __tt6, 0x31);
row7 = _mm256_permute2f128_ps(__tt3, __tt7, 0x31);
}
#endif
#ifdef FASTOR_AVX512F_IMPL
// 8x8 PD
inline void _MM_TRANSPOSE8_PD(__m512d &row0, __m512d &row1, __m512d &row2, __m512d &row3,
__m512d &row4, __m512d &row5, __m512d &row6, __m512d &row7)
{
__m512d __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7;
__m512d __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7;
FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {0, 8 , 1 , 9 , 4 , 12, 5 , 13};
FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {2, 10, 3 , 11, 6 , 14, 7 , 15};
FASTOR_ARCH_ALIGN constexpr int64_t idx3[8] = {0, 1 , 8 , 9 , 4 , 5 , 12, 13};
FASTOR_ARCH_ALIGN constexpr int64_t idx4[8] = {2, 3 , 10, 11, 6 , 7 , 14, 15};
FASTOR_ARCH_ALIGN constexpr int64_t idx5[8] = {4, 5 , 6 , 7 , 12, 13, 14, 15};
__m512i vidx1 = _mm512_load_epi64(idx1);
__m512i vidx2 = _mm512_load_epi64(idx2);
__m512i vidx3 = _mm512_load_epi64(idx3);
__m512i vidx4 = _mm512_load_epi64(idx4);
__m512i vidx5 = _mm512_load_epi64(idx5);
__t0 = _mm512_permutex2var_pd(row0, vidx1, row1);
__t1 = _mm512_permutex2var_pd(row0, vidx2, row1);
__t2 = _mm512_permutex2var_pd(row2, vidx1, row3);
__t3 = _mm512_permutex2var_pd(row2, vidx2, row3);
__t4 = _mm512_permutex2var_pd(row4, vidx1, row5);
__t5 = _mm512_permutex2var_pd(row4, vidx2, row5);
__t6 = _mm512_permutex2var_pd(row6, vidx1, row7);
__t7 = _mm512_permutex2var_pd(row6, vidx2, row7);
__tt0 = _mm512_permutex2var_pd(__t0, vidx3, __t2);
__tt1 = _mm512_permutex2var_pd(__t0, vidx4, __t2);
__tt2 = _mm512_permutex2var_pd(__t1, vidx3, __t3);
__tt3 = _mm512_permutex2var_pd(__t1, vidx4, __t3);
__tt4 = _mm512_permutex2var_pd(__t4, vidx3, __t6);
__tt5 = _mm512_permutex2var_pd(__t4, vidx4, __t6);
__tt6 = _mm512_permutex2var_pd(__t5, vidx3, __t7);
__tt7 = _mm512_permutex2var_pd(__t5, vidx4, __t7);
row0 = _mm512_insertf64x4(__tt0,_mm512_castpd512_pd256(__tt4),0x1);
row1 = _mm512_insertf64x4(__tt1,_mm512_castpd512_pd256(__tt5),0x1);
row2 = _mm512_insertf64x4(__tt2,_mm512_castpd512_pd256(__tt6),0x1);
row3 = _mm512_insertf64x4(__tt3,_mm512_castpd512_pd256(__tt7),0x1);
row4 = _mm512_permutex2var_pd(__tt0, vidx5, __tt4);
row5 = _mm512_permutex2var_pd(__tt1, vidx5, __tt5);
row6 = _mm512_permutex2var_pd(__tt2, vidx5, __tt6);
row7 = _mm512_permutex2var_pd(__tt3, vidx5, __tt7);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL)
// 16x16
FASTOR_INLINE void _MM_TRANSPOSE16_PS(const float * FASTOR_RESTRICT mat, float * FASTOR_RESTRICT matT)
{
__m512 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, ta, tb, tc, td, te, tf;
__m512 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, ra, rb, rc, rd, re, rf;
int mask;
FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {2, 3, 0, 1, 6, 7, 4, 5};
FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {1, 0, 3, 2, 5, 4, 7, 6};
FASTOR_ARCH_ALIGN constexpr int32_t idx3[16] = {1, 0, 3, 2, 5 ,4 ,7 ,6 ,9 ,8 , 11, 10, 13, 12 ,15, 14};
__m512i vidx1 = _mm512_load_epi64(idx1);
__m512i vidx2 = _mm512_load_epi64(idx2);
__m512i vidx3 = _mm512_load_epi32(idx3);
t0 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+0])), _mm256_loadu_ps(&mat[ 8*16+0]), 1);
t1 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+0])), _mm256_loadu_ps(&mat[ 9*16+0]), 1);
t2 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+0])), _mm256_loadu_ps(&mat[10*16+0]), 1);
t3 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+0])), _mm256_loadu_ps(&mat[11*16+0]), 1);
t4 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+0])), _mm256_loadu_ps(&mat[12*16+0]), 1);
t5 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+0])), _mm256_loadu_ps(&mat[13*16+0]), 1);
t6 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+0])), _mm256_loadu_ps(&mat[14*16+0]), 1);
t7 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+0])), _mm256_loadu_ps(&mat[15*16+0]), 1);
t8 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+8])), _mm256_loadu_ps(&mat[ 8*16+8]), 1);
t9 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+8])), _mm256_loadu_ps(&mat[ 9*16+8]), 1);
ta = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+8])), _mm256_loadu_ps(&mat[10*16+8]), 1);
tb = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+8])), _mm256_loadu_ps(&mat[11*16+8]), 1);
tc = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+8])), _mm256_loadu_ps(&mat[12*16+8]), 1);
td = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+8])), _mm256_loadu_ps(&mat[13*16+8]), 1);
te = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+8])), _mm256_loadu_ps(&mat[14*16+8]), 1);
tf = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+8])), _mm256_loadu_ps(&mat[15*16+8]), 1);
mask= 0xcc;
r0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t0), (__mmask8)mask, vidx1, _mm512_castps_pd(t4)));
r1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t1), (__mmask8)mask, vidx1, _mm512_castps_pd(t5)));
r2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t2), (__mmask8)mask, vidx1, _mm512_castps_pd(t6)));
r3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t3), (__mmask8)mask, vidx1, _mm512_castps_pd(t7)));
r8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t8), (__mmask8)mask, vidx1, _mm512_castps_pd(tc)));
r9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t9), (__mmask8)mask, vidx1, _mm512_castps_pd(td)));
ra = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ta), (__mmask8)mask, vidx1, _mm512_castps_pd(te)));
rb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tb), (__mmask8)mask, vidx1, _mm512_castps_pd(tf)));
mask= 0x33;
r4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t4), (__mmask8)mask, vidx1, _mm512_castps_pd(t0)));
r5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t5), (__mmask8)mask, vidx1, _mm512_castps_pd(t1)));
r6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t6), (__mmask8)mask, vidx1, _mm512_castps_pd(t2)));
r7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t7), (__mmask8)mask, vidx1, _mm512_castps_pd(t3)));
rc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tc), (__mmask8)mask, vidx1, _mm512_castps_pd(t8)));
rd = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(td), (__mmask8)mask, vidx1, _mm512_castps_pd(t9)));
re = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(te), (__mmask8)mask, vidx1, _mm512_castps_pd(ta)));
rf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tf), (__mmask8)mask, vidx1, _mm512_castps_pd(tb)));
mask = 0xaa;
t0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r0), (__mmask8)mask, vidx2, _mm512_castps_pd(r2)));
t1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r1), (__mmask8)mask, vidx2, _mm512_castps_pd(r3)));
t4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r4), (__mmask8)mask, vidx2, _mm512_castps_pd(r6)));
t5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r5), (__mmask8)mask, vidx2, _mm512_castps_pd(r7)));
t8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r8), (__mmask8)mask, vidx2, _mm512_castps_pd(ra)));
t9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r9), (__mmask8)mask, vidx2, _mm512_castps_pd(rb)));
tc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rc), (__mmask8)mask, vidx2, _mm512_castps_pd(re)));
td = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rd), (__mmask8)mask, vidx2, _mm512_castps_pd(rf)));
mask = 0x55;
t2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r2), (__mmask8)mask, vidx2, _mm512_castps_pd(r0)));
t3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r3), (__mmask8)mask, vidx2, _mm512_castps_pd(r1)));
t6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r6), (__mmask8)mask, vidx2, _mm512_castps_pd(r4)));
t7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r7), (__mmask8)mask, vidx2, _mm512_castps_pd(r5)));
ta = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ra), (__mmask8)mask, vidx2, _mm512_castps_pd(r8)));
tb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rb), (__mmask8)mask, vidx2, _mm512_castps_pd(r9)));
te = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(re), (__mmask8)mask, vidx2, _mm512_castps_pd(rc)));
tf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rf), (__mmask8)mask, vidx2, _mm512_castps_pd(rd)));
mask = 0xaaaa;
r0 = _mm512_mask_permutexvar_ps(t0, (__mmask16)mask, vidx3, t1);
r2 = _mm512_mask_permutexvar_ps(t2, (__mmask16)mask, vidx3, t3);
r4 = _mm512_mask_permutexvar_ps(t4, (__mmask16)mask, vidx3, t5);
r6 = _mm512_mask_permutexvar_ps(t6, (__mmask16)mask, vidx3, t7);
r8 = _mm512_mask_permutexvar_ps(t8, (__mmask16)mask, vidx3, t9);
ra = _mm512_mask_permutexvar_ps(ta, (__mmask16)mask, vidx3, tb);
rc = _mm512_mask_permutexvar_ps(tc, (__mmask16)mask, vidx3, td);
re = _mm512_mask_permutexvar_ps(te, (__mmask16)mask, vidx3, tf);
mask = 0x5555;
r1 = _mm512_mask_permutexvar_ps(t1, (__mmask16)mask, vidx3, t0);
r3 = _mm512_mask_permutexvar_ps(t3, (__mmask16)mask, vidx3, t2);
r5 = _mm512_mask_permutexvar_ps(t5, (__mmask16)mask, vidx3, t4);
r7 = _mm512_mask_permutexvar_ps(t7, (__mmask16)mask, vidx3, t6);
r9 = _mm512_mask_permutexvar_ps(t9, (__mmask16)mask, vidx3, t8);
rb = _mm512_mask_permutexvar_ps(tb, (__mmask16)mask, vidx3, ta);
rd = _mm512_mask_permutexvar_ps(td, (__mmask16)mask, vidx3, tc);
rf = _mm512_mask_permutexvar_ps(tf, (__mmask16)mask, vidx3, te);
_mm512_storeu_ps(&matT[ 0*16], r0);
_mm512_storeu_ps(&matT[ 1*16], r1);
_mm512_storeu_ps(&matT[ 2*16], r2);
_mm512_storeu_ps(&matT[ 3*16], r3);
_mm512_storeu_ps(&matT[ 4*16], r4);
_mm512_storeu_ps(&matT[ 5*16], r5);
_mm512_storeu_ps(&matT[ 6*16], r6);
_mm512_storeu_ps(&matT[ 7*16], r7);
_mm512_storeu_ps(&matT[ 8*16], r8);
_mm512_storeu_ps(&matT[ 9*16], r9);
_mm512_storeu_ps(&matT[10*16], ra);
_mm512_storeu_ps(&matT[11*16], rb);
_mm512_storeu_ps(&matT[12*16], rc);
_mm512_storeu_ps(&matT[13*16], rd);
_mm512_storeu_ps(&matT[14*16], re);
_mm512_storeu_ps(&matT[15*16], rf);
}
#endif
} // end of namespace internal
} // end of namespace Fastor
#endif // TRANSPOSE_KERNELS_H

182
noarch/include/Fastor/backend/voigt.h
View File

@@ -1,182 +0,0 @@
#ifndef VOIGT_H
#define VOIGT_H
#include "Fastor/tensor/Tensor.h"
// Conversion of tensors to symmetrised Voigt forms
namespace Fastor {
template<typename T, size_t ... Rest>
struct VoigtType;
template<typename T>
struct VoigtType<T,2,2,2,2> {
using return_type = Tensor<T,3,3>;
};
template<typename T>
struct VoigtType<T,3,3,3,3> {
using return_type = Tensor<T,6,6>;
};
template<typename T>
struct VoigtType<T,2,2,2> {
using return_type = Tensor<T,3,2>;
};
template<typename T>
struct VoigtType<T,3,3,3> {
using return_type = Tensor<T,6,3>;
};
template<typename T>
struct VoigtType<T,2,2> {
using return_type = Tensor<T,3>;
};
template<typename T>
struct VoigtType<T,3,3> {
using return_type = Tensor<T,6>;
};
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==4 && pack_prod<Rest...>::value == 16
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = a_data[0];
VoigtA[1] = a_data[3];
VoigtA[2] = 0.5*(a_data[1]+a_data[2]);
VoigtA[3] = VoigtA[1];
VoigtA[4] = a_data[15];
VoigtA[5] = 0.5*(a_data[13]+a_data[14]);
VoigtA[6] = VoigtA[2];
VoigtA[7] = VoigtA[5];
VoigtA[8] = 0.5*(a_data[5]+a_data[6]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==4 && pack_prod<Rest...>::value == 81
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = a_data[0];
VoigtA[1] = a_data[4];
VoigtA[2] = a_data[8];
VoigtA[3] = 0.5*(a_data[1]+a_data[3]);
VoigtA[4] = 0.5*(a_data[2]+a_data[6]);
VoigtA[5] = 0.5*(a_data[5]+a_data[7]);
VoigtA[6] = VoigtA[1];
VoigtA[7] = a_data[40];
VoigtA[8] = a_data[44];
VoigtA[9] = 0.5*(a_data[37]+a_data[39]);
VoigtA[10] = 0.5*(a_data[38]+a_data[42]);
VoigtA[11] = 0.5*(a_data[41]+a_data[43]);
VoigtA[12] = VoigtA[2];
VoigtA[13] = VoigtA[8];
VoigtA[14] = a_data[80];
VoigtA[15] = 0.5*(a_data[73]+a_data[75]);
VoigtA[16] = 0.5*(a_data[74]+a_data[78]);
VoigtA[17] = 0.5*(a_data[77]+a_data[79]);
VoigtA[18] = VoigtA[3];
VoigtA[19] = VoigtA[9];
VoigtA[20] = VoigtA[15];
VoigtA[21] = 0.5*(a_data[10]+a_data[12]);
VoigtA[22] = 0.5*(a_data[11]+a_data[15]);
VoigtA[23] = 0.5*(a_data[14]+a_data[16]);
VoigtA[24] = VoigtA[4];
VoigtA[25] = VoigtA[10];
VoigtA[26] = VoigtA[16];
VoigtA[27] = VoigtA[22];
VoigtA[28] = 0.5*(a_data[20]+a_data[24]);
VoigtA[29] = 0.5*(a_data[23]+a_data[25]);
VoigtA[30] = VoigtA[5];
VoigtA[31] = VoigtA[11];
VoigtA[32] = VoigtA[17];
VoigtA[33] = VoigtA[23];
VoigtA[34] = VoigtA[29];
VoigtA[35] = 0.5*(a_data[50]+a_data[52]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==3 && pack_prod<Rest...>::value == 8
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
// 3rd order tensor to 2nd order tensor
// 3rd order tensor should be symmetric in the first two indices
VoigtA[0] = e[0];
VoigtA[1] = e[1];
VoigtA[2] = e[6];
VoigtA[3] = e[7];
VoigtA[4] = 0.5*(e[2]+e[4]);
VoigtA[5] = 0.5*(e[3]+e[5]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==3 && pack_prod<Rest...>::value == 27
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
// 3rd order tensor to 2nd order tensor
// 3rd order tensor should be symmetric in the first two indices
VoigtA[0] = e[0];
VoigtA[1] = e[1];
VoigtA[2] = e[2];
VoigtA[3] = e[12];
VoigtA[4] = e[13];
VoigtA[5] = e[14];
VoigtA[6] = e[24];
VoigtA[7] = e[25];
VoigtA[8] = e[26];
VoigtA[9] = 0.5*(e[3]+e[9]);
VoigtA[10] = 0.5*(e[4]+e[10]);
VoigtA[11] = 0.5*(e[5]+e[11]);
VoigtA[12] = 0.5*(e[6]+e[18]);
VoigtA[13] = 0.5*(e[7]+e[19]);
VoigtA[14] = 0.5*(e[8]+e[20]);
VoigtA[15] = 0.5*(e[15]+e[21]);
VoigtA[16] = 0.5*(e[16]+e[22]);
VoigtA[17] = 0.5*(e[17]+e[23]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==2 && pack_prod<Rest...>::value == 4
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = e[0];
VoigtA[1] = e[3];
VoigtA[2] = 0.5*(e[1]+e[2]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==2 && pack_prod<Rest...>::value == 9
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = e[0];
VoigtA[1] = e[4];
VoigtA[2] = e[8];
VoigtA[3] = 0.5*(e[1]+e[3]);
VoigtA[4] = 0.5*(e[2]+e[6]);
VoigtA[5] = 0.5*(e[5]+e[7]);
}
template<typename T, size_t ... Rest>
FASTOR_INLINE auto voigt(const Tensor<T,Rest...> &a)
-> typename VoigtType<T,Rest...>::return_type {
T *a_data = a.data();
using ret_type = typename VoigtType<T,Rest...>::return_type;
ret_type voigt_a;
T *VoigtA = voigt_a.data();
_voigt<T,Rest...>(a_data,VoigtA);
return voigt_a;
}
}
#endif // VOIGT_H

359
noarch/include/Fastor/config/config.h
View File

@@ -1,359 +0,0 @@
/* This file is part of the FASTOR library. {{{
Copyright (c) 2020 Roman Poya
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
}}}*/
#ifndef CONFIG_H
#define CONFIG_H
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <iostream>
#include <stdexcept>
#include <string>
//------------------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------------------//
// Error out for pre-C++14 compiler versions
//------------------------------------------------------------------------------------------------//
#if defined(_MSC_VER)
#if _MSC_VER < 1920
#error FASTOR REQUIRES AN ISO C++14 COMPLIANT COMPILER
#endif
#elif defined(__GNUC__) || defined(__GNUG__)
#if __cplusplus < 201402L
#error FASTOR REQUIRES AN ISO C++14 COMPLIANT COMPILER
#endif
#endif
//------------------------------------------------------------------------------------------------//
// Compiler define macros
//------------------------------------------------------------------------------------------------//
#ifdef __INTEL_COMPILER
#define FASTOR_INTEL __INTEL_COMPILER_BUILD_DATE
#elif defined(__clang__) && defined(__apple_build_version__)
#define FASTOR_APPLECLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
#elif defined(__clang__)
#define FASTOR_CLANG (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
#elif defined(__GNUC__)
#define FASTOR_GCC (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
#elif defined(_MSC_VER)
#define FASTOR_MSVC _MSC_FULL_VER
#else
#define FASTOR_UNSUPPORTED_COMPILER 1
#endif
//------------------------------------------------------------------------------------------------//
// Operating system define macros
//------------------------------------------------------------------------------------------------//
#if defined(_WIN32)
#define FASTOR_WINDOWS32_OS 1
#endif
#if defined(_WIN64)
#define FASTOR_WINDOWS64_OS 1
#endif
#if defined(_WIN32) || defined(_WIN64)
#define FASTOR_WINDOWS_OS 1
#endif
#if defined(__unix__) || defined(__unix)
#define FASTOR_UNIX_OS 1
#endif
#if defined(__linux__)
#define FASTOR_LINUX_OS 1
#endif
#if defined(__APPLE__) || defined(__MACH__)
#define FASTOR_APPLE_OS 1
#endif
#if defined(__ANDROID__) || defined(ANDROID)
#define FASTOR_ANDROID_OS 1
#endif
#if defined(__CYGWIN__)
#define FASTOR_CYGWIN_OS 1
#endif
//------------------------------------------------------------------------------------------------//
// Determine CXX version
//------------------------------------------------------------------------------------------------//
#if defined(FASTOR_MSVC)
#if defined(_MSVC_LANG)
#if _MSVC_LANG == 199711L
#define FASTOR_CXX_VERSION 1998
#elif _MSVC_LANG == 201103L
#define FASTOR_CXX_VERSION 2011
#elif _MSVC_LANG == 201402L
#define FASTOR_CXX_VERSION 2014
#elif _MSVC_LANG == 201703L
#define FASTOR_CXX_VERSION 2017
#elif _MSVC_LANG > 201703L
#define FASTOR_CXX_VERSION 2020
#endif
#endif
#else
#if defined(__cplusplus)
#if __cplusplus == 199711L
#define FASTOR_CXX_VERSION 1998
#elif __cplusplus == 201103L
#define FASTOR_CXX_VERSION 2011
#elif __cplusplus == 201402L
#define FASTOR_CXX_VERSION 2014
#elif __cplusplus == 201703L
#define FASTOR_CXX_VERSION 2017
#elif __cplusplus > 201703L
#define FASTOR_CXX_VERSION 2020
#endif
#endif
#endif
//------------------------------------------------------------------------------------------------//
// Force inline and no-inline macros
//------------------------------------------------------------------------------------------------//
#if defined(__GNUC__) || defined(__GNUG__)
#define FASTOR_INLINE inline __attribute__((always_inline))
#define FASTOR_NOINLINE __attribute__((noinline))
#elif defined(_MSC_VER)
#define FASTOR_INLINE __forceinline
#define FASTOR_NOINLINE __declspec(noinline)
#endif
#if defined(__GNUC__) || defined(__GNUG__)
#define FASTOR_RESTRICT __restrict__
#elif defined(_MSC_VER)
#define FASTOR_RESTRICT __restrict
#endif
// Traditional inline which works will and helps the compiler
// eliminate a lot of code
#define FASTOR_HINT_INLINE inline
//------------------------------------------------------------------------------------------------//
// C++17 [if constexpr] define
//------------------------------------------------------------------------------------------------//
#if FASTOR_CXX_VERSION >= 2017
#define FASTOR_HAS_IF_CONSTEXPR 1
#define FASTOR_IF_CONSTEXPR if constexpr
#else
#define FASTOR_HAS_IF_CONSTEXPR 0
#define FASTOR_IF_CONSTEXPR if
#endif
//------------------------------------------------------------------------------------------------//
// Intrinsics defines
//------------------------------------------------------------------------------------------------//
#ifdef FASTOR_MSVC
#ifdef _M_IX86_FP
#if _M_IX86_FP >= 1
#ifndef __SSE__
#define __SSE__ 1
#endif
#endif
#if _M_IX86_FP >= 2
#ifndef __SSE2__
#define __SSE2__ 1
#endif
#endif
#elif defined(_M_AMD64)
#ifndef __SSE__
#define __SSE__ 1
#endif
#ifndef __SSE2__
#define __SSE2__ 1
#endif
#endif
#endif
#if defined(__MIC__)
#define FASTOR_MIC_IMPL 1
#endif
#if defined(__AVX512F__)
#define FASTOR_AVX512F_IMPL 1
#endif
#if defined(__AVX512CD__)
#define FASTOR_AVX512CD_IMPL 1
#endif
#if defined(__AVX512BW__)
#define FASTOR_AVX512BW_IMPL 1
#endif
#if defined(__AVX512DQ__)
#define FASTOR_AVX512DQ_IMPL 1
#endif
#if defined(__AVX512VL__)
#define FASTOR_AVX512VL_IMPL 1
#endif
#if defined(__AVX2__)
#define FASTOR_AVX2_IMPL 1
#endif
#if defined(__AVX__)
#define FASTOR_AVX_IMPL 1
#endif
#if defined(__SSE4_2__)
#define FASTOR_SSE4_2_IMPL 1
#endif
#if defined(__SSE4_1__)
#define FASTOR_SSE4_1_IMPL 1
#endif
#if defined(__SSE3__)
#define FASTOR_SSE3_IMPL 1
#endif
#if defined(__SSSE3__)
#define FASTOR_SSSE3_IMPL 1
#endif
#if defined(__SSE2__)
#define FASTOR_SSE2_IMPL 1
#endif
#if defined(__FMA4__)
#define FASTOR_FMA4_IMPL 1
#endif
#if defined(__FMA__)
#define FASTOR_FMA_IMPL 1
#endif
// #if !defined(__FMA__) && defined(__AVX2__)
// #define __FMA__ 1
// #endif
// Get around MSVC issue
#if defined(FASTOR_WINDOWS_OS) || defined(FASTOR_MSVC)
#if defined(FASTOR_AVX512F_IMPL)
#if !defined(FASTOR_SSE2_IMPL)
#define FASTOR_SSE2_IMPL 1
#endif
#if !defined(FASTOR_SSE3_IMPL)
#define FASTOR_SSE3_IMPL 1
#endif
#if !defined(FASTOR_SSSE3_IMPL)
#define FASTOR_SSSE3_IMPL 1
#endif
#if !defined(FASTOR_SSE4_1_IMPL)
#define FASTOR_SSE4_1_IMPL 1
#endif
#if !defined(FASTOR_SSE4_2_IMPL)
#define FASTOR_SSE4_2_IMPL 1
#endif
#if !defined(FASTOR_AVX_IMPL)
#define FASTOR_AVX_IMPL 1
#endif
#if !defined(FASTOR_AVX2_IMPL)
#define FASTOR_AVX2_IMPL 1
#endif
#elif defined(FASTOR_AVX_IMPL) || defined(FASTOR_AVX2_IMPL)
#if !defined(FASTOR_SSE2_IMPL)
#define FASTOR_SSE2_IMPL 1
#endif
#if !defined(FASTOR_SSE3_IMPL)
#define FASTOR_SSE3_IMPL 1
#endif
#if !defined(FASTOR_SSSE3_IMPL)
#define FASTOR_SSSE3_IMPL 1
#endif
#if !defined(FASTOR_SSE4_1_IMPL)
#define FASTOR_SSE4_1_IMPL 1
#endif
#if !defined(FASTOR_SSE4_2_IMPL)
#define FASTOR_SSE4_2_IMPL 1
#endif
#endif
#endif
#if defined(FASTOR_AVX512F_IMPL) || defined(FASTOR_AVX512CD_IMPL) || defined(FASTOR_AVX512BW_IMPL) || defined(FASTOR_AVX512DQ_IMPL) || defined(FASTOR_AVX512VL_IMPL)
#define FASTOR_AVX512_IMPL 1
#endif
#if defined(FASTOR_AVX2_IMPL) && !defined(FASTOR_AVX_IMPL)
#define FASTOR_AVX_IMPL 1
#endif
#if defined(FASTOR_SSE2_IMPL) || defined(FASTOR_SSSE3_IMPL) || defined(FASTOR_SSE3_IMPL) || defined(FASTOR_SSE4_1_IMPL) || defined(FASTOR_SSE4_2_IMPL)
#define FASTOR_SSE_IMPL 1
#endif
#if !defined(FASTOR_MIC_IMPL) && !defined(FASTOR_AVX512_IMPL) && !defined(FASTOR_AVX_IMPL) && !defined(FASTOR_SSE_IMPL)
#define FASTOR_SCALAR_IMPL 1
#endif
#ifdef FASTOR_SSE2_IMPL
#include <emmintrin.h>
#endif
#ifdef FASTOR_SSE3_IMPL
#include <pmmintrin.h>
#endif
#ifdef FASTOR_SSSE3_IMPL
#include <tmmintrin.h>
#endif
#ifdef FASTOR_SSE4_1_IMPL
#include <smmintrin.h>
#endif
#ifdef FASTOR_SSE4_2_IMPL
#include <nmmintrin.h>
#endif
#ifdef FASTOR_AVX_IMPL
#include <immintrin.h>
#endif
// Mask loading
//------------------------------------------------------------------------------------------------//
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL)
#define FASTOR_HAS_AVX512_MASKS 1
#endif
//------------------------------------------------------------------------------------------------//
// Horizontal add
//------------------------------------------------------------------------------------------------//
#if defined(FASTOR_AVX512F_IMPL)
#if defined(FASTOR_INTEL)
#define FASTOR_HAS_AVX512_REDUCE_ADD 1
#elif defined (FASTOR_GCC) && __GNUC__ >= 7
#define FASTOR_HAS_AVX512_REDUCE_ADD 1
#elif defined (FASTOR_GCC) && __clang_major__ >= 4
#define FASTOR_HAS_AVX512_REDUCE_ADD 1
#endif
#endif
//------------------------------------------------------------------------------------------------//
// AVX512 abs
//------------------------------------------------------------------------------------------------//
#if defined(FASTOR_AVX512F_IMPL)
#if defined(FASTOR_INTEL)
#define FASTOR_HAS_AVX512_ABS 1
#elif defined (FASTOR_GCC) && __GNUC__ >= 7 && __GNUC_MINOR__ >= 4
#define FASTOR_HAS_AVX512_ABS 1
#elif defined (FASTOR_GCC) && __clang_major__ >= 4
#define FASTOR_HAS_AVX512_ABS 1
#endif
#endif
//------------------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------------------//
#include "Fastor/config/macros.h"
//------------------------------------------------------------------------------------------------//
#endif // CONFIG_H

71
noarch/include/Fastor/config/cpuid.h
View File

@@ -1,71 +0,0 @@
/* This file is part of the FASTOR library. {{{
Copyright (c) 2020 Roman Poya
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
}}}*/
#ifndef CPUID_H
#define CPUID_H
#ifdef _WIN32
#include <limits.h>
#include <intrin.h>
typedef unsigned __int32 uint32_t;
#else
#include <stdint.h>
#endif
namespace Fastor {
class CPUID {
uint32_t regs[4];
public:
explicit CPUID(unsigned i) {
#ifdef _WIN32
__cpuid((int *)regs, (int)i);
#else
asm volatile
("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3])
: "a" (i), "c" (0));
// ECX is set to zero for CPUID function 4
#endif
}
const uint32_t& EAX() const {return regs[0];} // cpu base frequency
const uint32_t& EBX() const {return regs[1];} // cpu max frequency
const uint32_t& ECX() const {return regs[2];} // bus (reference) frequency
const uint32_t& EDX() const {return regs[3];}
};
// Usage:
// CPUID cpuID(0);
// std::string vendor;
// vendor += std::string((const char *)&cpuID.EBX(), 4);
// vendor += std::string((const char *)&cpuID.EDX(), 4);
// vendor += std::string((const char *)&cpuID.ECX(), 4);
// std::cout << "CPU vendor = " << vendor << std::endl;
} // Fastor
#endif // CPUID_H

292
noarch/include/Fastor/config/macros.h
View File

@@ -1,292 +0,0 @@
/* This file is part of the FASTOR library. {{{
Copyright (c) 2020 Roman Poya
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
}}}*/
#ifndef FASTOR_MACROS_H
#define FASTOR_MACROS_H
/* This file contains only the set of macros that Fastor defines and can be used or altered by the user
*/
//------------------------------------------------------------------------------------------------//
// Compiler version
//------------------------------------------------------------------------------------------------//
#define FASTOR_MAJOR 0
#define FASTOR_MINOR 6
#define FASTOR_PATCHLEVEL 4
#define FASTOR_VERSION (FASTOR_MAJOR * 10000 + FASTOR_MINOR * 100 + FASTOR_PATCHLEVEL)
//------------------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------------------//
// ICC's default option is fast anyway (i.e. -fp-model fast=1)
// but it does not define the __FAST_MATH__ macro
#if defined(__FAST_MATH__)
#define FASTOR_FAST_MATH
// Use the following for unsafe math
// Only for GCC & Clang, activates fast div/rcp
//#define FASTOR_UNSAFE_MATH
#endif
// Runtime checks - off by default under release
//------------------------------------------------------------------------------------------------//
#ifndef FASTOR_ENABLE_RUNTIME_CHECKS
#define FASTOR_ENABLE_RUNTIME_CHECKS 0
#endif
//------------------------------------------------------------------------------------------------//
// Bounds and shape checking - ON by default under debug
//------------------------------------------------------------------------------------------------//
#if !defined(NDEBUG) || FASTOR_ENABLE_RUNTIME_CHECKS
#ifndef FASTOR_BOUNDS_CHECK
#define FASTOR_BOUNDS_CHECK 1
#endif
#ifndef FASTOR_SHAPE_CHECK
#define FASTOR_SHAPE_CHECK 1
#endif
#else
#ifndef FASTOR_BOUNDS_CHECK
#define FASTOR_BOUNDS_CHECK 0
#endif
#ifndef FASTOR_SHAPE_CHECK
#define FASTOR_SHAPE_CHECK 0
#endif
#endif
//------------------------------------------------------------------------------------------------//
// Aliasing
//------------------------------------------------------------------------------------------------//
#ifndef FASTOR_NO_ALIAS
#define FASTOR_NO_ALIAS 0
#endif
//------------------------------------------------------------------------------------------------//
// Macros used throughout Fastor
//------------------------------------------------------------------------------------------------//
//#define FASTOR_COPY_EXPR
//#define FASTOR_DONT_VECTORISE
//#define FASTOR_DONT_PERFORM_OP_MIN
//#define FASTOR_USE_OLD_OUTER
//#define FASTOR_USE_OLD_INTRINSICS
//#define FASTOR_USE_HADD
//#define FASTOR_USE_VECTORISED_EXPR_ASSIGN // to use vectorised expression assignment
//#define FASTOR_ZERO_INITIALISE
//#define FASTOR_USE_OLD_NDVIEWS
//#define FASTOR_DISPATCH_DIV_TO_MUL_EXPR // change BINARY_DIV_OP to BINARY_MUL_OP for Expression/Number
//#define FASTOR_DISABLE_SPECIALISED_CTR
//FASTOR_MATMUL_OUTER_BLOCK_SIZE 2
//FASTOR_MATMUL_INNER_BLOCK_SIZE 2
//FASTOR_TRANS_OUTER_BLOCK_SIZE 2
//FASTOR_TRANS_INNER_BLOCK_SIZE 2
#ifndef FASTOR_BLAS_SWITCH_MATRIX_SIZE
#define FASTOR_BLAS_SWITCH_MATRIX_SIZE 16
#endif
// FASTOR_NIL
//------------------------------------------------------------------------------------------------//
#define FASTOR_NIL 0
//------------------------------------------------------------------------------------------------//
// Bit sizes
//------------------------------------------------------------------------------------------------//
#define FASTOR_AVX512_BITSIZE 512
#define FASTOR_AVX_BITSIZE 256
#define FASTOR_SSE_BITSIZE 128
#define FASTOR_DOUBLE_BITSIZE (sizeof(double)*8)
#define FASTOR_SINGLE_BITSIZE (sizeof(float)*8)
#ifndef FASTOR_SCALAR_BITSIZE
#define FASTOR_SCALAR_BITSIZE FASTOR_DOUBLE_BITSIZE
#endif
//------------------------------------------------------------------------------------------------//
// Alignment
//------------------------------------------------------------------------------------------------//
#ifndef FASTOR_MEMORY_ALIGNMENT_VALUE
#if defined(FASTOR_AVX512_IMPL)
#define FASTOR_MEMORY_ALIGNMENT_VALUE 64
#elif defined(FASTOR_AVX_IMPL)
#define FASTOR_MEMORY_ALIGNMENT_VALUE 32
#elif defined(FASTOR_SSE_IMPL)
#define FASTOR_MEMORY_ALIGNMENT_VALUE 16
#else
#define FASTOR_MEMORY_ALIGNMENT_VALUE 8
#endif
#endif
/* User controllable alignment for Fastor containers */
#define FASTOR_ALIGN alignas((FASTOR_MEMORY_ALIGNMENT_VALUE))
/* Strict non-controllable alignment for Fastor's internal use */
#define FASTOR_ARCH_ALIGN alignas((FASTOR_MEMORY_ALIGNMENT_VALUE))
// Conservative alignment for SIMD
#if defined(FASTOR_DONT_ALIGN) || defined(FASTOR_DONT_VECTORISE)
#define FASTOR_ALIGNED false
#else
#define FASTOR_ALIGNED true
#endif
#define FASTOR_ISALIGNED(POINTER, BYTE_COUNT) \
(((uintptr_t)(const void *)(POINTER)) % (BYTE_COUNT) == 0)
//------------------------------------------------------------------------------------------------//
// Assertions
//------------------------------------------------------------------------------------------------//
namespace Fastor {
// Strong unconditional assert
FASTOR_INLINE void FASTOR_EXIT_ASSERT(bool cond, const std::string &msg="") {
if (cond==false) {
throw std::runtime_error(msg);
}
}
#if !defined(NDEBUG) || FASTOR_ENABLE_RUNTIME_CHECKS
#define FASTOR_ASSERT(COND, MESSAGE) FASTOR_EXIT_ASSERT(COND, MESSAGE)
#else
#define FASTOR_ASSERT(COND, MESSAGE)
#endif
// Warn
FASTOR_INLINE void FASTOR_WARN(bool cond, const std::string &x) {
if (cond==false) {
std::cout << x << std::endl;
}
}
} // end of namespace Fastor
//------------------------------------------------------------------------------------------------//
// Warnings
//------------------------------------------------------------------------------------------------//
// Static warn
#ifndef FASTOR_NO_STATIC_WARNING
#if defined(__GNUC__)
#define FASTOR_DEPRECATE(foo, msg) foo __attribute__((deprecated(msg)))
#elif defined(_MSC_VER)
#define FASTOR_DEPRECATE(foo, msg) __declspec(deprecated(msg)) foo
#else
#error FASTOR STATIC WARNING DOES NOT SUPPORT THIS COMPILER
#endif
#define FASTOR_CAT(x,y) FASTOR_CAT1_(x,y)
#define FASTOR_CAT1_(x,y) x##y
namespace Fastor {
namespace useless
{
struct true_type {};
struct false_type {};
template <int test> struct converter : public true_type {};
template <> struct converter<0> : public false_type {};
}
#define FASTOR_STATIC_WARN(cond, msg) \
struct FASTOR_CAT(static_warning,__LINE__) { \
FASTOR_DEPRECATE(void _(::useless::false_type const& ),msg) {}; \
void _(::useless::true_type const& ) {}; \
FASTOR_CAT(static_warning,__LINE__)() {_(::useless::converter<(cond)>());} \
}
} // end of namespace Fastor
#endif // FASTOR_NO_STATIC_WARNING
//------------------------------------------------------------------------------------------------//
// Token to string
//------------------------------------------------------------------------------------------------//
#define FASTOR_TOSTRING_(X) #X
#define FASTOR_TOSTRING(X) FASTOR_TOSTRING_(X)
//------------------------------------------------------------------------------------------------//
// asm comment
//------------------------------------------------------------------------------------------------//
#define FASTOR_ASM(STR) asm(STR ::)
//------------------------------------------------------------------------------------------------//
// unused
//------------------------------------------------------------------------------------------------//
namespace Fastor {
// clobber
template <typename T> void unused(T &&x) {
#ifndef _WIN32
asm("" ::"m"(x));
#endif
}
template <typename T, typename ... U> void unused(T&& x, U&& ...y) { unused(x); unused(y...); }
} // end of namespace Fastor
//------------------------------------------------------------------------------------------------//
// SIMD round-down
//------------------------------------------------------------------------------------------------//
#define ROUND_DOWN2(x, s) ((x) & ~((s)-1))
#define ROUND_DOWN(x, s) ROUND_DOWN2(x,s)
//------------------------------------------------------------------------------------------------//
// FASTOR CONSTRUCTS
//------------------------------------------------------------------------------------------------//
#include <cstdint>
namespace Fastor {
using FASTOR_INDEX = size_t;
using Int64 = int64_t;
using DEFAULT_FLOAT_TYPE = double;
using DFT = DEFAULT_FLOAT_TYPE;
using FASTOR_VINDEX = volatile size_t;
constexpr int RowMajor = 0;
constexpr int ColumnMajor = 1;
constexpr int FASTOR_Symmetric = -100;
constexpr int FASTOR_NonSymmetric = -101;
constexpr int FASTOR_AntiSymmetric = -102;
constexpr int FASTOR_Identity = -103;
constexpr int FASTOR_One = -104;
constexpr int FASTOR_Zero = -105;
constexpr int FASTOR_ThreeD = -150;
constexpr int FASTOR_TwoD = -151;
constexpr int FASTOR_PlaneStrain = -152;
constexpr int FASTOR_PlaneStress = -153;
constexpr int FASTOR_Voigt = -106;
constexpr int DepthFirst = -200;
constexpr int NoDepthFirst = -201;
constexpr double PRECI_TOL = 1e-14;
}
//------------------------------------------------------------------------------------------------//
#endif // FASTOR_MACROS_H

274
noarch/include/Fastor/experimental/SingleValueTensor.h
View File

@@ -1,274 +0,0 @@
#ifndef SINGLEVALUE_TENSOR_H
#define SINGLEVALUE_TENSOR_H
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/TensorIO.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/meta/tensor_meta.h"
#include <limits>
namespace Fastor {
template<typename T, size_t ...Rest>
class SingleValueTensor : public AbstractTensor<SingleValueTensor<T,Rest...>,sizeof...(Rest)> {
public:
using scalar_type = T;
using simd_vector_type = choose_best_simd_vector_t<T>;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = SingleValueTensor<T,Rest...>;
using dimension_t = std::integral_constant<FASTOR_INDEX, sizeof...(Rest)>;
static constexpr FASTOR_INLINE FASTOR_INDEX rank() {return sizeof...(Rest);}
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return pack_prod<Rest...>::value;}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX dim) const {
#if FASTOR_SHAPE_CHECK
FASTOR_ASSERT(dim>=0 && dim < sizeof...(Rest), "TENSOR SHAPE MISMATCH");
#endif
const FASTOR_INDEX DimensionHolder[sizeof...(Rest)] = {Rest...};
return DimensionHolder[dim];
}
template<typename U=int>
SingleValueTensor(U num) : _data{(T)num} {}
SingleValueTensor(const SingleValueTensor<T,Rest...> &a) : _data{(T)a.data()[0]} {}
FASTOR_INLINE SingleValueTensor(const AbstractTensor<SingleValueTensor<T,Rest...>,sizeof...(Rest)>& src_) : _data{T(0)} {
}
constexpr FASTOR_INLINE T* data() const { return const_cast<T*>(this->_data.data());}
// Index retriever
//----------------------------------------------------------------------------------------------------------//
template<typename U>
FASTOR_INLINE int get_mem_index(U index) const {
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT((index>=0 && index<size()), "INDEX OUT OF BOUNDS");
#endif
return index;
}
template<typename... Args, typename std::enable_if<sizeof...(Args)==dimension_t::value &&
is_arithmetic_pack<Args...>::value,bool>::type =0>
FASTOR_INLINE int get_flat_index(Args ... args) const {
#if FASTOR_BOUNDS_CHECK
int largs[sizeof...(Args)] = {args...};
constexpr int DimensionHolder[dimension_t::value] = {Rest...};
for (int i=0; i<dimension_t::value; ++i) {
if (largs[i]==-1) largs[i] += DimensionHolder[i];
assert( (largs[i]>=0 && largs[i]<DimensionHolder[i]) && "INDEX OUT OF BOUNDS");
}
#endif
return 0;
}
FASTOR_INLINE int get_flat_index(const std::array<int, dimension_t::value> &as) const {
#if FASTOR_BOUNDS_CHECK
constexpr std::array<size_t,dimension_t::value> products_ = nprods_views<Index<Rest...>,
typename std_ext::make_index_sequence<dimension_t::value>::type>::values;
int index = 0;
for (int i=0; i<dimension_t::value; ++i) {
index += products_[i]*as[i];
}
FASTOR_ASSERT((index>=0 && index<size()), "INDEX OUT OF BOUNDS");
#endif
return 0;
}
//----------------------------------------------------------------------------------------------------------//
// Scalar indexing const
//----------------------------------------------------------------------------------------------------------//
#undef SCALAR_INDEXING_CONST_H
#include <Fastor/tensor/ScalarIndexing.h>
#define SCALAR_INDEXING_CONST_H
//----------------------------------------------------------------------------------------------------------//
// Expression templates evaluators
//----------------------------------------------------------------------------------------------------------//
#undef TENSOR_EVALUATOR_H
#include "Fastor/tensor/TensorEvaluator.h"
#define TENSOR_EVALUATOR_H
//----------------------------------------------------------------------------------------------------------//
// Tensor methods
//----------------------------------------------------------------------------------------------------------//
#undef TENSOR_METHODS_CONST_H
#include "Fastor/tensor/TensorMethods.h"
#define TENSOR_METHODS_CONST_H
//----------------------------------------------------------------------------------------------------------//
// Converters
//----------------------------------------------------------------------------------------------------------//
#undef PODCONVERTERS_H
#include "Fastor/tensor/PODConverters.h"
#define PODCONVERTERS_H
//----------------------------------------------------------------------------------------------------------//
// Cast method
//----------------------------------------------------------------------------------------------------------//
template<typename U>
FASTOR_INLINE SingleValueTensor<U,Rest...> cast() const {
SingleValueTensor<U,Rest...> out(static_cast<U>(_data[0]));
return out;
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
private:
const FASTOR_ALIGN std::array<T,1> _data;
//----------------------------------------------------------------------------------------------------------//
};
// template<typename T, size_t ...Rest>
// constexpr const T SingleValueTensor<T,Rest...>::_data[pack_prod<Rest...>::value];
template<typename T, size_t ... Rest>
struct tensor_type_finder<SingleValueTensor<T,Rest...>> {
using type = SingleValueTensor<T,Rest...>;
};
template<typename T, size_t ... Rest>
struct scalar_type_finder<SingleValueTensor<T,Rest...>> {
using type = T;
};
FASTOR_MAKE_OS_STREAM_TENSOR0(SingleValueTensor)
FASTOR_MAKE_OS_STREAM_TENSOR1(SingleValueTensor)
FASTOR_MAKE_OS_STREAM_TENSOR2(SingleValueTensor)
FASTOR_MAKE_OS_STREAM_TENSORn(SingleValueTensor)
template<typename T, size_t M, size_t N>
FASTOR_INLINE SingleValueTensor<T,N,M> transpose(const SingleValueTensor<T,M,N> &a) {
return SingleValueTensor<T,N,M>(a(0,0));
}
template<typename T, size_t M>
T trace(const SingleValueTensor<T,M,M> &a) {
return M*a(0,0);
}
template<typename T, size_t M>
FASTOR_INLINE T determinant(const SingleValueTensor<T,M,M> &a) {
// determinant of a single value tensor is 0
return 0.;
}
template<typename T, size_t M, size_t N>
FASTOR_INLINE double norm(const SingleValueTensor<T,M,N> &a) {
return a(0,0)*std::sqrt(double(M*N));
}
template<typename T, size_t I>
FASTOR_INLINE Tensor<T,I,I> inverse(const SingleValueTensor<T,I,I> &a) {
// A single value tensor is not invertible
Tensor<T,I,I> out(std::numeric_limits<T>::quiet_NaN());
return out;
}
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE Tensor<T,M,N> matmul(const Tensor<T,M,K> &a, const SingleValueTensor<T,K,N> &b) {
using V = SIMDVector<T,DEFAULT_ABI>;
Tensor<T,M,N> out;
T *out_data = out.data();
const T *a_data = a.data();
const T b_value = b(0,0);
for (size_t i=0; i<M; ++i) {
V vec_out;
size_t j=0;
for (; j<ROUND_DOWN(K,V::Size); j+=V::Size) {
vec_out += V(&a_data[i*K+j])*b_value;
}
T out_value = 0.;
for (; j<K; j++) {
out_value += a_data[i*K+j]*b_value;
}
out_value += vec_out.sum();
V out_vec_value(out_value);
j=0;
for (; j<ROUND_DOWN(N,V::Size); j+=V::Size) {
out_vec_value.store(&out_data[i*N+j],false);
}
for (; j<N; ++j) {
out_data[i*N+j] = out_value;
}
}
return out;
}
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE Tensor<T,M,N> matmul(const SingleValueTensor<T,M,K> &a, const Tensor<T,K,N> &b) {
return transpose(matmul(transpose(b),transpose(a)));
}
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE SingleValueTensor<T,M,N> matmul(const SingleValueTensor<T,M,K> &a, const SingleValueTensor<T,K,N> &b) {
const T a_value = a(0,0);
const T b_value = b(0,0);
// matmul is just this
SingleValueTensor<T,M,N> out(a_value*b_value*K);
// Not necessary
// using V = SIMDVector<T,DEFAULT_ABI>;
// V vec_out;
// size_t j=0;
// for (; j<ROUND_DOWN(K,V::Size); j+=V::Size) {
// vec_out = vec_out + V(a_value)*b_value;
// }
// T out_value = 0.;
// for (; j<K; j++) {
// out_value += a_value*b_value;
// }
// out_value += vec_out.sum();
// SingleValueTensor<T,M,N> out(out_value);
return out;
}
// This one is almost like a compile time einsum
template<class Index_I, class Index_J, typename T, size_t ... Rest0, size_t ... Rest1>
FASTOR_INLINE
typename contraction_impl<typename concat_<Index_I,Index_J>::type,SingleValueTensor<T,Rest0...,Rest1...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)+sizeof...(Rest1)>::type>::type
einsum(const SingleValueTensor<T,Rest0...> &a, const SingleValueTensor<T,Rest1...> &b) {
static_assert(einsum_index_checker<typename concat_<Index_I,Index_J>::type>::value,
"INDICES FOR EINSUM FUNCTION CANNOT APPEAR MORE THAN TWICE. USE CONTRACTION INSTEAD");
std::array<size_t,Index_I::NoIndices> idx0; std::copy_n(Index_I::_IndexHolder,Index_I::NoIndices,idx0.begin());
std::array<size_t,Index_J::NoIndices> idx1; std::copy_n(Index_J::_IndexHolder,Index_J::NoIndices,idx1.begin());
std::array<size_t,Index_I::NoIndices> dims0 = {Rest0...};
// n^2 but it is okay as this is a small loop with compile time spans
size_t total = 1;
for (int i=0; i<idx0.size(); ++i) {
for (int j=0; j<idx1.size(); ++j) {
if (idx0[i]==idx1[j]) {
total *= dims0[i];
}
}
}
const T a_value = a.eval_s(0);
const T b_value = b.eval_s(0);
const T out_value = total*a_value*b_value;
using OutTensor = typename contraction_impl<typename concat_<Index_I,Index_J>::type,SingleValueTensor<T,Rest0...,Rest1...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)+sizeof...(Rest1)>::type>::type;
OutTensor out(out_value);
return out;
}
} // end of namespace Fastor
#endif // SINGLEVALUE_TENSOR_H

247
noarch/include/Fastor/expressions/binary_ops/binary_add_op.h
View File

@@ -1,247 +0,0 @@
#ifndef BINARY_ADD_OP_H
#define BINARY_ADD_OP_H
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryAddOp: public AbstractTensor<BinaryAddOp<TLhs, TRhs, DIM0>,DIM0> {
private:
expression_t<TLhs> _lhs;
expression_t<TRhs> _rhs;
public:
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<BinaryAddOp<TLhs, TRhs, DIM0>>::type;
using simd_vector_type = binary_op_simd_vector_t<BinaryAddOp<TLhs, TRhs, DIM0> >;
using simd_abi_type = typename simd_vector_type::abi_type;
FASTOR_INLINE BinaryAddOp(expression_t<TLhs> inlhs, expression_t<TRhs> inrhs) : _lhs((inlhs)), _rhs((inrhs)) {}
FASTOR_INLINE FASTOR_INDEX size() const {return helper_size<TLhs,TRhs>();}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");
#endif
return _rhs.size();
}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension<TLhs,TRhs>(i);}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");
#endif
return _rhs.dimension(i);
}
constexpr FASTOR_INLINE expression_t<TLhs> lhs() const {return _lhs;}
constexpr FASTOR_INLINE expression_t<TRhs> rhs() const {return _rhs;}
// Generic version of eval
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
// Delay evaluation using a helper function to fully inform BinaryOp about _lhs and _rhs
return helper<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return _lhs.template eval<U>(i) + _rhs.template eval<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return (U)_lhs + _rhs.template eval<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return _lhs.template eval<U>(i) + (U)_rhs;
}
// scalar based
template<typename U>
FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return helper_s<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<U>(i) + _rhs.template eval_s<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return (U)_lhs + _rhs.template eval_s<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<U>(i) + (U)_rhs;
}
// for 2D tensors
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval<U>(i,j) + _rhs.template eval<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return (U)_lhs + _rhs.template eval<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval<U>(i,j) + (U)_rhs;
}
// scalar based (for 2D tensors)
template<typename U>
FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper_s<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<U>(i,j) + _rhs.template eval_s<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return (U)_lhs + _rhs.template eval_s<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<U>(i,j) + (U)_rhs;
}
// for nD tensors
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,DIM0> &as) const {
return thelper<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<U>(as) + _rhs.template teval<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return (U)_lhs + _rhs.template teval<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<U>(as) + (U)_rhs;
}
// scalar based (for nD tensors)
template<typename U>
FASTOR_INLINE U teval_s(const std::array<int,DIM0> &as) const {
return thelper_s<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<U>(as) + _rhs.template teval_s<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return (U)_lhs + _rhs.template teval_s<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<U>(as) + (U)_rhs;
}
};
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryAddOp<TLhs, TRhs, DIM0> operator+(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryAddOp<TLhs, TRhs, DIM0>(_lhs.self(), _rhs.self());
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryAddOp<TLhs, TRhs, DIM0> operator+(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {
return BinaryAddOp<TLhs, TRhs, DIM0>(_lhs.self(), bb);
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryAddOp<TLhs, TRhs, DIM0> operator+(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryAddOp<TLhs, TRhs, DIM0>(bb,_rhs.self());
}
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value &&
DIM0!=DIM1,bool>::type = 0 >
FASTOR_INLINE BinaryAddOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>
operator+(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM1> &_rhs) {
return BinaryAddOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(_lhs.self(), _rhs.self());
}
} // end of namespace Fastor
#endif // BINARY_ADD_OP_H

156
noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_assignment.h
View File

@@ -1,156 +0,0 @@
#ifndef BINARY_ARITHMETIC_ASSIGNMENT_H
#define BINARY_ARITHMETIC_ASSIGNMENT_H
#include "Fastor/expressions/binary_ops/binary_arithmetic_ops.h"
#include "Fastor/tensor/Aliasing.h"
namespace Fastor {
// Create assign for all binrary arithmetic ops
#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(NAME, ASSIGN_TYPE, OP_ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && !(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>), bool> = false >\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && (requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>), bool> = false >\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\
assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && !requires_evaluation_v<TRhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && requires_evaluation_v<TRhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
assign ##ASSIGN_TYPE (dst.self(), src.lhs());\
assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && is_primitive_v_<TRhs> && !requires_evaluation_v<TLhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && is_primitive_v_<TRhs> && requires_evaluation_v<TLhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\
assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs());\
}\
// Create assign_add, assign_sub for BinaryAddOp and BinarySubOp
#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(NAME, ASSIGN_TYPE, OP_ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && !(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>), bool> = false >\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && (requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>), bool> = false >\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
if (!does_alias(dst.self(),src.rhs().self())) {\
assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\
assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\
}\
else{\
const Derived tmp(dst.self());\
assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\
assign ##OP_ASSIGN_TYPE (dst.self(), tmp);\
}\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && !requires_evaluation_v<TRhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<is_primitive_v_<TLhs> && !is_primitive_v_<TRhs> && requires_evaluation_v<TRhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
assign ##ASSIGN_TYPE (dst.self(), src.lhs());\
assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs().self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && is_primitive_v_<TRhs> && !requires_evaluation_v<TLhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!is_primitive_v_<TLhs> && is_primitive_v_<TRhs> && requires_evaluation_v<TLhs>, bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
assign ##ASSIGN_TYPE (dst.self(), src.lhs().self());\
assign ##OP_ASSIGN_TYPE (dst.self(), src.rhs());\
}\
// Create assign_add, assign_sub for BinaryMulOp and BinaryDivOp
// Create assign_mul, assign_div for all binary ops
#define FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(NAME, ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>),bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>),bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
using result_type = typename Binary ##NAME ## Op<TLhs, TRhs, OtherDIM>::result_type;\
const result_type a(src.self());\
trivial_assign ##ASSIGN_TYPE (dst.self(), a);\
}\
// assign
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Add, , _add)
// assign_add
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Add, _add, _add)
// assign_sub
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Add, _sub, _sub)
// assign_mul
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Add, _mul)
// assign_div
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Add, _div)
// assign
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Sub, , _sub)
// assign_add
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Sub, _add, _sub)
// assign_sub
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_1(Sub, _sub, _add)
// assign_mul
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Sub, _mul)
// assign_div
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Sub, _div)
// assign
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Mul, , _mul)
// assign_add
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _add)
// assign_sub
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _sub)
// assign_mul
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _mul)
// assign_div
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Mul, _div)
// assign
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_0(Div, , _div)
// assign_add
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _add)
// assign_sub
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _sub)
// assign_mul
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _mul)
// assign_div
FASTOR_MAKE_BINARY_ARITHMETIC_ASSIGNMENT_2(Div, _div)
} // end of namespace Fastor
#endif // BINARY_ARITHMETIC_ASSIGNMENT_H

229
noarch/include/Fastor/expressions/binary_ops/binary_arithmetic_ops.h
View File

@@ -1,229 +0,0 @@
#ifndef BINARY_ARITHMETIC_OP_H
#define BINARY_ARITHMETIC_OP_H
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
#define FASTOR_MAKE_BINARY_ARITHMETIC_OPS(OP, NAME, EVAL_TYPE) \
template<typename TLhs, typename TRhs, size_t DIM0>\
struct Binary ##NAME ## Op: public AbstractTensor<Binary ##NAME ## Op<TLhs, TRhs, DIM0>,DIM0> {\
expression_t<TLhs> _lhs;\
expression_t<TRhs> _rhs;\
public:\
static constexpr FASTOR_INDEX Dimension = DIM0;\
static constexpr FASTOR_INDEX rank() {return DIM0;}\
using scalar_type = typename scalar_type_finder<Binary ##NAME ## Op<TLhs, TRhs, DIM0>>::type;\
using simd_vector_type = binary_op_simd_vector_t< Binary ##NAME ## Op<TLhs, TRhs, DIM0> >;\
using simd_abi_type = typename simd_vector_type::abi_type;\
using result_type = binary_arithmetic_result_t< Binary ##NAME ## Op<TLhs, TRhs, DIM0> >;\
FASTOR_INLINE Binary ##NAME ## Op(expression_t<TLhs> inlhs, expression_t<TRhs> inrhs) : _lhs(inlhs), _rhs(inrhs) {}\
FASTOR_INLINE FASTOR_INDEX size() const {return helper_size<TLhs,TRhs>();}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<LExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\
template<class LExpr, class RExpr,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {\
FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");\
return _rhs.size();\
}\
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension<TLhs,TRhs>(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<LExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\
FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");\
return _rhs.dimension(i);\
}\
FASTOR_INLINE expression_t<TLhs> lhs() const {return _lhs;}\
FASTOR_INLINE expression_t<TRhs> rhs() const {return _rhs;}\
template<typename U>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i) const {\
return helper<TLhs,TRhs,U>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {\
return _lhs.template eval<EVAL_TYPE>(i) OP _rhs.template eval<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {\
return _lhs.template eval<EVAL_TYPE>(i) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\
return helper_s<TLhs,TRhs,U>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\
return _lhs.template eval_s<EVAL_TYPE>(i) OP _rhs.template eval_s<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval_s<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\
return _lhs.template eval_s<EVAL_TYPE>(i) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return helper<TLhs,TRhs,U>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval<EVAL_TYPE>(i,j) OP _rhs.template eval<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval<EVAL_TYPE>(i,j) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return helper_s<TLhs,TRhs,U>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval_s<EVAL_TYPE>(i,j) OP _rhs.template eval_s<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval_s<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval_s<EVAL_TYPE>(i,j) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> teval(const std::array<int,DIM0> &as) const {\
return thelper<TLhs,TRhs,U>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return _lhs.template teval<EVAL_TYPE>(as) OP _rhs.template teval<EVAL_TYPE>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return (EVAL_TYPE)_lhs OP _rhs.template teval<EVAL_TYPE>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return _lhs.template teval<EVAL_TYPE>(as) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE EVAL_TYPE teval_s(const std::array<int,DIM0> &as) const {\
return thelper_s<TLhs,TRhs,U>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {\
return _lhs.template teval_s<EVAL_TYPE>(as) OP _rhs.template teval_s<EVAL_TYPE>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {\
return (EVAL_TYPE)_lhs OP _rhs.template teval_s<U>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {\
return _lhs.template teval_s<EVAL_TYPE>(as) OP (EVAL_TYPE)_rhs;\
}\
};\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<!is_primitive_v_<TLhs> &&\
!is_primitive_v_<TRhs>,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, DIM0> operator OP(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM0> &_rhs) {\
return Binary ##NAME ## Op<TLhs, TRhs, DIM0>(_lhs.self(), _rhs.self());\
}\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<!is_primitive_v_<TLhs> &&\
is_primitive_v_<TRhs>,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, DIM0> operator OP(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {\
return Binary ##NAME ## Op<TLhs, TRhs, DIM0>(_lhs.self(), bb);\
}\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<is_primitive_v_<TLhs> &&\
!is_primitive_v_<TRhs>,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, DIM0> operator OP(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {\
return Binary ##NAME ## Op<TLhs, TRhs, DIM0>(bb,_rhs.self());\
}\
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,\
typename std::enable_if<!is_primitive_v_<TLhs> &&\
!is_primitive_v_<TRhs> &&\
DIM0!=DIM1,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, meta_min<DIM0,DIM1>::value>\
operator OP(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM1> &_rhs) {\
return Binary ##NAME ## Op<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(_lhs.self(), _rhs.self());\
}\
// Dispatch based on type of expressions not the tensor
FASTOR_MAKE_BINARY_ARITHMETIC_OPS(+, Add, scalar_type)
FASTOR_MAKE_BINARY_ARITHMETIC_OPS(-, Sub, scalar_type)
FASTOR_MAKE_BINARY_ARITHMETIC_OPS(*, Mul, scalar_type)
// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(/, Div, scalar_type) // Dont create div as it is a special case
// Dispatch based on the type of tensor and not the expression
// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(+, Add, U)
// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(-, Sub, U)
// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(*, Mul, U)
// FASTOR_MAKE_BINARY_ARITHMETIC_OPS(/, Div, U)
}
#endif // BINARY_ARITHMETIC_OP_H

262
noarch/include/Fastor/expressions/binary_ops/binary_cmp_ops.h
View File

@@ -1,262 +0,0 @@
#ifndef BINARY_CMP_OPS
#define BINARY_CMP_OPS
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
#define FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(OP, NAME, EVAL_TYPE) \
template<typename TLhs, typename TRhs, size_t DIM0>\
struct BinaryCmpOp##NAME: public AbstractTensor<BinaryCmpOp##NAME<TLhs, TRhs, DIM0>,DIM0> {\
expression_t<TLhs> _lhs;\
expression_t<TRhs> _rhs;\
static constexpr FASTOR_INDEX Dimension = DIM0;\
static constexpr FASTOR_INDEX rank() {return DIM0;}\
using scalar_type = typename scalar_type_finder<BinaryCmpOp##NAME<TLhs, TRhs, DIM0>>::type;\
using result_type = to_bool_tensor_t<binary_arithmetic_result_t<BinaryCmpOp##NAME<TLhs, TRhs, DIM0>>>;\
using simd_vector_type = binary_op_simd_vector_t<BinaryCmpOp##NAME<TLhs, TRhs, DIM0> >;\
using simd_abi_type = typename simd_vector_type::abi_type;\
using ABI = simd_abi::fixed_size<SIMDVector<scalar_type,simd_abi_type>::Size>;\
using UU = bool /*this needs to change to U once masks are implemented*/;\
FASTOR_INLINE BinaryCmpOp##NAME(expression_t<TLhs> inlhs, expression_t<TRhs> inrhs) : _lhs(inlhs), _rhs(inrhs) {}\
FASTOR_INLINE FASTOR_INDEX size() const {return helper_size<TLhs,TRhs>();}\
template<class LExpr, class RExpr,\
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\
template<class LExpr, class RExpr,\
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\
template<class LExpr, class RExpr,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {\
return _rhs.size();\
}\
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension<TLhs,TRhs>(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\
return _rhs.dimension(i);\
}\
FASTOR_INLINE expression_t<TLhs> lhs() const {return _lhs;}\
FASTOR_INLINE expression_t<TRhs> rhs() const {return _rhs;}\
template<typename U>\
FASTOR_INLINE SIMDVector<UU,ABI> eval(FASTOR_INDEX i) const {\
return helper<TLhs,TRhs,U>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,ABI> helper(FASTOR_INDEX i) const {\
return _lhs.template eval<EVAL_TYPE>(i) OP _rhs.template eval<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,ABI> helper(FASTOR_INDEX i) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,ABI> helper(FASTOR_INDEX i) const {\
return _lhs.template eval<EVAL_TYPE>(i) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE UU eval_s(FASTOR_INDEX i) const {\
return helper_s<TLhs,TRhs,U>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\
return _lhs.template eval_s<EVAL_TYPE>(i) OP _rhs.template eval_s<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval_s<EVAL_TYPE>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU helper_s(FASTOR_INDEX i) const {\
return _lhs.template eval_s<EVAL_TYPE>(i) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<UU,ABI> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return helper<TLhs,TRhs,U>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,ABI> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval<EVAL_TYPE>(i,j) OP _rhs.template eval<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,ABI> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,ABI> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval<EVAL_TYPE>(i,j) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE UU eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return helper_s<TLhs,TRhs,U>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval_s<EVAL_TYPE>(i,j) OP _rhs.template eval_s<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return (EVAL_TYPE)_lhs OP _rhs.template eval_s<EVAL_TYPE>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return _lhs.template eval_s<EVAL_TYPE>(i,j) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<UU,simd_abi_type> teval(const std::array<int,DIM0> &as) const {\
return thelper<TLhs,TRhs,U>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return _lhs.template teval<EVAL_TYPE>(as) OP _rhs.template teval<EVAL_TYPE>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return (EVAL_TYPE)_lhs OP _rhs.template teval<EVAL_TYPE>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE SIMDVector<UU,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return _lhs.template teval<EVAL_TYPE>(as) OP (EVAL_TYPE)_rhs;\
}\
template<typename U>\
FASTOR_INLINE UU teval_s(const std::array<int,DIM0> &as) const {\
return thelper_s<TLhs,TRhs,U>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU thelper_s(const std::array<int,DIM0> &as) const {\
return _lhs.template teval_s<EVAL_TYPE>(as) OP _rhs.template teval_s<EVAL_TYPE>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<std::is_arithmetic<LExpr>::value &&\
!std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU thelper_s(const std::array<int,DIM0> &as) const {\
return (EVAL_TYPE)_lhs OP _rhs.template teval_s<EVAL_TYPE>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&\
std::is_arithmetic<RExpr>::value,bool>::type = 0>\
FASTOR_INLINE UU thelper_s(const std::array<int,DIM0> &as) const {\
return _lhs.template teval_s<EVAL_TYPE>(as) OP (EVAL_TYPE)_rhs;\
}\
};\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&\
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >\
FASTOR_INLINE BinaryCmpOp##NAME<TLhs, TRhs, DIM0> operator OP(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM0> &_rhs) {\
return BinaryCmpOp##NAME<TLhs, TRhs, DIM0>(_lhs.self(), _rhs.self());\
}\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&\
std::is_arithmetic<TRhs>::value,bool>::type = 0 >\
FASTOR_INLINE BinaryCmpOp##NAME<TLhs, TRhs, DIM0> operator OP(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {\
return BinaryCmpOp##NAME<TLhs, TRhs, DIM0>(_lhs.self(), bb);\
}\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<std::is_arithmetic<TLhs>::value &&\
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >\
FASTOR_INLINE BinaryCmpOp##NAME<TLhs, TRhs, DIM0> operator OP(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {\
return BinaryCmpOp##NAME<TLhs, TRhs, DIM0>(bb,_rhs.self());\
}\
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,\
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&\
!std::is_arithmetic<TRhs>::value &&\
DIM0!=DIM1,bool>::type = 0 >\
FASTOR_INLINE BinaryCmpOp##NAME<TLhs, TRhs, meta_min<DIM0,DIM1>::value>\
operator OP(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM1> &_rhs) {\
return BinaryCmpOp##NAME<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(_lhs.self(), _rhs.self());\
}\
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(== ,EQ, scalar_type)
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(!= ,NEQ,scalar_type)
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(< ,LT, scalar_type)
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(> ,GT, scalar_type)
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(<= ,LE, scalar_type)
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(>= ,GE, scalar_type)
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(&& ,AND,scalar_type)
FASTOR_MAKE_BINARY_CMP_TENSOR_OPS_(|| ,OR, scalar_type)
#define FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(NAME, ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>),bool> = false >\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const BinaryCmpOp ##NAME <TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>),bool> = false >\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const BinaryCmpOp ##NAME <TLhs, TRhs, OtherDIM> &src) {\
using lhs_type = typename get_binary_arithmetic_result_type<TLhs>::type;\
using rhs_type = typename get_binary_arithmetic_result_type<TRhs>::type;\
const lhs_type a(src.lhs());\
const rhs_type b(src.rhs());\
trivial_assign ##ASSIGN_TYPE (dst.self(), BinaryCmpOp ##NAME <lhs_type, rhs_type, OtherDIM>(a,b));\
}\
#define FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS(ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(EQ, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(NEQ, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(LT, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(GT, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(LE, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(GE, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(AND, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENT(OR, ASSIGN_TYPE)\
FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( )
FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _add)
FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _sub)
FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _mul)
FASTOR_MAKE_BINARY_CMP_ASSIGNMENTS( _div)
} // end of namespace Fastor
#endif // BINARY_CMP_OPS

306
noarch/include/Fastor/expressions/binary_ops/binary_div_op.h
View File

@@ -1,306 +0,0 @@
#ifndef BINARY_DIV_OP_H
#define BINARY_DIV_OP_H
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
// Dispatch based on type of expressions not the tensor
#define FASTOR_BD_OP_EVAL_TYPE scalar_type
// Dispatch based on the type of tensor and not the expression
// #define FASTOR_BD_OP_EVAL_TYPE U
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryDivOp: public AbstractTensor<BinaryDivOp<TLhs, TRhs, DIM0>,DIM0> {
private:
expression_t<TLhs> _lhs;
expression_t<TRhs> _rhs;
public:
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<BinaryDivOp<TLhs, TRhs, DIM0>>::type;
using simd_vector_type = binary_op_simd_vector_t<BinaryDivOp<TLhs, TRhs, DIM0> >;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = binary_arithmetic_result_t< BinaryDivOp<TLhs, TRhs, DIM0> >;
FASTOR_INLINE BinaryDivOp(expression_t<TLhs> inlhs, expression_t<TRhs> inrhs) : _lhs((inlhs)), _rhs((inrhs)) {}
FASTOR_INLINE FASTOR_INDEX size() const {return helper_size<TLhs,TRhs>();}
template<class LExpr, class RExpr,
typename std::enable_if<is_primitive_v_<LExpr>,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<is_primitive_v_<RExpr>,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");
#endif
return _rhs.size();
}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension<TLhs,TRhs>(i);}
template<class LExpr, class RExpr,
typename std::enable_if<is_primitive_v_<LExpr>,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<is_primitive_v_<RExpr>,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");
#endif
return _rhs.dimension(i);
}
constexpr FASTOR_INLINE expression_t<TLhs> lhs() const {return _lhs;}
constexpr FASTOR_INLINE expression_t<TRhs> rhs() const {return _rhs;}
template<typename U>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i) const {
return helper<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {
#ifndef FASTOR_UNSAFE_MATH
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i) / _rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i);
#else
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i) * rcp(_rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i));
#endif
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {
#ifndef FASTOR_UNSAFE_MATH
return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i);
#else
return (FASTOR_BD_OP_EVAL_TYPE)_lhs * rcp(_rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i));
#endif
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {
#ifndef FASTOR_UNSAFE_MATH
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i) / (FASTOR_BD_OP_EVAL_TYPE)_rhs;
#else
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i) * rcp(SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type>(_rhs));
#endif
}
// scalar based
template<typename U>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE eval_s(FASTOR_INDEX i) const {
return helper_s<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i) / _rhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const {
return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i) / (FASTOR_BD_OP_EVAL_TYPE)_rhs;
}
// for 2D tensors
template<typename U>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
#ifndef FASTOR_UNSAFE_MATH
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j) / _rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j);
#else
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j) * rcp(_rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j));
#endif
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
#ifndef FASTOR_UNSAFE_MATH
return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j);
#else
return (FASTOR_BD_OP_EVAL_TYPE)_lhs * rcp(_rhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j));
#endif
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
#ifndef FASTOR_UNSAFE_MATH
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j) / (FASTOR_BD_OP_EVAL_TYPE)_rhs;
#else
return _lhs.template eval<FASTOR_BD_OP_EVAL_TYPE>(i,j) * rcp(SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type>(_rhs));
#endif
}
// scalar based (for 2D tensors)
template<typename U>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper_s<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i,j) / _rhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<FASTOR_BD_OP_EVAL_TYPE>(i,j) / (FASTOR_BD_OP_EVAL_TYPE)_rhs;
}
// for nD tensors
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,DIM0> &as) const {
return thelper<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<FASTOR_BD_OP_EVAL_TYPE>(as) / _rhs.template teval<FASTOR_BD_OP_EVAL_TYPE>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template teval<FASTOR_BD_OP_EVAL_TYPE>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE SIMDVector<FASTOR_BD_OP_EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<FASTOR_BD_OP_EVAL_TYPE>(as) / (FASTOR_BD_OP_EVAL_TYPE)_rhs;
}
// scalar based (for nD tensors)
template<typename U>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE teval_s(const std::array<int,DIM0> &as) const {
return thelper_s<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<FASTOR_BD_OP_EVAL_TYPE>(as) / _rhs.template teval_s<FASTOR_BD_OP_EVAL_TYPE>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<is_primitive_v_<LExpr> &&
!is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {
return (FASTOR_BD_OP_EVAL_TYPE)_lhs / _rhs.template teval_s<FASTOR_BD_OP_EVAL_TYPE>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!is_primitive_v_<LExpr> &&
is_primitive_v_<RExpr>,bool>::type = 0>
FASTOR_INLINE FASTOR_BD_OP_EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<FASTOR_BD_OP_EVAL_TYPE>(as) / (FASTOR_BD_OP_EVAL_TYPE)_rhs;
}
};
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!is_primitive_v_<TLhs> &&
!is_primitive_v_<TRhs>,bool>::type = 0 >
FASTOR_INLINE BinaryDivOp<TLhs, TRhs, DIM0> operator/(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryDivOp<TLhs, TRhs, DIM0>(_lhs.self(), _rhs.self());
}
#ifndef FASTOR_DISPATCH_DIV_TO_MUL_EXPR
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!is_primitive_v_<TLhs> &&
is_primitive_v_<TRhs>,bool>::type = 0 >
FASTOR_INLINE BinaryDivOp<TLhs, TRhs, DIM0> operator/(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {
return BinaryDivOp<TLhs, TRhs, DIM0>(_lhs.self(), bb);
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<is_primitive_v_<TLhs> &&
!is_primitive_v_<TRhs>,bool>::type = 0 >
FASTOR_INLINE BinaryDivOp<TLhs, TRhs, DIM0> operator/(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryDivOp<TLhs, TRhs, DIM0>(bb,_rhs.self());
}
#else
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!is_primitive_v_<TLhs> &&
is_primitive_v_<TRhs> && !std::is_integral<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryMulOp<TLhs, TRhs, DIM0> operator/(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {
return BinaryMulOp<TLhs, TRhs, DIM0>(
_lhs.self(),
static_cast<typename scalar_type_finder<TLhs>::type>(1)/static_cast<typename scalar_type_finder<TLhs>::type>(bb));
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<is_primitive_v_<TLhs> && !std::is_integral<TRhs>::value &&
!is_primitive_v_<TRhs>,bool>::type = 0 >
FASTOR_INLINE BinaryMulOp<TLhs, TRhs, DIM0> operator/(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryMulOp<TLhs, TRhs, DIM0>(
static_cast<typename scalar_type_finder<TLhs>::type>(1)/static_cast<typename scalar_type_finder<TLhs>::type>(bb),
_rhs.self());
}
// Special case for integral types
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!is_primitive_v_<TLhs> &&
is_primitive_v_<TRhs> && std::is_integral<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryDivOp<TLhs, TRhs, DIM0> operator/(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {
return BinaryDivOp<TLhs, TRhs, DIM0>(_lhs.self(), bb);
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<is_primitive_v_<TLhs> && std::is_integral<TRhs>::value &&
!is_primitive_v_<TRhs>,bool>::type = 0 >
FASTOR_INLINE BinaryDivOp<TLhs, TRhs, DIM0> operator/(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryDivOp<TLhs, TRhs, DIM0>(bb,_rhs.self());
}
#endif
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
enable_if_t_<!is_arithmetic_v_<TLhs> &&
!is_arithmetic_v_<TRhs> &&
DIM0!=DIM1,bool> = false >
FASTOR_INLINE BinaryDivOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>
operator/(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM1> &_rhs) {
return BinaryDivOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(_lhs.self(), _rhs.self());
}
}
#endif // BINARY_DIV_OP_H

261
noarch/include/Fastor/expressions/binary_ops/binary_math_ops.h
View File

@@ -1,261 +0,0 @@
#ifndef BINARY_MATH_OP_H
#define BINARY_MATH_OP_H
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
#define FASTOR_MAKE_BINARY_MATH_OPS(OP_NAME, SIMD_OP, OP, NAME, EVAL_TYPE) \
template<typename TLhs, typename TRhs, size_t DIM0>\
struct Binary ##NAME ## Op: public AbstractTensor<Binary ##NAME ## Op<TLhs, TRhs, DIM0>,DIM0> {\
expression_t<TLhs> _lhs;\
expression_t<TRhs> _rhs;\
public:\
static constexpr FASTOR_INDEX Dimension = DIM0;\
static constexpr FASTOR_INDEX rank() {return DIM0;}\
using scalar_type = typename scalar_type_finder<Binary ##NAME ## Op<TLhs, TRhs, DIM0>>::type;\
using simd_vector_type = binary_op_simd_vector_t< Binary ##NAME ## Op<TLhs, TRhs, DIM0> >;\
using simd_abi_type = typename simd_vector_type::abi_type;\
using result_type = binary_arithmetic_result_t< Binary ##NAME ## Op<TLhs, TRhs, DIM0> >;\
FASTOR_INLINE Binary ##NAME ## Op(expression_t<TLhs> inlhs, expression_t<TRhs> inrhs) : _lhs(inlhs), _rhs(inrhs) {}\
FASTOR_INLINE FASTOR_INDEX size() const {return helper_size<TLhs,TRhs>();}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<LExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}\
template<class LExpr, class RExpr,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_size() const {\
FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");\
return _rhs.size();\
}\
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension<TLhs,TRhs>(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<LExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}\
template<class LExpr, class RExpr,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type =0 >\
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {\
FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");\
return _rhs.dimension(i);\
}\
FASTOR_INLINE expression_t<TLhs> lhs() const {return _lhs;}\
FASTOR_INLINE expression_t<TRhs> rhs() const {return _rhs;}\
template<typename U>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i) const {\
return helper<TLhs,TRhs,U>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {\
return SIMD_OP(_lhs.template eval<EVAL_TYPE>(i), _rhs.template eval<EVAL_TYPE>(i));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {\
return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template eval<EVAL_TYPE>(i));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i) const {\
return SIMD_OP(_lhs.template eval<EVAL_TYPE>(i), (EVAL_TYPE)_rhs);\
}\
template<typename U>\
FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\
return helper_s<TLhs,TRhs,U>(i);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\
return OP(_lhs.template eval_s<EVAL_TYPE>(i), _rhs.template eval_s<EVAL_TYPE>(i));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\
return OP((EVAL_TYPE)_lhs, _rhs.template eval_s<EVAL_TYPE>(i));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i) const {\
return OP(_lhs.template eval_s<EVAL_TYPE>(i), (EVAL_TYPE)_rhs);\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return helper<TLhs,TRhs,U>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return SIMD_OP(_lhs.template eval<EVAL_TYPE>(i,j), _rhs.template eval<EVAL_TYPE>(i,j));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template eval<EVAL_TYPE>(i,j));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return SIMD_OP(_lhs.template eval<EVAL_TYPE>(i,j), (EVAL_TYPE)_rhs);\
}\
template<typename U>\
FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return helper_s<TLhs,TRhs,U>(i,j);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return OP(_lhs.template eval_s<EVAL_TYPE>(i,j), _rhs.template eval_s<EVAL_TYPE>(i,j));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return OP((EVAL_TYPE)_lhs, _rhs.template eval_s<EVAL_TYPE>(i,j));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return OP(_lhs.template eval_s<EVAL_TYPE>(i,j), (EVAL_TYPE)_rhs);\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> teval(const std::array<int,DIM0> &as) const {\
return thelper<TLhs,TRhs,U>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return SIMD_OP(_lhs.template teval<EVAL_TYPE>(as), _rhs.template teval<EVAL_TYPE>(as));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return SIMD_OP((EVAL_TYPE)_lhs, _rhs.template teval<EVAL_TYPE>(as));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {\
return SIMD_OP(_lhs.template teval<EVAL_TYPE>(as), (EVAL_TYPE)_rhs);\
}\
template<typename U>\
FASTOR_INLINE EVAL_TYPE teval_s(const std::array<int,DIM0> &as) const {\
return thelper_s<TLhs,TRhs,U>(as);\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {\
return OP(_lhs.template teval_s<EVAL_TYPE>(as), _rhs.template teval_s<EVAL_TYPE>(as));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<is_primitive_v_<LExpr> &&\
!is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {\
return OP((EVAL_TYPE)_lhs, _rhs.template teval_s<U>(as));\
}\
template<typename LExpr, typename RExpr, typename U,\
typename std::enable_if<!is_primitive_v_<LExpr> &&\
is_primitive_v_<RExpr>,bool>::type = 0>\
FASTOR_INLINE EVAL_TYPE thelper_s(const std::array<int,DIM0> &as) const {\
return OP(_lhs.template teval_s<EVAL_TYPE>(as), (EVAL_TYPE)_rhs);\
}\
};\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<!is_primitive_v_<TLhs> &&\
!is_primitive_v_<TRhs>,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, DIM0> OP_NAME(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM0> &_rhs) {\
return Binary ##NAME ## Op<TLhs, TRhs, DIM0>(_lhs.self(), _rhs.self());\
}\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<!is_primitive_v_<TLhs> &&\
is_primitive_v_<TRhs>,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, DIM0> OP_NAME(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {\
return Binary ##NAME ## Op<TLhs, TRhs, DIM0>(_lhs.self(), bb);\
}\
template<typename TLhs, typename TRhs, size_t DIM0,\
typename std::enable_if<is_primitive_v_<TLhs> &&\
!is_primitive_v_<TRhs>,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, DIM0> OP_NAME(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {\
return Binary ##NAME ## Op<TLhs, TRhs, DIM0>(bb,_rhs.self());\
}\
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,\
typename std::enable_if<!is_primitive_v_<TLhs> &&\
!is_primitive_v_<TRhs> &&\
DIM0!=DIM1,bool>::type = 0 >\
FASTOR_INLINE Binary ##NAME ## Op<TLhs, TRhs, meta_min<DIM0,DIM1>::value>\
OP_NAME(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM1> &_rhs) {\
return Binary ##NAME ## Op<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(_lhs.self(), _rhs.self());\
}\
// Dispatch based on type of expressions not the tensor
FASTOR_MAKE_BINARY_MATH_OPS(min, min, std::min, Min, scalar_type)
FASTOR_MAKE_BINARY_MATH_OPS(max, max, std::max, Max, scalar_type)
FASTOR_MAKE_BINARY_MATH_OPS(pow, pow, std::pow, Pow, scalar_type)
FASTOR_MAKE_BINARY_MATH_OPS(atan2, atan2, std::atan2, Atan2, scalar_type)
FASTOR_MAKE_BINARY_MATH_OPS(hypot, hypot, std::hypot, Hypot, scalar_type)
// Dispatch based on the type of tensor and not the expression
// FASTOR_MAKE_BINARY_MATH_OPS(min, min, std::min, Min, U)
// FASTOR_MAKE_BINARY_MATH_OPS(max, max, std::max, Max, U)
// FASTOR_MAKE_BINARY_MATH_OPS(pow, pow, std::pow, Pow, U)
// FASTOR_MAKE_BINARY_MATH_OPS(atan2, atan2, std::atan2, Atan2, U)
// FASTOR_MAKE_BINARY_MATH_OPS(hypot, hypot, std::hypot, Hypot, U)
// Create assignment for all binary math_ops
#define FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<!(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>),bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,\
enable_if_t_<(requires_evaluation_v<TLhs> || requires_evaluation_v<TRhs>),bool> = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs, TRhs, OtherDIM> &src) {\
using result_type = typename Binary ##NAME ## Op<TLhs, TRhs, OtherDIM>::result_type;\
const result_type a(src.self());\
trivial_assign ##ASSIGN_TYPE (dst.self(), a);\
}\
#define FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(NAME)\
FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, )\
FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _add)\
FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _sub)\
FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _mul)\
FASTOR_MAKE_BINARY_MATH_ASSIGNMENT(NAME, _div)\
FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Min)
FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Max)
FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Pow)
FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Atan2)
FASTOR_MAKE_BINARY_MATH_ASSIGNMENTS(Hypot)
}
#endif // BINARY_MATH_OP_H

245
noarch/include/Fastor/expressions/binary_ops/binary_mul_op.h
View File

@@ -1,245 +0,0 @@
#ifndef BINARY_MUL_OP_H
#define BINARY_MUL_OP_H
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryMulOp: public AbstractTensor<BinaryMulOp<TLhs, TRhs, DIM0>,DIM0> {
private:
expression_t<TLhs> _lhs;
expression_t<TRhs> _rhs;
public:
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<BinaryMulOp<TLhs, TRhs, DIM0>>::type;
using simd_vector_type = binary_op_simd_vector_t<BinaryMulOp<TLhs, TRhs, DIM0> >;
using simd_abi_type = typename simd_vector_type::abi_type;
FASTOR_INLINE BinaryMulOp(expression_t<TLhs> inlhs, expression_t<TRhs> inrhs) : _lhs((inlhs)), _rhs((inrhs)) {}
FASTOR_INLINE FASTOR_INDEX size() const {return helper_size<TLhs,TRhs>();}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");
#endif
return _rhs.size();
}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension<TLhs,TRhs>(i);}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");
#endif
return _rhs.dimension(i);
}
constexpr FASTOR_INLINE expression_t<TLhs> lhs() const {return _lhs;}
constexpr FASTOR_INLINE expression_t<TRhs> rhs() const {return _rhs;}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
return helper<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return _lhs.template eval<U>(i) * _rhs.template eval<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return (U)_lhs * _rhs.template eval<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return _lhs.template eval<U>(i) * (U)_rhs;
}
// scalar based
template<typename U>
FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return helper_s<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<U>(i) * _rhs.template eval_s<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return (U)_lhs * _rhs.template eval_s<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<U>(i) * (U)_rhs;
}
// for 2D tensors
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval<U>(i,j) * _rhs.template eval<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return (U)_lhs * _rhs.template eval<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval<U>(i,j) * (U)_rhs;
}
// scalar based (for 2D tensors)
template<typename U>
FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper_s<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<U>(i,j) * _rhs.template eval_s<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return (U)_lhs * _rhs.template eval_s<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<U>(i,j) * (U)_rhs;
}
// for nD tensors
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,DIM0> &as) const {
return thelper<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<U>(as) * _rhs.template teval<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return (U)_lhs * _rhs.template teval<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<U>(as) * (U)_rhs;
}
// scalar based (for nD tensors)
template<typename U>
FASTOR_INLINE U teval_s(const std::array<int,DIM0> &as) const {
return thelper_s<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<U>(as) * _rhs.template teval_s<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return (U)_lhs * _rhs.template teval_s<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<U>(as) * (U)_rhs;
}
};
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryMulOp<TLhs, TRhs, DIM0> operator*(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryMulOp<TLhs, TRhs, DIM0>(_lhs.self(), _rhs.self());
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryMulOp<TLhs, TRhs, DIM0> operator*(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {
return BinaryMulOp<TLhs, TRhs, DIM0>(_lhs.self(), bb);
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinaryMulOp<TLhs, TRhs, DIM0> operator*(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinaryMulOp<TLhs, TRhs, DIM0>(bb,_rhs.self());
}
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value &&
DIM0!=DIM1,bool>::type = 0 >
FASTOR_INLINE BinaryMulOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>
operator*(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM1> &_rhs) {
return BinaryMulOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(_lhs.self(), _rhs.self());
}
}
#endif // BINARY_MUL_OP_H

246
noarch/include/Fastor/expressions/binary_ops/binary_sub_op.h
View File

@@ -1,246 +0,0 @@
#ifndef BINARY_SUB_OP_H
#define BINARY_SUB_OP_H
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinarySubOp: public AbstractTensor<BinarySubOp<TLhs, TRhs, DIM0>,DIM0> {
private:
expression_t<TLhs> _lhs;
expression_t<TRhs> _rhs;
public:
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<BinarySubOp<TLhs, TRhs, DIM0>>::type;
using simd_vector_type = binary_op_simd_vector_t<BinarySubOp<TLhs, TRhs, DIM0> >;
using simd_abi_type = typename simd_vector_type::abi_type;
FASTOR_INLINE BinarySubOp(expression_t<TLhs> inlhs, expression_t<TRhs> inrhs) : _lhs((inlhs)), _rhs((inrhs)) {}
FASTOR_INLINE FASTOR_INDEX size() const {return helper_size<TLhs,TRhs>();}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _rhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {return _lhs.size();}
template<class LExpr, class RExpr,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_size() const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.size()==_lhs.size(),"EXPRESSION SIZE MISMATCH");
#endif
return _rhs.size();
}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return helper_dimension<TLhs,TRhs>(i);}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<LExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _rhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {return _lhs.dimension(i);}
template<class LExpr, class RExpr,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type =0 >
FASTOR_INLINE FASTOR_INDEX helper_dimension(FASTOR_INDEX i) const {
#ifndef NDEBUG
FASTOR_ASSERT(_rhs.dimension(i)==_lhs.dimension(i),"EXPRESSION SHAPE MISMATCH");
#endif
return _rhs.dimension(i);
}
constexpr FASTOR_INLINE expression_t<TLhs> lhs() const {return _lhs;}
constexpr FASTOR_INLINE expression_t<TRhs> rhs() const {return _rhs;}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
return helper<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return _lhs.template eval<U>(i) - _rhs.template eval<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return (U)_lhs - _rhs.template eval<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i) const {
return _lhs.template eval<U>(i) - (U)_rhs;
}
// scalar based
template<typename U>
FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return helper_s<TLhs,TRhs,U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<U>(i) - _rhs.template eval_s<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return (U)_lhs - _rhs.template eval_s<U>(i);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i) const {
return _lhs.template eval_s<U>(i) - (U)_rhs;
}
// for 2D tensors
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval<U>(i,j) - _rhs.template eval<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return (U)_lhs - _rhs.template eval<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> helper(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval<U>(i,j) - (U)_rhs;
}
// scalar based (for 2D tensors)
template<typename U>
FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return helper_s<TLhs,TRhs,U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<U>(i,j) - _rhs.template eval_s<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return (U)_lhs - _rhs.template eval_s<U>(i,j);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U helper_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _lhs.template eval_s<U>(i,j) - (U)_rhs;
}
// for nD tensors
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,DIM0> &as) const {
return thelper<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<U>(as) - _rhs.template teval<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return (U)_lhs - _rhs.template teval<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE SIMDVector<U,simd_abi_type> thelper(const std::array<int,DIM0> &as) const {
return _lhs.template teval<U>(as) - (U)_rhs;
}
// scalar based (for nD tensors)
template<typename U>
FASTOR_INLINE U teval_s(const std::array<int,DIM0> &as) const {
return thelper_s<TLhs,TRhs,U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<U>(as) - _rhs.template teval_s<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<std::is_arithmetic<LExpr>::value &&
!std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return (U)_lhs - _rhs.template teval_s<U>(as);
}
template<typename LExpr, typename RExpr, typename U,
typename std::enable_if<!std::is_arithmetic<LExpr>::value &&
std::is_arithmetic<RExpr>::value,bool>::type = 0>
FASTOR_INLINE U thelper_s(const std::array<int,DIM0> &as) const {
return _lhs.template teval_s<U>(as) - (U)_rhs;
}
};
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinarySubOp<TLhs, TRhs, DIM0> operator-(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinarySubOp<TLhs, TRhs, DIM0>(_lhs.self(), _rhs.self());
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinarySubOp<TLhs, TRhs, DIM0> operator-(const AbstractTensor<TLhs,DIM0> &_lhs, TRhs bb) {
return BinarySubOp<TLhs, TRhs, DIM0>(_lhs.self(), bb);
}
template<typename TLhs, typename TRhs, size_t DIM0,
typename std::enable_if<std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value,bool>::type = 0 >
FASTOR_INLINE BinarySubOp<TLhs, TRhs, DIM0> operator-(TLhs bb, const AbstractTensor<TRhs,DIM0> &_rhs) {
return BinarySubOp<TLhs, TRhs, DIM0>(bb,_rhs.self());
}
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
typename std::enable_if<!std::is_arithmetic<TLhs>::value &&
!std::is_arithmetic<TRhs>::value &&
DIM0!=DIM1,bool>::type = 0 >
FASTOR_INLINE BinarySubOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>
operator-(const AbstractTensor<TLhs,DIM0> &_lhs, const AbstractTensor<TRhs,DIM1> &_rhs) {
return BinarySubOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(_lhs.self(), _rhs.self());
}
}
#endif // BINARY_SUB_OP_H

327
noarch/include/Fastor/expressions/expression_traits.h
View File

@@ -1,327 +0,0 @@
#ifndef EXPRESSION_TRAITS_H
#define EXPRESSION_TRAITS_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include <type_traits>
namespace Fastor {
//------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_expression {
static constexpr bool value = false;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct is_expression<UnaryExpr<Expr,DIM>> {
static constexpr bool value = true;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct is_expression<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = true;
};
template<typename Derived>
static constexpr bool is_expression_v = is_expression<Derived>::value;
//------------------------------------------------------------------------------------------------//
// Expression binder - by value or ref
//------------------------------------------------------------------------------------------------//
template<class T>
struct expression_binder_type {
#ifndef FASTOR_COPY_EXPR
using type = conditional_t_<is_expression_v<T> || is_arithmetic_v_<T>, const T, const T&>;
#else
using type = const T;
#endif
};
template<class T>
using expression_t = typename expression_binder_type<T>::type;
//------------------------------------------------------------------------------------------------//
template<typename T, size_t ... Rest>
class Tensor;
template<typename T, size_t ... Rest>
class TensorMap;
template<typename T, size_t ... Rest>
class SingleValueTensor;
// traits
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_unary_bool_op {
static constexpr bool value = false;
};
template<typename Expr, size_t DIM>
struct is_unary_bool_op<UnaryNotOp<Expr,DIM>> {
static constexpr bool value = true;
};
template<typename Expr, size_t DIM>
struct is_unary_bool_op<UnaryIsinfOp<Expr,DIM>> {
static constexpr bool value = true;
};
template<typename Expr, size_t DIM>
struct is_unary_bool_op<UnaryIsnanOp<Expr,DIM>> {
static constexpr bool value = true;
};
template<typename Expr, size_t DIM>
struct is_unary_bool_op<UnaryIsfiniteOp<Expr,DIM>> {
static constexpr bool value = true;
};
template<size_t ... Rest>
struct is_unary_bool_op<Tensor<bool,Rest...>> {
static constexpr bool value = true;
};
template<size_t ... Rest>
struct is_unary_bool_op<TensorMap<bool,Rest...>> {
static constexpr bool value = true;
};
template<size_t ... Rest>
struct is_unary_bool_op<SingleValueTensor<bool,Rest...>> {
static constexpr bool value = true;
};
template<typename Derived>
static constexpr bool is_unary_bool_op_v = is_unary_bool_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_binary_cmp_op {
static constexpr bool value = false;
};
#define FASTOR_MAKE_IS_BINARY_CMP_OP(NAME) \
template<typename TLhs, typename TRhs, size_t DIMS>\
struct is_binary_cmp_op<BinaryCmpOp##NAME <TLhs,TRhs,DIMS>> {\
static constexpr bool value = true;\
};\
FASTOR_MAKE_IS_BINARY_CMP_OP(EQ)
FASTOR_MAKE_IS_BINARY_CMP_OP(NEQ)
FASTOR_MAKE_IS_BINARY_CMP_OP(LT)
FASTOR_MAKE_IS_BINARY_CMP_OP(GT)
FASTOR_MAKE_IS_BINARY_CMP_OP(LE)
FASTOR_MAKE_IS_BINARY_CMP_OP(GE)
FASTOR_MAKE_IS_BINARY_CMP_OP(AND)
FASTOR_MAKE_IS_BINARY_CMP_OP(OR)
template<size_t ... Rest>
struct is_binary_cmp_op<Tensor<bool,Rest...>> {
static constexpr bool value = true;
};
template<size_t ... Rest>
struct is_binary_cmp_op<TensorMap<bool,Rest...>> {
static constexpr bool value = true;
};
template<size_t ... Rest>
struct is_binary_cmp_op<SingleValueTensor<bool,Rest...>> {
static constexpr bool value = true;
};
template<typename Derived>
static constexpr bool is_binary_cmp_op_v = is_binary_cmp_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
/* Is boolean expresssion */
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_boolean_expression {
static constexpr bool value = is_unary_bool_op<Derived>::value || is_binary_cmp_op<Derived>::value;
};
template<typename Derived>
static constexpr bool is_boolean_expression_v = is_boolean_expression<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_tensor_view {
static constexpr bool value = false;
};
template<typename T, size_t DIMS, size_t M, size_t N, size_t ...Rest>
struct is_tensor_view<TensorViewExpr<Tensor<T,M,N,Rest...>,DIMS>> {
static constexpr bool value = true;
};
template<typename T, size_t DIMS, size_t M, size_t N, size_t ...Rest>
struct is_tensor_view<TensorConstViewExpr<Tensor<T,M,N,Rest...>,DIMS>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_tensor_view {
static constexpr bool value = is_tensor_view<Derived>::value ? true : false;
};
template<typename T, size_t DIMS, size_t M, size_t N, size_t ...Rest>
struct has_tensor_view<TensorViewExpr<Tensor<T,M,N,Rest...>,DIMS>> {
static constexpr bool value = true;
};
template<typename T, size_t DIMS, size_t M, size_t N, size_t ...Rest>
struct has_tensor_view<TensorConstViewExpr<Tensor<T,M,N,Rest...>,DIMS>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_tensor_view<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_tensor_view<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_tensor_view<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_tensor_view<TRhs>::value || has_tensor_view<TLhs>::value;
};
template<typename Derived>
static constexpr bool has_tensor_view_v = has_tensor_view<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_tensor_fixed_view_2d {
static constexpr bool value = false;
};
template<typename T, size_t M, size_t N, typename Seq0, typename Seq1>
struct is_tensor_fixed_view_2d<TensorFixedViewExpr2D<Tensor<T,M,N>,Seq0,Seq1,2>> {
static constexpr bool value = true;
};
template<typename T, size_t M, size_t N, typename Seq0, typename Seq1>
struct is_tensor_fixed_view_2d<TensorConstFixedViewExpr2D<Tensor<T,M,N>,Seq0,Seq1,2>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_tensor_fixed_view_2d {
static constexpr bool value = is_tensor_fixed_view_2d<Derived>::value ? true : false;
};
template<typename T, size_t M, size_t N, typename Seq0, typename Seq1>
struct has_tensor_fixed_view_2d<TensorFixedViewExpr2D<Tensor<T,M,N>,Seq0,Seq1,2>> {
static constexpr bool value = true;
};
template<typename T, size_t M, size_t N, typename Seq0, typename Seq1>
struct has_tensor_fixed_view_2d<TensorConstFixedViewExpr2D<Tensor<T,M,N>,Seq0,Seq1,2>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_tensor_fixed_view_2d<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_tensor_fixed_view_2d<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_tensor_fixed_view_2d<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_tensor_fixed_view_2d<TRhs>::value || has_tensor_fixed_view_2d<TLhs>::value;
};
template<typename Derived>
static constexpr bool has_tensor_fixed_view_2d_v = has_tensor_fixed_view_2d<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_tensor_fixed_view_nd {
static constexpr bool value = false;
};
template<typename T, size_t ...Rest, typename ...Fseq>
struct is_tensor_fixed_view_nd<TensorFixedViewExprnD<Tensor<T,Rest...>,Fseq...>> {
static constexpr bool value = true;
};
template<typename T, size_t ...Rest, typename ...Fseq>
struct is_tensor_fixed_view_nd<TensorConstFixedViewExprnD<Tensor<T,Rest...>,Fseq...>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_tensor_fixed_view_nd {
static constexpr bool value = is_tensor_fixed_view_nd<Derived>::value ? true : false;
};
template<typename T, size_t ...Rest, typename ...Fseq>
struct has_tensor_fixed_view_nd<TensorFixedViewExprnD<Tensor<T,Rest...>,Fseq...>> {
static constexpr bool value = true;
};
template<typename T, size_t ...Rest, typename ...Fseq>
struct has_tensor_fixed_view_nd<TensorConstFixedViewExprnD<Tensor<T,Rest...>,Fseq...>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_tensor_fixed_view_nd<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_tensor_fixed_view_nd<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_tensor_fixed_view_nd<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_tensor_fixed_view_nd<TRhs>::value || has_tensor_fixed_view_nd<TLhs>::value;
};
template<typename Derived>
static constexpr bool has_tensor_fixed_view_nd_v = has_tensor_fixed_view_nd<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------------------//
template<typename T, typename T2 = void>
struct get_binary_arithmetic_result_type;
template<typename T>
struct get_binary_arithmetic_result_type<T, enable_if_t_<is_primitive_v_<T> > > {
using type = T;
};
template<typename T>
struct get_binary_arithmetic_result_type<T, enable_if_t_<!is_primitive_v_<T> > > {
using type = typename T::result_type;
};
template<class Derived>
struct binary_arithmetic_result_type;
template<template<typename,typename,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIM0>
struct binary_arithmetic_result_type< BinaryExpr<TLhs,TRhs,DIM0> > {
using type = conditional_t_<!is_primitive_v_<TLhs>,
typename get_binary_arithmetic_result_type<TLhs>::type,
typename get_binary_arithmetic_result_type<TRhs>::type >;
};
template<class Derived>
using binary_arithmetic_result_t = typename binary_arithmetic_result_type<Derived>::type;
//------------------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------------------//
template<typename T, typename T2 = void>
struct get_binary_op_simd_vector_type;
template<typename T>
struct get_binary_op_simd_vector_type<T, enable_if_t_<is_primitive_v_<T> > > {
using type = T;
};
template<typename T>
struct get_binary_op_simd_vector_type<T, enable_if_t_<!is_primitive_v_<T> > > {
using type = typename T::simd_vector_type;
};
template<class Derived>
struct binary_op_simd_vector_type;
template<template<typename,typename,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIM0>
struct binary_op_simd_vector_type< BinaryExpr<TLhs,TRhs,DIM0> > {
using type = conditional_t_<!is_primitive_v_<TLhs>,
typename get_binary_op_simd_vector_type<TLhs>::type,
typename get_binary_op_simd_vector_type<TRhs>::type >;
};
template<class Derived>
using binary_op_simd_vector_t = typename binary_op_simd_vector_type<Derived>::type;
//------------------------------------------------------------------------------------------------//
template<class T>
struct scalar_type_finder;
} // end of namespace Fastor
#endif // EXPRESSION_TRAITS_H

27
noarch/include/Fastor/expressions/expressions.h
View File

@@ -1,27 +0,0 @@
#ifndef EXPRESSIONS_H
#define EXPRESSIONS_H
#include "Fastor/expressions/binary_ops/binary_arithmetic_ops.h"
// #include "Fastor/expressions/binary_ops/binary_add_op.h"
// #include "Fastor/expressions/binary_ops/binary_sub_op.h"
// #include "Fastor/expressions/binary_ops/binary_mul_op.h"
#include "Fastor/expressions/binary_ops/binary_div_op.h"
#include "Fastor/expressions/binary_ops/binary_arithmetic_assignment.h"
#include "Fastor/expressions/binary_ops/binary_math_ops.h"
#include "Fastor/expressions/binary_ops/binary_cmp_ops.h"
#include "Fastor/expressions/unary_ops/unary_math_ops.h"
#include "Fastor/expressions/unary_ops/unary_bool_ops.h"
#include "Fastor/expressions/linalg_ops/linalg_ops.h"
#include "Fastor/expressions/views/tensor_fixed_views_1d.h"
#include "Fastor/expressions/views/tensor_fixed_views_2d.h"
#include "Fastor/expressions/views/tensor_fixed_views_nd.h"
#include "Fastor/expressions/views/tensor_random_views.h"
#include "Fastor/expressions/views/tensor_filter_views.h"
#include "Fastor/expressions/views/tensor_diag_views.h"
#include "Fastor/expressions/views/tensor_views.h"
#include "Fastor/expressions/views/tensor_views_assignment.h"
#endif // EXPRESSIONS_H

191
noarch/include/Fastor/expressions/linalg_ops/binary_cross_op.h
View File

@@ -1,191 +0,0 @@
#ifndef BINARY_CROSS_OP_H
#define BINARY_CROSS_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/backend/tensor_cross.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
// classic vector cross product
template<typename T, size_t I, enable_if_t_<I==3,bool> = false>
FASTOR_INLINE Tensor<T,I> cross(const Tensor<T,I> &a, const Tensor<T,I> &b) {
Tensor<T,I> out;
_vector_crossproduct<T,I>(a.data(),b.data(),out.data());
return out;
}
template<typename T, size_t I, enable_if_t_<I==2,bool> = false>
FASTOR_INLINE Tensor<T,I+1> cross(const Tensor<T,I> &a, const Tensor<T,I> &b) {
Tensor<T,I+1> out;
_vector_crossproduct<T,I>(a.data(),b.data(),out.data());
return out;
}
// Tensor cross product of two 2nd order tensors
template<typename T, size_t I, size_t J, typename std::enable_if<I==3 && J==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J> cross(const Tensor<T,I,J> &b, const Tensor<T,I,J> &a) {
Tensor<T,I,J> out;
_crossproduct<T,I,I,J>(a.data(),b.data(),out.data());
return out;
}
template<typename T, size_t I, size_t J, typename std::enable_if<I==2 && J==2,bool>::type=0>
FASTOR_INLINE Tensor<T,I+1,J+1> cross(const Tensor<T,I,J> &b, const Tensor<T,I,J> &a) {
Tensor<T,I+1,J+1> out;
_crossproduct<T,I,I,J>(a.data(),b.data(),out.data());
return out;
}
template<int Plane, typename T, size_t I, size_t J, typename std::enable_if<I==2 && J==2 && Plane==FASTOR_PlaneStrain,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J> cross(const Tensor<T,I,J> &b, const Tensor<T,I,J> &a) {
// Plane strain case
Tensor<T,I,J> out;
Tensor<T,I+1,J+1> a3, b3;
a3(0,0) = a(0,0);
a3(0,1) = a(0,1);
a3(1,0) = a(1,0);
a3(1,1) = a(1,1);
a3(2,2) = 1;
b3(0,0) = b(0,0);
b3(0,1) = b(0,1);
b3(1,0) = b(1,0);
b3(1,1) = b(1,1);
b3(2,2) = 1;
_crossproduct<T,FASTOR_PlaneStrain>(a.data(),b.data(),out.data());
return out;
}
// Tensor cross product of a vector with 2nd order tensor
template<typename T, size_t I, size_t J, typename std::enable_if<I==3 && J==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J> cross(const Tensor<T,I> &b, const Tensor<T,I,J> &a) {
Tensor<T,I,J> out;
_crossproduct<T,I,1,J>(a.data(),b.data(),out.data());
return out;
}
template<typename T, size_t I, size_t J, typename std::enable_if<I==2 && J==2,bool>::type=0>
FASTOR_INLINE Tensor<T,I+1,J+1> cross(const Tensor<T,I> &b, const Tensor<T,I,J> &a) {
Tensor<T,I+1,J+1> out;
_crossproduct<T,I,1,J>(a.data(),b.data(),out.data());
return out;
}
// Tensor cross product of a 2nd order tensor with a vector
template<typename T, size_t I, size_t J, typename std::enable_if<I==3 && J==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J> cross(const Tensor<T,I,J> &b, const Tensor<T,J> &a) {
Tensor<T,I,J> out;
_crossproduct<T,I,J,1>(a.data(),b.data(),out.data());
return out;
}
template<typename T, size_t I, size_t J, typename std::enable_if<I==2 && J==2,bool>::type=0>
FASTOR_INLINE Tensor<T,I+1,J+1> cross(const Tensor<T,I,J> &b, const Tensor<T,J> &a) {
Tensor<T,I+1,J+1> out;
_crossproduct<T,I,J,1>(a.data(),b.data(),out.data());
return out;
}
// Tensor cross product of a 3rd order tensors
template<typename T, size_t I, size_t J, size_t K, size_t L, size_t M, size_t N,
typename std::enable_if<I==3 && J==3 && K==3 && L==3 && M==3 && N==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,3,3,N> cross(const Tensor<T,I,J,K> &A, const Tensor<T,L,M,N> &B) {
Tensor<T,I,3,3,N> C;
_crossproduct<T,I,J,K,L,M,N>(A.data(),B.data(),C.data());
return C;
}
// Tensor cross product of a 4th order tensors
template<typename T, size_t I, size_t J, size_t K, size_t L, size_t M, size_t N, size_t O, size_t P,
typename std::enable_if<I==3 && J==3 && K==3 && L==3 && M==3 && N==3 && O==3 && P==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J,3,3,O,P> cross(const Tensor<T,I,J,K,L> &A, const Tensor<T,M,N,O,P> &B) {
Tensor<T,I,J,3,3,O,P> C;
_crossproduct<T,I,J,K,L,M,N,O,P>(A.data(),B.data(),C.data());
return C;
}
// Tensor cross product of a 4th order tensor with 2nd order tensor
template<typename T, size_t I, size_t J, size_t K, size_t L, size_t M, size_t N,
typename std::enable_if<I==3 && J==3 && K==3 && L==3 && M==3 && N==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J,K,L> cross(const Tensor<T,I,J,K,L> &A, const Tensor<T,M,N> &B) {
Tensor<T,I,J,K,L> C;
_crossproduct42<T,I,J,K,L,M,N>(A.data(),B.data(),C.data());
return C;
}
// Tensor cross product of a 2nd order tensor with 4th order tensor
template<typename T, size_t I, size_t J, size_t K, size_t L, size_t M, size_t N,
typename std::enable_if<I==3 && J==3 && K==3 && L==3 && M==3 && N==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J,K,L> cross(const Tensor<T,M,N> &A, const Tensor<T,I,J,K,L> &B) {
Tensor<T,I,J,K,L> C;
_crossproduct24<T,I,J,K,L,M,N>(B.data(),A.data(),C.data());
return C;
}
// Tensor cross product of a 3rd order tensor with 2nd order tensor
template<typename T, size_t I, size_t J, size_t K, size_t L, size_t M,
typename std::enable_if<I==3 && J==3 && K==3 && L==3 && M==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J,K> cross(const Tensor<T,I,J,K> &A, const Tensor<T,L,M> &B) {
Tensor<T,I,J,K> C;
_crossproduct32<T,I,J,K,L,M>(A.data(),B.data(),C.data());
return C;
}
// Tensor cross product of a 2nd order tensor with 3rd order tensor
template<typename T, size_t I, size_t J, size_t K, size_t L, size_t M,
typename std::enable_if<I==3 && J==3 && K==3 && L==3 && M==3,bool>::type=0>
FASTOR_INLINE Tensor<T,I,J,K> cross(const Tensor<T,L,M> &A, const Tensor<T,I,J,K> &B) {
Tensor<T,I,J,K> C;
_crossproduct23<T,I,J,K,L,M>(B.data(),A.data(),C.data());
return C;
}
#if FASTOR_CXX_VERSION >= 2014
// Tensor cross product for generic expressions
template<typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<is_tensor_v<Derived0> && !is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
auto
cross(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using rhs_type = typename Derived1::result_type;
const rhs_type tmp_b(b);
return cross(a.self(),tmp_b);
}
template<typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<!is_tensor_v<Derived0> && is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
auto
cross(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using lhs_type = typename Derived0::result_type;
const lhs_type tmp_a(a);
return cross(tmp_a,b.self());
}
template<typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<!is_tensor_v<Derived0> && !is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
auto
cross(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using lhs_type = typename Derived0::result_type;
using rhs_type = typename Derived1::result_type;
const lhs_type tmp_a(a);
const rhs_type tmp_b(b);
return cross(tmp_a,tmp_b);
}
#endif
} // end of namespace Fastor
#endif // BINARY_CROSS_OP_H

627
noarch/include/Fastor/expressions/linalg_ops/binary_matmul_op.h
View File

@@ -1,627 +0,0 @@
#ifndef BINARY_MATMUL_OP_H
#define BINARY_MATMUL_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/backend/inner.h"
#include "Fastor/backend/matmul/matmul.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryMatMulOp: public AbstractTensor<BinaryMatMulOp<TLhs, TRhs, DIM0>,DIM0> {
using lhs_expr_type = expression_t<TLhs>;
using rhs_expr_type = expression_t<TRhs>;
using lhs_type = typename TLhs::result_type;
using rhs_type = typename TRhs::result_type;
static constexpr FASTOR_INDEX lhs_rank = lhs_type::dimension_t::value;
static constexpr FASTOR_INDEX rhs_rank = rhs_type::dimension_t::value;
static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>;
static constexpr FASTOR_INDEX K = get_tensor_dimension_v<1,lhs_type>;
static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,rhs_type>;
static constexpr FASTOR_INDEX K_other = get_tensor_dimension_v<0,rhs_type>;
static constexpr FASTOR_INDEX flop_count = M*N*K;
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<BinaryMatMulOp<TLhs, TRhs, DIM0>>::type;
// does not matter which one as matmul bypasses this
using simd_vector_type = typename TLhs::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = conditional_t_<lhs_rank == 1, // vector-matrix
Tensor<scalar_type,N> ,
conditional_t_<rhs_rank == 1, // matrix-vector
Tensor<scalar_type,M>,
Tensor<scalar_type,M,N> // matrix-matrix
>
>;
FASTOR_INLINE BinaryMatMulOp(lhs_expr_type inlhs, rhs_expr_type inrhs) : _lhs(inlhs), _rhs(inrhs) {
static_assert(lhs_rank <=2 && rhs_rank <=2, "EXPRESSIONS FOR MATRIX MULTIPLICATION HAVE TO BE 2-DIMENSIONAL");
static_assert(K == K_other, "INVALID MATMUL OPERANDS. COLUMNS(A)!=ROWS(B)");
}
constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M*N;}
constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i==0 ? M : N;}
constexpr FASTOR_INLINE lhs_expr_type lhs() const {return _lhs;}
constexpr FASTOR_INLINE rhs_expr_type rhs() const {return _rhs;}
private:
lhs_expr_type _lhs;
rhs_expr_type _rhs;
};
template<typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
enable_if_t_<!is_arithmetic_v_<TLhs> &&!is_arithmetic_v_<TRhs>,bool> = 0 >
constexpr FASTOR_INLINE BinaryMatMulOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>
operator %(const AbstractTensor<TLhs,DIM0> &lhs, const AbstractTensor<TRhs,DIM1> &rhs) {
return BinaryMatMulOp<TLhs, TRhs, meta_min<DIM0,DIM1>::value>(lhs.self(), rhs.self());
}
// helper dispatcher functions
namespace internal {
// till streaming gemm is implemented
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _gemm(const T alpha, const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, const T beta, T * FASTOR_RESTRICT c) {
FASTOR_ARCH_ALIGN T tmp[M*N];
if (beta == 0) {
// non-streaming
_matmul<T,M,K,N>(a,b,tmp);
for (size_t i = 0; i<M*N; ++i)
c[i] = alpha * tmp[i];
}
else {
// streaming
_matmul<T,M,K,N>(a,b,tmp);
for (size_t i = 0; i<M*N; ++i)
c[i] = alpha * tmp[i] + beta*c[i];
}
}
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _gemm_mul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
FASTOR_ARCH_ALIGN T tmp[M*N];
_matmul<T,M,K,N>(a,b,tmp);
for (size_t i = 0; i<M*N; ++i)
c[i] *= tmp[i];
}
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _gemm_div(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
FASTOR_ARCH_ALIGN T tmp[M*N];
_matmul<T,M,K,N>(a,b,tmp);
for (size_t i = 0; i<M*N; ++i)
c[i] /= tmp[i];
}
// matmul - matvec overloads
template<typename T, size_t I, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher(const Tensor<T,I,J> &a, const Tensor<T,J,K> &b, Tensor<T,I,K> &out) {
FASTOR_IF_CONSTEXPR(J==1) {
_dyadic<T,I,K>(a.data(),b.data(),out.data());
}
else FASTOR_IF_CONSTEXPR(I==1 && J!=1 && K==1) {
out.data()[0] = _inner<T,J>(a.data(),b.data());
}
else {
_matmul<T,I,J,K>(a.data(),b.data(),out.data());
}
}
template<typename T, size_t I, size_t J>
FASTOR_INLINE void matmul_dispatcher(const Tensor<T,I,J> &a, const Tensor<T,J> &b, Tensor<T,I> &out) {
_matmul<T,I,J,1>(a.data(),b.data(),out.data());
}
template<typename T, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher(const Tensor<T,J> &a, const Tensor<T,J,K> &b, Tensor<T,K> &out) {
_matmul<T,1,J,K>(a.data(),b.data(),out.data());
}
template<typename T, size_t I, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor<T,I,J> &a, const Tensor<T,J,K> &b, const T beta, Tensor<T,I,K> &out) {
_gemm<T,I,J,K>(alpha,a.data(),b.data(),beta,out.data());
}
template<typename T, size_t I, size_t J>
FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor<T,I,J> &a, const Tensor<T,J> &b, const T beta, Tensor<T,I> &out) {
_gemm<T,I,J,1>(alpha,a.data(),b.data(),beta,out.data());
}
template<typename T, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher(const T alpha, const Tensor<T,J> &a, const Tensor<T,J,K> &b, const T beta, Tensor<T,K> &out) {
_gemm<T,1,J,K>(alpha,a.data(),b.data(),beta,out.data());
}
template<typename T, size_t I, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher_mul(const Tensor<T,I,J> &a, const Tensor<T,J,K> &b, Tensor<T,I,K> &out) {
_gemm_mul<T,I,J,K>(a.data(),b.data(),out.data());
}
template<typename T, size_t I, size_t J>
FASTOR_INLINE void matmul_dispatcher_mul(const Tensor<T,I,J> &a, const Tensor<T,J> &b, Tensor<T,I> &out) {
_gemm_mul<T,I,J,1>(a.data(),b.data(),out.data());
}
template<typename T, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher_mul(const Tensor<T,J> &a, const Tensor<T,J,K> &b, Tensor<T,K> &out) {
_gemm_mul<T,1,J,K>(a.data(),b.data(),out.data());
}
template<typename T, size_t I, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher_div(const Tensor<T,I,J> &a, const Tensor<T,J,K> &b, Tensor<T,I,K> &out) {
_gemm_div<T,I,J,K>(a.data(),b.data(),out.data());
}
template<typename T, size_t I, size_t J>
FASTOR_INLINE void matmul_dispatcher_div(const Tensor<T,I,J> &a, const Tensor<T,J> &b, Tensor<T,I> &out) {
_gemm_div<T,I,J,1>(a.data(),b.data(),out.data());
}
template<typename T, size_t J, size_t K>
FASTOR_INLINE void matmul_dispatcher_div(const Tensor<T,J> &a, const Tensor<T,J,K> &b, Tensor<T,K> &out) {
_gemm_div<T,1,J,K>(a.data(),b.data(),out.data());
}
} // internal
// For tensors
template<typename T, size_t I, size_t J, size_t K>
FASTOR_INLINE Tensor<T,I,K> matmul(const Tensor<T,I,J> &a, const Tensor<T,J,K> &b) {
Tensor<T,I,K> out;
FASTOR_IF_CONSTEXPR(J==1) {
_dyadic<T,I,K>(a.data(),b.data(),out.data());
}
else FASTOR_IF_CONSTEXPR(I==1 && J!=1 && K==1) {
out.data()[0] = _inner<T,J>(a.data(),b.data());
}
else {
_matmul<T,I,J,K>(a.data(),b.data(),out.data());
}
return out;
}
template<typename T, size_t I, size_t J>
FASTOR_INLINE Tensor<T,I> matmul(const Tensor<T,I,J> &a, const Tensor<T,J> &b) {
Tensor<T,I> out;
_matmul<T,I,J,1>(a.data(),b.data(),out.data());
return out;
}
template<typename T, size_t J, size_t K>
FASTOR_INLINE Tensor<T,K> matmul(const Tensor<T,J> &a, const Tensor<T,J,K> &b) {
Tensor<T,K> out;
_matmul<T,1,J,K>(a.data(),b.data(),out.data());
return out;
}
// Generic matmul function for AbstractTensor types are provided here
template<typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<is_less_equal_v_<DIM0,2> && is_less_equal_v_<DIM1,2>
&& !is_tensor_v<Derived0> && !is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
conditional_t_<Derived0::result_type::dimension_t::value == 1,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<1,typename Derived1::result_type> >,
conditional_t_<Derived1::result_type::dimension_t::value == 1,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<0,typename Derived0::result_type> >,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<0,typename Derived0::result_type> ,
get_tensor_dimension_v<1,typename Derived1::result_type> >
>
>
matmul(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using lhs_type = typename Derived0::result_type;
using rhs_type = typename Derived1::result_type;
const lhs_type tmp_a(a);
const rhs_type tmp_b(b);
return matmul(tmp_a,tmp_b);
}
template<typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<is_less_equal_v_<DIM0,2> && is_less_equal_v_<DIM1,2>
&& !is_tensor_v<Derived0> && is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
conditional_t_<Derived0::result_type::dimension_t::value == 1,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<1,typename Derived1::result_type> >,
conditional_t_<Derived1::result_type::dimension_t::value == 1,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<0,typename Derived0::result_type> >,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<0,typename Derived0::result_type> ,
get_tensor_dimension_v<1,typename Derived1::result_type> >
>
>
matmul(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using lhs_type = typename Derived0::result_type;
const lhs_type tmp_a(a);
return matmul(tmp_a,b.self());
}
template<typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<is_less_equal_v_<DIM0,2> && is_less_equal_v_<DIM1,2>
&& is_tensor_v<Derived0> && !is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
conditional_t_<Derived0::result_type::dimension_t::value == 1,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<1,typename Derived1::result_type> >,
conditional_t_<Derived1::result_type::dimension_t::value == 1,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<0,typename Derived0::result_type> >,
Tensor<typename Derived0::scalar_type,
get_tensor_dimension_v<0,typename Derived0::result_type> ,
get_tensor_dimension_v<1,typename Derived1::result_type> >
>
>
matmul(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using rhs_type = typename Derived1::result_type;
const rhs_type tmp_b(b);
return matmul(a.self(),tmp_b);
}
// triangular matmul functions
template<typename LhsType = UpLoType::General, typename RhsType = UpLoType::General, typename T, size_t M, size_t K, size_t N>
Tensor<T,M,N> tmatmul(const Tensor<T,M,K> &a, const Tensor<T,K,N> &b) {
Tensor<T,M,N> out;
_tmatmul<T,M,K,N,LhsType,RhsType>(a.data(),b.data(),out.data());
return out;
}
template<typename LhsType = UpLoType::General, typename RhsType = UpLoType::General, typename T, size_t I, size_t J>
FASTOR_INLINE Tensor<T,I> tmatmul(const Tensor<T,I,J> &a, const Tensor<T,J> &b) {
Tensor<T,I> out;
_tmatmul<T,I,J,1,LhsType,RhsType>(a.data(),b.data(),out.data());
return out;
}
template<typename LhsType = UpLoType::General, typename RhsType = UpLoType::General, typename T, size_t J, size_t K>
FASTOR_INLINE Tensor<T,K> tmatmul(const Tensor<T,J> &a, const Tensor<T,J,K> &b) {
Tensor<T,K> out;
_tmatmul<T,1,J,K,LhsType,RhsType>(a.data(),b.data(),out.data());
return out;
}
#if FASTOR_CXX_VERSION >= 2014
// tmatmul for generic expressions
template<typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<is_less_equal_v_<DIM0,2> && is_less_equal_v_<DIM1,2>
&& !is_tensor_v<Derived0> && !is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
decltype(auto)
tmatmul(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using lhs_type = typename Derived0::result_type;
using rhs_type = typename Derived1::result_type;
const lhs_type tmp_a(a);
const rhs_type tmp_b(b);
return tmatmul<LhsType,RhsType>(tmp_a,tmp_b);
}
template<typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<is_less_equal_v_<DIM0,2> && is_less_equal_v_<DIM1,2>
&& !is_tensor_v<Derived0> && is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
decltype(auto)
tmatmul(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using lhs_type = typename Derived0::result_type;
const lhs_type tmp_a(a);
return tmatmul<LhsType,RhsType>(tmp_a,b.self());
}
template<typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename Derived0, size_t DIM0, typename Derived1, size_t DIM1,
enable_if_t_<is_less_equal_v_<DIM0,2> && is_less_equal_v_<DIM1,2>
&& is_tensor_v<Derived0> && !is_tensor_v<Derived1>,bool> = 0 >
FASTOR_INLINE
decltype(auto)
tmatmul(const AbstractTensor<Derived0,DIM0> &a, const AbstractTensor<Derived1,DIM1> &b) {
using rhs_type = typename Derived1::result_type;
const rhs_type tmp_b(b);
return tmatmul<LhsType,RhsType>(a.self(),tmp_b);
}
#endif
// assignments
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
// dst = matmul(src.lhs().self(),src.rhs().self()); // this makes a copy for dst, compiler emits a memcpy
internal::matmul_dispatcher(src.lhs().self(),src.rhs().self(),dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
lhs_t a(src.lhs().self());
// dst = matmul(a,src.rhs().self()); // this makes a copy for dst, compiler emits a memcpy
internal::matmul_dispatcher(a,src.rhs().self(),dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
rhs_t b(src.rhs().self());
// dst = matmul(src.lhs().self(),b); // this makes a copy for dst, compiler emits a memcpy
internal::matmul_dispatcher(src.lhs().self(),b,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
lhs_t a(src.lhs().self());
rhs_t b(src.rhs().self());
// dst = matmul(a,b); // this makes a copy for dst, compiler emits a memcpy
internal::matmul_dispatcher(a,b,dst.self());
}
// assignments add
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
internal::matmul_dispatcher((T)1,src.lhs().self(),src.rhs().self(),(T)1,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
lhs_t a(src.lhs().self());
internal::matmul_dispatcher((T)1,a,src.rhs().self(),(T)1,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
rhs_t b(src.rhs().self());
internal::matmul_dispatcher((T)1,src.lhs().self(),b,(T)1,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
lhs_t a(src.lhs().self());
rhs_t b(src.rhs().self());
internal::matmul_dispatcher((T)1,a,b,(T)1,dst.self());
}
// assignments sub
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
internal::matmul_dispatcher((T)-1,src.lhs().self(),src.rhs().self(),(T)1,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
lhs_t a(src.lhs().self());
internal::matmul_dispatcher((T)-1,a,src.rhs().self(),(T)1,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
rhs_t b(src.rhs().self());
internal::matmul_dispatcher((T)-1,src.lhs().self(),b,(T)1,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using T = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::scalar_type;
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
lhs_t a(src.lhs().self());
rhs_t b(src.rhs().self());
internal::matmul_dispatcher((T)-1,a,b,(T)1,dst.self());
}
// assignments mul
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
internal::matmul_dispatcher_mul(src.lhs().self(),src.rhs().self(),dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
lhs_t a(src.lhs().self());
internal::matmul_dispatcher_mul(a,src.rhs().self(),dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
rhs_t b(src.rhs().self());
internal::matmul_dispatcher_mul(src.lhs().self(),b,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
lhs_t a(src.lhs().self());
rhs_t b(src.rhs().self());
internal::matmul_dispatcher_mul(a,b,dst.self());
}
// assignments div
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
internal::matmul_dispatcher_div(src.lhs().self(),src.rhs().self(),dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
lhs_t a(src.lhs().self());
internal::matmul_dispatcher_div(a,src.rhs().self(),dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
rhs_t b(src.rhs().self());
internal::matmul_dispatcher_div(src.lhs().self(),b,dst.self());
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM,
typename std::enable_if<!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_expr_type>> &&
!is_tensor_v<remove_all_t<typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_expr_type>>, bool >::type = false>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<TLhs, TRhs, OtherDIM> &src) {
using lhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::lhs_type;
using rhs_t = typename BinaryMatMulOp<TLhs, TRhs, OtherDIM>::rhs_type;
lhs_t a(src.lhs().self());
rhs_t b(src.rhs().self());
internal::matmul_dispatcher_div(a,b,dst.self());
}
// recursive greedy-like
template<typename Derived, size_t DIM, typename TLhs, typename TRhs0, typename TRhs1, size_t OtherDIM>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<BinaryMatMulOp<TLhs, TRhs0, OtherDIM>, TRhs1, OtherDIM> &src) {
FASTOR_IF_CONSTEXPR(BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::flop_count > BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::flop_count)
{
using result_t = typename BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>(src.lhs().rhs().self(),src.rhs().self()));
assign(dst, BinaryMatMulOp<TLhs, result_t, OtherDIM>(src.lhs().lhs().self(),tmp));
}
else
{
using result_t = typename BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TLhs, TRhs0, OtherDIM>(src.lhs().lhs().self(),src.lhs().rhs().self()));
assign(dst, BinaryMatMulOp<result_t, TRhs1, OtherDIM>(tmp,src.rhs().self()));
}
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs0, typename TRhs1, size_t OtherDIM>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<BinaryMatMulOp<TLhs, TRhs0, OtherDIM>, TRhs1, OtherDIM> &src) {
FASTOR_IF_CONSTEXPR(BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::flop_count > BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::flop_count)
{
using result_t = typename BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>(src.lhs().rhs().self(),src.rhs().self()));
assign_add(dst, BinaryMatMulOp<TLhs, result_t, OtherDIM>(src.lhs().lhs().self(),tmp));
}
else
{
using result_t = typename BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TLhs, TRhs0, OtherDIM>(src.lhs().lhs().self(),src.lhs().rhs().self()));
assign_add(dst, BinaryMatMulOp<result_t, TRhs1, OtherDIM>(tmp,src.rhs().self()));
}
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs0, typename TRhs1, size_t OtherDIM>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<BinaryMatMulOp<TLhs, TRhs0, OtherDIM>, TRhs1, OtherDIM> &src) {
FASTOR_IF_CONSTEXPR(BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::flop_count > BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::flop_count)
{
using result_t = typename BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>(src.lhs().rhs().self(),src.rhs().self()));
assign_sub(dst, BinaryMatMulOp<TLhs, result_t, OtherDIM>(src.lhs().lhs().self(),tmp));
}
else
{
using result_t = typename BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TLhs, TRhs0, OtherDIM>(src.lhs().lhs().self(),src.lhs().rhs().self()));
assign_sub(dst, BinaryMatMulOp<result_t, TRhs1, OtherDIM>(tmp,src.rhs().self()));
}
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs0, typename TRhs1, size_t OtherDIM>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<BinaryMatMulOp<TLhs, TRhs0, OtherDIM>, TRhs1, OtherDIM> &src) {
FASTOR_IF_CONSTEXPR(BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::flop_count > BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::flop_count)
{
using result_t = typename BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>(src.lhs().rhs().self(),src.rhs().self()));
assign_mul(dst, BinaryMatMulOp<TLhs, result_t, OtherDIM>(src.lhs().lhs().self(),tmp));
}
else
{
using result_t = typename BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TLhs, TRhs0, OtherDIM>(src.lhs().lhs().self(),src.lhs().rhs().self()));
assign_mul(dst, BinaryMatMulOp<result_t, TRhs1, OtherDIM>(tmp,src.rhs().self()));
}
}
template<typename Derived, size_t DIM, typename TLhs, typename TRhs0, typename TRhs1, size_t OtherDIM>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const BinaryMatMulOp<BinaryMatMulOp<TLhs, TRhs0, OtherDIM>, TRhs1, OtherDIM> &src) {
FASTOR_IF_CONSTEXPR(BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::flop_count > BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::flop_count)
{
using result_t = typename BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TRhs0, TRhs1, OtherDIM>(src.lhs().rhs().self(),src.rhs().self()));
assign_div(dst, BinaryMatMulOp<TLhs, result_t, OtherDIM>(src.lhs().lhs().self(),tmp));
}
else
{
using result_t = typename BinaryMatMulOp<TLhs, TRhs0, OtherDIM>::result_type;
result_t tmp;
assign(tmp, BinaryMatMulOp<TLhs, TRhs0, OtherDIM>(src.lhs().lhs().self(),src.lhs().rhs().self()));
assign_div(dst, BinaryMatMulOp<result_t, TRhs1, OtherDIM>(tmp,src.rhs().self()));
}
}
} // end of namespace Fastor
#endif // BINARY_MATMUL_OP_H

197
noarch/include/Fastor/expressions/linalg_ops/binary_solve_op.h
View File

@@ -1,197 +0,0 @@
#ifndef BINARY_SOLVE_OP_H
#define BINARY_SOLVE_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/binary_matmul_op.h"
#include "Fastor/expressions/linalg_ops/unary_inv_op.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/unary_piv_op.h"
namespace Fastor {
// Solving using LU decomposition is in the LU module [unary_lu_op]
// For tensors
// Solve - no pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t I,
enable_if_t_< SType == SolveCompType::SimpleInv, bool> = false>
FASTOR_INLINE Tensor<T,I> solve(const Tensor<T,I,I> &A, const Tensor<T,I> &b) {
return matmul(inverse<InvCompType::SimpleInv>(A),b);
}
// Solve - pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t I,
enable_if_t_< SType == SolveCompType::SimpleInvPiv, bool> = false>
FASTOR_INLINE Tensor<T,I> solve(const Tensor<T,I,I> &A, const Tensor<T,I> &b) {
// We need to post multiply - swap columns using row permutation vector
Tensor<size_t,I> p;
pivot_inplace(A,p);
auto tmp(apply_pivot(A,p));
Tensor<T,I,I> invA = inverse<InvCompType::SimpleInv>(tmp);
return matmul(reconstruct_colwise(invA,p),b);
// // matrix version
// Tensor<T,I,I> p;
// pivot_inplace(A,p);
// auto tmp(apply_pivot(A,p));
// Tensor<T,I,I> invA = inverse<InvCompType::SimpleInv>(tmp);
// return matmul(matmul(invA,p),b);
}
// Multiple right hand sides
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t I, size_t J,
enable_if_t_< SType == SolveCompType::SimpleInv, bool> = false>
FASTOR_INLINE Tensor<T,I,J> solve(const Tensor<T,I,I> &A, const Tensor<T,I,J> &B) {
return matmul(inverse(A),B);
}
// Solve - pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t I, size_t J,
enable_if_t_< SType == SolveCompType::SimpleInvPiv, bool> = false>
FASTOR_INLINE Tensor<T,I,J> solve(const Tensor<T,I,I> &A, const Tensor<T,I,J> &B) {
Tensor<size_t,I> p;
pivot_inplace(A,p);
auto tmp(apply_pivot(A,p));
Tensor<T,I,I> invA = inverse<InvCompType::SimpleInv>(tmp);
return matmul(reconstruct(invA,p),B);
}
// For expressions
template<SolveCompType SType = SolveCompType::SimpleInv,
typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
enable_if_t_< is_tensor_v<TLhs> && is_tensor_v<TRhs> && DIM0==2,bool> = false>
FASTOR_INLINE
conditional_t_<TRhs::result_type::dimension_t::value == 1,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>
>,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>,
get_tensor_dimension_v<1,typename TRhs::result_type>
>
>
solve(const AbstractTensor<TLhs,DIM0> &lhs, const AbstractTensor<TRhs,DIM1> &rhs) {
using lhs_type = typename TLhs::result_type;
using rhs_type = typename TRhs::result_type;
constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>;
constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>;
constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>;
static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX");
static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)");
return solve<SType>(lhs.self(),rhs.self());
}
template<SolveCompType SType = SolveCompType::SimpleInv,
typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
enable_if_t_< !is_tensor_v<TLhs> && is_tensor_v<TRhs> && DIM0==2,bool> = false>
FASTOR_INLINE
conditional_t_<TRhs::result_type::dimension_t::value == 1,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>
>,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>,
get_tensor_dimension_v<1,typename TRhs::result_type>
>
>
solve(const AbstractTensor<TLhs,DIM0> &lhs, const AbstractTensor<TRhs,DIM1> &rhs) {
using lhs_type = typename TLhs::result_type;
using rhs_type = typename TRhs::result_type;
constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>;
constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>;
constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>;
static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX");
static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)");
const lhs_type A(lhs.self());
return solve<SType>(A,rhs.self());
}
template<SolveCompType SType = SolveCompType::SimpleInv,
typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
enable_if_t_< is_tensor_v<TLhs> && !is_tensor_v<TRhs> && DIM0==2,bool> = false>
FASTOR_INLINE
conditional_t_<TRhs::result_type::dimension_t::value == 1,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>
>,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>,
get_tensor_dimension_v<1,typename TRhs::result_type>
>
>
solve(const AbstractTensor<TLhs,DIM0> &lhs, const AbstractTensor<TRhs,DIM1> &rhs) {
using lhs_type = typename TLhs::result_type;
using rhs_type = typename TRhs::result_type;
constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>;
constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>;
constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>;
static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX");
static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)");
const rhs_type b(rhs.self());
return solve<SType>(lhs.self(),b);
}
template<SolveCompType SType = SolveCompType::SimpleInv,
typename TLhs, typename TRhs, size_t DIM0, size_t DIM1,
enable_if_t_< !is_tensor_v<TLhs> && !is_tensor_v<TRhs> && DIM0==2,bool> = false>
FASTOR_INLINE
conditional_t_<TRhs::result_type::dimension_t::value == 1,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>
>,
Tensor<
typename TLhs::scalar_type,
get_tensor_dimension_v<0,typename TLhs::result_type>,
get_tensor_dimension_v<1,typename TRhs::result_type>
>
>
solve(const AbstractTensor<TLhs,DIM0> &lhs, const AbstractTensor<TRhs,DIM1> &rhs) {
using lhs_type = typename TLhs::result_type;
using rhs_type = typename TRhs::result_type;
constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,lhs_type>;
constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,lhs_type>;
constexpr FASTOR_INDEX M_other = get_tensor_dimension_v<0,rhs_type>;
static_assert(M==N, "LHS EXPRESSION FOR SOLVE HAS TO BE A SQUARE MATRIX");
static_assert(M == M_other, "INVALID SOLVE OPERANDS. IN Ax=b ROWS(A)!=ROWS(b)");
const lhs_type A(lhs.self());
const rhs_type b(rhs.self());
return solve<SType>(A,b);
}
} // end of namespace Fastor
#endif // BINARY_SOLVE_OP_H

66
noarch/include/Fastor/expressions/linalg_ops/linalg_computation_types.h
View File

@@ -1,66 +0,0 @@
#ifndef LINALG_COMPUTATION_TYPES_H
#define LINALG_COMPUTATION_TYPES_H
namespace Fastor {
// Pivot types
enum class PivType : int
{
V = 0, /* The permutation vector */
M, /* The permutatoin matrix */
};
// QR factorisation computation types
enum class QRCompType : int
{
MGSR = 0, /* Modified Gram-Schmidt Row-wise */
MGSRPiv, /* Modified Gram-Schmidt Row-wise with pivot */
HHR /* House Holder Reflections */
};
// LU factorisation computation types
enum class LUCompType : int
{
BlockLU = 0, /* Block LU decomposition wiht no pivot */
BlockLUPiv, /* Block LU decomposition wiht pivot */
SimpleLU, /* Simple LU decomposition wiht no pivot */
SimpleLUPiv, /* Simple LU decomposition wiht pivot */
};
// Determinant computation type
enum class DetCompType : int
{
Simple = 0, /* Using simple hand-optimised calculations */
LU, /* Using LU factorisation */
QR, /* Using QR factorisation */
};
// Inverse computation type
enum class InvCompType : int
{
SimpleInv = 0,/* Using simple hand-optimised calculations */
SimpleInvPiv, /* Simple with pivot */
BlockLU, /* Using block LU factorisation */
BlockLUPiv, /* Using block LU factorisation with pivot */
SimpleLU, /* Using simple LU factorisation */
SimpleLUPiv, /* Using simple LU factorisation with pivot */
};
// Solve computation type
enum class SolveCompType : int
{
SimpleInv = 0, /* Using optimised inversion */
SimpleInvPiv, /* Using optimised inversion with pivot */
BlockLU, /* Using block LU factorisation */
BlockLUPiv, /* Using block LU factorisation with pivot */
SimpleLU, /* Using simple LU factorisation */
SimpleLUPiv, /* Using simple LU factorisation with pivot */
QR, /* Using QR factorisation */
Chol, /* Using Cholesky factorisation */
};
} // end of namespace Fastor
#endif // LINALG_COMPUTATION_TYPES_H

24
noarch/include/Fastor/expressions/linalg_ops/linalg_ops.h
View File

@@ -1,24 +0,0 @@
#ifndef LINALG_OPS_H
#define LINALG_OPS_H
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/binary_matmul_op.h"
#include "Fastor/expressions/linalg_ops/unary_piv_op.h"
#include "Fastor/expressions/linalg_ops/unary_lu_op.h"
#include "Fastor/expressions/linalg_ops/binary_solve_op.h"
#include "Fastor/expressions/linalg_ops/unary_trans_op.h"
#include "Fastor/expressions/linalg_ops/unary_ctrans_op.h"
#include "Fastor/expressions/linalg_ops/unary_adj_op.h"
#include "Fastor/expressions/linalg_ops/unary_cof_op.h"
#include "Fastor/expressions/linalg_ops/unary_inv_op.h"
#include "Fastor/expressions/linalg_ops/unary_trace_op.h"
#include "Fastor/expressions/linalg_ops/unary_norm_op.h"
#include "Fastor/expressions/linalg_ops/unary_qr_op.h"
#include "Fastor/expressions/linalg_ops/unary_det_op.h"
#include "Fastor/expressions/linalg_ops/unary_svd_op.h"
#include "Fastor/expressions/linalg_ops/binary_cross_op.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
#endif // LINALG_OPS_H

273
noarch/include/Fastor/expressions/linalg_ops/linalg_traits.h
View File

@@ -1,273 +0,0 @@
#ifndef LINALG_TRAITS_H
#define LINALG_TRAITS_H
#include "Fastor/expressions/linalg_ops/binary_matmul_op.h"
#include "Fastor/expressions/linalg_ops/unary_trans_op.h"
#include "Fastor/expressions/linalg_ops/unary_ctrans_op.h"
#include "Fastor/expressions/linalg_ops/unary_adj_op.h"
#include "Fastor/expressions/linalg_ops/unary_cof_op.h"
#include "Fastor/expressions/linalg_ops/unary_inv_op.h"
#include "Fastor/expressions/linalg_ops/binary_solve_op.h"
#include <type_traits>
namespace Fastor {
// Is a binary matmul expression
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_binary_matmul_op {
static constexpr bool value = false;
};
template<typename T, size_t ... Rest0, size_t ... Rest1, size_t DIM>
struct is_binary_matmul_op<BinaryMatMulOp<Tensor<T,Rest0...>,Tensor<T,Rest1...>,DIM>> {
static constexpr bool value = true;
};
template<typename Derived0, typename Derived1, size_t DIM>
struct is_binary_matmul_op<BinaryMatMulOp<Derived0,Derived1,DIM>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_binary_matmul_op {
static constexpr bool value = is_binary_matmul_op<Derived>::value ? true : false;
};
template<typename T, size_t ... Rest0, size_t ... Rest1, size_t DIM>
struct has_binary_matmul_op<BinaryMatMulOp<Tensor<T,Rest0...>,Tensor<T,Rest1...>,DIM>> {
static constexpr bool value = true;
};
template<typename Derived0, typename Derived1, size_t DIM>
struct has_binary_matmul_op<BinaryMatMulOp<Derived0,Derived1,DIM>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_binary_matmul_op<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_binary_matmul_op<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_binary_matmul_op<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_binary_matmul_op<TRhs>::value || has_binary_matmul_op<TLhs>::value;
};
// helper
template<typename Derived>
static constexpr bool is_binary_matmul_op_v = is_binary_matmul_op<Derived>::value;
template<typename Derived>
static constexpr bool has_binary_matmul_op_v = has_binary_matmul_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
// Is unary trans expression
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_unary_trans_op {
static constexpr bool value = false;
};
template<typename Derived, size_t DIM>
struct is_unary_trans_op<UnaryTransOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_unary_trans_op {
static constexpr bool value = is_unary_trans_op<Derived>::value ? true : false;
};
template<typename Derived, size_t DIM>
struct has_unary_trans_op<UnaryTransOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_unary_trans_op<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_unary_trans_op<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_unary_trans_op<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_unary_trans_op<TRhs>::value || has_unary_trans_op<TLhs>::value;
};
// helper
template<typename Derived>
static constexpr bool is_unary_trans_op_v = is_unary_trans_op<Derived>::value;
template<typename Derived>
static constexpr bool has_unary_trans_op_v = has_unary_trans_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
// Is unary ctrans expression
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_unary_ctrans_op {
static constexpr bool value = false;
};
template<typename Derived, size_t DIM>
struct is_unary_ctrans_op<UnaryCTransOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_unary_ctrans_op {
static constexpr bool value = is_unary_ctrans_op<Derived>::value ? true : false;
};
template<typename Derived, size_t DIM>
struct has_unary_ctrans_op<UnaryCTransOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_unary_ctrans_op<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_unary_ctrans_op<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_unary_ctrans_op<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_unary_ctrans_op<TRhs>::value || has_unary_ctrans_op<TLhs>::value;
};
// helper
template<typename Derived>
static constexpr bool is_unary_ctrans_op_v = is_unary_ctrans_op<Derived>::value;
template<typename Derived>
static constexpr bool has_unary_ctrans_op_v = has_unary_ctrans_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
// Is unary adj expression
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_unary_adj_op {
static constexpr bool value = false;
};
template<typename Derived, size_t DIM>
struct is_unary_adj_op<UnaryAdjOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_unary_adj_op {
static constexpr bool value = is_unary_adj_op<Derived>::value ? true : false;
};
template<typename Derived, size_t DIM>
struct has_unary_adj_op<UnaryAdjOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_unary_adj_op<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_unary_adj_op<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_unary_adj_op<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_unary_adj_op<TRhs>::value || has_unary_adj_op<TLhs>::value;
};
// helper
template<typename Derived>
static constexpr bool is_unary_adj_op_v = is_unary_adj_op<Derived>::value;
template<typename Derived>
static constexpr bool has_unary_adj_op_v = has_unary_adj_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
// Is unary adj expression
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_unary_cof_op {
static constexpr bool value = false;
};
template<typename Derived, size_t DIM>
struct is_unary_cof_op<UnaryCofOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_unary_cof_op {
static constexpr bool value = is_unary_cof_op<Derived>::value ? true : false;
};
template<typename Derived, size_t DIM>
struct has_unary_cof_op<UnaryCofOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_unary_cof_op<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_unary_cof_op<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_unary_cof_op<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_unary_cof_op<TRhs>::value || has_unary_cof_op<TLhs>::value;
};
// helper
template<typename Derived>
static constexpr bool is_unary_cof_op_v = is_unary_cof_op<Derived>::value;
template<typename Derived>
static constexpr bool has_unary_cof_op_v = has_unary_cof_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
// Is unary inv expression
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct is_unary_inv_op {
static constexpr bool value = false;
};
template<typename Derived, size_t DIM>
struct is_unary_inv_op<UnaryInvOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<typename Derived>
struct has_unary_inv_op {
static constexpr bool value = is_unary_inv_op<Derived>::value ? true : false;
};
template<typename Derived, size_t DIM>
struct has_unary_inv_op<UnaryInvOp<Derived,DIM>> {
static constexpr bool value = true;
};
template<template<typename,size_t> class UnaryExpr, typename Expr, size_t DIM>
struct has_unary_inv_op<UnaryExpr<Expr,DIM>> {
static constexpr bool value = has_unary_inv_op<Expr>::value;
};
template<template<class,class,size_t> class BinaryExpr, typename TLhs, typename TRhs, size_t DIMS>
struct has_unary_inv_op<BinaryExpr<TLhs,TRhs,DIMS>> {
static constexpr bool value = has_unary_inv_op<TRhs>::value || has_unary_inv_op<TLhs>::value;
};
// helper
template<typename Derived>
static constexpr bool is_unary_inv_op_v = is_unary_inv_op<Derived>::value;
template<typename Derived>
static constexpr bool has_unary_inv_op_v = has_unary_inv_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
// Is a linear algebra expression
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct has_linalg_op {
static constexpr bool value = has_binary_matmul_op<Derived>::value || has_unary_trans_op<Derived>::value ||
has_unary_ctrans_op<Derived>::value || has_unary_adj_op<Derived>::value ||
has_unary_cof_op<Derived>::value || has_unary_inv_op<Derived>::value;
};
// helper
template<typename Derived>
static constexpr bool has_linalg_op_v = has_linalg_op<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
// Requires immediate evaluation
//----------------------------------------------------------------------------------------------------------//
template<typename Derived>
struct requires_evaluation {
static constexpr bool value = has_linalg_op<Derived>::value;
};
// helper
template<typename Derived>
static constexpr bool requires_evaluation_v = requires_evaluation<Derived>::value;
//----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // LINALG_TRAITS_H

160
noarch/include/Fastor/expressions/linalg_ops/unary_adj_op.h
View File

@@ -1,160 +0,0 @@
#ifndef UNARY_ADJ_OP_H
#define UNARY_ADJ_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/adjoint.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename Expr, size_t DIM0>
struct UnaryAdjOp: public AbstractTensor<UnaryAdjOp<Expr, DIM0>,DIM0> {
using expr_type = expression_t<Expr>;
using result_type = typename Expr::result_type;
static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>;
static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>;
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<UnaryAdjOp<Expr, DIM0>>::type;
using simd_vector_type = typename Expr::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
FASTOR_INLINE UnaryAdjOp(expr_type inexpr) : _expr(inexpr) {
static_assert(M==N, "MATRIX MUST BE SQUARE");
}
FASTOR_INLINE FASTOR_INDEX size() const {return M*N;}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;}
constexpr FASTOR_INLINE expr_type expr() const {return _expr;}
private:
expr_type _expr;
};
template<typename Expr, size_t DIM0>
FASTOR_INLINE UnaryAdjOp<Expr, DIM0>
adj(const AbstractTensor<Expr,DIM0> &src) {
return UnaryAdjOp<Expr, DIM0>(src.self());
}
namespace internal {
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,4>,bool> = false>
FASTOR_INLINE void adjoint_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
_adjoint<T,M>(in.data(),out.data());
}
} // internal
// For tensors
template<typename T, size_t I>
FASTOR_INLINE Tensor<T,I,I> adjoint(const Tensor<T,I,I> &a) {
Tensor<T,I,I> out;
_adjoint<T,I>(a.data(),out.data());
return out;
}
// For high order tensors
template<typename T, size_t ... Rest, typename std::enable_if<sizeof...(Rest)>=3,bool>::type=0>
FASTOR_INLINE Tensor<T,Rest...>
adjoint(const Tensor<T,Rest...> &a) {
constexpr size_t remaining_product = last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::remaining_product;
constexpr size_t I = get_value<sizeof...(Rest)-1,Rest...>::value;
constexpr size_t J = get_value<sizeof...(Rest),Rest...>::value;
static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME");
Tensor<T,Rest...> out;
T *a_data = a.data();
T *out_data = out.data();
for (size_t i=0; i<remaining_product; ++i) {
_adjoint<T,J,J>(a_data+i*J*J,out_data+i*J*J);
}
return out;
}
// Adjoint for generic expressions is provided here
template<typename Derived, size_t DIM>
FASTOR_INLINE
typename Derived::result_type
adjoint(const AbstractTensor<Derived,DIM> &src) {
// If we are here Derived is already an expression
using result_type = typename Derived::result_type;
const result_type tmp(src.self());
return adjoint(tmp);
}
// assignments
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const UnaryAdjOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
const result_type& tmp = evaluate(src.expr().self());
internal::adjoint_dispatcher(tmp,dst.self());
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const UnaryAdjOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::adjoint_dispatcher(tmp,tmp_inv);
trivial_assign_add(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const UnaryAdjOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::adjoint_dispatcher(tmp,tmp_inv);
trivial_assign_sub(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const UnaryAdjOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::adjoint_dispatcher(tmp,tmp_inv);
trivial_assign_mul(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const UnaryAdjOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::adjoint_dispatcher(tmp,tmp_inv);
trivial_assign_div(dst.self(),tmp_inv);
}
} // end of namespace Fastor
#endif // UNARY_ADJ_OP_H

160
noarch/include/Fastor/expressions/linalg_ops/unary_cof_op.h
View File

@@ -1,160 +0,0 @@
#ifndef UNARY_COF_OP_H
#define UNARY_COF_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/cofactor.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename Expr, size_t DIM0>
struct UnaryCofOp: public AbstractTensor<UnaryCofOp<Expr, DIM0>,DIM0> {
using expr_type = expression_t<Expr>;
using result_type = typename Expr::result_type;
static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>;
static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>;
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<UnaryCofOp<Expr, DIM0>>::type;
using simd_vector_type = typename Expr::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
FASTOR_INLINE UnaryCofOp(expr_type inexpr) : _expr(inexpr) {
static_assert(M==N, "MATRIX MUST BE SQUARE");
}
FASTOR_INLINE FASTOR_INDEX size() const {return M*N;}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;}
constexpr FASTOR_INLINE expr_type expr() const {return _expr;}
private:
expr_type _expr;
};
template<typename Expr, size_t DIM0>
FASTOR_INLINE UnaryCofOp<Expr, DIM0>
cof(const AbstractTensor<Expr,DIM0> &src) {
return UnaryCofOp<Expr, DIM0>(src.self());
}
namespace internal {
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,4>,bool> = false>
FASTOR_INLINE void cofactor_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
_cofactor<T,M>(in.data(),out.data());
}
} // internal
// For tensors
template<typename T, size_t I>
FASTOR_INLINE Tensor<T,I,I> cofactor(const Tensor<T,I,I> &a) {
Tensor<T,I,I> out;
_cofactor<T,I>(a.data(),out.data());
return out;
}
// For high order tensors
template<typename T, size_t ... Rest, typename std::enable_if<sizeof...(Rest)>=3,bool>::type=0>
FASTOR_INLINE Tensor<T,Rest...>
cofactor(const Tensor<T,Rest...> &a) {
constexpr size_t remaining_product = last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::remaining_product;
constexpr size_t I = get_value<sizeof...(Rest)-1,Rest...>::value;
constexpr size_t J = get_value<sizeof...(Rest),Rest...>::value;
static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME");
Tensor<T,Rest...> out;
T *a_data = a.data();
T *out_data = out.data();
for (size_t i=0; i<remaining_product; ++i) {
_cofactor<T,J,J>(a_data+i*J*J,out_data+i*J*J);
}
return out;
}
// Cofactor for generic expressions is provided here
template<typename Derived, size_t DIM>
FASTOR_INLINE
typename Derived::result_type
cofactor(const AbstractTensor<Derived,DIM> &src) {
// If we are here Derived is already an expression
using result_type = typename Derived::result_type;
const result_type tmp(src.self());
return cofactor(tmp);
}
// assignments
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const UnaryCofOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
const result_type& tmp = evaluate(src.expr().self());
internal::cofactor_dispatcher(tmp,dst.self());
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const UnaryCofOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::cofactor_dispatcher(tmp,tmp_inv);
trivial_assign_add(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const UnaryCofOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::cofactor_dispatcher(tmp,tmp_inv);
trivial_assign_sub(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const UnaryCofOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::cofactor_dispatcher(tmp,tmp_inv);
trivial_assign_mul(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const UnaryCofOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::cofactor_dispatcher(tmp,tmp_inv);
trivial_assign_div(dst.self(),tmp_inv);
}
} // end of namespace Fastor
#endif // UNARY_COF_OP_H

178
noarch/include/Fastor/expressions/linalg_ops/unary_ctrans_op.h
View File

@@ -1,178 +0,0 @@
#ifndef UNARY_CTRANS_OP_H
#define UNARY_CTRANS_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename Expr, size_t DIM0>
struct UnaryCTransOp: public AbstractTensor<UnaryCTransOp<Expr, DIM0>,DIM0> {
using expr_type = expression_t<Expr>;
static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,typename Expr::result_type>;
static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,typename Expr::result_type>;
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<UnaryCTransOp<Expr, DIM0>>::type;
using simd_vector_type = typename Expr::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = Tensor<scalar_type,N,M>;
FASTOR_INLINE UnaryCTransOp(expr_type inexpr) : _expr(inexpr) {
}
FASTOR_INLINE FASTOR_INDEX size() const {return M*N;}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i == 0 ? N : M;}
constexpr FASTOR_INLINE expr_type expr() const {return _expr;}
private:
expr_type _expr;
};
/* Tensor conjugate transpose returning a tensor expression */
template<typename Expr, size_t DIM0>
FASTOR_INLINE UnaryCTransOp<Expr, DIM0>
ctrans(const AbstractTensor<Expr,DIM0> &src) {
return UnaryCTransOp<Expr, DIM0>(src.self());
}
/* Backend implementation */
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _ctranspose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) {
for (size_t j=0; j<N; ++j)
for (size_t i=0; i< M; ++i)
out[j*M+i] = conj(a[i*N+j]);
}
/* Tensor conjugate transpose immediately returning a tensor */
template<typename T, size_t I, size_t J>
FASTOR_INLINE Tensor<T,J,I> ctranspose(const Tensor<T,I,J> &a) {
Tensor<T,J,I> out;
_ctranspose<T,I,J>(a.data(),out.data());
return out;
}
/* Tensor conjugate transpose for higher order tensors immediately returning a tensor */
template<typename T, size_t ... Rest, typename std::enable_if<sizeof...(Rest)>=3,bool>::type=0>
FASTOR_INLINE Tensor<T,Rest...>
ctranspose(const Tensor<T,Rest...> &a) {
constexpr size_t remaining_product = last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::remaining_product;
constexpr size_t I = get_value<sizeof...(Rest)-1,Rest...>::value;
constexpr size_t J = get_value<sizeof...(Rest),Rest...>::value;
static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME");
Tensor<T,Rest...> out;
T *a_data = a.data();
T *out_data = out.data();
for (size_t i=0; i<remaining_product; ++i) {
_ctranspose<T,J,J>(a_data+i*J*J,out_data+i*J*J);
}
return out;
}
// Conjagate transpose for generic expressions
template<typename Expr, size_t DIM0>
FASTOR_INLINE
Tensor<
typename scalar_type_finder<Expr>::type,
get_tensor_dimension_v<1,typename Expr::result_type>,
get_tensor_dimension_v<0,typename Expr::result_type>>
ctranspose(const AbstractTensor<Expr,DIM0> &src) {
// If we are here Expr is already an expression
using result_type = typename Expr::result_type;
const result_type tmp(src.self());
return ctranspose(tmp);
}
// assignments
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const UnaryCTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
const result_type& tmp = evaluate(src.expr().self());
using T = typename UnaryCTransOp<Expr, OtherDIM>::scalar_type;
static constexpr size_t M = UnaryCTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryCTransOp<Expr, OtherDIM>::N;
_ctranspose<T,M,N>(tmp.data(),dst.self().data());
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const UnaryCTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryCTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryCTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryCTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryCTransOp<Expr, OtherDIM>::N;
_ctranspose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_add(dst.self(),tmp_trans);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const UnaryCTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryCTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryCTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryCTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryCTransOp<Expr, OtherDIM>::N;
_ctranspose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_sub(dst.self(),tmp_trans);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const UnaryCTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryCTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryCTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryCTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryCTransOp<Expr, OtherDIM>::N;
_ctranspose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_mul(dst.self(),tmp_trans);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const UnaryCTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryCTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryCTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryCTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryCTransOp<Expr, OtherDIM>::N;
_ctranspose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_div(dst.self(),tmp_trans);
}
} // end of namespace Fastor
#endif // UNARY_CTRANS_OP_H

86
noarch/include/Fastor/expressions/linalg_ops/unary_det_op.h
View File

@@ -1,86 +0,0 @@
#ifndef UNARY_DET_OP_H
#define UNARY_DET_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/backend/determinant.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
#include <cmath>
namespace Fastor {
// Computing determinant using QR/LU decompositions are in those respective modules
// For tensors
template<DetCompType DetType = DetCompType::Simple, typename T, size_t M,
enable_if_t_<is_less_equal_v_<M,4UL> && DetType==DetCompType::Simple,bool> = false>
FASTOR_INLINE T determinant(const Tensor<T,M,M> &a) {
return _det<T,M,M>(static_cast<const T *>(a.data()));
}
template<DetCompType DetType = DetCompType::Simple, typename T, size_t M,
enable_if_t_<is_greater_v_<M,4UL> && DetType == DetCompType::Simple,bool> = false>
FASTOR_INLINE T determinant(const Tensor<T,M,M> &a) {
// Dispatch to LU
return determinant<DetCompType::LU>(a);
}
// For high order tensors
template<DetCompType DetType = DetCompType::Simple,
typename T, size_t ... Rest, enable_if_t_<sizeof...(Rest)>=3 && DetType == DetCompType::Simple,bool> = false>
FASTOR_INLINE
typename last_matrix_extracter<Tensor<T,Rest...>, typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::type
determinant(const Tensor<T,Rest...> &a) {
using OutTensor = typename last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::type;
constexpr size_t remaining_product = last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::remaining_product;
constexpr size_t I = get_value<sizeof...(Rest)-1,Rest...>::value;
constexpr size_t J = get_value<sizeof...(Rest),Rest...>::value;
static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME");
OutTensor out;
T *a_data = a.data();
T *out_data = out.data();
for (size_t i=0; i<remaining_product; ++i) {
out_data[i] = _det<T,J,J>(static_cast<const T *>(a_data+i*J*J));
}
return out;
}
// Find determinant of a tensor expression - dispatches to determinant for tensors
template<DetCompType DetType = DetCompType::Simple, typename Derived, size_t DIMS>
FASTOR_INLINE typename Derived::scalar_type determinant(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out = evaluate(src);
return determinant<DetType>(out);
}
template<DetCompType DetType = DetCompType::Simple, typename Derived, size_t DIMS>
FASTOR_INLINE typename Derived::scalar_type det(const AbstractTensor<Derived,DIMS> &_src) {
return determinant<DetType>(_src.self());
}
template<DetCompType DetType = DetCompType::Simple, typename Derived, size_t DIMS>
FASTOR_INLINE typename Derived::scalar_type absdet(const AbstractTensor<Derived,DIMS> &_src) {
return std::abs(determinant<DetType>(_src.self()));
}
template<DetCompType DetType = DetCompType::Simple, typename Derived, size_t DIMS>
FASTOR_INLINE typename Derived::scalar_type logdet(const AbstractTensor<Derived,DIMS> &_src) {
return std::log(std::abs(determinant<DetType>(_src.self())));
}
} // end of namespace Fastor
#endif // UNARY_DET_OP_H

731
noarch/include/Fastor/expressions/linalg_ops/unary_inv_op.h
View File

@@ -1,731 +0,0 @@
#ifndef UNARY_INV_OP_H
#define UNARY_INV_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/inverse.h"
#include "Fastor/backend/lut_inverse.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/unary_piv_op.h"
namespace Fastor {
template<typename Expr, size_t DIM0>
struct UnaryInvOp: public AbstractTensor<UnaryInvOp<Expr, DIM0>,DIM0> {
using expr_type = expression_t<Expr>;
using result_type = typename Expr::result_type;
static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>;
static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>;
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<UnaryInvOp<Expr, DIM0>>::type;
using simd_vector_type = typename Expr::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
FASTOR_INLINE UnaryInvOp(expr_type inexpr) : _expr(inexpr) {
static_assert(M==N, "MATRIX MUST BE SQUARE");
}
constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M*N;}
constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;}
constexpr FASTOR_INLINE expr_type expr() const {return _expr;}
private:
expr_type _expr;
};
template<typename Expr, size_t DIM0>
FASTOR_INLINE UnaryInvOp<Expr, DIM0>
inv(const AbstractTensor<Expr,DIM0> &src) {
return UnaryInvOp<Expr, DIM0>(src.self());
}
// Upper triangular block matrix inversion
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
namespace internal {
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,4>,bool> = false>
FASTOR_INLINE void ut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
_inverse<T,M>(in.data(),out.data());
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,4> && is_less_equal_v_<M,8>,bool> = false>
FASTOR_INLINE void ut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
constexpr size_t N = 4UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
// Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
ut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,M-N> inv_d;
ut_inverse_dispatcher(d, inv_d);
Tensor<T,N,M-N> b_invd = matmul(b, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = -matmul(inv_a, b_invd);
out(fseq<N,M>(),fseq<0,N>()) = 0;
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,8> && is_less_equal_v_<M,16>,bool> = false>
FASTOR_INLINE void ut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
constexpr size_t N = 8UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
// Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
ut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,M-N> inv_d;
ut_inverse_dispatcher(d, inv_d);
Tensor<T,N,M-N> b_invd = matmul(b, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = -matmul(inv_a, b_invd);
out(fseq<N,M>(),fseq<0,N>()) = 0;
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,16> && is_less_equal_v_<M,32>,bool> = false>
FASTOR_INLINE void ut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 16UL; // start size
constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
// Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
ut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,M-N> inv_d;
ut_inverse_dispatcher(d, inv_d);
Tensor<T,N,M-N> b_invd = matmul(b, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = -matmul(inv_a, b_invd);
out(fseq<N,M>(),fseq<0,N>()) = 0;
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,32> && is_less_equal_v_<M,64>,bool> = false>
FASTOR_INLINE void ut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 32UL; // start size
constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
// Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
ut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,M-N> inv_d;
ut_inverse_dispatcher(d, inv_d);
Tensor<T,N,M-N> b_invd = tmatmul<UpLoType::General,UpLoType::Upper>(b, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = -tmatmul<UpLoType::Upper,UpLoType::General>(inv_a, b_invd);
out(fseq<N,M>(),fseq<0,N>()) = 0;
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,64> && is_less_equal_v_<M,128>,bool> = false>
FASTOR_INLINE void ut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 64UL; // start size
constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
// Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
ut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,M-N> inv_d;
ut_inverse_dispatcher(d, inv_d);
Tensor<T,N,M-N> b_invd = tmatmul<UpLoType::General,UpLoType::Upper>(b, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = -tmatmul<UpLoType::Upper,UpLoType::General>(inv_a, b_invd);
out(fseq<N,M>(),fseq<0,N>()) = 0;
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,128> && is_less_equal_v_<M,256>,bool> = false>
FASTOR_INLINE void ut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 128UL; // start size
constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
// Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
ut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,M-N> inv_d;
ut_inverse_dispatcher(d, inv_d);
Tensor<T,N,M-N> b_invd = tmatmul<UpLoType::General,UpLoType::Upper>(b, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = -tmatmul<UpLoType::Upper,UpLoType::General>(inv_a, b_invd);
out(fseq<N,M>(),fseq<0,N>()) = 0;
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // internal
// Lower uni-triangular block matrix inversion
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
namespace internal {
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,4>,bool> = false>
FASTOR_INLINE void lut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// This restricts the recursive block to lower uni-triangualar matrices
_lowunitri_inverse<T,M>(in.data(),out.data());
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,4> && is_less_equal_v_<M,8>,bool> = false>
FASTOR_INLINE void lut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
constexpr size_t N = 4UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
// Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
lut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> inv_d;
lut_inverse_dispatcher(d, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = 0;
out(fseq<N,M>(),fseq<0,N>()) = -matmul(inv_d, c_inva);
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,8> && is_less_equal_v_<M,16>,bool> = false>
FASTOR_INLINE void lut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
constexpr size_t N = 8UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
// Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
lut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> inv_d;
lut_inverse_dispatcher(d, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = 0;
out(fseq<N,M>(),fseq<0,N>()) = -matmul(inv_d, c_inva);
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,16> && is_less_equal_v_<M,32>,bool> = false>
FASTOR_INLINE void lut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 16UL; // start size
constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
// Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
lut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> inv_d;
lut_inverse_dispatcher(d, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = 0;
out(fseq<N,M>(),fseq<0,N>()) = -matmul(inv_d, c_inva);
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,32> && is_less_equal_v_<M,64>,bool> = false>
FASTOR_INLINE void lut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 32UL; // start size
constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
// Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
lut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = tmatmul<UpLoType::General,UpLoType::Lower>(c, inv_a);
Tensor<T,M-N,M-N> inv_d;
lut_inverse_dispatcher(d, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = 0;
out(fseq<N,M>(),fseq<0,N>()) = -tmatmul<UpLoType::Lower,UpLoType::General>(inv_d, c_inva);
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,64> && is_less_equal_v_<M,128>,bool> = false>
FASTOR_INLINE void lut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 64UL; // start size
constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
// Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
lut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = tmatmul<UpLoType::General,UpLoType::Lower>(c, inv_a);
Tensor<T,M-N,M-N> inv_d;
lut_inverse_dispatcher(d, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = 0;
out(fseq<N,M>(),fseq<0,N>()) = -tmatmul<UpLoType::Lower,UpLoType::General>(inv_d, c_inva);
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,128> && is_less_equal_v_<M,256>,bool> = false>
FASTOR_INLINE void lut_inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 128UL; // start size
constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
// Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
lut_inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = tmatmul<UpLoType::General,UpLoType::Lower>(c, inv_a);
Tensor<T,M-N,M-N> inv_d;
lut_inverse_dispatcher(d, inv_d);
out(fseq<0,N>(),fseq<0,N>()) = inv_a;
out(fseq<0,N>(),fseq<N,M>()) = 0;
out(fseq<N,M>(),fseq<0,N>()) = -tmatmul<UpLoType::Lower,UpLoType::General>(inv_d, c_inva);
out(fseq<N,M>(),fseq<N,M>()) = inv_d;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // internal
// General recursive block matrix inversion
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
namespace internal {
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,4>,bool> = false>
FASTOR_INLINE void inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
_inverse<T,M>(in.data(),out.data());
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,4> && is_less_equal_v_<M,8>,bool> = false>
FASTOR_INLINE void inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
constexpr size_t N = 4UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a = inverse(a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> block_bb = inverse(static_cast<Tensor<T,M-N,M-N>>(d - matmul(c_inva, b)));
Tensor<T,N,M-N> inva_b = matmul(inv_a, b);
Tensor<T,M-N,N> bb_c_inva = matmul(block_bb, c_inva);
Tensor<T,N ,N > block_aa = inv_a + matmul(inva_b, bb_c_inva);
Tensor<T,N ,M-N> block_ab = -matmul(inva_b, block_bb);
Tensor<T,M-N,N > block_ba = -bb_c_inva;
out(fseq<0,N>(),fseq<0,N>()) = block_aa;
out(fseq<0,N>(),fseq<N,M>()) = block_ab;
out(fseq<N,M>(),fseq<0,N>()) = block_ba;
out(fseq<N,M>(),fseq<N,M>()) = block_bb;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,8> && is_less_equal_v_<M,16>,bool> = false>
FASTOR_INLINE void inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
constexpr size_t N = 8UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> block_bb;
inverse_dispatcher(static_cast<Tensor<T,M-N,M-N>>(d - matmul(c_inva, b)),block_bb);
Tensor<T,N,M-N> inva_b = matmul(inv_a, b);
Tensor<T,M-N,N> bb_c_inva = matmul(block_bb, c_inva);
Tensor<T,N ,N > block_aa = inv_a + matmul(inva_b, bb_c_inva);
Tensor<T,N ,M-N> block_ab = -matmul(inva_b, block_bb);
Tensor<T,M-N,N > block_ba = -bb_c_inva;
out(fseq<0,N>(),fseq<0,N>()) = block_aa;
out(fseq<0,N>(),fseq<N,M>()) = block_ab;
out(fseq<N,M>(),fseq<0,N>()) = block_ba;
out(fseq<N,M>(),fseq<N,M>()) = block_bb;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,16> && is_less_equal_v_<M,32>,bool> = false>
FASTOR_INLINE void inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 16UL; // start size
constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> block_bb;
inverse_dispatcher(static_cast<Tensor<T,M-N,M-N>>(d - matmul(c_inva, b)),block_bb);
Tensor<T,N,M-N> inva_b = matmul(inv_a, b);
Tensor<T,M-N,N> bb_c_inva = matmul(block_bb, c_inva);
Tensor<T,N ,N > block_aa = inv_a + matmul(inva_b, bb_c_inva);
Tensor<T,N ,M-N> block_ab = -matmul(inva_b, block_bb);
Tensor<T,M-N,N > block_ba = -bb_c_inva;
out(fseq<0,N>(),fseq<0,N>()) = block_aa;
out(fseq<0,N>(),fseq<N,M>()) = block_ab;
out(fseq<N,M>(),fseq<0,N>()) = block_ba;
out(fseq<N,M>(),fseq<N,M>()) = block_bb;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,32> && is_less_equal_v_<M,64>,bool> = false>
FASTOR_INLINE void inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 32UL; // start size
constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> block_bb;
inverse_dispatcher(static_cast<Tensor<T,M-N,M-N>>(d - matmul(c_inva, b)),block_bb);
Tensor<T,N,M-N> inva_b = matmul(inv_a, b);
Tensor<T,M-N,N> bb_c_inva = matmul(block_bb, c_inva);
Tensor<T,N ,N > block_aa = inv_a + matmul(inva_b, bb_c_inva);
Tensor<T,N ,M-N> block_ab = -matmul(inva_b, block_bb);
Tensor<T,M-N,N > block_ba = -bb_c_inva;
out(fseq<0,N>(),fseq<0,N>()) = block_aa;
out(fseq<0,N>(),fseq<N,M>()) = block_ab;
out(fseq<N,M>(),fseq<0,N>()) = block_ba;
out(fseq<N,M>(),fseq<N,M>()) = block_bb;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,64> && is_less_equal_v_<M,128>,bool> = false>
FASTOR_INLINE void inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 64UL; // start size
constexpr size_t N = (M / 32UL * 32UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> block_bb;
inverse_dispatcher(static_cast<Tensor<T,M-N,M-N>>(d - matmul(c_inva, b)),block_bb);
Tensor<T,N,M-N> inva_b = matmul(inv_a, b);
Tensor<T,M-N,N> bb_c_inva = matmul(block_bb, c_inva);
Tensor<T,N ,N > block_aa = inv_a + matmul(inva_b, bb_c_inva);
Tensor<T,N ,M-N> block_ab = -matmul(inva_b, block_bb);
Tensor<T,M-N,N > block_ba = -bb_c_inva;
out(fseq<0,N>(),fseq<0,N>()) = block_aa;
out(fseq<0,N>(),fseq<N,M>()) = block_ab;
out(fseq<N,M>(),fseq<0,N>()) = block_ba;
out(fseq<N,M>(),fseq<N,M>()) = block_bb;
}
template<typename T, size_t M, enable_if_t_<is_greater_v_<M,128> && is_less_equal_v_<M,256>,bool> = false>
FASTOR_INLINE void inverse_dispatcher(const Tensor<T,M,M> &in, Tensor<T,M,M>& out) {
// constexpr size_t N = 128UL; // start size
constexpr size_t N = (M / 64UL * 64UL) / 2UL; // start size
Tensor<T,N ,N > a = in(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> b = in(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> c = in(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> d = in(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> inv_a;
inverse_dispatcher(a, inv_a);
Tensor<T,M-N,N> c_inva = matmul(c, inv_a);
Tensor<T,M-N,M-N> block_bb;
inverse_dispatcher(static_cast<Tensor<T,M-N,M-N>>(d - matmul(c_inva, b)),block_bb);
Tensor<T,N,M-N> inva_b = matmul(inv_a, b);
Tensor<T,M-N,N> bb_c_inva = matmul(block_bb, c_inva);
Tensor<T,N ,N > block_aa = inv_a + matmul(inva_b, bb_c_inva);
Tensor<T,N ,M-N> block_ab = -matmul(inva_b, block_bb);
Tensor<T,M-N,N > block_ba = -bb_c_inva;
out(fseq<0,N>(),fseq<0,N>()) = block_aa;
out(fseq<0,N>(),fseq<N,M>()) = block_ab;
out(fseq<N,M>(),fseq<0,N>()) = block_ba;
out(fseq<N,M>(),fseq<N,M>()) = block_bb;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // internal
// Inversion using LU decomposition is in the LU module [unary_lu_op]
// For tensors
// SimpleInv - no pivot
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t M, enable_if_t_<is_less_equal_v_<M,4UL> && InvType == InvCompType::SimpleInv,bool> = false>
FASTOR_INLINE Tensor<T,M,M> inverse(const Tensor<T,M,M> &a) {
Tensor<T,M,M> out;
_inverse<T,M>(a.data(),out.data());
return out;
}
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t M, enable_if_t_<is_greater_v_<M,4UL> && InvType == InvCompType::SimpleInv,bool> = false>
FASTOR_INLINE Tensor<T,M,M> inverse(const Tensor<T,M,M> &in) {
Tensor<T,M,M> out;
internal::inverse_dispatcher(in,out);
return out;
}
// SimpleInv pivot
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t M, enable_if_t_<InvType == InvCompType::SimpleInvPiv,bool> = false>
FASTOR_INLINE Tensor<T,M,M> inverse(const Tensor<T,M,M> &in) {
Tensor<T,M,M> out;
// We need to post multiply - swap columns using row permutation vector
Tensor<size_t,M> P;
pivot_inplace(in,P);
auto A(apply_pivot(in,P));
internal::inverse_dispatcher(A,out);
return reconstruct_colwise(out,P);
// // matrix version
// Tensor<T,M,M> P;
// pivot_inplace(in,P);
// auto A(apply_pivot(in,P));
// internal::inverse_dispatcher(A,out);
// return matmul(out,P);
}
// For high order tensors
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t ... Rest, enable_if_t_<sizeof...(Rest)>=3 && InvType == InvCompType::SimpleInv,bool> = false >
FASTOR_INLINE Tensor<T,Rest...>
inverse(const Tensor<T,Rest...> &a) {
constexpr size_t remaining_product = last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::remaining_product;
constexpr size_t I = get_value<sizeof...(Rest)-1,Rest...>::value;
constexpr size_t J = get_value<sizeof...(Rest),Rest...>::value;
static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME");
Tensor<T,Rest...> out;
T *a_data = a.data();
T *out_data = out.data();
for (size_t i=0; i<remaining_product; ++i) {
T det = _det<T,J,J>(static_cast<const T *>(a_data+i*J*J));
_inverse<T,J>(a_data+i*J*J,out_data+i*J*J);
}
return out;
}
// Inverse for generic expressions is provided here
template<InvCompType InvType = InvCompType::SimpleInv,
typename Derived, size_t DIM>
FASTOR_INLINE
typename Derived::result_type
inverse(const AbstractTensor<Derived,DIM> &src) {
// If we are here Derived is already an expression
using result_type = typename Derived::result_type;
const result_type tmp(src.self());
return inverse<InvType>(tmp);
}
// Inverse of lower uni-triangular matrices
template<InvCompType InvType = InvCompType::SimpleInv, typename ULType = UpLoType::General,
typename T, size_t M, enable_if_t_<InvType == InvCompType::SimpleInv && is_same_v_<ULType,UpLoType::UniLower>,bool> = false>
FASTOR_INLINE Tensor<T,M,M> tinverse(const Tensor<T,M,M> &in) {
Tensor<T,M,M> out;
internal::lut_inverse_dispatcher(in,out);
return out;
}
// Inverse of lower uni-triangular matrices
template<InvCompType InvType = InvCompType::SimpleInv, typename ULType = UpLoType::General,
typename T, size_t M, enable_if_t_<InvType == InvCompType::SimpleInv && is_same_v_<ULType,UpLoType::Upper>,bool> = false>
FASTOR_INLINE Tensor<T,M,M> tinverse(const Tensor<T,M,M> &in) {
Tensor<T,M,M> out;
internal::ut_inverse_dispatcher(in,out);
return out;
}
// Inverse for generic expressions of lower uni-triangular matrices is provided here
template<InvCompType InvType = InvCompType::SimpleInv, typename ULType = UpLoType::General,
typename Derived, size_t DIM>
FASTOR_INLINE
typename Derived::result_type
tinverse(const AbstractTensor<Derived,DIM> &src) {
// If we are here Derived is already an expression
using result_type = typename Derived::result_type;
const result_type tmp(src.self());
return tinverse<InvType,ULType>(tmp);
}
// assignments
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const UnaryInvOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
const result_type& tmp = evaluate(src.expr().self());
internal::inverse_dispatcher(tmp,dst.self());
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const UnaryInvOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::inverse_dispatcher(tmp,tmp_inv);
trivial_assign_add(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const UnaryInvOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::inverse_dispatcher(tmp,tmp_inv);
trivial_assign_sub(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const UnaryInvOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::inverse_dispatcher(tmp,tmp_inv);
trivial_assign_mul(dst.self(),tmp_inv);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const UnaryInvOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
result_type tmp_inv;
internal::inverse_dispatcher(tmp,tmp_inv);
trivial_assign_div(dst.self(),tmp_inv);
}
} // end of namespace Fastor
#endif // UNARY_INV_OP_H

970
noarch/include/Fastor/expressions/linalg_ops/unary_lu_op.h
View File

@@ -1,970 +0,0 @@
#ifndef UNARY_LU_OP_H
#define UNARY_LU_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/inner.h"
#include "Fastor/backend/lufact.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/unary_piv_op.h"
namespace Fastor {
namespace internal {
/* Compile time recursive loop with inner for forward substitution of b/B given the lower unitriangular matrix L.
The following meta functions implements L * y = b for single or multiple right sides
*/
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
template <size_t from, size_t to>
struct forward_subs_impl {
template<typename T, size_t M>
static FASTOR_INLINE void do_single_rhs(const Tensor<T,M,M> &L, const Tensor<T,M> &b, Tensor<T,M> &y) {
y(from) = b(from) - _inner<T,from>(&L.data()[from*M],y.data());
forward_subs_impl<from+1,to>::do_single_rhs(L, b, y);
}
template<typename T, size_t M>
static FASTOR_INLINE void do_single_rhs_pivot(const Tensor<T,M,M> &L, const Tensor<T,M> &b, const Tensor<size_t,M> &p, Tensor<T,M> &y) {
y(from) = b(p(from)) - _inner<T,from>(&L.data()[from*M],y.data());
forward_subs_impl<from+1,to>::do_single_rhs_pivot(L, b, p, y);
}
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor<T,M,M> &L, const Tensor<T,M,N> &B, Tensor<T,M> &y, Tensor<T,M,N> &X) {
y(from) = B(from,j) - _inner<T,from>(&L.data()[from*M],y.data());
X(from,j) = y(from);
forward_subs_impl<from+1,to>::do_multi_rhs(j, L, B, y, X);
}
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void do_multi_rhs_pivot(const size_t j, const Tensor<T,M,M> &L, const Tensor<T,M,N> &B,
const Tensor<size_t,M> &p, Tensor<T,M> &y, Tensor<T,M,N> &X) {
y(from) = B(p(from),j) - _inner<T,from>(&L.data()[from*M],y.data());
X(from,j) = y(from);
forward_subs_impl<from+1,to>::do_multi_rhs_pivot(j, L, B, p, y, X);
}
};
template <size_t from>
struct forward_subs_impl<from,from> {
template<typename T, size_t M>
static FASTOR_INLINE void do_single_rhs(const Tensor<T,M,M> &L, const Tensor<T,M> &b, Tensor<T,M> &y) {
y(from) = b(from) - _inner<T,from>(&L.data()[from*M],y.data());
}
template<typename T, size_t M>
static FASTOR_INLINE void do_single_rhs_pivot(const Tensor<T,M,M> &L, const Tensor<T,M> &b, const Tensor<size_t,M> &p, Tensor<T,M> &y) {
y(from) = b(p(from)) - _inner<T,from>(&L.data()[from*M],y.data());
}
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor<T,M,M> &L, const Tensor<T,M,N> &B, Tensor<T,M> &y, Tensor<T,M,N> &X) {
y(from) = B(from,j) - _inner<T,from>(&L.data()[from*M],y.data());
X(from,j) = y(from);
}
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void do_multi_rhs_pivot(const size_t j, const Tensor<T,M,M> &L, const Tensor<T,M,N> &B,
const Tensor<size_t,M> &p, Tensor<T,M> &y, Tensor<T,M,N> &X) {
y(from) = B(p(from),j) - _inner<T,from>(&L.data()[from*M],y.data());
X(from,j) = y(from);
}
};
template<typename T, size_t M>
FASTOR_INLINE Tensor<T,M> forward_subs(const Tensor<T,M,M> &L, const Tensor<T,M> &b) {
Tensor<T,M> y(0);
forward_subs_impl<0,M-1>::do_single_rhs(L, b, y);
#if 0
// The run-time loop version
// Solve for L * y = b
for (size_t i=0; i< M; ++i) {
T value = 0;
for (size_t k=0; k<i; ++k) {
value += L(i,k)*y(k);
}
y(i) = b(i) - value;
}
#endif
return y;
}
template<typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> forward_subs(const Tensor<T,M,M> &L, const Tensor<T,M,N> &B) {
// We keep a separate output tensor X from y [y is columns of X]
// to avoid strided access in X for the inner product
Tensor<T,M,N> X;
for (size_t j=0; j < N; ++j) {
Tensor<T,M> y(0);
forward_subs_impl<0,M-1>::do_multi_rhs(j, L, B, y, X);
}
#if 0
// The run-time loop version - X needs to be zeroed out for this version
for (size_t j=0; j < N; ++j) {
Tensor<T,M> y(0);
// Solve for L * y = b
for (size_t i=0; i< M; ++i) {
T value = 0;
for (size_t k=0; k<i; ++k) {
value += L(i,k)*y(k);
}
y(i) = B(i,j) - value;
X(i,j) = y(i);
}
}
#endif
return X;
}
template<typename T, size_t M>
FASTOR_INLINE Tensor<T,M> forward_subs(const Tensor<T,M,M> &L, const Tensor<size_t,M> &p, const Tensor<T,M> &b) {
Tensor<T,M> y(0);
forward_subs_impl<0,M-1>::do_single_rhs_pivot(L, b, p, y);
#if 0
// The run-time loop version
// Solve for L * y = b
for (size_t i=0; i< M; ++i) {
T value = 0;
for (size_t k=0; k<i; ++k) {
value += L(i,k)*y(k);
}
y(i) = b(p(i)) - value;
}
#endif
return y;
}
template<typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> forward_subs(const Tensor<T,M,M> &L, const Tensor<size_t,M> &p, const Tensor<T,M,N> &B) {
// We keep a separate output tensor X from y [y is columns of X]
// to avoid strided access in X for the inner product
Tensor<T,M,N> X;
for (size_t j=0; j < N; ++j) {
Tensor<T,M> y(0);
forward_subs_impl<0,M-1>::do_multi_rhs_pivot(j, L, B, p, y, X);
}
#if 0
// The run-time loop version - X needs to be zeroed out for this version
for (size_t j=0; j < N; ++j) {
Tensor<T,M> y(0);
// Solve for L * y = b
for (size_t i=0; i< M; ++i) {
T value = 0;
for (size_t k=0; k<i; ++k) {
value += L(i,k)*y(k);
}
y(i) = B(p(i),j) - value;
X(i,j) = y(i);
}
}
#endif
return X;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
/* Compile time recursive loop with inner for backward substitution of y/Y given the upper triangular matrix U.
The following meta functions implements U * x = y for single or multiple right sides
*/
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
template <int from, int to>
struct backward_subs_impl {
template<typename T, size_t M>
static FASTOR_INLINE void do_single_rhs(const Tensor<T,M,M> &U, const Tensor<T,M> &y, Tensor<T,M> &x) {
constexpr int idx = (int)M - from;
const T value = _inner<T,idx>(&U.data()[from*M+from],&x.data()[from]);
x(from) = ( y(from) - value) / U(from, from);
backward_subs_impl<from-1,to>::do_single_rhs(U, y, x);
}
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor<T,M,M> &U, const Tensor<T,M,N> &Y, Tensor<T,M> &x, Tensor<T,M,N> &X) {
constexpr int idx = (int)M - from;
const T value = _inner<T,idx>(&U.data()[from*M+from],&x.data()[from]);
x(from) = ( Y(from,j) - value ) / U(from, from);
X(from,j) = x(from);
backward_subs_impl<from-1,to>::do_multi_rhs(j, U, Y, x, X);
}
};
template <>
struct backward_subs_impl<0,0> {
template<typename T, size_t M>
static FASTOR_INLINE void do_single_rhs(const Tensor<T,M,M> &U, const Tensor<T,M> &y, Tensor<T,M> &x) {
constexpr int idx = (int)M;
const T value = _inner<T,idx>(&U.data()[0*M+0],&x.data()[0]);
x(0) = ( y(0) - value) / U(0, 0);
}
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void do_multi_rhs(const size_t j, const Tensor<T,M,M> &U, const Tensor<T,M,N> &Y, Tensor<T,M> &x, Tensor<T,M,N> &X) {
constexpr int idx = (int)M;
const T value = _inner<T,idx>(&U.data()[0*M+0],&x.data()[0]);
x(0) = ( Y(0,j) - value ) / U(0, 0);
X(0,j) = x(0);
}
};
template<typename T, size_t M>
FASTOR_INLINE Tensor<T,M> backward_subs(const Tensor<T,M,M> &U, const Tensor<T,M> &y) {
// We keep a separate output tensor X from x [x is columns of X]
// to avoid strided access in X for the inner product
Tensor<T,M> x(0);
backward_subs_impl<int(M)-1,0>::do_single_rhs(U, y, x);
#if 0
// The run-time loop version
// Solve for of U * x = y
for (int i= int(M) - 1; i>=0; --i) {
T value = 0;
for (int k=i; k<int(M); ++k) {
value += U(i,k)*x(k);
}
x(i) = (y(i) - value) / U(i, i);
}
#endif
return x;
}
template<typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> backward_subs(const Tensor<T,M,M> &U, const Tensor<T,M,N> &Y) {
// We keep a separate output tensor X from x [x is columns of X]
// to avoid strided access in X for the inner product
Tensor<T,M,N> X;
for (size_t j=0; j < N; ++j) {
Tensor<T,M> x(0);
backward_subs_impl<int(M)-1,0>::do_multi_rhs(j, U, Y, x, X);
}
#if 0
// The run-time loop version - X needs to be zeroed out for this version
Tensor<T,M,N> X(0);
for (size_t j=0; j < 1; ++j) {
// Solve for of U * x = y
for (int i= int(M) - 1; i>=0; --i) {
T value = 0;
for (int k=i; k<int(M); ++k) {
value += U(i,k)*X(k,j);
}
X(i,j) = (Y(i,j) - value) / U(i, i);
}
}
#endif
return X;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
/* Simple LU factorisation without pivoting */
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,8>,bool> = false>
FASTOR_INLINE void lu_simple_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
_lufact<T,M>(A.data(),L.data(),U.data());
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,8>,bool> = false>
FASTOR_INLINE void lu_simple_dispatcher(const Tensor<T,M,M>& A1, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
L.fill(0);
U.fill(0);
for (size_t j = 0; j < M; ++j) {
L(j, j) = 1;
for (size_t i = 0; i <= j; ++i) {
T value = A1(i, j);
for (size_t k = 0; k < i; ++k) {
value -= L(i, k) * U(k, j);
}
U(i, j) = value;
}
for (size_t i = j; i < M; ++i) {
T value = A1(i, j);
for (size_t k = 0; k < j; ++k) {
value -= L(i, k) * U(k, j);
}
value /= U(j, j);
L(i, j) = value;
}
}
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// Recursive non-modifying LU using matmul/outer product
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
/* This in essence implements the following loop but with static views
as dynamic views cannot be dispatched to matmul or a tensor cannot be
constructed to be sent to matmul
for(size_t j=0; j<M-1; ++j) {
L(seq(j+1,M),j) /= L(j,j);
L(seq(j+1,M),seq(j+1,M)) -= L(seq(j+1,M),j) % L(j,seq(j+1,M));
}
This is a non-modifying version that fills L and U directly and avoids
the need for extracting L and U later
The pre-requisite is that L should be Identity and U be a copy of A before
the recursive routine starts
*/
template<size_t from, size_t to>
struct recursive_lu_impl {
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void Do(Tensor<T,M,N> &L, Tensor<T,M,N> &U) {
L(fseq<from+1,M>(),fix<from>) = U(fseq<from+1,M>(),fix<from>) / U.data()[from*N+from];
U(fseq<from+1,M>(),fix<from>) = 0;
U(fseq<from+1,M>(),fseq<from+1,M>()) -= matmul(L(fseq<from+1,M>(),fix<from>), U(fix<from>,fseq<from+1,M>()));
recursive_lu_impl<from+1,to>::Do(L, U);
}
};
template<size_t from>
struct recursive_lu_impl<from,from> {
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void Do(Tensor<T,M,N> &L, Tensor<T,M,N> &U) {
L(fseq<from+1,M>(),fix<from>) = U(fseq<from+1,M>(),fix<from>) / U.data()[from*N+from];
U(fseq<from+1,M>(),fix<from>) = 0;
U(fseq<from+1,M>(),fseq<from+1,M>()) -= matmul(L(fseq<from+1,M>(),fix<from>), U(fix<from>,fseq<from+1,M>()));
}
};
template <typename T, size_t M, enable_if_t_<is_equal_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_lu_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
L(0,0) = 1; U(0,0) = A(0,0);
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_lu_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
// Requires M >=2
U = A;
L.eye2();
recursive_lu_impl<0,M-2>::Do(L, U);
}
#if 0
// Recursive in-place LU using matmul/outer product
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
/* This in essence implements the following loop but with static views
as dynamic views cannot be dispatched to matmul or a tensor cannot be
constructed to be sent to matmul
for(size_t j=0; j<M-1; ++j) {
L(seq(j+1,M),j) /= L(j,j);
L(seq(j+1,M),seq(j+1,M)) -= L(seq(j+1,M),j) % L(j,seq(j+1,M));
}
This is a self-modifying [in-place] version - if instead of a copy of
(LU in this case) we pass the matrix itself it will decompose it in to
an LU. Extracting L and U after this recursive decomposition is done
almost beats the purpose performance wise
*/
template<size_t from, size_t to>
struct recursive_lu_impl_inplace {
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void Do(Tensor<T,M,N> &LU) {
LU(fseq<from+1,M>(),fix<from>) /= LU.data()[from*N+from];
LU(fseq<from+1,M>(),fseq<from+1,M>()) -= matmul(LU(fseq<from+1,M>(),fix<from>), LU(fix<from>,fseq<from+1,M>()));
recursive_lu_impl_inplace<from+1,to>::Do(LU);
}
};
template<size_t from>
struct recursive_lu_impl_inplace<from,from> {
template<typename T, size_t M, size_t N>
static FASTOR_INLINE void Do(Tensor<T,M,N> &LU) {
LU(fseq<from+1,M>(),fix<from>) /= LU.data()[from*N+from];
LU(fseq<from+1,M>(),fseq<from+1,M>()) -= matmul(LU(fseq<from+1,M>(),fix<from>), LU(fix<from>,fseq<from+1,M>()));
}
};
template <typename T, size_t M, enable_if_t_<is_equal_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_inplace_lu_dispatcher(Tensor<T,M,M>& A) {
return;
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_inplace_lu_dispatcher(Tensor<T,M,M>& A) {
// Requires M >=2
recursive_lu_impl_inplace<0,M-2>::Do(A);
}
template <typename T, size_t M, enable_if_t_<is_equal_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& LU) {
LU(0,0) = A(0,0);
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& LU) {
// Requires M >=2
LU =A;
recursive_lu_impl_inplace<0,M-2>::Do(LU);
}
template <typename T, size_t M, enable_if_t_<is_equal_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
L(0,0) = 1; U(0,0) = A(0,0);
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,1>,bool> = false>
FASTOR_INLINE void recursive_inplace_lu_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
// Requires M >=2
Tensor<T,M,M> LU(A);
recursive_lu_impl_inplace<0,M-2>::Do(LU);
// Extracting L and U after this recursive decomposition is done
// almost beats the purpose performance wise
for (size_t i=0; i<M; ++i) {
L(i,i) = 1;
}
for (size_t i=0; i<M; ++i) {
for (size_t j=0; j<i; ++j) {
L(i,j) = LU(i,j);
}
}
for (size_t i=0; i<M; ++i) {
for (size_t j=i; j<M; ++j) {
U(i,j) = LU(i,j);
}
}
}
#endif
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
/* Block LU factorisation without pivoting */
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,8>,bool> = false>
FASTOR_INLINE void lu_block_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
_lufact<T,M>(A.data(),L.data(),U.data());
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,8> && is_less_equal_v_<M,32>,bool> = false>
FASTOR_INLINE void lu_block_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
recursive_lu_dispatcher(A, L, U);
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,32> && is_less_equal_v_<M,64>,bool> = false>
FASTOR_INLINE void lu_block_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
// We will compute LU decomposition block-wise assuming that A11 and A22 are invertible
//
// [A11 A12] [L11 0] [U11 U12]
// [A21 A22] [L21 L22] [0 U22]
//
// This results in
//
// A11 = L11 * U11
// A12 = L11 * U12
// A21 = L21 * U11
// A22 = L21 * U12 - L22 * U22
//
// Hence we need to do LU factorisation once for A11 and once for A22
// This is to avoid odd sizes for instance for size 35 we would
// want to do 35 = 16 + 19 rather than 35 = 32 + 3 if the start size was 32
constexpr size_t N = (M / 8UL * 8UL) / 2UL; // start size
Tensor<T,N ,N > A11 = A(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> A12 = A(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> A21 = A(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> A22 = A(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> L11(0), U11(0);
lu_block_dispatcher(A11, L11, U11);
// Solve for U12 = {L11}^(-1)*A12
Tensor<T,N ,M-N> U12 = tmatmul<UpLoType::Lower,UpLoType::General>(tinverse<InvCompType::SimpleInv, UpLoType::UniLower>(L11),A12);
// Solve for L21 = A21*{U11}^(-1)
Tensor<T,M-N, N> L21 = tmatmul<UpLoType::General,UpLoType::Upper>(A21,tinverse<InvCompType::SimpleInv, UpLoType::Upper>(U11));
Tensor<T,M-N,M-N> S = A22 - matmul(L21,U12);
Tensor<T,M-N,M-N> L22(0), U22(0);
lu_block_dispatcher(S, L22, U22);
L(fseq<0,N>(),fseq<0,N>()) = L11;
// L(fseq<0,N>(),fseq<N,M>()) = 0;
L(fseq<N,M>(),fseq<0,N>()) = L21;
L(fseq<N,M>(),fseq<N,M>()) = L22;
U(fseq<0,N>(),fseq<0,N>()) = U11;
U(fseq<0,N>(),fseq<N,M>()) = U12;
// U(fseq<N,M>(),fseq<0,N>()) = 0;
U(fseq<N,M>(),fseq<N,M>()) = U22;
}
// Conditional dispatch
namespace useless {
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,0> && is_less_equal_v_<M,64>,bool> = false>
FASTOR_INLINE void lu_block_simple_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
lu_block_dispatcher(A, L, U);
return;
}
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,64>,bool> = false>
FASTOR_INLINE void lu_block_simple_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
// lu_simple_dispatcher(A, L, U);
recursive_lu_dispatcher(A, L, U);
return;
}
} // useless
/* For sizes greater than 64 we tile differently to avoid too many recursions
*/
template <typename T, size_t M, enable_if_t_<is_greater_v_<M,64>,bool> = false>
FASTOR_INLINE void lu_block_dispatcher(const Tensor<T,M,M>& A, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
// We will compute LU decomposition block-wise assuming that A11 and A22 are invertible
//
// [A11 A12] [L11 0] [U11 U12]
// [A21 A22] [L21 L22] [0 U22]
//
// This results in
//
// A11 = L11 * U11
// A12 = L11 * U12
// A21 = L21 * U11
// A22 = L21 * U12 - L22 * U22
//
// Hence we need to do LU factorisation once for A11 and once for A22
// This is to avoid odd sizes for instance for size 65 we would
// want to do 65 = 32 + 33 rather than 65 = 64 + 1 if the start size was 64
constexpr size_t N = (M / 16UL * 16UL) / 2UL; // start size
Tensor<T,N ,N > A11 = A(fseq<0,N>(),fseq<0,N>());
Tensor<T,N ,M-N> A12 = A(fseq<0,N>(),fseq<N,M>());
Tensor<T,M-N, N> A21 = A(fseq<N,M>(),fseq<0,N>());
Tensor<T,M-N,M-N> A22 = A(fseq<N,M>(),fseq<N,M>());
Tensor<T,N,N> L11(0), U11(0);
useless::lu_block_simple_dispatcher(A11, L11, U11);
// Solve for U12 = {L11}^(-1)*A12
Tensor<T,N ,M-N> U12 = tmatmul<UpLoType::Lower,UpLoType::General>(tinverse<InvCompType::SimpleInv, UpLoType::UniLower>(L11),A12);
// Ideally use forward_subs but its iterative nature makes it less efficient than tmatmul
// Tensor<T,N ,M-N> U12 = forward_subs(L11,A12);
// Solve for L21 = A21*{U11}^(-1)
Tensor<T,M-N, N> L21 = tmatmul<UpLoType::General,UpLoType::Upper>(A21,tinverse<InvCompType::SimpleInv, UpLoType::Upper>(U11));
// Not quite performant as we can't avoid the matmul here
// Tensor<T,N ,N > I; I.eye2();
// Tensor<T,M-N, N> L21 = matmul(A21, backward_subs(U11, I));
Tensor<T,M-N,M-N> S = A22 - matmul(L21,U12);
Tensor<T,M-N,M-N> L22(0), U22(0);
useless::lu_block_simple_dispatcher(S, L22, U22);
L(fseq<0,N>(),fseq<0,N>()) = L11;
// L(fseq<0,N>(),fseq<N,M>()) = 0;
L(fseq<N,M>(),fseq<0,N>()) = L21;
L(fseq<N,M>(),fseq<N,M>()) = L22;
U(fseq<0,N>(),fseq<0,N>()) = U11;
U(fseq<0,N>(),fseq<N,M>()) = U12;
// U(fseq<N,M>(),fseq<0,N>()) = 0;
U(fseq<N,M>(),fseq<N,M>()) = U22;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // internal
/* Block LU factorisation overloads */
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// BlockLU - no pivot
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<is_tensor_v<Expr> && LUType == LUCompType::BlockLU,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
L.fill(0);
U.fill(0);
internal::lu_block_dispatcher(src.self(),L,U);
}
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<!is_tensor_v<Expr> && LUType == LUCompType::BlockLU,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
L.fill(0);
U.fill(0);
typename Expr::result_type tmp(src.self());
internal::lu_block_dispatcher(tmp,L,U);
}
// BlockLU - vector pivot
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<is_tensor_v<Expr> && LUType == LUCompType::BlockLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<size_t,M>& P) {
L.fill(0);
U.fill(0);
pivot_inplace(src.self(),P);
auto A(apply_pivot(src.self(),P));
internal::lu_block_dispatcher(A,L,U);
}
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<!is_tensor_v<Expr> && LUType == LUCompType::BlockLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<size_t,M>& P) {
L.fill(0);
U.fill(0);
typename Expr::result_type A(src.self());
pivot_inplace(A,P);
// Modify A as A is a temporary anyway
apply_pivot_inplace(A,P);
internal::lu_block_dispatcher(A,L,U);
}
// BlockLU - matrix pivot
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<is_tensor_v<Expr> && LUType == LUCompType::BlockLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<T,M,M>& P) {
L.fill(0);
U.fill(0);
pivot_inplace(src.self(),P);
auto A(apply_pivot(src.self(),P));
internal::lu_block_dispatcher(A,L,U);
}
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<!is_tensor_v<Expr> && LUType == LUCompType::BlockLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<T,M,M>& P) {
L.fill(0);
U.fill(0);
typename Expr::result_type A(src.self());
pivot_inplace(A,P);
// Modify A as A is a temporary anyway
apply_pivot_inplace(A,P);
internal::lu_block_dispatcher(A,L,U);
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
/* Simple LU factorisation overloads */
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// SimpleLU - no pivot
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<is_tensor_v<Expr> && LUType == LUCompType::SimpleLU,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
internal::lu_simple_dispatcher(src.self(),L,U);
}
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<!is_tensor_v<Expr> && LUType == LUCompType::SimpleLU,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U) {
typename Expr::result_type tmp(src.self());
internal::lu_simple_dispatcher(tmp,L,U);
}
// SimpleLU - vector pivot
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<is_tensor_v<Expr> && LUType == LUCompType::SimpleLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<size_t,M>& P) {
pivot_inplace(src.self(),P);
auto A(apply_pivot(src.self(),P));
internal::lu_simple_dispatcher(A,L,U);
}
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<!is_tensor_v<Expr> && LUType == LUCompType::SimpleLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<size_t,M>& P) {
typename Expr::result_type A(src.self());
pivot_inplace(A,P);
// Modify A as A is a temporary anyway
apply_pivot_inplace(A,P);
internal::lu_simple_dispatcher(A,L,U);
}
// SimpleLU - matrix pivot
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<is_tensor_v<Expr> && LUType == LUCompType::SimpleLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<T,M,M>& P) {
pivot_inplace(src.self(),P);
auto A(apply_pivot(src.self(),P));
internal::lu_simple_dispatcher(A,L,U);
}
template<LUCompType LUType = LUCompType::BlockLU, typename Expr, size_t DIM0, typename T, size_t M,
enable_if_t_<!is_tensor_v<Expr> && LUType == LUCompType::SimpleLUPiv,bool> = false>
FASTOR_INLINE
void
lu(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,M>& L, Tensor<T,M,M>& U, Tensor<T,M,M>& P) {
typename Expr::result_type A(src.self());
pivot_inplace(A,P);
// Modify A as A is a temporary anyway
apply_pivot_inplace(A,P);
internal::lu_simple_dispatcher(A,L,U);
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// Inversion using LU
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
namespace internal {
template<typename T, size_t M>
FASTOR_INLINE Tensor<T,M,M> get_lu_inverse(const Tensor<T,M,M> &L, const Tensor<T,M,M> &U) {
// We will solve for multiple RHS [B = I]
// Loop over columns of B = I
Tensor<T,M,M> I; I.eye2();
Tensor<T,M,M> Y = forward_subs(L, I);
Tensor<T,M,M> X = backward_subs(U, Y);
return X;
}
template<typename T, size_t M>
FASTOR_INLINE Tensor<T,M,M> get_lu_inverse(Tensor<T,M,M> &L, const Tensor<T,M,M> &U, const Tensor<size_t,M> &p) {
// We will solve for multiple RHS [B = I]
// Loop over columns of B = I
Tensor<T,M,M> I; I.eye2();
Tensor<T,M,M> Y = forward_subs(L, p, I);
Tensor<T,M,M> X = backward_subs(U, Y);
return X;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // internal
// SimpleLU
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t M, enable_if_t_<InvType == InvCompType::SimpleLU,bool> = false>
FASTOR_INLINE Tensor<T,M,M> inverse(const Tensor<T,M,M> &A) {
Tensor<T,M,M> L, U;
lu<LUCompType::SimpleLU>(A, L, U);
return internal::get_lu_inverse(L, U);
}
// BlockLU
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t M, enable_if_t_<InvType == InvCompType::BlockLU,bool> = false>
FASTOR_INLINE Tensor<T,M,M> inverse(const Tensor<T,M,M> &A) {
Tensor<T,M,M> L, U;
lu<LUCompType::BlockLU>(A, L, U);
return internal::get_lu_inverse(L, U);
}
// SimpleLUPiv
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t M, enable_if_t_<InvType == InvCompType::SimpleLUPiv,bool> = false>
FASTOR_INLINE Tensor<T,M,M> inverse(const Tensor<T,M,M> &A) {
Tensor<T,M,M> L, U;
Tensor<size_t,M> p;
lu<LUCompType::SimpleLUPiv>(A, L, U, p);
return internal::get_lu_inverse(L, U, p);
}
// BlockLUPiv
template<InvCompType InvType = InvCompType::SimpleInv,
typename T, size_t M, enable_if_t_<InvType == InvCompType::BlockLUPiv,bool> = false>
FASTOR_INLINE Tensor<T,M,M> inverse(const Tensor<T,M,M> &A) {
Tensor<T,M,M> L, U;
Tensor<size_t,M> p;
lu<LUCompType::BlockLUPiv>(A, L, U, p);
return internal::get_lu_inverse(L, U, p);
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// Solving linear system of equations using LU
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
namespace internal {
template<typename T, size_t M>
FASTOR_INLINE Tensor<T,M> get_lu_solve(const Tensor<T,M,M> &L, const Tensor<T,M,M> &U, const Tensor<T,M> &b) {
Tensor<T,M> y = forward_subs(L, b);
Tensor<T,M> x = backward_subs(U, y);
return x;
}
template<typename T, size_t M>
FASTOR_INLINE Tensor<T,M> get_lu_solve(Tensor<T,M,M> &L, const Tensor<T,M,M> &U, const Tensor<size_t,M> &p, const Tensor<T,M> &b) {
Tensor<T,M> y = forward_subs(L, p, b);
Tensor<T,M> x = backward_subs(U, y);
return x;
}
// Multiple RHS
template<typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> get_lu_solve(const Tensor<T,M,M> &L, const Tensor<T,M,M> &U, const Tensor<T,M,N> &B) {
Tensor<T,M,N> Y = forward_subs(L, B);
Tensor<T,M,N> X = backward_subs(U, Y);
return X;
}
template<typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> get_lu_solve(const Tensor<T,M,M> &L, const Tensor<T,M,M> &U, const Tensor<size_t,M> &p, const Tensor<T,M,N> &B) {
Tensor<T,M,N> Y = forward_subs(L, p, B);
Tensor<T,M,N> X = backward_subs(U, Y);
return X;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // internal
// Single RHS
// SimpleLU - no pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M,
enable_if_t_< SType == SolveCompType::SimpleLU, bool> = false>
FASTOR_INLINE Tensor<T,M> solve(const Tensor<T,M,M> &A, const Tensor<T,M> &b) {
Tensor<T,M,M> L, U;
lu<LUCompType::SimpleLU>(A, L, U);
return internal::get_lu_solve(L, U, b);
}
// SimpleLU - pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M,
enable_if_t_< SType == SolveCompType::SimpleLUPiv, bool> = false>
FASTOR_INLINE Tensor<T,M> solve(const Tensor<T,M,M> &A, const Tensor<T,M> &b) {
Tensor<T,M,M> L, U;
Tensor<size_t,M> p;
lu<LUCompType::SimpleLUPiv>(A, L, U, p);
return internal::get_lu_solve(L, U, p, b);
}
// BlockLU - no pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M,
enable_if_t_< SType == SolveCompType::BlockLU, bool> = false>
FASTOR_INLINE Tensor<T,M> solve(const Tensor<T,M,M> &A, const Tensor<T,M> &b) {
Tensor<T,M,M> L, U;
lu<LUCompType::BlockLU>(A, L, U);
return internal::get_lu_solve(L, U, b);
}
// BlockLU - pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M,
enable_if_t_< SType == SolveCompType::BlockLUPiv, bool> = false>
FASTOR_INLINE Tensor<T,M> solve(const Tensor<T,M,M> &A, const Tensor<T,M> &b) {
Tensor<T,M,M> L, U;
Tensor<size_t,M> p;
lu<LUCompType::BlockLUPiv>(A, L, U, p);
return internal::get_lu_solve(L, U, p, b);
}
// Multiple RHS
// SimpleLU - no pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M, size_t N,
enable_if_t_< SType == SolveCompType::SimpleLU, bool> = false>
FASTOR_INLINE Tensor<T,M,N> solve(const Tensor<T,M,M> &A, const Tensor<T,M,N> &B) {
Tensor<T,M,M> L, U;
lu<LUCompType::SimpleLU>(A, L, U);
return internal::get_lu_solve(L, U, B);
}
// SimpleLU - pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M, size_t N,
enable_if_t_< SType == SolveCompType::SimpleLUPiv, bool> = false>
FASTOR_INLINE Tensor<T,M,N> solve(const Tensor<T,M,M> &A, const Tensor<T,M,N> &B) {
Tensor<T,M,M> L, U;
Tensor<size_t,M> p;
lu<LUCompType::SimpleLUPiv>(A, L, U, p);
return internal::get_lu_solve(L, U, p, B);
}
// SimpleLU - no pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M, size_t N,
enable_if_t_< SType == SolveCompType::BlockLU, bool> = false>
FASTOR_INLINE Tensor<T,M,N> solve(const Tensor<T,M,M> &A, const Tensor<T,M,N> &B) {
Tensor<T,M,M> L, U;
lu<LUCompType::BlockLU>(A, L, U);
return internal::get_lu_solve(L, U, B);
}
// SimpleLU - pivot
template<SolveCompType SType = SolveCompType::SimpleInv, typename T, size_t M, size_t N,
enable_if_t_< SType == SolveCompType::BlockLUPiv, bool> = false>
FASTOR_INLINE Tensor<T,M,N> solve(const Tensor<T,M,M> &A, const Tensor<T,M,N> &B) {
Tensor<T,M,M> L, U;
Tensor<size_t,M> p;
lu<LUCompType::BlockLUPiv>(A, L, U, p);
return internal::get_lu_solve(L, U, p, B);
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// Computing determinant using LU
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
template<DetCompType DetType = DetCompType::Simple, typename T, size_t M,
enable_if_t_<DetType == DetCompType::LU,bool> = false>
FASTOR_INLINE T determinant(const Tensor<T,M,M> &A) {
int nswaps = internal::count_swaps(A) % 2UL == 0 ? 1 : -1;
Tensor<T,M,M> L, U;
Tensor<size_t,M> p;
lu<LUCompType::BlockLUPiv>(A, L, U, p);
return product(diag(U)) * nswaps;
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // UNARY_LU_OP_H

106
noarch/include/Fastor/expressions/linalg_ops/unary_norm_op.h
View File

@@ -1,106 +0,0 @@
#ifndef UNARY_NORM_OP_H
#define UNARY_NORM_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/backend/norm.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
namespace Fastor {
// For tensors
template<typename T, enable_if_t_<is_arithmetic_v_<T>,bool> = false>
FASTOR_INLINE T norm(const T &a) {
return std::abs(a);
}
template<typename T, size_t ... Rest>
FASTOR_INLINE T norm(const Tensor<T,Rest...> &a) {
if (sizeof...(Rest) == 0)
return *a.data();
return _norm<T,pack_prod<Rest...>::value>(a.data());
}
// For generic expressions
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type norm(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using T = typename Derived::scalar_type;
using V = choose_best_simd_vector_t<T>;
T _scal=0;
#ifdef FASTOR_AVX512_IMPL
V omm0, omm1, omm2, omm3, omm4, omm5, omm6, omm7;
#else
V omm0, omm1, omm2, omm3;
#endif
FASTOR_INDEX i = 0;
// With AVX utilises all the 16 registers but hurts the performance
// due to spill if eval has created temporary registers so only
// activated for AVX512
#ifdef FASTOR_AVX512_IMPL
for (; i < ROUND_DOWN(src.size(),8*V::Size); i+=8*V::Size) {
const auto smm0 = src.template eval<T>(i);
const auto smm1 = src.template eval<T>(i+V::Size);
const auto smm2 = src.template eval<T>(i+2*V::Size);
const auto smm3 = src.template eval<T>(i+3*V::Size);
const auto smm4 = src.template eval<T>(i+4*V::Size);
const auto smm5 = src.template eval<T>(i+5*V::Size);
const auto smm6 = src.template eval<T>(i+6*V::Size);
const auto smm7 = src.template eval<T>(i+7*V::Size);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
omm2 = fmadd(smm2,smm2,omm2);
omm3 = fmadd(smm3,smm3,omm3);
omm4 = fmadd(smm4,smm4,omm4);
omm5 = fmadd(smm5,smm5,omm5);
omm6 = fmadd(smm6,smm6,omm6);
omm7 = fmadd(smm7,smm7,omm7);
}
#endif
for (; i < ROUND_DOWN(src.size(),4*V::Size); i+=4*V::Size) {
const auto smm0 = src.template eval<T>(i);
const auto smm1 = src.template eval<T>(i+V::Size);
const auto smm2 = src.template eval<T>(i+2*V::Size);
const auto smm3 = src.template eval<T>(i+3*V::Size);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
omm2 = fmadd(smm2,smm2,omm2);
omm3 = fmadd(smm3,smm3,omm3);
}
for (; i < ROUND_DOWN(src.size(),2*V::Size); i+=2*V::Size) {
const auto smm0 = src.template eval<T>(i);
const auto smm1 = src.template eval<T>(i+V::Size);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
}
for (; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) {
const auto smm0 = src.template eval<T>(i);
omm0 = fmadd(smm0,smm0,omm0);
}
for (; i < src.size(); ++i) {
const auto smm0 = src.template eval_s<T>(i);
_scal += smm0*smm0;
}
#ifdef FASTOR_AVX512_IMPL
return sqrts( (omm0 + omm1 + omm2 + omm3 + omm4 + omm5 + omm6 + omm7).sum() + _scal);
#else
return sqrts( (omm0 + omm1 + omm2 + omm3).sum() + _scal);
#endif
}
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type norm(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out(src);
return norm(out);
}
} // end of namespace Fastor
#endif // UNARY_NORM_OP_H

409
noarch/include/Fastor/expressions/linalg_ops/unary_piv_op.h
View File

@@ -1,409 +0,0 @@
#ifndef UNARY_PIV_OP_H
#define UNARY_PIV_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include <algorithm>
namespace Fastor {
namespace internal {
template<typename T, size_t M, size_t N>
FASTOR_INLINE size_t count_swaps(const Tensor<T,M,N>& A) {
size_t count = 0;
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A(i, j)) > std::abs(A(max_index, j)))
max_index = i;
}
if (j != max_index)
count++;
}
return count;
}
}
template<PivType PType = PivType::V, typename T, size_t M, size_t N,
enable_if_t_<PType == PivType::M, bool> = false>
FASTOR_INLINE Tensor<T,M,N> pivot(const Tensor<T,M,N>& A) {
Tensor<size_t,M> perm;
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A(i, j)) > std::abs(A(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
Tensor<T,M,N> P(0);
for (size_t i = 0; i < M; ++i)
P(i, perm(i)) = 1;
return P;
}
template<PivType PType = PivType::V, typename T, size_t M, size_t N,
enable_if_t_<PType == PivType::V, bool> = false>
FASTOR_INLINE Tensor<size_t,M> pivot(const Tensor<T,M,N>& A) {
Tensor<size_t,M> perm;
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A(i, j)) > std::abs(A(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
return perm;
}
// For generic expressions
template<PivType PType = PivType::V, typename Derived, size_t DIM,
enable_if_t_<PType == PivType::M && requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
Tensor<typename Derived::scalar_type,
get_tensor_dimension_v<0,typename Derived::result_type>,
get_tensor_dimension_v<1,typename Derived::result_type>
>
pivot(const AbstractTensor<Derived,DIM>& src) {
typename Derived::result_type tmp(src.self());
return pivot<PType>(tmp);
}
template<PivType PType = PivType::M, typename Derived, size_t DIM,
enable_if_t_<PType == PivType::M && !requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
Tensor<typename Derived::scalar_type,
get_tensor_dimension_v<0,typename Derived::result_type>,
get_tensor_dimension_v<1,typename Derived::result_type>
>
pivot(const AbstractTensor<Derived,DIM>& src) {
static_assert(DIM==2, "TENSOR MUST BE SQUARE FOR PIVOT COMPUTATION");
using T = typename Derived::scalar_type;
using result_type = typename Derived::result_type;
constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>;
constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>;
const Derived &A = src.self();
Tensor<size_t,M> perm;
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A.template eval_s<T>(i, j)) > std::abs(A.template eval_s<T>(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
Tensor<T,M,N> P(0);
for (size_t i = 0; i < M; ++i)
P(i, perm(i)) = 1;
return P;
}
template<PivType PType = PivType::V, typename Derived, size_t DIM,
enable_if_t_<PType == PivType::V && requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
Tensor<size_t,
get_tensor_dimension_v<0,typename Derived::result_type>
>
pivot(const AbstractTensor<Derived,DIM>& src) {
typename Derived::result_type tmp(src.self());
return pivot<PType>(tmp);
}
template<PivType PType = PivType::M, typename Derived, size_t DIM,
enable_if_t_<PType == PivType::V && !requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
Tensor<size_t,
get_tensor_dimension_v<0,typename Derived::result_type>
>
pivot(const AbstractTensor<Derived,DIM>& src) {
static_assert(DIM==2, "TENSOR MUST BE SQUARE FOR PIVOT COMPUTATION");
using T = typename Derived::scalar_type;
using result_type = typename Derived::result_type;
constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,result_type>;
constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,result_type>;
const Derived &A = src.self();
Tensor<size_t,M> perm;
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A.template eval_s<T>(i, j)) > std::abs(A.template eval_s<T>(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
return perm;
}
/* In place versions - Given an evaluated pivot tensor populates it
*/
template<typename T, size_t M, size_t N>
FASTOR_INLINE void pivot_inplace(const Tensor<T,M,N>& A, Tensor<size_t,M>& perm) {
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A(i, j)) > std::abs(A(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
}
template<typename T, size_t M, size_t N>
FASTOR_INLINE void pivot_inplace(const Tensor<T,M,N>& A, Tensor<T,M,N> &P) {
Tensor<size_t,M> perm;
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A(i, j)) > std::abs(A(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
P.fill(0);
for (size_t i = 0; i < M; ++i)
P(i, perm(i)) = 1;
}
template<typename Derived, size_t DIM, size_t M,
enable_if_t_<requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
void
pivot_inplace(const AbstractTensor<Derived,DIM>& src, Tensor<size_t,M> &perm) {
typename Derived::result_type tmp(src.self());
pivot_inplace(tmp,perm);
}
template<typename Derived, size_t DIM, size_t M,
enable_if_t_<!requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
void
pivot_inplace(const AbstractTensor<Derived,DIM>& src, Tensor<size_t,M> &perm) {
static_assert(DIM==2, "TENSOR MUST BE 2D FOR PIVOT COMPUTATION");
using T = typename Derived::scalar_type;
const Derived &A = src.self();
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A.template eval_s<T>(i, j)) > std::abs(A.template eval_s<T>(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
}
template<typename Derived, size_t DIM, typename T, size_t M, size_t N,
enable_if_t_<requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
void
pivot_inplace(const AbstractTensor<Derived,DIM>& src, Tensor<T,M,N> &P) {
typename Derived::result_type tmp(src.self());
pivot_inplace(tmp,P);
}
template<typename Derived, size_t DIM, typename T, size_t M, size_t N,
enable_if_t_<!requires_evaluation_v<Derived>, bool> = false>
FASTOR_INLINE
void
pivot_inplace(const AbstractTensor<Derived,DIM>& src, Tensor<T,M,N> &P) {
static_assert(DIM==2, "TENSOR MUST BE 2D FOR PIVOT COMPUTATION");
const Derived &A = src.self();
Tensor<size_t,M> perm;
perm.iota();
for (size_t j = 0; j < M; ++j) {
size_t max_index = j;
for (size_t i = j; i < M; ++i) {
// std::abs is necessary only for complex valued numbers
if (std::abs(A.template eval_s<T>(i, j)) > std::abs(A.template eval_s<T>(max_index, j)))
max_index = i;
}
if (j != max_index)
std::swap(perm(j), perm(max_index));
}
for (size_t i = 0; i < M; ++i)
P(i, perm(i)) = 1;
}
/* Apply a pivot on a tensor/matrix
Applying pivot can only work on evaluated tensors and not expression
as non-evaluated expression do not have storage/data
*/
template <typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> apply_pivot(const Tensor<T,M,N>& A, const Tensor<size_t,M>& P) {
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
if (P(i) != i) {
std::copy_n(&A.data()[P(i)*N],N,&copyA.data()[i*N]);
}
}
return copyA;
}
template <typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> apply_pivot(const Tensor<T,M,N>& A, const Tensor<T,M,N>& P) {
// The output tensor is just matmul(P,A), but we are going to avoid
// the call to matmul
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1));
size_t p = std::distance(&P.data()[i*N],it);
// Swap row p and i
if (p != i) {
std::copy_n(&A.data()[p*N],N,&copyA.data()[i*N]);
}
}
return copyA;
}
template <typename T, size_t M, size_t N>
FASTOR_INLINE void apply_pivot_inplace(Tensor<T,M,N>& A, const Tensor<size_t,M>& P) {
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
if (P(i) != i) {
std::copy_n(&copyA.data()[P(i)*N],N,&A.data()[i*N]);
}
}
}
template <typename T, size_t M, size_t N>
FASTOR_INLINE void apply_pivot_inplace(Tensor<T,M,N>& A, const Tensor<T,M,N>& P) {
// The output tensor is just matmul(P,A), but we are going to avoid
// the call to matmul
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1));
size_t p = std::distance(&P.data()[i*N],it);
// Swap row p and i
if (p != i) {
std::copy_n(&copyA.data()[p*N],N,&A.data()[i*N]);
}
}
}
/* Reconstructing the matrix back from a pivoted factorisation
Reconstructs from LU/QR/QL etc
*/
template <typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> reconstruct(const Tensor<T,M,N>& L, Tensor<T,M,N>& U) {
return matmul(L,U);
}
/* Reconstructing the matrix back from a pivoted factorisation
Reconstructs from an PLU decomposition given P, L, U where P is
integral permutation vector
A = {P}^(-1)*L*U
*/
template <typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> reconstruct(const Tensor<T,M,N>& L, Tensor<T,M,N>& U, const Tensor<size_t,M>& P) {
Tensor<T,M,N> A = matmul(L,U);
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
if (P(i) != i) {
std::copy_n(&copyA.data()[i*N],N,&A.data()[P(i)*N]);
}
}
return A;
}
/* Reconstructing the matrix back from a pivoted factorisation
Reconstructs from an PLU decomposition given P, L, U
A = {P}^(-1)*L*U
*/
template <typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> reconstruct(const Tensor<T,M,N>& L, Tensor<T,M,N>& U, const Tensor<T,M,N>& P) {
// To avoid computing the inverse of P in {P}^(-1)*L*U
Tensor<T,M,N> A = matmul(L,U);
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
auto it = std::find(&P.data()[i*N],&P.data()[i*N+N],T(1));
size_t p = std::distance(&P.data()[i*N],it);
if (p != i) {
std::copy_n(&copyA.data()[i*N],N,&A.data()[p*N]);
}
}
return A;
}
/* Reconstructing the matrix back from a pivot only
*/
template <typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> reconstruct(const Tensor<T,M,N>& A, const Tensor<size_t,M>& P) {
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
if (P(i) != i) {
std::copy_n(&A.data()[i*N],N,&copyA.data()[P(i)*N]);
}
}
return copyA;
}
/* Reconstructing the matrix back from a pivot only - column-wise or rather post multiplication
used for computing inverse
*/
template <typename T, size_t M, size_t N>
FASTOR_INLINE Tensor<T,M,N> reconstruct_colwise(const Tensor<T,M,N>& A, const Tensor<size_t,M>& P) {
Tensor<T,M,N> copyA(A);
for (size_t i=0; i< M; ++i) {
if (P(i) != i) {
copyA(all,P(i)) = A(all,i);
}
}
return copyA;
}
} // end of namespace Fastor
#endif // UNARY_PIV_OP_H

198
noarch/include/Fastor/expressions/linalg_ops/unary_qr_op.h
View File

@@ -1,198 +0,0 @@
#ifndef UNARY_QR_OP_H
#define UNARY_QR_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/unary_piv_op.h"
namespace Fastor {
namespace internal {
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
/* Modified Gram-Schmidt Row-wise [MGSR] QR factorisation
that provides numerical stabilitiy.
This simple implementation provided for now is just for
convenience and not tuned for performance. Given that Tensor
types have fixed dimensions the compiler typically does a good
job
See:
Thomas Jakobs et. al. "Performance and energy consumption of the SIMD
Gram–Schmidt process for vector orthogonalization"
for implementation details including AVX/AVX512 implementation. A straight-
forward explicit vectorisation of this does not yield good performance for
row-major tensors, as step 1-2 need gather instructions and step 3-4 require
dynamic masking to gain performance
*/
template<typename T, size_t M, size_t N>
FASTOR_INLINE void qr_mgsr_dispatcher(const Tensor<T,M,N> &A0, Tensor<T,M,N>& Q, Tensor<T,M,N>& R) {
// Copy incoming tensor as we will modify A
Tensor<T,M,N> A(A0);
// Zero out
R.fill(0);
for (size_t i=0; i< N; ++i) {
// step 1
T R_ii = 0;
for (size_t k=0; k< M; ++k) {
R_ii += A(k,i)*A(k,i);
}
R_ii = sqrts(R_ii);
R(i,i) = R_ii;
// step 2
for (size_t k=0; k< M; ++k) {
Q(k,i) = A(k,i) / R_ii;
}
// step 3
for (size_t k=0; k< M; ++k) {
for (size_t j=i+1; j<N; ++j) {
R(i,j) += Q(k,i) * A(k,j);
}
}
// step 4
for (size_t k=0; k< M; ++k) {
for (size_t j=i+1; j<N; ++j) {
A(k,j) -= Q(k,i) * R(i,j);
}
}
}
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // internal
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// QR MGSR - no pivot
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<is_tensor_v<Expr> && QRType == QRCompType::MGSR,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R) {
internal::qr_mgsr_dispatcher(src.self(),Q,R);
}
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<!is_tensor_v<Expr> && QRType == QRCompType::MGSR,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R) {
Tensor<T,M,N> A(src.self());
internal::qr_mgsr_dispatcher(A,Q,R);
}
// QR MGSR - vector pivot
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<is_tensor_v<Expr> && QRType == QRCompType::MGSRPiv,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R, Tensor<size_t,M> &P) {
pivot_inplace(src.self(),P);
auto A(apply_pivot(src.self(),P));
internal::qr_mgsr_dispatcher(A,Q,R);
}
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<!is_tensor_v<Expr> && QRType == QRCompType::MGSRPiv,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R, Tensor<size_t,M> &P) {
typename Expr::result_type A(src.self());
pivot_inplace(A,P);
// Modify A as A is a temporary anyway
apply_pivot_inplace(A,P);
internal::qr_mgsr_dispatcher(A,Q,R);
}
// QR MGSR - matrix pivot
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<is_tensor_v<Expr> && QRType == QRCompType::MGSRPiv,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R, Tensor<T,M,N> &P) {
pivot_inplace(src.self(),P);
auto A(apply_pivot(src.self(),P));
internal::qr_mgsr_dispatcher(A,Q,R);
}
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<!is_tensor_v<Expr> && QRType == QRCompType::MGSRPiv,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R, Tensor<T,M,N> &P) {
typename Expr::result_type A(src.self());
pivot_inplace(A,P);
// Modify A as A is a temporary anyway
apply_pivot_inplace(A,P);
internal::qr_mgsr_dispatcher(A,Q,R);
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// QR householder reflections etc
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<QRType != QRCompType::MGSR,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R) {
static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET");
}
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<QRType != QRCompType::MGSRPiv,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R, Tensor<size_t,M> &P) {
static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET");
}
template<QRCompType QRType = QRCompType::MGSR, typename Expr, size_t DIM0, typename T, size_t M, size_t N,
enable_if_t_<QRType != QRCompType::MGSRPiv,bool> = false>
FASTOR_INLINE
void
qr(const AbstractTensor<Expr,DIM0> &src, Tensor<T,M,N> &Q, Tensor<T,N,M> &R, Tensor<T,M,N> &P) {
static_assert(QRType==QRCompType::MGSR, "QR FACTORISATION USING HOUSEHOLDER REFLECTIONS IS NOT IMPLEMENETED YET");
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
// Computing determinant using QR
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
template<DetCompType DetType = DetCompType::Simple, typename T, size_t M,
enable_if_t_<DetType == DetCompType::QR,bool> = false>
FASTOR_INLINE T determinant(const Tensor<T,M,M> &a) {
Tensor<T,M,M> Q, R;
qr(a, Q, R);
return product(diag(R));
}
//-----------------------------------------------------------------------------------------------------------//
//-----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // UNARY_QR_OP_H

163
noarch/include/Fastor/expressions/linalg_ops/unary_svd_op.h
View File

@@ -1,163 +0,0 @@
#ifndef UNARY_SVD_OP_H
#define UNARY_SVD_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/inner.h"
#include "Fastor/backend/lufact.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_computation_types.h"
#include "Fastor/expressions/linalg_ops/unary_det_op.h"
#include "Fastor/expressions/linalg_ops/unary_trans_op.h"
namespace Fastor {
// SVD
template<typename T, size_t M, enable_if_t_<M==2, bool> = false >
FASTOR_INLINE void svd(const Tensor<T,M,M> &A, Tensor<T,M,M> &U, Tensor<T,M,M> &S, Tensor<T,M,M> &V) {
constexpr T Epsilon_v = std::numeric_limits<T>::epsilon();
const T f00 = A(0, 0);
const T f01 = A(0, 1);
const T f10 = A(1, 0);
const T f11 = A(1, 1);
// If matrix is diagonal, SVD is trivial
if (std::abs(f01 - f10) < Epsilon_v && std::abs(f01) < Epsilon_v)
{
// Compute U
U(0,0) = f00 < 0 ? -1. : 1.;
U(0,1) = 0.;
U(1,0) = 0.;
U(1,1) = f11 < 0. ? -1. : 1.;
// Compute S
S(0,0) = std::abs(f00);
S(0,1) = 0;
S(1,0) = 0;
S(1,1) = std::abs(f11);
// Compute V
V.eye2();
}
// Otherwise, we need to compute A^T*A
else
{
T j = f00 * f00 + f01 * f01;
T k = f10 * f10 + f11 * f11;
T v_c = f00 * f10 + f01 * f11;
// Check to see if A^T*A is diagonal
if (std::abs(v_c) < Epsilon_v)
{
// Compute S
T s1 = std::sqrt(j);
T s2 = std::abs(j - k) < Epsilon_v ? s1 : std::sqrt(k);
S(0,0) = s1;
S(0,1) = 0;
S(1,0) = 0;
S(1,1) = s2;
// Compute U
U.eye2();
// Compute V
V(0,0) = f00 / s1;
V(0,1) = f10 / s2;
V(1,0) = f01 / s1;
V(1,1) = f11 / s2;
}
// Otherwise, solve quadratic equation for eigenvalues
else
{
T jmk = j - k;
T jpk = j + k;
T root = std::sqrt(jmk * jmk + 4. * v_c * v_c);
T eig1 = (jpk + root) * 0.5;
T eig2 = (jpk - root) * 0.5;
// Compute S
T s1 = std::sqrt(eig1);
T s2 = std::abs(root) < Epsilon_v ? s1 : ( eig2 > 0 ? std::sqrt(eig2) : Epsilon_v);
S(0,0) = s1;
S(0,1) = 0;
S(1,0) = 0;
S(1,1) = s2;
// Compute U - use eigenvectors of A^T*A as U
T v_s = eig1 - j;
T len = std::max(std::sqrt(v_s * v_s + v_c * v_c), Epsilon_v);
v_c /= len;
v_s /= len;
U(0,0) = v_c;
U(0,1) = -v_s;
U(1,0) = v_s;
U(1,1) = v_c;
// Compute V - as A * U / s
const T cc = (f00 * v_c + f10 * v_s) / s1;
const T cs = (f01 * v_c + f11 * v_s) / s1;
if (std::abs(s2) > Epsilon_v)
{
V(0,0) = cc;
V(0,1) = (f10* v_c - f00 * v_s) / s2;
V(1,0) = cs;
V(1,1) = (f11 * v_c - f01 * v_s) / s2;
}
else
{
V(0,0) = cc;
V(0,1) = cs;
V(1,0) = cs;
V(1,1) = -cc;
}
}
}
}
// Signed SVD
template<typename T, size_t M>
FASTOR_INLINE void ssvd(const Tensor<T,M,M> &A, Tensor<T,M,M> &U, Tensor<T,M,M> &S, Tensor<T,M,M> &V) {
// Same as above but avoiding the L matrix
svd(A, U, S, V);
// See where to pull the reflection out of
const T detU = determinant(U);
const T detV = determinant(V);
if (detU >= 0 && detV >= 0)
{
// No reflection svd == svd_rv, return
return;
}
Tensor<T, M, M> L = matmul(U, transpose(V));
const T lastColumn = determinant(L);
if (detU < 0 && detV > 0)
{
U(all, M - 1) *= lastColumn;
}
else if (detU > 0 && detV < 0)
{
V(all, M - 1) *= lastColumn;
}
// Push the reflection to the diagonal
S(M - 1, M - 1) *= lastColumn;
}
//-----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // UNARY_SVD_OP_H

75
noarch/include/Fastor/expressions/linalg_ops/unary_trace_op.h
View File

@@ -1,75 +0,0 @@
#ifndef UNARY_TRACE_OP_H
#define UNARY_TRACE_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
namespace Fastor {
// For tensors
template<typename T, size_t I>
FASTOR_INLINE T trace(const Tensor<T,I,I> &a) {
return _trace<T,I,I>(static_cast<const T *>(a.data()));
}
// For high order tensors
template<typename T, size_t ... Rest, typename std::enable_if<sizeof...(Rest)>=3,bool>::type=0>
FASTOR_INLINE
typename last_matrix_extracter<Tensor<T,Rest...>, typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::type
trace(const Tensor<T,Rest...> &a) {
using OutTensor = typename last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::type;
constexpr size_t remaining_product = last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::remaining_product;
constexpr size_t I = get_value<sizeof...(Rest)-1,Rest...>::value;
constexpr size_t J = get_value<sizeof...(Rest),Rest...>::value;
static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME");
OutTensor out;
T *a_data = a.data();
T *out_data = out.data();
for (size_t i=0; i<remaining_product; ++i) {
out_data[i] = _trace<T,J,J>(static_cast<const T *>(a_data+i*J*J));
}
return out;
}
// Trace for generic expressions
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type trace(const AbstractTensor<Derived,DIMS> &_src) {
const Derived& src = _src.self();
using T = typename Derived::scalar_type;
using tensor_type = typename Derived::result_type;
constexpr size_t M = get_tensor_dimensions<tensor_type>::dims[0];
constexpr size_t N = get_tensor_dimensions<tensor_type>::dims[1];
static_assert(DIMS==2,"TENSOR EXPRESSION SHOULD BE UNIFORM (SQUARE)");
static_assert(M==N,"TENSOR EXPRESSION SHOULD BE TWO DIMENSIONAL");
FASTOR_INDEX i;
T _scal=0;
for (i = 0; i < M; ++i) {
_scal += src.template eval_s<T>(i*(N+1));
}
return _scal;
}
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type trace(const AbstractTensor<Derived,DIMS> &_src) {
using result_type = typename Derived::result_type;
const result_type out(_src.self());
return trace(out);
}
} // end of namespace Fastor
#endif // UNARY_TRACE_OP_H

173
noarch/include/Fastor/expressions/linalg_ops/unary_trans_op.h
View File

@@ -1,173 +0,0 @@
#ifndef UNARY_TRANS_OP_H
#define UNARY_TRANS_OP_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/transpose/transpose.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/tensor/AbstractTensor.h"
#include "Fastor/tensor/Aliasing.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/tensor/TensorTraits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
template<typename Expr, size_t DIM0>
struct UnaryTransOp: public AbstractTensor<UnaryTransOp<Expr, DIM0>,DIM0> {
using expr_type = expression_t<Expr>;
static constexpr FASTOR_INDEX M = get_tensor_dimension_v<0,typename Expr::result_type>;
static constexpr FASTOR_INDEX N = get_tensor_dimension_v<1,typename Expr::result_type>;
static constexpr FASTOR_INDEX Dimension = DIM0;
static constexpr FASTOR_INDEX rank() {return DIM0;}
using scalar_type = typename scalar_type_finder<UnaryTransOp<Expr, DIM0>>::type;
using simd_vector_type = typename Expr::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = Tensor<scalar_type,N,M>;
FASTOR_INLINE UnaryTransOp(expr_type inexpr) : _expr(inexpr) {
}
FASTOR_INLINE FASTOR_INDEX size() const {return M*N;}
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return i == 0 ? N : M;}
constexpr FASTOR_INLINE expr_type expr() const {return _expr;}
private:
expr_type _expr;
};
/* Tensor transpose returning a tensor expression */
template<typename Expr, size_t DIM0>
FASTOR_INLINE UnaryTransOp<Expr, DIM0>
trans(const AbstractTensor<Expr,DIM0> &src) {
return UnaryTransOp<Expr, DIM0>(src.self());
}
/* Tensor transpose immediately returning a tensor */
template<typename T, size_t I, size_t J>
FASTOR_INLINE Tensor<T,J,I> transpose(const Tensor<T,I,J> &a) {
Tensor<T,J,I> out;
_transpose<T,I,J>(a.data(),out.data());
return out;
}
/* Tensor transpose for higher order tensors immediately returning a tensor */
template<typename T, size_t ... Rest, typename std::enable_if<sizeof...(Rest)>=3,bool>::type=0>
FASTOR_INLINE Tensor<T,Rest...>
transpose(const Tensor<T,Rest...> &a) {
constexpr size_t remaining_product = last_matrix_extracter<Tensor<T,Rest...>,
typename std_ext::make_index_sequence<sizeof...(Rest)-2>::type>::remaining_product;
constexpr size_t I = get_value<sizeof...(Rest)-1,Rest...>::value;
constexpr size_t J = get_value<sizeof...(Rest),Rest...>::value;
static_assert(I==J,"THE LAST TWO DIMENSIONS OF TENSOR MUST BE THE SAME");
Tensor<T,Rest...> out;
T *a_data = a.data();
T *out_data = out.data();
for (size_t i=0; i<remaining_product; ++i) {
_transpose<T,J,J>(a_data+i*J*J,out_data+i*J*J);
}
return out;
}
// Transpose for generic expressions
template<typename Expr, size_t DIM0>
FASTOR_INLINE
Tensor<
typename scalar_type_finder<Expr>::type,
get_tensor_dimension_v<1,typename Expr::result_type>,
get_tensor_dimension_v<0,typename Expr::result_type>>
transpose(const AbstractTensor<Expr,DIM0> &src) {
// If we are here Expr is already an expression
using result_type = typename Expr::result_type;
const result_type tmp(src.self());
return transpose(tmp);
}
// assignments
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign(AbstractTensor<Derived,DIM> &dst, const UnaryTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
const result_type& tmp = evaluate(src.expr().self());
using T = typename UnaryTransOp<Expr, OtherDIM>::scalar_type;
static constexpr size_t M = UnaryTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryTransOp<Expr, OtherDIM>::N;
_transpose<T,M,N>(tmp.data(),dst.self().data());
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_add(AbstractTensor<Derived,DIM> &dst, const UnaryTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryTransOp<Expr, OtherDIM>::N;
_transpose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_add(dst.self(),tmp_trans);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_sub(AbstractTensor<Derived,DIM> &dst, const UnaryTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryTransOp<Expr, OtherDIM>::N;
_transpose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_sub(dst.self(),tmp_trans);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_mul(AbstractTensor<Derived,DIM> &dst, const UnaryTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryTransOp<Expr, OtherDIM>::N;
_transpose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_mul(dst.self(),tmp_trans);
}
template<typename Derived, size_t DIM, typename Expr, size_t OtherDIM>
FASTOR_INLINE void assign_div(AbstractTensor<Derived,DIM> &dst, const UnaryTransOp<Expr, OtherDIM> &src) {
using result_type = typename Expr::result_type;
// no copies if expr is a tensor
const result_type& tmp = evaluate(src.expr().self());
// one copy for the inverse
using T = typename UnaryTransOp<Expr, OtherDIM>::scalar_type;
using result_t = typename UnaryTransOp<Expr, OtherDIM>::result_type;
result_t tmp_trans;
static constexpr size_t M = UnaryTransOp<Expr, OtherDIM>::M;
static constexpr size_t N = UnaryTransOp<Expr, OtherDIM>::N;
_transpose<T,M,N>(tmp.data(),tmp_trans.data());
trivial_assign_div(dst.self(),tmp_trans);
}
} // end of namespace Fastor
#endif // UNARY_TRANS_OP_H

101
noarch/include/Fastor/expressions/unary_ops/unary_bool_ops.h
View File

@@ -1,101 +0,0 @@
#ifndef UNARY_BOOL_OP_H
#define UNARY_BOOL_OP_H
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/simd_math/simd_math.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
// All unary bool ops
#define FASTOR_MAKE_UNARY_BOOL_OPS(OP_NAME, SIMD_OP, SCALAR_OP, STRUCT_NAME, EVAL_TYPE)\
template<typename Expr, size_t DIM0>\
struct Unary ##STRUCT_NAME ## Op: public AbstractTensor<Unary ##STRUCT_NAME ## Op<Expr, DIM0>,DIM0> {\
private:\
expression_t<Expr> _expr;\
public:\
using scalar_type = typename Expr::scalar_type;\
using simd_vector_type = typename Expr::simd_vector_type;\
using simd_abi_type = typename simd_vector_type::abi_type;\
using result_type = to_bool_tensor_t<typename Expr::result_type>;\
using result_scalar_type = typename result_type::scalar_type;\
using result_simd_abi_type = typename result_type::simd_abi_type;\
using result_simd_vector_type = typename result_type::simd_vector_type;\
using UU = bool /*this needs to change to U once masks are implemented*/;\
using ABI = simd_abi::fixed_size<SIMDVector<EVAL_TYPE,simd_abi_type>::Size>;\
static constexpr FASTOR_INDEX Dimension = DIM0;\
static constexpr FASTOR_INDEX rank() {return DIM0;}\
FASTOR_INLINE FASTOR_INDEX size() const {return _expr.size();}\
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _expr.dimension(i);}\
Unary ##STRUCT_NAME ## Op(expression_t<Expr> inexpr) : _expr(inexpr) {}\
FASTOR_INLINE expression_t<Expr> expr() const {return _expr;}\
template<typename U=scalar_type>\
FASTOR_INLINE SIMDVector<UU,ABI> eval(FASTOR_INDEX i) const {\
return SIMD_OP(_expr.template eval<EVAL_TYPE>(i));\
}\
template<typename U=scalar_type>\
FASTOR_INLINE UU eval_s(FASTOR_INDEX i) const {\
return SCALAR_OP(_expr.template eval_s<EVAL_TYPE>(i));\
}\
template<typename U=scalar_type>\
FASTOR_INLINE SIMDVector<UU,ABI> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return SIMD_OP(_expr.template eval<EVAL_TYPE>(i,j));\
}\
template<typename U=scalar_type>\
FASTOR_INLINE UU eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return SCALAR_OP(_expr.template eval_s<EVAL_TYPE>(i,j));\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<UU,ABI> teval(const std::array<int,DIM0> &as) const {\
return SIMD_OP(_expr.template teval<EVAL_TYPE>(as));\
}\
template<typename U>\
FASTOR_INLINE UU teval_s(const std::array<int,DIM0> &as) const {\
return SCALAR_OP(_expr.template teval_s<EVAL_TYPE>(as));\
}\
};\
template<typename Expr, size_t DIM0,\
typename std::enable_if<!std::is_arithmetic<Expr>::value,bool>::type = 0 >\
FASTOR_INLINE Unary ##STRUCT_NAME ## Op<Expr, DIM0> OP_NAME(const AbstractTensor<Expr,DIM0> &_expr) {\
return Unary ##STRUCT_NAME ## Op<Expr, DIM0>(_expr.self());\
}\
FASTOR_MAKE_UNARY_BOOL_OPS(operator!, ! , ! , Not , scalar_type)
FASTOR_MAKE_UNARY_BOOL_OPS(isinf , isinf , std::isinf , Isinf , scalar_type)
FASTOR_MAKE_UNARY_BOOL_OPS(isnan , isnan , std::isnan , Isnan , scalar_type)
FASTOR_MAKE_UNARY_BOOL_OPS(isfinite, isfinite, std::isfinite, Isfinite, scalar_type)
#define FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM,\
typename std::enable_if<requires_evaluation_v<OtherDerived>,bool>::type = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Unary ##NAME ## Op<OtherDerived,OtherDIM> &src) {\
using result_type = typename OtherDerived::result_type;\
const result_type tmp(src.expr().self());\
trivial_assign ##ASSIGN_TYPE (dst.self(), OP(tmp));\
}\
template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM,\
typename std::enable_if<!requires_evaluation_v<OtherDerived>,bool>::type = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Unary ##NAME ## Op<OtherDerived,OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
// only assignment
FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(! , Not , )
FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isinf , Isinf , )
FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isnan , Isnan , )
FASTOR_MAKE_UNARY_BOOL_OP_ASSIGNMENT(isfinite, Isfinite, )
} // end of namespace Fastor
#endif // UNARY_BOOL_OP_H

209
noarch/include/Fastor/expressions/unary_ops/unary_math_ops.h
View File

@@ -1,209 +0,0 @@
#ifndef UNARY_MATH_OP_H
#define UNARY_MATH_OP_H
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/simd_math/simd_math.h"
#include "Fastor/tensor/Tensor.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
#include "Fastor/expressions/expression_traits.h"
namespace Fastor {
// All unary math ops
#define FASTOR_MAKE_UNARY_MATH_OPS(OP_NAME, SIMD_OP, SCALAR_OP, STRUCT_NAME, EVAL_TYPE)\
template<typename Expr, size_t DIM0>\
struct Unary ##STRUCT_NAME ## Op: public AbstractTensor<Unary ##STRUCT_NAME ## Op<Expr, DIM0>,DIM0> {\
private:\
expression_t<Expr> _expr;\
public:\
using scalar_type = typename Expr::scalar_type;\
using simd_vector_type = typename Expr::simd_vector_type;\
using simd_abi_type = typename simd_vector_type::abi_type;\
using result_type = typename Expr::result_type;\
static constexpr FASTOR_INDEX Dimension = DIM0;\
static constexpr FASTOR_INDEX rank() {return DIM0;}\
FASTOR_INLINE FASTOR_INDEX size() const {return _expr.size();}\
FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return _expr.dimension(i);}\
Unary ##STRUCT_NAME ## Op(expression_t<Expr> inexpr) : _expr(inexpr) {}\
FASTOR_INLINE expression_t<Expr> expr() const {return _expr;}\
template<typename U=scalar_type>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i) const {\
return SIMD_OP(_expr.template eval<EVAL_TYPE>(i));\
}\
template<typename U=scalar_type>\
FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i) const {\
return SCALAR_OP(_expr.template eval_s<EVAL_TYPE>(i));\
}\
template<typename U=scalar_type>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return SIMD_OP(_expr.template eval<EVAL_TYPE>(i,j));\
}\
template<typename U=scalar_type>\
FASTOR_INLINE EVAL_TYPE eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {\
return SCALAR_OP(_expr.template eval_s<EVAL_TYPE>(i,j));\
}\
template<typename U>\
FASTOR_INLINE SIMDVector<EVAL_TYPE,simd_abi_type> teval(const std::array<int,DIM0> &as) const {\
return SIMD_OP(_expr.template teval<EVAL_TYPE>(as));\
}\
template<typename U>\
FASTOR_INLINE EVAL_TYPE teval_s(const std::array<int,DIM0> &as) const {\
return SCALAR_OP(_expr.template teval_s<EVAL_TYPE>(as));\
}\
};\
template<typename Expr, size_t DIM0,\
typename std::enable_if<!std::is_arithmetic<Expr>::value,bool>::type = 0 >\
FASTOR_INLINE Unary ##STRUCT_NAME ## Op<Expr, DIM0> OP_NAME(const AbstractTensor<Expr,DIM0> &_expr) {\
return Unary ##STRUCT_NAME ## Op<Expr, DIM0>(_expr.self());\
}\
FASTOR_MAKE_UNARY_MATH_OPS(operator+, , , Add, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(operator-, -, -, Sub, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(abs, abs, std::abs, Abs, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(sqrt, sqrt, sqrts, Sqrt, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(cbrt, cbrt, std::cbrt, Cbrt, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(exp, exp, std::exp, Exp, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(exp2, exp2, std::exp2, Exp2, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(expm1, expm1, std::expm1, Expm1, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(log, log, std::log, Log, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(log10, log10, std::log10, Log10, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(log2, log2, std::log2, Log2, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(log1p, log1p, std::log1p, Log1p, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(sin, sin, std::sin, Sin, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(cos, cos, std::cos, Cos, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(tan, tan, std::tan, Tan, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(asin, asin, std::asin, Asin, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(acos, acos, std::acos, Acos, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(atan, atan, std::atan, Atan, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(sinh, sinh, std::sinh, Sinh, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(cosh, cosh, std::cosh, Cosh, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(tanh, tanh, std::tanh, Tanh, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(asinh, asinh, std::asinh, Asinh, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(acosh, acosh, std::acosh, Acosh, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(atanh, atanh, std::atanh, Atanh, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(erf, erf, std::erf, Erf, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(tgamma, tgamma, std::tgamma, Tgamma, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(lgamma, lgamma, std::lgamma, Lgamma, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(ceil, ceil, std::ceil, Ceil, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(round, round, std::round, Round, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(floor, floor, std::floor, Floor, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(trunc, trunc, std::trunc, Trunc, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(conj, conj, std::conj, Conj, scalar_type)
FASTOR_MAKE_UNARY_MATH_OPS(arg , arg , std::arg , Arg , scalar_type)
#define FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM,\
typename std::enable_if<requires_evaluation_v<OtherDerived>,bool>::type = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Unary ##NAME ## Op<OtherDerived,OtherDIM> &src) {\
assign ##ASSIGN_TYPE (dst.self(), src.expr().self());\
trivial_assign(dst.self(), OP(dst.self()));\
}\
template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM,\
typename std::enable_if<!requires_evaluation_v<OtherDerived>,bool>::type = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Unary ##NAME ## Op<OtherDerived,OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
// only assignment
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT( , Add, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(-, Sub, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(abs, Abs, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sqrt, Sqrt, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cbrt, Cbrt, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(exp, Exp, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(exp2, Exp2, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(expm1, Expm1, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log, Log, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log10, Log10, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log2, Log2, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(log1p, Log1p, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sin, Sin, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cos, Cos, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tan, Tan, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(asin, Asin, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(acos, Acos, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(atan, Atan, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(sinh, Sinh, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(cosh, Cosh, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tanh, Tanh, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(asinh, Asinh, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(acosh, Acosh, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(atanh, Atanh, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(erf, Erf, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(tgamma, Tgamma, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(lgamma, Lgamma, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(ceil, Ceil, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(round, Round, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(floor, Floor, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(trunc, Trunc, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(conj, Conj, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENT(arg , Arg , )
// arithmetic assignments
#define FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(OP, NAME, ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM,\
typename std::enable_if<requires_evaluation_v<OtherDerived>,bool>::type = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Unary ##NAME ## Op<OtherDerived,OtherDIM> &src) {\
using result_type = typename Unary ##NAME ## Op<OtherDerived,OtherDIM>::result_type;\
const result_type tmp(src.expr().self());\
trivial_assign ##ASSIGN_TYPE (dst.self(), OP(tmp));\
}\
template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM,\
typename std::enable_if<!requires_evaluation_v<OtherDerived>,bool>::type = false>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const Unary ##NAME ## Op<OtherDerived,OtherDIM> &src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
#define FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT( , Add, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(-, Sub, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(abs, Abs, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sqrt, Sqrt, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cbrt, Cbrt, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(exp, Exp, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(exp2, Exp2, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(expm1, Expm1, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log, Log, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log10, Log10, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log2, Log2, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(log1p, Log1p, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sin, Sin, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cos, Cos, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tan, Tan, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(asin, Asin, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(acos, Acos, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(atan, Atan, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(sinh, Sinh, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(cosh, Cosh, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tanh, Tanh, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(asinh, Asinh, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(acosh, Acosh, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(atanh, Atanh, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(erf, Erf, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(tgamma, Tgamma, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(lgamma, Lgamma, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(ceil, Ceil, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(round, Round, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(floor, Floor, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(trunc, Trunc, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(conj, Conj, ASSIGN_TYPE)\
FASTOR_MAKE_UNARY_MATH_OP_ARITHMETIC_ASSIGNMENT(arg , Arg , ASSIGN_TYPE)\
// FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, )
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _add)
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _sub)
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _mul)
FASTOR_MAKE_UNARY_MATH_OP_ASSIGNMENTS(OP, NAME, _div)
} // end of namespace Fastor
#endif // UNARY_MATH_OP_H

317
noarch/include/Fastor/expressions/views/tensor_diag_views.h
View File

@@ -1,317 +0,0 @@
#ifndef TENSOR_DIAG_VIEWS_H
#define TENSOR_DIAG_VIEWS_H
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/TensorMap.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
namespace Fastor {
template<template<typename,size_t...> class TensorType, typename T, size_t M, size_t N, size_t DIM>
struct TensorDiagViewExpr<TensorType<T,M,N>,DIM> : public AbstractTensor<TensorDiagViewExpr<TensorType<T,M,N>,DIM>,DIM> {
private:
TensorType<T,M,N>& _expr;
bool _does_alias = false;
constexpr FASTOR_INLINE Tensor<T,M,N> get_tensor() const {return _expr;};
public:
using scalar_type = T;
using simd_vector_type = typename Tensor<T,M,N>::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = Tensor<T,M>;
using V = SIMDVector<T,simd_abi_type>;
static constexpr FASTOR_INDEX Dimension = 1;
static constexpr FASTOR_INDEX Stride = simd_vector_type::Size;
static constexpr FASTOR_INDEX rank() {return 1;}
constexpr FASTOR_INLINE FASTOR_INDEX size() const {return M;}
constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return M;}
constexpr const TensorType<T,M,N>& expr() const {return _expr;}
FASTOR_INLINE TensorViewExpr<TensorType<T,M,N>,2>& noalias() {
_does_alias = true;
return *this;
}
FASTOR_INLINE TensorDiagViewExpr(Tensor<T,M,N> &_ex) : _expr(_ex) {
static_assert(M==N, "MATRIX MUST BE SQUARE FOR DIAGONAL VIEW");
}
// Copy assignment
//---------------------------------------------------------------------------------//
FASTOR_INLINE void operator=(const TensorDiagViewExpr<TensorType<T,M,N>,DIM>& other_src) {
// Diagonal views don't alias
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
// Get around recursive inlining issue
const result_type tmp(other_src);
this->operator=(tmp);
return;
}
#endif
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(other_src.dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) = other_src.template eval_s<T>(i,i);
}
}
// AbstractTensor binders
//---------------------------------------------------------------------------------//
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorDiagViewExpr<TensorType<T,M,N>,2>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) = other_src.template eval_s<T>(i);
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator+=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator+=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator+=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorDiagViewExpr<TensorType<T,M,N>,2>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator+=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) += other_src.template eval_s<T>(i);
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator-=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator-=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator-=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorDiagViewExpr<TensorType<T,M,N>,2>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator-=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) -= other_src.template eval_s<T>(i);
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator*=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator*=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator*=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorDiagViewExpr<TensorType<T,M,N>,2>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator*=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) *= other_src.template eval_s<T>(i);
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator/=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator/=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator/=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorDiagViewExpr<TensorType<T,M,N>,2>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator/=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) /= other_src.template eval_s<T>(i);
}
}
// Scalar binders
//---------------------------------------------------------------------------------//
template<typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator=(U num) {
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) = num;
}
}
template<typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator+=(U num) {
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) += num;
}
}
template<typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator-=(U num) {
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) -= num;
}
}
template<typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator*=(U num) {
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) *= num;
}
}
template<typename U, enable_if_t_<is_arithmetic_v_<U> && !is_integral_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator/=(U num) {
T inum = T(1) / T(num);
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) *= inum;
}
}
template<typename U, enable_if_t_<is_arithmetic_v_<U> && is_integral_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator/=(U num) {
for (FASTOR_INDEX i = 0; i < M; ++i) {
_expr(i,i) /= num;
}
}
// Evaluators
//---------------------------------------------------------------------------------//
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX idx) const {
V _vec;
std::array<int,V::Size> inds;
for (auto j=0; j<V::Size; ++j) {
const size_t it = idx+j;
inds[j] = it*N+it;
}
vector_setter(_vec,_expr.data(),inds);
return _vec;
}
template<typename U=T>
FASTOR_INLINE U eval_s(FASTOR_INDEX idx) const {
return _expr(idx,idx);
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
V _vec;
vector_setter(_vec,_expr.data(),i*N+j);
return _vec;
}
template<typename U=T>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _expr(i,j);
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,2>& as) const {
V _vec;
vector_setter(_vec,_expr.data(),as[0]*N+as[1]);
return _vec;
}
template<typename U=T>
constexpr FASTOR_INLINE U teval_s(const std::array<int,2>& as) const {
return _expr.eval_s(as);
}
};
// Actual definition of diag function
// Note that like other views the input can only be an evaluated tensor
template<typename T, size_t M, size_t N>
TensorDiagViewExpr<Tensor<T,M,N>,2> diag(Tensor<T,M,N> &a) {
return TensorDiagViewExpr<Tensor<T,M,N>,2>(a);
}
template<typename T, size_t M, size_t N>
TensorDiagViewExpr<TensorMap<T,M,N>,2> diag(TensorMap<T,M,N> &a) {
return TensorDiagViewExpr<TensorMap<T,M,N>,2>(a);
}
} // end of namespace Fastor
#endif

281
noarch/include/Fastor/expressions/views/tensor_filter_views.h
View File

@@ -1,281 +0,0 @@
#ifndef TENSOR_FILTER_VIEWS_H
#define TENSOR_FILTER_VIEWS_H
#include "Fastor/tensor/Tensor.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
namespace Fastor {
template<typename T, size_t ... Rest, size_t DIMS>
struct TensorFilterViewExpr<Tensor<T,Rest...>,Tensor<bool,Rest...>,DIMS>:
public AbstractTensor<TensorFilterViewExpr<Tensor<T,Rest...>,Tensor<bool,Rest...>,DIMS>,DIMS> {
private:
Tensor<T,Rest...> &_expr;
const Tensor<bool,Rest...> &fl_expr;
bool _does_alias = false;
constexpr FASTOR_INLINE Tensor<T,Rest...> get_tensor() const {return _expr;}
public:
using scalar_type = T;
using simd_vector_type = typename Tensor<T,Rest...>::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = Tensor<T,Rest...>;
static constexpr FASTOR_INDEX Dimension = DIMS;
static constexpr FASTOR_INDEX Stride = simd_vector_type::Size;
static constexpr FASTOR_INDEX rank() {return DIMS;}
constexpr FASTOR_INLINE FASTOR_INDEX size() const {return pack_prod<Rest...>::value;}
constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) const {return fl_expr.dimension(i);}
constexpr const Tensor<T,Rest...>& expr() const {return _expr;}
FASTOR_INLINE TensorFilterViewExpr<Tensor<T,Rest...>,Tensor<bool,Rest...>,DIMS>& noalias() {
_does_alias = true;
return *this;
}
constexpr FASTOR_INLINE TensorFilterViewExpr(Tensor<T,Rest...> &_ex, const Tensor<bool,Rest...> &_it) : _expr(_ex), fl_expr(_it) {
static_assert(sizeof...(Rest)==DIMS, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
}
// In place operators
//------------------------------------------------------------------------------------//
void operator=(const TensorFilterViewExpr<Tensor<T,Rest...>,Tensor<bool,Rest...>,DIMS> &src) {
#ifndef NDEBUG
FASTOR_ASSERT(src.size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(src.dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] = src.template eval_s<T>(i);
}
}
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
void operator=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
const typename Derived::result_type& tmp = evaluate(src.self());
this->operator=(tmp);
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
void operator=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
#ifndef NDEBUG
FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(src.self().dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] = src.self().template eval_s<T>(i);
}
}
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
void operator+=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
const typename Derived::result_type& tmp = evaluate(src.self());
this->operator+=(tmp);
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
void operator+=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
#ifndef NDEBUG
FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(src.self().dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] += src.self().template eval_s<T>(i);
}
}
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
void operator-=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
const typename Derived::result_type& tmp = evaluate(src.self());
this->operator-=(tmp);
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
void operator-=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
#ifndef NDEBUG
FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(src.self().dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] -= src.self().template eval_s<T>(i);
}
}
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
void operator*=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
const typename Derived::result_type& tmp = evaluate(src.self());
this->operator*=(tmp);
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
void operator*=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
#ifndef NDEBUG
FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(src.self().dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] *= src.self().template eval_s<T>(i);
}
}
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
void operator/=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
const typename Derived::result_type& tmp = evaluate(src.self());
this->operator/=(tmp);
}
template<typename Derived, size_t OTHER_DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
void operator/=(const AbstractTensor<Derived,OTHER_DIMS> &src) {
#ifndef NDEBUG
FASTOR_ASSERT(src.self().size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(src.self().dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] /= src.self().template eval_s<T>(i);
}
}
}
//------------------------------------------------------------------------------------//
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
void operator=(U num) {
T tnum = (T)num;
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] = tnum;
}
}
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
void operator+=(U num) {
T tnum = (T)num;
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] += tnum;
}
}
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
void operator-=(U num) {
T tnum = (T)num;
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] -= tnum;
}
}
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
void operator*=(U num) {
T tnum = (T)num;
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] *= tnum;
}
}
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U> && !is_integral_v_<U>,bool> = false>
void operator/=(U num) {
T tnum = T(1)/(T)num;
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] *= tnum;
}
}
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U> && is_integral_v_<U>,bool> = false>
void operator/=(U num) {
T tnum = (T)num;
for (FASTOR_INDEX i = 0; i <size(); i++) {
if (fl_expr.eval_s(i)) {
_expr.data()[i] /= tnum;
}
}
}
//------------------------------------------------------------------------------------//
//------------------------------------------------------------------------------------//
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
constexpr size_t _Stride = SIMDVector<T,simd_abi_type>::Size;
SIMDVector<U,simd_abi_type> _vec;
U inds[_Stride];
for (FASTOR_INDEX j=0; j<_Stride; ++j)
inds[j] = fl_expr.eval_s(i+j) ? _expr.eval_s(i+j) : 0;
_vec.load(inds,false);
return _vec;
}
template<typename U=T>
FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return fl_expr.eval_s(i) ? _expr.eval_s(i) : 0;
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX k) const {
constexpr size_t _Stride = SIMDVector<T,simd_abi_type>::Size;
SIMDVector<U,simd_abi_type> _vec;
U inds[_Stride];
for (FASTOR_INDEX j=0; j<_Stride; ++j)
inds[j] = fl_expr.eval_s(i,k+j) ? _expr.eval_s(i,k+j) : 0;
_vec.load(inds,false);
return _vec;
}
template<typename U=T>
FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX k) const {
return fl_expr.eval_s(i,k) ? _expr.eval_s(i,k) : 0;
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,DIMS>& as) const {
constexpr size_t _Stride = SIMDVector<T,simd_abi_type>::Size;
SIMDVector<U,simd_abi_type> _vec;
U inds[_Stride];
for (FASTOR_INDEX j=0; j<_Stride; ++j)
inds[j] = fl_expr.teval_s(as) ? _expr.teval_s(as) : 0;
_vec.load(inds,false);
return _vec;
}
template<typename U=T>
FASTOR_INLINE U teval_s(const std::array<int,DIMS>& as) const {
return fl_expr.teval_s(as) ? _expr.teval_s(as) : 0;
}
//------------------------------------------------------------------------------------//
};
} // end of namespace Fastor
#endif // TENSOR_FILTER_VIEWS_H

681
noarch/include/Fastor/expressions/views/tensor_fixed_views_1d.h
View File

@@ -1,681 +0,0 @@
#ifndef TENSOR_FIXED_VIEWS_1D_H
#define TENSOR_FIXED_VIEWS_1D_H
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
namespace Fastor {
// 1D const fixed views
//----------------------------------------------------------------------------------------------//
template<typename T, size_t N, int F0, int L0, int S0>
struct TensorConstFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1> :
public AbstractTensor<TensorConstFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>,1> {
private:
const Tensor<T,N> &_expr;
public:
using scalar_type = T;
using simd_vector_type = typename Tensor<T,N>::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = Tensor<T,range_detector<F0,L0,S0>::value>;
static constexpr FASTOR_INDEX Dimension = 1;
static constexpr FASTOR_INDEX Stride = simd_vector_type::Size;
static constexpr FASTOR_INLINE FASTOR_INDEX size() { return range_detector<F0,L0,S0>::value;}
static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) {return range_detector<F0,L0,S0>::value;}
constexpr const Tensor<T,N>& expr() const {return _expr;}
constexpr FASTOR_INLINE TensorConstFixedViewExpr1D(const Tensor<T,N> &_ex) : _expr(_ex) {}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),S0*i+F0,S0);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return _expr.data()[S0*i+F0];
}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),S0*(i+j)+F0,S0);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _expr.data()[S0*(i+j)+F0];
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,1>& as) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),S0*as[0]+F0,S0);
return _vec;
}
template<typename U=T>
constexpr FASTOR_INLINE U teval_s(const std::array<int,1>& as) const {
return _expr.data()[S0*as[0]+F0];
}
};
// 1D non-const fixed views
//----------------------------------------------------------------------------------------------//
template<typename T, size_t N, int F0, int L0, int S0>
struct TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1> :
public AbstractTensor<TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>,1> {
private:
Tensor<T,N> &_expr;
bool _does_alias = false;
constexpr FASTOR_INLINE Tensor<T,N> get_tensor() const {return _expr;}
public:
using scalar_type = T;
using simd_vector_type = typename Tensor<T,N>::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using result_type = Tensor<T,range_detector<F0,L0,S0>::value>;
static constexpr FASTOR_INDEX Dimension = 1;
static constexpr FASTOR_INDEX Stride = simd_vector_type::Size;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return range_detector<F0,L0,S0>::value;}
static constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX i) {return range_detector<F0,L0,S0>::value;}
constexpr const Tensor<T,N>& expr() const {return _expr;}
FASTOR_INLINE TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>& noalias() {
_does_alias = true;
return *this;
}
constexpr FASTOR_INLINE TensorFixedViewExpr1D(Tensor<T,N> &_ex) : _expr(_ex) {}
//----------------------------------------------------------------------------------//
FASTOR_HINT_INLINE void operator=(const TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1> &other_src) {
#ifndef FASTOR_NO_ALIAS
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>(tmp_this_tensor);
// Assign other to temporary
tmp = other_src;
// assign temporary to this
this->operator=(tmp);
return;
}
#endif
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(other_src.dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
_vec_other.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] = other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] = _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] = other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] = other_src.template eval_s<T>(i);
}
#endif
}
}
//----------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------//
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator=(const AbstractTensor<Derived,DIMS> &other) {
#ifndef FASTOR_NO_ALIAS
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR(is_boolean_expression_v<Derived>) {
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] = other_src.template eval_s<T>(i);
}
return;
}
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
_vec_other.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] = other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] = _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] = other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] = other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator+=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator+=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator+=(const AbstractTensor<Derived,DIMS> &other) {
#ifndef FASTOR_NO_ALIAS
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator+=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) + other_src.template eval<T>(i);
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] += other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] += _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] += other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] += other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator-=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator-=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator-=(const AbstractTensor<Derived,DIMS> &other) {
#ifndef FASTOR_NO_ALIAS
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator-=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) - other_src.template eval<T>(i);
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] -= other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] -= _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] -= other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] -= other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator*=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator*=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator*=(const AbstractTensor<Derived,DIMS> &other) {
#ifndef FASTOR_NO_ALIAS
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) * other_src.template eval<T>(i);
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] *= other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] *= _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] *= other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] *= other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator/=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator/=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator/=(const AbstractTensor<Derived,DIMS> &other) {
#ifndef FASTOR_NO_ALIAS
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorFixedViewExpr1D<Tensor<T,N>,fseq<F0,L0,S0>,1>(tmp_this_tensor);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator/=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) / other_src.template eval<T>(i);
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] /= other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] /= _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] /= other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] /= other_src.template eval_s<T>(i);
}
#endif
}
}
//----------------------------------------------------------------------------------//
// Scalar binders
//----------------------------------------------------------------------------------//
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
_vec_other.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] = num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] = _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] = num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] = num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator+=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) + _vec_other;
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] += num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] += _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] += num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] += num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator-=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) - _vec_other;
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] -= num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] -= _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] -= num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] -= num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator*=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) * _vec_other;
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] *= num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] *= _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] *= num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] *= num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U> && !is_integral_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator/=(U num) {
T inum = T(1) / static_cast<T>(num);
simd_vector_type _vec_other(inum);
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) * _vec_other;
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] *= inum;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] *= _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] *= inum;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] *= inum;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U> && is_integral_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator/=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR (S0 == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) / _vec_other;
_vec.store(&_data[i+F0],false);
}
for (; i <size(); i++) {
_data[i+F0] /= num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<SIMDVector<T,simd_abi_type>::Size; ++j) {
_data[S0*(i+j) + F0] /= _vec_other[j];
}
}
for (; i <size(); i++) {
_data[S0*i+F0] /= num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
_data[S0*i+F0] /= num;
}
#endif
}
}
//----------------------------------------------------------------------------------//
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),S0*i+F0,S0);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return _expr.data()[S0*i+F0];
}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),S0*(i+j)+F0,S0);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _expr.data()[S0*(i+j)+F0];
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,1>& as) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),S0*as[0]+F0,S0);
return _vec;
}
template<typename U=T>
constexpr FASTOR_INLINE U teval_s(const std::array<int,1>& as) const {
return _expr.data()[S0*as[0]+F0];
}
};
}
#endif // TENSOR_FIXED_VIEWS_1D_H

1121
noarch/include/Fastor/expressions/views/tensor_fixed_views_2d.h
View File

File diff suppressed because it is too large Load Diff

1645
noarch/include/Fastor/expressions/views/tensor_fixed_views_nd.h
View File

File diff suppressed because it is too large Load Diff

1152
noarch/include/Fastor/expressions/views/tensor_random_views.h
View File

File diff suppressed because it is too large Load Diff

10
noarch/include/Fastor/expressions/views/tensor_views.h
View File

@@ -1,10 +0,0 @@
#ifndef TENSOR_VIEWS_H
#define TENSOR_VIEWS_H
#include "Fastor/expressions/views/tensor_views_1d.h"
#include "Fastor/expressions/views/tensor_views_2d.h"
#include "Fastor/expressions/views/tensor_views_nd.h"
#include "Fastor/expressions/views/tensor_views_assignment.h"
#endif // TENSOR_VIEWS_H

739
noarch/include/Fastor/expressions/views/tensor_views_1d.h
View File

@@ -1,739 +0,0 @@
#ifndef TENSOR_VIEWS_1D_H
#define TENSOR_VIEWS_1D_H
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/Ranges.h"
#include "Fastor/expressions/linalg_ops/linalg_traits.h"
namespace Fastor {
// 1D const views
//-------------------------------------------------------------------------------------//
template<typename T, size_t N>
struct TensorConstViewExpr<Tensor<T,N>,1>: public AbstractTensor<TensorConstViewExpr<Tensor<T,N>,1>,1> {
private:
const Tensor<T,N> &_expr;
seq _seq;
public:
using scalar_type = T;
using simd_vector_type = typename Tensor<T,N>::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using V = simd_vector_type;
using result_type = Tensor<T,N>;
static constexpr FASTOR_INDEX Dimension = 1;
static constexpr FASTOR_INDEX Stride = simd_vector_type::Size;
static constexpr FASTOR_INDEX rank() {return 1;}
constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq.size();}
constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return _seq.size();}
constexpr const Tensor<T,N>& expr() const {return _expr;}
FASTOR_INLINE TensorConstViewExpr(const Tensor<T,N> &_ex, const seq &_s) : _expr(_ex), _seq(_s) {
if (_seq._last < 0) _seq._last += N + /*including the end point*/ 1;
if (_seq._first < 0) _seq._first += N + /*including the end point*/ 1;
}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),i*_seq._step+_seq._first,_seq._step);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return _expr.data()[i*_seq._step+_seq._first];
}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),(i+j)*_seq._step+_seq._first,_seq._step);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _expr.data()[(i+j)*_seq._step+_seq._first];
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,1>& as) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),as[0]*_seq._step+_seq._first,_seq._step);
return _vec;
}
template<typename U=T>
constexpr FASTOR_INLINE U teval_s(const std::array<int,1>& as) const {
return _expr.data()[as[0]*_seq._step+_seq._first];
}
};
// 1D non-const views
//-------------------------------------------------------------------------------------//
template<typename T, size_t N>
struct TensorViewExpr<Tensor<T,N>,1>: public AbstractTensor<TensorViewExpr<Tensor<T,N>,1>,1> {
private:
Tensor<T,N> &_expr;
seq _seq;
bool _does_alias = false;
constexpr FASTOR_INLINE Tensor<T,N> get_tensor() const {return _expr;}
public:
using scalar_type = T;
using simd_vector_type = typename Tensor<T,N>::simd_vector_type;
using simd_abi_type = typename simd_vector_type::abi_type;
using V = simd_vector_type;
using result_type = Tensor<T,N>;
static constexpr FASTOR_INDEX Dimension = 1;
static constexpr FASTOR_INDEX Stride = simd_vector_type::Size;
static constexpr FASTOR_INDEX rank() {return 1;}
constexpr FASTOR_INLINE FASTOR_INDEX size() const {return _seq.size();}
constexpr FASTOR_INLINE FASTOR_INDEX dimension(FASTOR_INDEX ) const {return _seq.size();}
constexpr const Tensor<T,N>& expr() const {return _expr;}
FASTOR_INLINE TensorViewExpr<Tensor<T,N>,1>& noalias() {
_does_alias = true;
return *this;
}
FASTOR_INLINE TensorViewExpr(Tensor<T,N> &_ex, const seq &_s) : _expr(_ex), _seq(_s) {
if (_seq._last < 0) _seq._last += N + /*including the end point*/ 1;
if (_seq._first < 0) _seq._first += N + /*including the end point*/ 1;
}
// View evalution operators
// Copy assignment operators [Needed in addition to generic AbstractTensor overload]
//----------------------------------------------------------------------------------//
FASTOR_HINT_INLINE void operator=(const TensorViewExpr<Tensor<T,N>,1> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorViewExpr<Tensor<T,N>,1>(tmp_this_tensor,_seq);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator=(tmp);
return;
}
#endif
#ifndef NDEBUG
FASTOR_ASSERT(other.size()==this->size(), "TENSOR SIZE MISMATCH");
// Check if shape of tensors match - for this 1D case for loop unnecessary
for (FASTOR_INDEX i=0; i<Dimension; ++i) {
FASTOR_ASSERT(other.dimension(i)==dimension(i), "TENSOR SHAPE MISMATCH");
}
#endif
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = other.template eval<T>(i);
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] = other.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other.template eval<T>(i);
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] = _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] = other.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] = other.template eval_s<T>(i);
}
#endif
}
}
//----------------------------------------------------------------------------------//
// AbstractTensor binders
//----------------------------------------------------------------------------------//
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorViewExpr<Tensor<T,N>,1>(tmp_this_tensor,_seq);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
FASTOR_IF_CONSTEXPR(is_boolean_expression_v<Derived>) {
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] = other_src.template eval_s<T>(i);
}
return;
}
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = other_src.template eval<T>(i);
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] = other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] = _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] = other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] = other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator+=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator+=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator+=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorViewExpr<Tensor<T,N>,1>(tmp_this_tensor,_seq);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator+=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) + other_src.template eval<T>(i);
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] += other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] += _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] += other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] += other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator-=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator-=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator-=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorViewExpr<Tensor<T,N>,1>(tmp_this_tensor,_seq);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator-=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) - other_src.template eval<T>(i);
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] -= other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] -= _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] -= other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] -= other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator*=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator*=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator*=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorViewExpr<Tensor<T,N>,1>(tmp_this_tensor,_seq);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator*=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) * other_src.template eval<T>(i);
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] *= other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] *= _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] *= other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] *= other_src.template eval_s<T>(i);
}
#endif
}
}
template<typename Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator/=(const AbstractTensor<Derived,DIMS> &other) {
const typename Derived::result_type& tmp = evaluate(other.self());
this->operator/=(tmp);
}
template<typename Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_HINT_INLINE void operator/=(const AbstractTensor<Derived,DIMS> &other) {
#if !(FASTOR_NO_ALIAS)
if (_does_alias) {
_does_alias = false;
// Evaluate this into a temporary
auto tmp_this_tensor = get_tensor();
auto tmp = TensorViewExpr<Tensor<T,N>,1>(tmp_this_tensor,_seq);
// Assign other to temporary
tmp = other;
// assign temporary to this
this->operator/=(tmp);
return;
}
#endif
const Derived& other_src = other.self();
#ifndef NDEBUG
FASTOR_ASSERT(other_src.size()==this->size(), "TENSOR SIZE MISMATCH");
#endif
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) / other_src.template eval<T>(i);
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] /= other_src.template eval_s<T>(i);
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec_other = other_src.template eval<T>(i);
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] /= _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] /= other_src.template eval_s<T>(i);
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] /= other_src.template eval_s<T>(i);
}
#endif
}
}
//----------------------------------------------------------------------------------//
// scalar binders
//----------------------------------------------------------------------------------//
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
_vec_other.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] = num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] = _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] = num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] = num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator+=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) + _vec_other;
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] += num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] += _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] += num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] += num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator-=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) - _vec_other;
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] -= num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] -= _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] -= num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] -= num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator*=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) * _vec_other;
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] *= num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] *= _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] *= num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] *= num;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U> && !is_integral_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator/=(U num) {
T inum = T(1) / static_cast<T>(num);
simd_vector_type _vec_other(inum);
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) * _vec_other;
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] *= inum;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] *= _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] *= inum;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] *= inum;
}
#endif
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U> && is_integral_v_<U>,bool> = false>
FASTOR_HINT_INLINE void operator/=(U num) {
simd_vector_type _vec_other(static_cast<T>(num));
T *FASTOR_RESTRICT _data = _expr.data();
if (_seq._step == 1) {
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
auto _vec = this->template eval<T>(i) / _vec_other;
_vec.store(&_data[i+_seq._first],false);
}
for (; i <size(); i++) {
_data[i+_seq._first] /= num;
}
}
else {
#ifdef FASTOR_USE_VECTORISED_EXPR_ASSIGN
FASTOR_INDEX i;
for (i = 0; i <ROUND_DOWN(size(),Stride); i+=Stride) {
for (auto j=0; j<simd_vector_type::Size; ++j) {
auto idx = (i+j)*_seq._step+_seq._first;
_data[idx] /= _vec_other[j];
}
}
for (; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] /= num;
}
#else
for (FASTOR_INDEX i = 0; i <size(); i++) {
auto idx = i*_seq._step+_seq._first;
_data[idx] /= num;
}
#endif
}
}
//----------------------------------------------------------------------------------//
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),i*_seq._step+_seq._first,_seq._step);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i) const {
return _expr.data()[i*_seq._step+_seq._first];
}
template<typename U>
FASTOR_INLINE SIMDVector<U,simd_abi_type> eval(FASTOR_INDEX i, FASTOR_INDEX j) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),(i+j)*_seq._step+_seq._first,_seq._step);
return _vec;
}
template<typename U>
constexpr FASTOR_INLINE U eval_s(FASTOR_INDEX i, FASTOR_INDEX j) const {
return _expr.data()[(i+j)*_seq._step+_seq._first];
}
template<typename U=T>
FASTOR_INLINE SIMDVector<U,simd_abi_type> teval(const std::array<int,1>& as) const {
SIMDVector<U,simd_abi_type> _vec;
vector_setter(_vec,_expr.data(),as[0]*_seq._step+_seq._first,_seq._step);
return _vec;
}
template<typename U=T>
constexpr FASTOR_INLINE U teval_s(const std::array<int,1>& as) const {
return _expr.data()[as[0]*_seq._step+_seq._first];
}
};
} // end of namespace Fastor
#endif // TENSOR_VIEWS_1D_H

1163
noarch/include/Fastor/expressions/views/tensor_views_2d.h
View File

File diff suppressed because it is too large Load Diff

72
noarch/include/Fastor/expressions/views/tensor_views_assignment.h
View File

@@ -1,72 +0,0 @@
#ifndef TENSOR_VIEWS_ASSIGNMENT_H
#define TENSOR_VIEWS_ASSIGNMENT_H
#include "Fastor/expressions/views/tensor_views.h"
#include "Fastor/expressions/views/tensor_fixed_views_1d.h"
#include "Fastor/expressions/views/tensor_fixed_views_2d.h"
#include "Fastor/expressions/views/tensor_random_views.h"
namespace Fastor {
#define FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(ASSIGN_TYPE)\
template<typename Derived, size_t DIM, typename TensorType, typename Seq>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorConstFixedViewExpr1D<TensorType,Seq,1>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename Seq>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorFixedViewExpr1D<TensorType,Seq,1>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename Seq0, typename Seq1>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorConstFixedViewExpr2D<TensorType,Seq0,Seq1,2>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename Seq0, typename Seq1>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorFixedViewExpr2D<TensorType,Seq0,Seq1,2>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename ... Fseq>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorConstFixedViewExprnD<TensorType,Fseq...>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename ... Fseq>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorFixedViewExprnD<TensorType,Fseq...>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, size_t OtherDIM>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorConstViewExpr<TensorType,OtherDIM>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, size_t OtherDIM>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorViewExpr<TensorType,OtherDIM>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename TensorIndexType>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorConstRandomViewExpr<TensorType,TensorIndexType,DIM>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename TensorIndexType>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorRandomViewExpr<TensorType,TensorIndexType,DIM>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, typename TensorIndexType>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorFilterViewExpr<TensorType,TensorIndexType,DIM>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
template<typename Derived, size_t DIM, typename TensorType, size_t OtherDIM>\
FASTOR_INLINE void assign ##ASSIGN_TYPE (AbstractTensor<Derived,DIM> &dst, const TensorDiagViewExpr<TensorType,OtherDIM>& src) {\
trivial_assign ##ASSIGN_TYPE (dst.self(), src.self());\
}\
FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT( )
FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_add)
FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_sub)
FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_mul)
FASTOR_MAKE_ALL_TENSOR_VIEWS_ASSIGNMENT(_div)
} // end of namespace Fastor
#endif // TENSOR_VIEWS_ASSIGNMENT_H

1680
noarch/include/Fastor/expressions/views/tensor_views_nd.h
View File

File diff suppressed because it is too large Load Diff

1133
noarch/include/Fastor/meta/einsum_meta.h
View File

File diff suppressed because it is too large Load Diff

344
noarch/include/Fastor/meta/meta.h
View File

@@ -1,344 +0,0 @@
#ifndef META_H_
#define META_H_
#include <type_traits>
#include <complex>
namespace Fastor {
//----------------------------------------------------------------------------------------------------------//
template< bool B, class T = void >
using enable_if_t_ = typename std::enable_if<B,T>::type;
template< bool B, class T = void >
using disable_if_t_ = typename std::enable_if<!B,T>::type;
template< bool B, class T, class F >
using conditional_t_ = typename std::conditional<B,T,F>::type;
template< class T, class U >
constexpr bool is_same_v_ = std::is_same<T, U>::value;
template< class T >
constexpr bool is_fundamental_v_ = std::is_fundamental<T>::value;
template< class T >
constexpr bool is_arithmetic_v_ = std::is_arithmetic<T>::value;
template< class T >
constexpr bool is_integral_v_ = std::is_integral<T>::value;
template< class T >
constexpr bool is_floating_v_ = std::is_floating_point<T>::value;
template< class T >
constexpr bool is_array_v_ = std::is_array<T>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
// If BLAS compatible type
template< class T > struct is_numeric { static constexpr bool value = false; };
template<> struct is_numeric<float> { static constexpr bool value = true; };
template<> struct is_numeric<double> { static constexpr bool value = true; };
template<> struct is_numeric<std::complex<float>> { static constexpr bool value = true; };
template<> struct is_numeric<std::complex<double>> { static constexpr bool value = true; };
template< class T >
constexpr bool is_numeric_v_ = is_numeric<T>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
// If complex type
template< class T > struct is_complex { static constexpr bool value = false; };
template< class T > struct is_complex<std::complex<T>> { static constexpr bool value = true; };
template< class T >
constexpr bool is_complex_v_ = is_complex<T>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
// If a type is std::is_fundamental + std::complex<float/double> + any type that specialises this trait class
// This class is provided because the behaviour of any code that specialises std::is_fundamental/arithmetic/
// integral/floating_point is undefined
template< class T > struct is_primitive {
static constexpr bool value = std::is_fundamental<T>::value || is_numeric<T>::value;
};
template< class T >
constexpr bool is_primitive_v_ = is_primitive<T>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
// Checks if all parameters in a variadic parameter pack are arithmetic
template<class ... T>
struct is_arithmetic_pack;
template<class T, class ... Ts>
struct is_arithmetic_pack<T,Ts...> {
static constexpr bool value = std::is_arithmetic<T>::value && is_arithmetic_pack<Ts...>::value;
};
template<class T>
struct is_arithmetic_pack<T> { static constexpr bool value = std::is_arithmetic<T>::value; };
template<class ... T>
static constexpr bool is_arithmetic_pack_v = is_arithmetic_pack<T...>::value;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
// Remove cv-qualified and their refs
template<typename T> struct remove_cvref_ { using type = T; };
template<typename T> struct remove_cvref_<const T> { using type = typename remove_cvref_<T>::type; };
template<typename T> struct remove_cvref_<T const&> { using type = typename remove_cvref_<T>::type; };
template<typename T> struct remove_cvref_<T&> { using type = typename remove_cvref_<T>::type; };
template<typename T> struct remove_cvref_<volatile T> { using type = typename remove_cvref_<T>::type; };
template<typename T> struct remove_cvref_<T volatile&> { using type = typename remove_cvref_<T>::type; };
template<typename T>
using remove_cv_ref_t = typename remove_cvref_<T>::type;
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<typename T> struct remove_all { using type = T; };
template<typename T> struct remove_all<const T> { using type = typename remove_all<T>::type; };
template<typename T> struct remove_all<volatile T> { using type = typename remove_all<T>::type; };
template<typename T> struct remove_all<T&> { using type = typename remove_all<T>::type; };
template<typename T> struct remove_all<T const&> { using type = typename remove_all<T>::type; };
template<typename T> struct remove_all<T volatile&> { using type = typename remove_all<T>::type; };
template<typename T> struct remove_all<T*> { using type = typename remove_all<T>::type; };
template<typename T> struct remove_all<T const*> { using type = typename remove_all<T>::type; };
template<typename T> struct remove_all<T volatile*> { using type = typename remove_all<T>::type; };
template<typename T>
using remove_all_t = typename remove_all<T>::type;
//----------------------------------------------------------------------------------------------------------//
// Sum/multiply reduce all elements of a variadic pack
//----------------------------------------------------------------------------------------------------------//
// Sum
template<size_t...Rest> struct pack_add;
template<size_t Head, size_t ...Rest>
struct pack_add<Head, Rest...> { static const size_t value = Head+pack_add<Rest...>::value;};
template<> struct pack_add<> { static const size_t value = 0;};
// Multiply
template<size_t...Rest> struct pack_prod;
template<size_t Head, size_t ...Rest>
struct pack_prod<Head, Rest...> { static const size_t value = Head*pack_prod<Rest...>::value;};
template<> struct pack_prod<> { static const size_t value = 1;};
// Multiply first n elements
template<size_t Idx, size_t ... Rest> struct prod_nel;
template<size_t Idx, size_t First, size_t ... Rest>
struct prod_nel<Idx, First, Rest...> { static const size_t value = prod_nel<Idx-1, Rest...>::value;};
template<size_t First, size_t ... Rest>
struct prod_nel<1, First, Rest...> { static const size_t value = pack_prod<Rest...>::value;};
//! Partial product of a sequence up to an index up_to
template<class T, T N>
constexpr
inline T partial_prod(const T (&ind)[N], T up_to, T num=0) {
return num == up_to ? ind[num] : ind[num]*partial_prod(ind, up_to, num+1);
}
//! Partial product of a sequence up to an index num from the end
template<class T, T N>
constexpr
inline T partial_prod_reverse(const T (&ind)[N], T up_to, T num=N-1) {
return num == up_to ? ind[num] : ind[num]*partial_prod_reverse(ind, up_to, num-1);
}
template<typename T>
constexpr T size_proder_(T one){return one.size();}
template<typename T>
constexpr T size_proder_(T one, T two){return one.size()*two.size();}
template<typename T, typename ... Ts>
constexpr T size_proder_(T one, T two, Ts ... ts) {return _proder_(_proder_(one,two),ts...);}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<size_t Idx, size_t ... Rest>
struct get_value;
template<size_t Idx, size_t First, size_t ... Rest>
struct get_value<Idx, First, Rest...> {
static const size_t value = get_value<Idx-1, Rest...>::value;
};
template<size_t First, size_t ... Rest>
struct get_value<1,First,Rest...> {
static const size_t value = First;
};
template<size_t Idx>
// Work around to avoid compiler errors
struct get_value<Idx> {
static const size_t value = 0;
};
template <size_t N, typename... Args>
constexpr inline auto get_index(Args&&... as)
-> decltype(std::get<N>(std::forward_as_tuple(std::forward<Args>(as)...))) {
return std::get<N>(std::forward_as_tuple(std::forward<Args>(as)...));
}
template<int N, typename... Ts>
using get_nth_type = typename std::tuple_element<N, std::tuple<Ts...>>::type;
template<size_t Idx, size_t ... Rest>
struct get_all {
static const size_t indices[sizeof...(Rest)];
};
//----------------------------------------------------------------------------------------------------------//
// comparitors
//----------------------------------------------------------------------------------------------------------//
template<size_t I, size_t J>
struct is_equal {
static constexpr bool value = I == J;
};
template<size_t I, size_t J>
static constexpr bool is_equal_v_ = is_equal<I,J>::value;
template<size_t I, size_t J>
struct is_less {
static constexpr bool value = (I < J);
};
template<size_t I, size_t J>
static constexpr bool is_less_v_ = is_less<I,J>::value;
template<size_t I, size_t J>
struct is_less_equal {
static constexpr bool value = I <= J;
};
template<size_t I, size_t J>
static constexpr bool is_less_equal_v_ = is_less_equal<I,J>::value;
template<size_t I, size_t J>
struct is_greater {
static constexpr bool value = I > J;
};
template<size_t I, size_t J>
static constexpr bool is_greater_v_ = is_greater<I,J>::value;
template<size_t I, size_t J>
struct is_greater_equal {
static constexpr bool value = I >= J;
};
template<size_t I, size_t J>
static constexpr bool is_greater_equal_v_ = is_greater_equal<I,J>::value;
//----------------------------------------------------------------------------------------------------------//
// min-max
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<size_t ... rest>
struct meta_min;
template<size_t m, size_t n, size_t ... rest>
struct meta_min<m,n,rest...> {
static constexpr size_t pval = meta_min<m,n>::value;
static const size_t value = (pval <= meta_min<pval,rest...>::value) ?
pval : meta_min<pval,rest...>::value;
};
template <size_t m, size_t n>
struct meta_min<m,n> {
static const size_t value = (m<=n) ? m : n;
};
template<size_t ... rest>
struct meta_max;
template<size_t m, size_t n, size_t ... rest>
struct meta_max<m,n,rest...> {
static constexpr size_t pval = meta_max<m,n>::value;
static const size_t value = (pval >= meta_max<pval,rest...>::value) ?
pval : meta_max<pval,rest...>::value;
};
template<size_t m, size_t n>
struct meta_max<m,n> {
static const size_t value = (m>=n) ? m : n;
};
namespace internal {
// namespace to avoid clash with MSVC macros
template<typename T> constexpr FASTOR_INLINE T min_(const T a, const T b) {return a < b ? a : b;}
template<typename T> constexpr FASTOR_INLINE T max_(const T a, const T b) {return a > b ? a : b;}
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<size_t ... rest>
struct meta_argmin;
template<size_t m, size_t n, size_t ... rest>
struct meta_argmin<m,n,rest...> {
static constexpr size_t pval = meta_min<m,n>::value;
static const size_t value = (pval <= meta_min<pval,rest...>::value) ?
meta_argmin<m,n>::value : meta_argmin<pval,rest...>::value+1;
};
template<size_t m, size_t n>
struct meta_argmin<m,n> {
static const size_t value = (m<n) ? 0 : 1;
};
template<size_t ... rest>
struct meta_argmax;
template<size_t m, size_t n, size_t ... rest>
struct meta_argmax<m,n,rest...> {
static constexpr size_t pval = meta_max<m,n>::value;
static const size_t value = (pval >= meta_max<pval,rest...>::value) ?
meta_argmax<m,n>::value : meta_argmax<pval,rest...>::value+1;
};
template<size_t m, size_t n>
struct meta_argmax<m,n> {
static const size_t value = (m>n) ? 0 : 1;
};
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
// square/cube
//----------------------------------------------------------------------------------------------------------//
namespace internal {
template<size_t Val> struct meta_square { static constexpr size_t value = Val*Val;};
template<size_t Val> struct meta_cube { static constexpr size_t value = Val*Val*Val;};
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
//square root of integers at compile time, use like meta_sqrt<36>::ret
template<int Y,
int InfX = 0,
int SupX = ((Y==1) ? 1 : Y/2),
bool Done = ((SupX-InfX)<=1 ? true : ((SupX*SupX <= Y) && ((SupX+1)*(SupX+1) > Y))) >
// use ?: instead of || just to shut up a gcc 4.3 warning
class meta_sqrt
{
enum {
MidX = (InfX+SupX)/2,
TakeInf = MidX*MidX > Y ? 1 : 0,
NewInf = int(TakeInf) ? InfX : int(MidX),
NewSup = int(TakeInf) ? int(MidX) : SupX
};
public:
enum { ret = meta_sqrt<Y,NewInf,NewSup>::ret };
};
template<int Y, int InfX, int SupX>
class meta_sqrt<Y, InfX, SupX, true>
{
public: enum { ret = (SupX*SupX <= Y) ? SupX : InfX };
};
//----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // META_H_

620
noarch/include/Fastor/meta/opmin_meta.h
View File

@@ -1,620 +0,0 @@
#ifndef OPMIN_META_H
#define OPMIN_META_H
#ifndef FASTOR_DONT_PERFORM_OP_MIN
#include "tensor_meta.h"
#include "einsum_meta.h"
namespace Fastor {
// Cost model for by-pair tensor contraction
//------------------------------------------------------------------------------------------------------------//
template<class Ind0, class Ind1, class Tensor0, class Tensor1, class Seq>
struct pair_flop_cost;
template<class T, size_t... Idx0, size_t... Idx1, size_t ...Rest0, size_t ...Rest1, size_t... ss>
struct pair_flop_cost<Index<Idx0...>,Index<Idx1...>,Tensor<T,Rest0...>,Tensor<T,Rest1...>,std_ext::index_sequence<ss...>> {
static constexpr size_t ind0[sizeof...(Idx0)] = {Idx0... };
static constexpr size_t ind1[sizeof...(Idx1)] = {Idx1... };
static constexpr int nums1[sizeof...(Rest1)] = {Rest1... };
static constexpr size_t cost_tensor0 = pack_prod<Rest0...>::value;
static constexpr size_t remaining_cost = pack_prod<retrieve_value(ind0,ind1,nums1,ss)...>::value;
static constexpr size_t value = cost_tensor0*remaining_cost;
using resulting_tensor = typename get_resuling_tensor<Index<Idx0...>,Index<Idx1...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>>::type;
using resulting_index = typename get_resuling_index<Index<Idx0...>,Index<Idx1...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>>::type;
};
template<class T, size_t... Idx0, size_t ...Rest0, size_t... ss>
struct pair_flop_cost<Index<Idx0...>,Index<>,Tensor<T,Rest0...>,Tensor<T>,std_ext::index_sequence<ss...>> {
static constexpr size_t value = pack_prod<Rest0...>::value;
};
template<class T, size_t... Idx1, size_t ...Rest1, size_t... ss>
struct pair_flop_cost<Index<>,Index<Idx1...>,Tensor<T>,Tensor<T,Rest1...>,std_ext::index_sequence<ss...>> {
static constexpr size_t value = pack_prod<Rest1...>::value;
};
template<class T, size_t... ss>
struct pair_flop_cost<Index<>,Index<>,Tensor<T>,Tensor<T>,std_ext::index_sequence<ss...>> {
static constexpr size_t value = 1;
};
//------------------------------------------------------------------------------------------------------------//
// Cost of triplet tensor contraction (single evaluation)
//------------------------------------------------------------------------------------------------------------//
template<class Ind0, class Ind1, class Ind2, class Tensor0, class Tensor1, class Tensor2>
struct single_evaluation_triplet_cost;
template<class T, size_t... Idx0, size_t... Idx1, size_t... Idx2, size_t ...Rest0, size_t ...Rest1, size_t ...Rest2>
struct single_evaluation_triplet_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>> {
using concat_tensor_01 = typename no_of_loops_to_set<Index<Idx0...>,Index<Idx1...>,Tensor<T,Rest0...>,Tensor<T,Rest1...>,
typename std_ext::make_index_sequence<no_of_unique<Idx0...,Idx1...>::value>::type>::type;
using concat_index_01 = typename no_of_loops_to_set<Index<Idx0...>,Index<Idx1...>,Tensor<T,Rest0...>,Tensor<T,Rest1...>,
typename std_ext::make_index_sequence<no_of_unique<Idx0...,Idx1...>::value>::type>::indices;
static constexpr size_t value = pair_flop_cost<concat_index_01,Index<Idx2...>,concat_tensor_01,Tensor<T,Rest2...>,
typename std_ext::make_index_sequence<sizeof...(Rest2)>::type>::value;
};
//------------------------------------------------------------------------------------------------------------//
// Cost model for triplet tensor network contraction
//------------------------------------------------------------------------------------------------------------//
template<class Ind0, class Ind1, class Ind2, class Tensor0, class Tensor1, class Tensor2>
struct triplet_flop_cost;
template<class T, size_t... Idx0, size_t... Idx1, size_t... Idx2, size_t ...Rest0, size_t ...Rest1, size_t ...Rest2>
struct triplet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>> {
// Perform depth-first search
//---------------------------------------------------------------------
// first two tensors contracted first
using resulting_tensor_0 = typename get_resuling_tensor<Index<Idx0...>,Index<Idx1...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>>::type;
using resulting_index_0 = typename get_resuling_index<Index<Idx0...>,Index<Idx1...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>>::type;
static constexpr size_t flop_count_01_0 = pair_flop_cost<Index<Idx0...>,Index<Idx1...>,Tensor<T,Rest0...>,Tensor<T,Rest1...>,
typename std_ext::make_index_sequence<sizeof...(Rest1)>::type>::value;
static constexpr size_t flop_count_01_1 = pair_flop_cost<resulting_index_0,Index<Idx2...>,resulting_tensor_0,Tensor<T,Rest2...>,
typename std_ext::make_index_sequence<sizeof...(Rest2)>::type>::value;
static constexpr size_t flop_count_01 = flop_count_01_0 + flop_count_01_1;
// first and last tensors contracted first
using resulting_tensor_1 = typename get_resuling_tensor<Index<Idx0...>,Index<Idx2...>,
Tensor<T,Rest0...>,Tensor<T,Rest2...>>::type;
using resulting_index_1 = typename get_resuling_index<Index<Idx0...>,Index<Idx2...>,
Tensor<T,Rest0...>,Tensor<T,Rest2...>>::type;
static constexpr size_t flop_count_02_0 = pair_flop_cost<Index<Idx0...>,Index<Idx2...>,Tensor<T,Rest0...>,Tensor<T,Rest2...>,
typename std_ext::make_index_sequence<sizeof...(Rest2)>::type>::value;
static constexpr size_t flop_count_02_1 = pair_flop_cost<resulting_index_1,Index<Idx1...>,resulting_tensor_1,Tensor<T,Rest1...>,
typename std_ext::make_index_sequence<sizeof...(Rest1)>::type>::value;
static constexpr size_t flop_count_02 = flop_count_02_0 + flop_count_02_1;
// second and last tensors contracted first
using resulting_tensor_2 = typename get_resuling_tensor<Index<Idx1...>,Index<Idx2...>,
Tensor<T,Rest1...>,Tensor<T,Rest2...>>::type;
using resulting_index_2 = typename get_resuling_index<Index<Idx1...>,Index<Idx2...>,
Tensor<T,Rest1...>,Tensor<T,Rest2...>>::type;
static constexpr size_t flop_count_12_0 = pair_flop_cost<Index<Idx1...>,Index<Idx2...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,
typename std_ext::make_index_sequence<sizeof...(Rest2)>::type>::value;
static constexpr size_t flop_count_12_1 = pair_flop_cost<resulting_index_2,Index<Idx0...>,resulting_tensor_2,Tensor<T,Rest0...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)>::type>::value;
static constexpr size_t flop_count_12 = flop_count_12_0 + flop_count_12_1;
static constexpr size_t flop_count_012 = single_evaluation_triplet_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>>::value;
static constexpr size_t min_cost = meta_min<flop_count_01,flop_count_02,flop_count_12,flop_count_012>::value;
static constexpr int which_variant = meta_argmin<flop_count_01,flop_count_02,flop_count_12,flop_count_012>::value;
// this is the overall resulting tensor and index from overall triplet contraction
using resulting_tensor = typename std::conditional<
which_variant==0,
typename get_resuling_tensor<resulting_index_0,Index<Idx2...>,resulting_tensor_0,Tensor<T,Rest2...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_tensor<Index<Idx1...>,resulting_index_1,Tensor<T,Rest1...>,resulting_tensor_1>::type,
typename get_resuling_tensor<Index<Idx0...>,resulting_index_2,Tensor<T,Rest0...>,resulting_tensor_2>::type
>::type
>::type;
using resulting_index = typename std::conditional<
which_variant==0,
typename get_resuling_index<resulting_index_0,Index<Idx2...>,resulting_tensor_0,Tensor<T,Rest2...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_index<Index<Idx1...>,resulting_index_1,Tensor<T,Rest1...>,resulting_tensor_1>::type,
typename get_resuling_index<Index<Idx0...>,resulting_index_2,Tensor<T,Rest0...>,resulting_tensor_2>::type
>::type
>::type;
};
//------------------------------------------------------------------------------------------------------------//
// Cost model for quartet tensor network contraction
//------------------------------------------------------------------------------------------------------------//
template<class Ind0, class Ind1, class Ind2, class Ind3, class Tensor0, class Tensor1, class Tensor2, class Tensor3>
struct quartet_flop_cost;
template<class T, size_t... Idx0, size_t... Idx1, size_t... Idx2, size_t... Idx3,
size_t ...Rest0, size_t ...Rest1, size_t ...Rest2, size_t ...Rest3>
struct quartet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx3...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>> {
// Perform depth-first search
//---------------------------------------------------------------------
// first three tensors contracted first
using triplet_cost_012 = triplet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>>;
using resulting_tensor_0 = typename triplet_cost_012::resulting_tensor;
using resulting_index_0 = typename triplet_cost_012::resulting_index;
static constexpr size_t flop_count_012 = triplet_cost_012::min_cost;
static constexpr size_t flop_count_012_3 = pair_flop_cost<resulting_index_0,Index<Idx3...>,resulting_tensor_0,Tensor<T,Rest3...>,
typename std_ext::make_index_sequence<sizeof...(Rest3)>::type>::value;
static constexpr size_t flop_count_0 = flop_count_012 + flop_count_012_3;
// first, second and last tensors contracted first
using triplet_cost_013 = triplet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx3...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest3...>>;
using resulting_tensor_1 = typename triplet_cost_013::resulting_tensor;
using resulting_index_1 = typename triplet_cost_013::resulting_index;
static constexpr size_t flop_count_013 = triplet_cost_013::min_cost;
static constexpr size_t flop_count_013_2 = pair_flop_cost<resulting_index_1,Index<Idx2...>,resulting_tensor_1,Tensor<T,Rest2...>,
typename std_ext::make_index_sequence<sizeof...(Rest2)>::type>::value;
static constexpr size_t flop_count_1 = flop_count_013 + flop_count_013_2;
// first, third and last tensors contracted first
using triplet_cost_023 = triplet_flop_cost<Index<Idx0...>,Index<Idx2...>,Index<Idx3...>,
Tensor<T,Rest0...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>>;
using resulting_tensor_2 = typename triplet_cost_023::resulting_tensor;
using resulting_index_2 = typename triplet_cost_023::resulting_index;
static constexpr size_t flop_count_023 = triplet_cost_023::min_cost;
static constexpr size_t flop_count_023_1 = pair_flop_cost<resulting_index_2,Index<Idx1...>,resulting_tensor_2,Tensor<T,Rest1...>,
typename std_ext::make_index_sequence<sizeof...(Rest1)>::type>::value;
static constexpr size_t flop_count_2 = flop_count_023 + flop_count_023_1;
// last three tensors contracted first
using triplet_cost_123 = triplet_flop_cost<Index<Idx1...>,Index<Idx2...>,Index<Idx3...>,
Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>>;
using resulting_tensor_3 = typename triplet_cost_123::resulting_tensor;
using resulting_index_3 = typename triplet_cost_123::resulting_index;
static constexpr size_t flop_count_123 = triplet_cost_123::min_cost;
static constexpr size_t flop_count_123_0 = pair_flop_cost<resulting_index_3,Index<Idx0...>,resulting_tensor_3,Tensor<T,Rest0...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)>::type>::value;
static constexpr size_t flop_count_3 = flop_count_123 + flop_count_123_0;
static constexpr size_t min_cost = meta_min<flop_count_0,flop_count_1,flop_count_2,flop_count_3>::value;
static constexpr int which_variant = meta_argmin<flop_count_0,flop_count_1,flop_count_2,flop_count_3>::value;
// this is the overall resulting tensor and index from overall quartet contraction
using resulting_tensor = typename std::conditional<
which_variant==0,
typename get_resuling_tensor<resulting_index_0,Index<Idx3...>,resulting_tensor_0,Tensor<T,Rest3...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_tensor<Index<Idx2...>,resulting_index_1,Tensor<T,Rest2...>,resulting_tensor_1>::type,
typename std::conditional<
which_variant==2,
typename get_resuling_tensor<Index<Idx1...>,resulting_index_2,Tensor<T,Rest1...>,resulting_tensor_2>::type,
typename get_resuling_tensor<Index<Idx0...>,resulting_index_3,Tensor<T,Rest0...>,resulting_tensor_3>::type
>::type
>::type
>::type;
using resulting_index = typename std::conditional<
which_variant==0,
typename get_resuling_index<resulting_index_0,Index<Idx3...>,resulting_tensor_0,Tensor<T,Rest3...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_index<Index<Idx2...>,resulting_index_1,Tensor<T,Rest2...>,resulting_tensor_1>::type,
typename std::conditional<
which_variant==2,
typename get_resuling_index<Index<Idx1...>,resulting_index_2,Tensor<T,Rest1...>,resulting_tensor_2>::type,
typename get_resuling_index<Index<Idx0...>,resulting_index_3,Tensor<T,Rest0...>,resulting_tensor_3>::type
>::type
>::type
>::type;
};
//------------------------------------------------------------------------------------------------------------//
// Cost model for quintet tensor contraction
//------------------------------------------------------------------------------------------------------------//
template<class Ind0, class Ind1, class Ind2, class Ind3, class Ind4,
class Tensor0, class Tensor1, class Tensor2, class Tensor3, class Tensor4>
struct quintet_flop_cost;
template<class T, size_t... Idx0, size_t... Idx1, size_t... Idx2, size_t... Idx3, size_t... Idx4,
size_t ...Rest0, size_t ...Rest1, size_t ...Rest2, size_t ...Rest3, size_t ...Rest4>
struct quintet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx3...>, Index<Idx4...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>> {
// Perform depth-first search
//---------------------------------------------------------------------
// first four tensors contracted first
using quartet_cost_0123 = quartet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx3...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>>;
using resulting_tensor_0 = typename quartet_cost_0123::resulting_tensor;
using resulting_index_0 = typename quartet_cost_0123::resulting_index;
static constexpr size_t flop_count_0123 = quartet_cost_0123::min_cost;
static constexpr size_t flop_count_0123_4 = pair_flop_cost<resulting_index_0,Index<Idx4...>,resulting_tensor_0,Tensor<T,Rest4...>,
typename std_ext::make_index_sequence<sizeof...(Rest4)>::type>::value;
static constexpr size_t flop_count_0 = flop_count_0123 + flop_count_0123_4;
// 1st, 2nd, 3rd, 5th tensors contracted first
using quartet_cost_0124 = quartet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx4...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest4...>>;
using resulting_tensor_1 = typename quartet_cost_0124::resulting_tensor;
using resulting_index_1 = typename quartet_cost_0124::resulting_index;
static constexpr size_t flop_count_0124 = quartet_cost_0124::min_cost;
static constexpr size_t flop_count_0124_3 = pair_flop_cost<resulting_index_1,Index<Idx3...>,resulting_tensor_1,Tensor<T,Rest3...>,
typename std_ext::make_index_sequence<sizeof...(Rest3)>::type>::value;
static constexpr size_t flop_count_1 = flop_count_0124 + flop_count_0124_3;
// 1st, 2nd, 4th, 5th tensors contracted first
using quartet_cost_0134 = quartet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx3...>,Index<Idx4...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>>;
using resulting_tensor_2 = typename quartet_cost_0134::resulting_tensor;
using resulting_index_2 = typename quartet_cost_0134::resulting_index;
static constexpr size_t flop_count_0134 = quartet_cost_0134::min_cost;
static constexpr size_t flop_count_0134_2 = pair_flop_cost<resulting_index_2,Index<Idx2...>,resulting_tensor_2,Tensor<T,Rest2...>,
typename std_ext::make_index_sequence<sizeof...(Rest2)>::type>::value;
static constexpr size_t flop_count_2 = flop_count_0134 + flop_count_0134_2;
// 1st, 3rd, 4th, 5th tensors contracted first
using quartet_cost_0234 = quartet_flop_cost<Index<Idx0...>,Index<Idx2...>,Index<Idx3...>,Index<Idx4...>,
Tensor<T,Rest0...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>>;
using resulting_tensor_3 = typename quartet_cost_0234::resulting_tensor;
using resulting_index_3 = typename quartet_cost_0234::resulting_index;
static constexpr size_t flop_count_0234 = quartet_cost_0234::min_cost;
static constexpr size_t flop_count_0234_1 = pair_flop_cost<resulting_index_3,Index<Idx1...>,resulting_tensor_3,Tensor<T,Rest1...>,
typename std_ext::make_index_sequence<sizeof...(Rest1)>::type>::value;
static constexpr size_t flop_count_3 = flop_count_0234 + flop_count_0234_1;
// last four tensors contracted first
using quartet_cost_1234 = quartet_flop_cost<Index<Idx1...>,Index<Idx2...>,Index<Idx3...>,Index<Idx4...>,
Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>>;
using resulting_tensor_4 = typename quartet_cost_1234::resulting_tensor;
using resulting_index_4 = typename quartet_cost_1234::resulting_index;
static constexpr size_t flop_count_1234 = quartet_cost_1234::min_cost;
static constexpr size_t flop_count_1234_0 = pair_flop_cost<resulting_index_4,Index<Idx0...>,resulting_tensor_4,Tensor<T,Rest0...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)>::type>::value;
static constexpr size_t flop_count_4 = flop_count_1234 + flop_count_1234_0;
static constexpr size_t min_cost = meta_min<flop_count_0,flop_count_1,flop_count_2,flop_count_3,flop_count_4>::value;
static constexpr int which_variant = meta_argmin<flop_count_0,flop_count_1,flop_count_2,flop_count_3,flop_count_4>::value;
// this is the overall resulting tensor and index from overall quintet contraction
using resulting_tensor = typename std::conditional<
which_variant==0,
typename get_resuling_tensor<resulting_index_0,Index<Idx4...>,resulting_tensor_0,Tensor<T,Rest4...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_tensor<Index<Idx3...>,resulting_index_1,Tensor<T,Rest3...>,resulting_tensor_1>::type,
typename std::conditional<
which_variant==2,
typename get_resuling_tensor<Index<Idx2...>,resulting_index_2,Tensor<T,Rest2...>,resulting_tensor_2>::type,
typename std::conditional<
which_variant==3,
typename get_resuling_tensor<Index<Idx1...>,resulting_index_3,Tensor<T,Rest1...>,resulting_tensor_3>::type,
typename get_resuling_tensor<Index<Idx0...>,resulting_index_4,Tensor<T,Rest0...>,resulting_tensor_4>::type
>::type
>::type
>::type
>::type;
using resulting_index = typename std::conditional<
which_variant==0,
typename get_resuling_index<resulting_index_0,Index<Idx4...>,resulting_tensor_0,Tensor<T,Rest4...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_index<Index<Idx3...>,resulting_index_1,Tensor<T,Rest3...>,resulting_tensor_1>::type,
typename std::conditional<
which_variant==2,
typename get_resuling_index<Index<Idx2...>,resulting_index_2,Tensor<T,Rest2...>,resulting_tensor_2>::type,
typename std::conditional<
which_variant==3,
typename get_resuling_index<Index<Idx1...>,resulting_index_3,Tensor<T,Rest1...>,resulting_tensor_3>::type,
typename get_resuling_index<Index<Idx0...>,resulting_index_4,Tensor<T,Rest0...>,resulting_tensor_4>::type
>::type
>::type
>::type
>::type;
};
//------------------------------------------------------------------------------------------------------------//
// Cost model for sixtet tensor contraction
//------------------------------------------------------------------------------------------------------------//
template<class Ind0, class Ind1, class Ind2, class Ind3, class Ind4, class Ind5,
class Tensor0, class Tensor1, class Tensor2, class Tensor3, class Tensor4, class Tensor5>
struct sixtet_flop_cost;
template<class T, size_t... Idx0, size_t... Idx1, size_t... Idx2, size_t... Idx3, size_t... Idx4, size_t... Idx5,
size_t ...Rest0, size_t ...Rest1, size_t ...Rest2, size_t ...Rest3, size_t ...Rest4, size_t ...Rest5>
struct sixtet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx3...>, Index<Idx4...>, Index<Idx5...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>,Tensor<T,Rest5...>> {
// Perform depth-first search
//---------------------------------------------------------------------
// first 5 tensors contracted first
using quintet_cost_01234 = quintet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx3...>,Index<Idx4...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>>;
using resulting_tensor_0 = typename quintet_cost_01234::resulting_tensor;
using resulting_index_0 = typename quintet_cost_01234::resulting_index;
static constexpr size_t flop_count_01234 = quintet_cost_01234::min_cost;
static constexpr size_t flop_count_01234_5 = pair_flop_cost<resulting_index_0,Index<Idx5...>,resulting_tensor_0,Tensor<T,Rest5...>,
typename std_ext::make_index_sequence<sizeof...(Rest5)>::type>::value;
static constexpr size_t flop_count_0 = flop_count_01234 + flop_count_01234_5;
// 1st, 2nd, 3rd, 4th, 6th tensors contracted first
using quintet_cost_01235 = quintet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx3...>,Index<Idx5...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest5...>>;
using resulting_tensor_1 = typename quintet_cost_01235::resulting_tensor;
using resulting_index_1 = typename quintet_cost_01235::resulting_index;
static constexpr size_t flop_count_01235 = quintet_cost_01235::min_cost;
static constexpr size_t flop_count_01235_4 = pair_flop_cost<resulting_index_1,Index<Idx4...>,resulting_tensor_1,Tensor<T,Rest4...>,
typename std_ext::make_index_sequence<sizeof...(Rest4)>::type>::value;
static constexpr size_t flop_count_1 = flop_count_01235 + flop_count_01235_4;
// 1st, 2nd, 3rd, 5th, 6th tensors contracted first
using quintet_cost_01245 = quintet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx2...>,Index<Idx4...>,Index<Idx5...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest4...>,Tensor<T,Rest5...>>;
using resulting_tensor_2 = typename quintet_cost_01245::resulting_tensor;
using resulting_index_2 = typename quintet_cost_01245::resulting_index;
static constexpr size_t flop_count_01245 = quintet_cost_01235::min_cost;
static constexpr size_t flop_count_01245_3 = pair_flop_cost<resulting_index_2,Index<Idx3...>,resulting_tensor_2,Tensor<T,Rest3...>,
typename std_ext::make_index_sequence<sizeof...(Rest3)>::type>::value;
static constexpr size_t flop_count_2 = flop_count_01245 + flop_count_01245_3;
// 1st, 2nd, 4th, 5th, 6th tensors contracted first
using quintet_cost_01345 = quintet_flop_cost<Index<Idx0...>,Index<Idx1...>,Index<Idx3...>,Index<Idx4...>,Index<Idx5...>,
Tensor<T,Rest0...>,Tensor<T,Rest1...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>,Tensor<T,Rest5...>>;
using resulting_tensor_3 = typename quintet_cost_01345::resulting_tensor;
using resulting_index_3 = typename quintet_cost_01345::resulting_index;
static constexpr size_t flop_count_01345 = quintet_cost_01235::min_cost;
static constexpr size_t flop_count_01345_2 = pair_flop_cost<resulting_index_3,Index<Idx2...>,resulting_tensor_3,Tensor<T,Rest2...>,
typename std_ext::make_index_sequence<sizeof...(Rest2)>::type>::value;
static constexpr size_t flop_count_3 = flop_count_01345 + flop_count_01345_2;
// 1st, 3rd, 4th, 5th, 6th tensors contracted first
using quintet_cost_02345 = quintet_flop_cost<Index<Idx0...>,Index<Idx2...>,Index<Idx3...>,Index<Idx4...>,Index<Idx5...>,
Tensor<T,Rest0...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>,Tensor<T,Rest5...>>;
using resulting_tensor_4 = typename quintet_cost_02345::resulting_tensor;
using resulting_index_4 = typename quintet_cost_02345::resulting_index;
static constexpr size_t flop_count_02345 = quintet_cost_01235::min_cost;
static constexpr size_t flop_count_02345_1 = pair_flop_cost<resulting_index_4,Index<Idx1...>,resulting_tensor_4,Tensor<T,Rest1...>,
typename std_ext::make_index_sequence<sizeof...(Rest1)>::type>::value;
static constexpr size_t flop_count_4 = flop_count_02345 + flop_count_02345_1;
// last 5 tensors contracted first
using quintet_cost_12345 = quintet_flop_cost<Index<Idx1...>,Index<Idx2...>,Index<Idx3...>,Index<Idx4...>,Index<Idx5...>,
Tensor<T,Rest1...>,Tensor<T,Rest2...>,Tensor<T,Rest3...>,Tensor<T,Rest4...>,Tensor<T,Rest5...>>;
using resulting_tensor_5 = typename quintet_cost_12345::resulting_tensor;
using resulting_index_5 = typename quintet_cost_12345::resulting_index;
static constexpr size_t flop_count_12345 = quintet_cost_01235::min_cost;
static constexpr size_t flop_count_12345_0 = pair_flop_cost<resulting_index_5,Index<Idx0...>,resulting_tensor_5,Tensor<T,Rest0...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)>::type>::value;
static constexpr size_t flop_count_5 = flop_count_12345 + flop_count_12345_0;
static constexpr size_t min_cost = meta_min<flop_count_0,flop_count_1,flop_count_2,flop_count_3,flop_count_4,flop_count_5>::value;
static constexpr int which_variant = meta_argmin<flop_count_0,flop_count_1,flop_count_2,flop_count_3,flop_count_4,flop_count_5>::value;
// this is the overall resulting tensor and index from overall sextet contraction
using resulting_tensor = typename std::conditional<
which_variant==0,
typename get_resuling_tensor<resulting_index_0,Index<Idx5...>,resulting_tensor_0,Tensor<T,Rest5...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_tensor<Index<Idx4...>,resulting_index_1,Tensor<T,Rest4...>,resulting_tensor_1>::type,
typename std::conditional<
which_variant==2,
typename get_resuling_tensor<Index<Idx3...>,resulting_index_2,Tensor<T,Rest3...>,resulting_tensor_2>::type,
typename std::conditional<
which_variant==3,
typename get_resuling_tensor<Index<Idx2...>,resulting_index_3,Tensor<T,Rest2...>,resulting_tensor_3>::type,
typename std::conditional<
which_variant==4,
typename get_resuling_tensor<Index<Idx1...>,resulting_index_4,Tensor<T,Rest1...>,resulting_tensor_4>::type,
typename get_resuling_tensor<Index<Idx0...>,resulting_index_5,Tensor<T,Rest0...>,resulting_tensor_5>::type
>::type
>::type
>::type
>::type
>::type;
using resulting_index = typename std::conditional<
which_variant==0,
typename get_resuling_index<resulting_index_0,Index<Idx5...>,resulting_tensor_0,Tensor<T,Rest5...>>::type,
typename std::conditional<
which_variant==1,
typename get_resuling_index<Index<Idx4...>,resulting_index_1,Tensor<T,Rest4...>,resulting_tensor_1>::type,
typename std::conditional<
which_variant==2,
typename get_resuling_index<Index<Idx3...>,resulting_index_2,Tensor<T,Rest3...>,resulting_tensor_2>::type,
typename std::conditional<
which_variant==3,
typename get_resuling_index<Index<Idx2...>,resulting_index_3,Tensor<T,Rest2...>,resulting_tensor_3>::type,
typename std::conditional<
which_variant==4,
typename get_resuling_index<Index<Idx1...>,resulting_index_4,Tensor<T,Rest1...>,resulting_tensor_4>::type,
typename get_resuling_index<Index<Idx0...>,resulting_index_5,Tensor<T,Rest0...>,resulting_tensor_5>::type
>::type
>::type
>::type
>::type
>::type;
};
//------------------------------------------------------------------------------------------------------------//
// einsum helper to extract the resulting index and the resulting tensor
//------------------------------------------------------------------------------------------------------------//
template<typename ...Ts>
struct einsum_helper;
template<class Ind0,
typename T, size_t ... Rest0>
struct einsum_helper<Ind0,Tensor<T,Rest0...>> {
using resulting_index = typename contraction_impl<Ind0, Tensor<T,Rest0...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)>::type>::indices;
using resulting_tensor = typename contraction_impl<Ind0, Tensor<T,Rest0...>,
typename std_ext::make_index_sequence<sizeof...(Rest0)>::type>::type;
};
template<class Ind0, class Ind1,
class Tensor0, class Tensor1>
struct einsum_helper<Ind0,Ind1,Tensor0,Tensor1> {
using resulting_index = typename get_resuling_index<Ind0,Ind1,Tensor0,Tensor1>::type;
using resulting_tensor = typename get_resuling_tensor<Ind0,Ind1,Tensor0,Tensor1>::type;
};
template<class Ind0, class Ind1, class Ind2,
class Tensor0, class Tensor1, class Tensor2>
struct einsum_helper<Ind0,Ind1,Ind2,Tensor0,Tensor1,Tensor2> {
using cost_model = triplet_flop_cost<Ind0,Ind1,Ind2,
Tensor0,Tensor1,Tensor2>;
using resulting_index = typename cost_model::resulting_index;
using resulting_tensor = typename cost_model::resulting_tensor;
};
template<class Ind0, class Ind1, class Ind2, class Ind3,
class Tensor0, class Tensor1, class Tensor2, class Tensor3>
struct einsum_helper<Ind0,Ind1,Ind2,Ind3,Tensor0,Tensor1,Tensor2,Tensor3> {
using cost_model = quartet_flop_cost<Ind0,Ind1,Ind2,Ind3,
Tensor0,Tensor1,Tensor2,Tensor3>;
using resulting_index = typename cost_model::resulting_index;
using resulting_tensor = typename cost_model::resulting_tensor;
};
template<class Ind0, class Ind1, class Ind2, class Ind3, class Ind4,
class Tensor0, class Tensor1, class Tensor2, class Tensor3, class Tensor4>
struct einsum_helper<Ind0,Ind1,Ind2,Ind3,Ind4,Tensor0,Tensor1,Tensor2,Tensor3,Tensor4> {
using cost_model = quintet_flop_cost<Ind0,Ind1,Ind2,Ind3,Ind4,
Tensor0,Tensor1,Tensor2,Tensor3,Tensor4>;
using resulting_index = typename cost_model::resulting_index;
using resulting_tensor = typename cost_model::resulting_tensor;
};
template<class Ind0, class Ind1, class Ind2, class Ind3, class Ind4, class Ind5,
class Tensor0, class Tensor1, class Tensor2, class Tensor3, class Tensor4, class Tensor5>
struct einsum_helper<Ind0,Ind1,Ind2,Ind3,Ind4,Ind5,Tensor0,Tensor1,Tensor2,Tensor3,Tensor4,Tensor5> {
using cost_model = sixtet_flop_cost<Ind0,Ind1,Ind2,Ind3,Ind4,Ind5,
Tensor0,Tensor1,Tensor2,Tensor3,Tensor4,Tensor5>;
using resulting_index = typename cost_model::resulting_index;
using resulting_tensor = typename cost_model::resulting_tensor;
};
//------------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // FASTOR_DONT_PERFORM_OP_MIN
#endif // OPMIN_META_H

209
noarch/include/Fastor/meta/tensor_meta.h
View File

@@ -1,209 +0,0 @@
#ifndef TENSOR_META_H
#define TENSOR_META_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
namespace Fastor {
//----------------------------------------------------------------------------------------------------------//
// UpLoType
namespace UpLoType {
struct General {};
struct Lower {};
struct UniLower {};
struct StrictlyLower {};
struct Upper {};
struct UniUpper {};
struct StrictlyUpper {};
struct Diagonal {};
struct BiDiagonal {};
struct TriDiagonal {};
struct BlockDiagonal {};
struct Symmetric {};
struct SymmetricPositiveDefinite {};
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template <size_t...T>
struct is_unique : std::integral_constant<bool, true> {};
template <size_t T, size_t U, size_t... VV>
struct is_unique<T, U, VV...> : std::integral_constant<bool, T != U && is_unique<T, VV...>::value> {};
template <size_t...T>
struct no_of_unique : std::integral_constant<size_t, 0> {};
template <size_t T, size_t... UU>
struct no_of_unique<T, UU...> : std::integral_constant<size_t, is_unique<T, UU...>::value + no_of_unique<UU...>::value> {};
//----------------------------------------------------------------------------------------------------------//
// Note that Intel's ICC 2017 does not support make_index_sequence and the following
// version also seems faster than c++14's built-in (for clang) on Linux systems
namespace std_ext // back port to c++11
{
template <size_t... Ints>
struct index_sequence
{
using type = index_sequence;
using value_type = size_t;
static constexpr std::size_t size() { return sizeof...(Ints); }
};
// --------------------------------------------------------------
template <class Sequence1, class Sequence2>
struct _merge_and_renumber;
template <size_t... I1, size_t... I2>
struct _merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
: index_sequence<I1..., (sizeof...(I1)+I2)...>
{ };
// --------------------------------------------------------------
template <size_t N>
struct make_index_sequence
: _merge_and_renumber<typename make_index_sequence<N/2>::type,
typename make_index_sequence<N - N/2>::type>
{ };
template<> struct make_index_sequence<0> : index_sequence<> { };
template<> struct make_index_sequence<1> : index_sequence<0> { };
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template <class... > struct typelist { };
template <class T, T ... Vals>
using typelist_c = typelist<std::integral_constant<T, Vals>...>;
template <class... > struct concat;
template <> struct concat<> { using type = typelist<>; };
template <class... Ts> struct concat<typelist<Ts...>> { using type = typelist<Ts...>; };
template <class... Ts, class... Us, class... Args>
struct concat<typelist<Ts...>, typelist<Us...>, Args...> : concat<typelist<Ts..., Us...>, Args...> { };
template <class T, class TL> struct filter_out;
template <class T, class... Ts> struct filter_out<T, typelist<Ts...>>
: concat< conditional_t_<is_same_v_<T, Ts>, typelist<>, typelist<Ts>>...> { };
template <class T, class TL>
using filter_out_t = typename filter_out<T, TL>::type;
template <class >
struct uniq;
template <class TL>
using uniq_t = typename uniq<TL>::type;
template <>
struct uniq<typelist<>> {
using type = typelist<>;
};
template <class T, class... Ts>
struct uniq<typelist<T, Ts...>>
: concat<typelist<T>, uniq_t<filter_out_t<T, typelist<Ts...>>>>
{ };
template <size_t N>
using size_t_ = std::integral_constant<size_t, N>;
template <class > struct length;
template <class T> using length_t = typename length<T>::type;
template <class... Ts>
struct length<typelist<Ts...>>
: size_t_<sizeof...(Ts)>
{ };
template <size_t... Ns>
using no_of_uniques = length_t<uniq_t<typelist<size_t_<Ns>...>>>;
//----------------------------------------------------------------------------------------------------------//
////////////////
template <class T, template <T...> class Z>
struct quote_c {
template <class... Ts>
using apply = Z<Ts::value...>;
};
template <class MFC, class TL>
struct apply_typelist;
template <class MFC, class TL>
using apply_typelist_t = typename apply_typelist<MFC, TL>::type;
template <class MFC, class... Ts>
struct apply_typelist<MFC, typelist<Ts...>> {
using type = typename MFC::template apply<Ts...>;
};
//----------------------------------------------------------------------------------------------------------//
// Check if indices appear more than twice [for Einstein summation]
//----------------------------------------------------------------------------------------------------------//
namespace useless {
template <typename T>
constexpr const T& ct_max(T const& t1, T const& t2) {
return t1 < t2 ? t2 : t1;
}
template <size_t S, size_t... Sizes>
struct count__;
template <size_t S>
struct count__<S> {
static constexpr size_t value = std::integral_constant<size_t, 0>::value;
};
template <size_t S1, size_t... Sizes>
struct count__<S1, S1, Sizes...> {
static constexpr size_t value = std::integral_constant<size_t, 1 + count__<S1, Sizes...>::value>::value;
};
template <size_t S1, size_t S2, size_t... Sizes>
struct count__<S1, S2, Sizes...> {
static constexpr size_t value = count__<S1, Sizes...>::value;
};
template <size_t...all>
struct max_count;
template <>
struct max_count<> {
static constexpr size_t value = std::integral_constant<size_t, 0>::value;
};
template <size_t S, size_t... Sizes>
struct max_count<S, Sizes...> {
static constexpr size_t value = std::integral_constant<size_t, ct_max(1 + count__<S, Sizes...>::value,
max_count<Sizes...>::value)>::value;
};
} // useless
template <size_t... Sizes>
struct no_more_than_two {
static constexpr size_t value = std::integral_constant<bool, useless::max_count<Sizes...>::value <= 2>::value;
};
//----------------------------------------------------------------------------------------------------------//
}
#endif // TENSOR_META_H

549
noarch/include/Fastor/simd_math/simd_math.h
View File

@@ -1,549 +0,0 @@
#ifndef SIMD_MATH_H
#define SIMD_MATH_H
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include <cmath>
// SHUT GCC6 -Wignored-attributes WARNINGS
#ifdef __GNUC__
#if __GNUC__==6
#pragma GCC diagnostic ignored "-Wignored-attributes"
#endif
#endif
namespace Fastor {
// minimum
//----------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> min(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::min(((T*)&a)[i],((T*)&b)[i]); }
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> min(const SIMDVector<T,ABI> &a, T b) {
return min(a,SIMDVector<T,ABI>(b));
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> min(T a, const SIMDVector<T,ABI> &b) {
return min(SIMDVector<T,ABI>(a),b);
}
#ifdef FASTOR_SSE2_IMPL
#ifdef FASTOR_SSE4_1_IMPL
template<>
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> min(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
return _mm_min_epi32(a.value,b.value);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL)
template<>
FASTOR_INLINE SIMDVector<int64_t,simd_abi::sse> min(const SIMDVector<int64_t,simd_abi::sse> &a, const SIMDVector<int64_t,simd_abi::sse> &b) {
return _mm_min_epi64(a.value,b.value);
}
#endif
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> min(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return _mm_min_ps(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> min(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return _mm_min_pd(a.value,b.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
#ifdef FASTOR_AVX2_IMPL
template<>
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> min(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
return _mm256_min_epi32(a.value,b.value);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL)
template<>
FASTOR_INLINE SIMDVector<int64_t,simd_abi::avx> min(const SIMDVector<int64_t,simd_abi::avx> &a, const SIMDVector<int64_t,simd_abi::avx> &b) {
return _mm256_min_epi64(a.value,b.value);
}
#endif
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> min(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return _mm256_min_ps(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> min(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return _mm256_min_pd(a.value,b.value);
}
#endif
#ifdef FASTOR_AVX512F_IMPL
template<>
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> min(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
return _mm512_min_epi32(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<int64_t,simd_abi::avx512> min(const SIMDVector<int64_t,simd_abi::avx512> &a, const SIMDVector<int64_t,simd_abi::avx512> &b) {
return _mm512_min_epi64(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> min(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return _mm512_min_ps(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> min(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return _mm512_min_pd(a.value,b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// maximum
//----------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> max(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::max(((T*)&a)[i],((T*)&b)[i]); }
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> max(const SIMDVector<T,ABI> &a, T b) {
return max(a,SIMDVector<T,ABI>(b));
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> max(T a, const SIMDVector<T,ABI> &b) {
return max(SIMDVector<T,ABI>(a),b);
}
#ifdef FASTOR_SSE2_IMPL
#ifdef FASTOR_SSE4_1_IMPL
template<>
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> max(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
return _mm_max_epi32(a.value,b.value);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL)
template<>
FASTOR_INLINE SIMDVector<int64_t,simd_abi::sse> max(const SIMDVector<int64_t,simd_abi::sse> &a, const SIMDVector<int64_t,simd_abi::sse> &b) {
return _mm_max_epi64(a.value,b.value);
}
#endif
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> max(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return _mm_max_ps(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> max(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return _mm_max_pd(a.value,b.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
#ifdef FASTOR_AVX2_IMPL
template<>
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> max(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
return _mm256_max_epi32(a.value,b.value);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512VL_IMPL)
template<>
FASTOR_INLINE SIMDVector<int64_t,simd_abi::avx> max(const SIMDVector<int64_t,simd_abi::avx> &a, const SIMDVector<int64_t,simd_abi::avx> &b) {
return _mm256_max_epi64(a.value,b.value);
}
#endif
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> max(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return _mm256_max_ps(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> max(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return _mm256_max_pd(a.value,b.value);
}
#endif
#ifdef FASTOR_AVX512F_IMPL
template<>
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> max(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
return _mm512_max_epi32(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<int64_t,simd_abi::avx512> max(const SIMDVector<int64_t,simd_abi::avx512> &a, const SIMDVector<int64_t,simd_abi::avx512> &b) {
return _mm512_max_epi64(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> max(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return _mm512_max_ps(a.value,b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> max(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return _mm512_max_pd(a.value,b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// ceil
//----------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> ceil(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::ceil(((T*)&a)[i]);}
return out;
}
#ifdef FASTOR_SSE4_1_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> ceil(const SIMDVector<float,simd_abi::sse> &a) {
return _mm_ceil_ps(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> ceil(const SIMDVector<double,simd_abi::sse> &a) {
return _mm_ceil_pd(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> ceil(const SIMDVector<float,simd_abi::avx> &a) {
return _mm256_ceil_ps(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> ceil(const SIMDVector<double,simd_abi::avx> &a) {
return _mm256_ceil_pd(a.value);
}
#endif
// Part of SVML
// #ifdef FASTOR_AVX512F_IMPL
// template<>
// FASTOR_INLINE SIMDVector<float,simd_abi::avx512> ceil(const SIMDVector<float,simd_abi::avx512> &a) {
// return _mm512_ceil_ps(a.value);
// }
// template<>
// FASTOR_INLINE SIMDVector<double,simd_abi::avx512> ceil(const SIMDVector<double,simd_abi::avx512> &a) {
// return _mm512_ceil_pd(a.value);
// }
// #endif
//----------------------------------------------------------------------------------------------------------//
// round
//----------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> round(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::round(((T*)&a)[i]);}
return out;
}
#ifdef FASTOR_SSE4_1_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> round(const SIMDVector<float,simd_abi::sse> &a) {
return _mm_round_ps(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) );
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> round(const SIMDVector<double,simd_abi::sse> &a) {
return _mm_round_pd(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) );
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> round(const SIMDVector<float,simd_abi::avx> &a) {
return _mm256_round_ps(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) );
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> round(const SIMDVector<double,simd_abi::avx> &a) {
return _mm256_round_pd(a.value, ( _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC ) );
}
#endif
//----------------------------------------------------------------------------------------------------------//
// floor
//----------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> floor(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::floor(((T*)&a)[i]);}
return out;
}
#ifdef FASTOR_SSE4_1_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> floor(const SIMDVector<float,simd_abi::sse> &a) {
return _mm_floor_ps(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> floor(const SIMDVector<double,simd_abi::sse> &a) {
return _mm_floor_pd(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> floor(const SIMDVector<float,simd_abi::avx> &a) {
return _mm256_floor_ps(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> floor(const SIMDVector<double,simd_abi::avx> &a) {
return _mm256_floor_pd(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// remaining math functions from STL
//----------------------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> exp(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::exp(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> exp2(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::exp2(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> expm1(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::expm1(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> log(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::log(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> log10(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::log10(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> log2(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::log2(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> log1p(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::log1p(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> pow(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i = 0; i < SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::pow(((T*)&a)[i], ((T*)&b)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> pow(const SIMDVector<T,ABI> &a, T b) {
return pow(a, SIMDVector<T,ABI>(b));
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> pow(T a, const SIMDVector<T,ABI> &b) {
return pow(SIMDVector<T,ABI>(a),b);
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> cbrt(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::cbrt(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> sin(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::sin(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> cos(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::cos(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> tan(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::tan(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> asin(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::asin(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> acos(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::acos(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> atan(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::atan(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> atan2(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i = 0; i < SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::atan2(((T*)&a)[i], ((T*)&b)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> atan2(const SIMDVector<T,ABI> &a, T b) {
return atan2(a, SIMDVector<T,ABI>(b));
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> atan2(T a, const SIMDVector<T,ABI> &b) {
return atan2(SIMDVector<T,ABI>(a),b);
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> sinh(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::sinh(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> cosh(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::cosh(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> tanh(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::tanh(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> asinh(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::asinh(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> acosh(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::acosh(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> atanh(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::atanh(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> erf(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::erf(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> tgamma(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::tgamma(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> lgamma(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::lgamma(((T*)&a)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> hypot(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i = 0; i < SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::hypot(((T*)&a)[i], ((T*)&b)[i]);}
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> hypot(const SIMDVector<T,ABI> &a, T b) {
return hypot(a, SIMDVector<T,ABI>(b));
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> hypot(T a, const SIMDVector<T,ABI> &b) {
return hypot(SIMDVector<T,ABI>(a),b);
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> trunc(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((T*)&out)[i] = std::trunc(((T*)&a)[i]);}
return out;
}
//----------------------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------------------//
// Boolean arithmetic
// ! or not
//----------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> operator!(const SIMDVector<T,ABI> &a) {
SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((bool*)&out)[i] = !(((T*)&a)[i]); }
return out;
}
//----------------------------------------------------------------------------------------------------------//
// isinf/nan/finite
//----------------------------------------------------------------------------------------------------------//
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> isinf(const SIMDVector<T,ABI> &a) {
SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((bool*)&out)[i] = std::isinf(((T*)&a)[i]); }
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> isnan(const SIMDVector<T,ABI> &a) {
SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((bool*)&out)[i] = std::isnan(((T*)&a)[i]); }
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> isfinite(const SIMDVector<T,ABI> &a) {
SIMDVector<bool,simd_abi::fixed_size<SIMDVector<T,ABI>::Size>> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; i++) { ((bool*)&out)[i] = std::isfinite(((T*)&a)[i]); }
return out;
}
//----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
// Include all backends
#include "Fastor/simd_math/sleef_backend.h"
#endif // SIMD_MATH_H

7
noarch/include/Fastor/simd_math/sleef_backend.h
View File

@@ -1,7 +0,0 @@
#ifndef SLEEF_BACKEND_H
#define SLEEF_BACKEND_H
#include "Fastor/simd_math/sleef_backend_u10.h"
#include "Fastor/simd_math/sleef_backend_u35.h"
#endif // SLEEF_BACKEND_H

932
noarch/include/Fastor/simd_math/sleef_backend_u10.h
View File

@@ -1,932 +0,0 @@
#ifndef SLEEF_BACKEND_U10_H
#define SLEEF_BACKEND_U10_H
#if defined(FASTOR_USE_SLEEF_U10) || defined(FASTOR_USE_SLEEF)
#include <sleef.h>
namespace Fastor {
// exp
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> exp(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_expf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> exp(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_expd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> exp(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_expf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> exp(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_expd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> exp(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_expf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> exp(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_expd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// exp2
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> exp2(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_exp2f4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> exp2(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_exp2d2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> exp2(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_exp2f8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> exp2(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_exp2d4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> exp2(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_exp2f16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> exp2(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_exp2d8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// expm1
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> expm1(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_expm1f4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> expm1(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_expm1d2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> expm1(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_expm1f8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> expm1(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_expm1d4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> expm1(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_expm1f16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> expm1(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_expm1d8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_logf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_logd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_logf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_logd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_logf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_logd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log10
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log10(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_log10f4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log10(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_log10d2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log10(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_log10f8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log10(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_log10d4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log10(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_log10f16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log10(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_log10d8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log2
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log2(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_log2f4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log2(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_log2d2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log2(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_log2f8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log2(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_log2d4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log2(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_log2f16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log2(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_log2d8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log1p
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log1p(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_log1pf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log1p(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_log1pd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log1p(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_log1pf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log1p(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_log1pd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log1p(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_log1pf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log1p(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_log1pd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// pow - other variants are automatically taken care off through the generic overloads
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> pow(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return Sleef_powf4_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> pow(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return Sleef_powd2_u10(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> pow(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_powf8_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> pow(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_powd4_u10(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> pow(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return Sleef_powf16_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> pow(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return Sleef_powd8_u10(a.value, b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// cbrt
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> cbrt(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_cbrtf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> cbrt(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_cbrtd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> cbrt(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_cbrtf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> cbrt(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_cbrtd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> cbrt(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_cbrtf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> cbrt(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_cbrtd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// sin
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> sin(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_sinf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> sin(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_sind2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> sin(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_sinf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> sin(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_sind4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> sin(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_sinf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> sin(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_sind8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// cos
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> cos(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_cosf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> cos(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_cosd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> cos(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_cosf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> cos(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_cosd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> cos(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_cosf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> cos(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_cosd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// tan
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> tan(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_tanf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> tan(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_tand2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> tan(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_tanf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> tan(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_tand4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> tan(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_tanf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> tan(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_tand8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// asin
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> asin(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_asinf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> asin(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_asind2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> asin(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_asinf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> asin(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_asind4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> asin(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_asinf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> asin(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_asind8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// acos
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> acos(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_acosf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> acos(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_acosd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> acos(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_acosf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> acos(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_acosd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> acos(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_acosf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> acos(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_acosd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// atan
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> atan(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_atanf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> atan(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_atand2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> atan(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_atanf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> atan(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_atand4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> atan(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_atanf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> atan(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_atand8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// atan2 - other variants are automatically taken care off through the generic overloads
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> atan2(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return Sleef_atan2f4_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> atan2(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return Sleef_atan2d2_u10(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> atan2(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_atan2f8_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> atan2(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_atan2d4_u10(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> atan2(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return Sleef_atan2f16_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> atan2(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return Sleef_atan2d8_u10(a.value, b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// sinh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> sinh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_sinhf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> sinh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_sinhd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> sinh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_sinhf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> sinh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_sinhd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> sinh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_sinhf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> sinh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_sinhd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// cosh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> cosh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_coshf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> cosh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_coshd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> cosh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_coshf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> cosh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_coshd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> cosh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_coshf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> cosh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_coshd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// tanh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> tanh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_tanhf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> tanh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_tanhd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> tanh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_tanhf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> tanh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_tanhd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> tanh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_tanhf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> tanh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_tanhd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// asinh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> asinh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_asinhf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> asinh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_asinhd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> asinh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_asinhf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> asinh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_asinhd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> asinh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_asinhf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> asinh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_asinhd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// acosh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> acosh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_acoshf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> acosh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_acoshd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> acosh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_acoshf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> acosh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_acoshd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> acosh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_acoshf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> acosh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_acoshd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// atanh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> atanh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_atanhf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> atanh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_atanhd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> atanh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_atanhf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> atanh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_atanhd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> atanh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_atanhf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> atanh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_atanhd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// erf
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> erf(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_erff4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> erf(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_erfd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> erf(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_erff8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> erf(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_erfd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> erf(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_erff16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> erf(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_erfd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// tgamma
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> tgamma(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_tgammaf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> tgamma(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_tgammad2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> tgamma(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_tgammaf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> tgamma(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_tgammad4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> tgamma(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_tgammaf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> tgamma(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_tgammad8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// lgamma
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> lgamma(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_lgammaf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> lgamma(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_lgammad2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> lgamma(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_lgammaf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> lgamma(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_lgammad4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> lgamma(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_lgammaf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> lgamma(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_lgammad8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// hypot - other variants are automatically taken care off through the generic overloads
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> hypot(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return Sleef_hypotf4_u05(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> hypot(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return Sleef_hypotd2_u05(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> hypot(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_hypotf8_u05avx2(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> hypot(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_hypotd4_u05avx2(a.value, b.value);
}
#elif defined(FASTOR_AVX_IMPL)
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> hypot(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_hypotf8_u05avx(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> hypot(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_hypotd4_u05avx(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX512F_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> hypot(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return Sleef_hypotf16_u05avx512f(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> hypot(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return Sleef_hypotd8_u05avx512f(a.value, b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // FASTOR_USE_SLEEF_U10
#endif // SLEEF_BACKEND_U10_H

933
noarch/include/Fastor/simd_math/sleef_backend_u35.h
View File

@@ -1,933 +0,0 @@
#ifndef SLEEF_BACKEND_U35_H
#define SLEEF_BACKEND_U35_H
// Not all functions have u35 variant implemented, so falling back to u10 in such cases
#ifdef FASTOR_USE_SLEEF_U35
#include <sleef.h>
namespace Fastor {
// exp
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> exp(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_expf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> exp(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_expd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> exp(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_expf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> exp(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_expd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> exp(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_expf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> exp(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_expd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// exp2
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> exp2(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_exp2f4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> exp2(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_exp2d2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> exp2(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_exp2f8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> exp2(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_exp2d4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> exp2(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_exp2f16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> exp2(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_exp2d8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// expm1
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> expm1(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_expm1f4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> expm1(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_expm1d2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> expm1(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_expm1f8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> expm1(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_expm1d4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> expm1(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_expm1f16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> expm1(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_expm1d8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_logf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_logd2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_logf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_logd4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_logf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_logd8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log10
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log10(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_log10f4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log10(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_log10d2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log10(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_log10f8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log10(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_log10d4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log10(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_log10f16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log10(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_log10d8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log2
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log2(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_log2f4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log2(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_log2d2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log2(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_log2f8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log2(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_log2d4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log2(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_log2f16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log2(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_log2d8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// log1p
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> log1p(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_log1pf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> log1p(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_log1pd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> log1p(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_log1pf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> log1p(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_log1pd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> log1p(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_log1pf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> log1p(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_log1pd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// pow - other variants are automatically taken care off through the generic overloads
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> pow(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return Sleef_powf4_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> pow(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return Sleef_powd2_u10(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> pow(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_powf8_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> pow(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_powd4_u10(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> pow(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return Sleef_powf16_u10(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> pow(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return Sleef_powd8_u10(a.value, b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// cbrt
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> cbrt(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_cbrtf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> cbrt(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_cbrtd2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> cbrt(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_cbrtf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> cbrt(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_cbrtd4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> cbrt(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_cbrtf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> cbrt(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_cbrtd8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// sin
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> sin(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_sinf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> sin(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_sind2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> sin(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_sinf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> sin(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_sind4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> sin(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_sinf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> sin(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_sind8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// cos
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> cos(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_cosf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> cos(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_cosd2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> cos(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_cosf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> cos(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_cosd4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> cos(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_cosf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> cos(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_cosd8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// tan
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> tan(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_tanf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> tan(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_tand2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> tan(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_tanf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> tan(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_tand4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> tan(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_tanf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> tan(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_tand8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// asin
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> asin(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_asinf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> asin(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_asind2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> asin(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_asinf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> asin(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_asind4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> asin(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_asinf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> asin(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_asind8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// acos
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> acos(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_acosf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> acos(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_acosd2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> acos(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_acosf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> acos(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_acosd4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> acos(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_acosf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> acos(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_acosd8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// atan
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> atan(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_atanf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> atan(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_atand2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> atan(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_atanf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> atan(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_atand4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> atan(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_atanf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> atan(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_atand8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// atan2 - other variants are automatically taken care off through the generic overloads
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> atan2(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return Sleef_atan2f4_u35(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> atan2(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return Sleef_atan2d2_u35(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> atan2(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_atan2f8_u35(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> atan2(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_atan2d4_u35(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> atan2(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return Sleef_atan2f16_u35(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> atan2(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return Sleef_atan2d8_u35(a.value, b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// sinh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> sinh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_sinhf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> sinh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_sinhd2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> sinh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_sinhf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> sinh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_sinhd4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> sinh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_sinhf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> sinh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_sinhd8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// cosh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> cosh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_coshf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> cosh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_coshd2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> cosh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_coshf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> cosh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_coshd4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> cosh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_coshf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> cosh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_coshd8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// tanh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> tanh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_tanhf4_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> tanh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_tanhd2_u35(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> tanh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_tanhf8_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> tanh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_tanhd4_u35(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> tanh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_tanhf16_u35(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> tanh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_tanhd8_u35(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// asinh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> asinh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_asinhf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> asinh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_asinhd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> asinh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_asinhf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> asinh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_asinhd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> asinh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_asinhf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> asinh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_asinhd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// acosh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> acosh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_acoshf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> acosh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_acoshd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> acosh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_acoshf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> acosh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_acoshd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> acosh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_acoshf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> acosh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_acoshd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// atanh
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> atanh(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_atanhf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> atanh(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_atanhd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> atanh(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_atanhf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> atanh(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_atanhd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> atanh(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_atanhf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> atanh(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_atanhd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// erf
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> erf(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_erff4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> erf(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_erfd2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> erf(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_erff8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> erf(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_erfd4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> erf(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_erff16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> erf(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_erfd8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// tgamma
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> tgamma(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_tgammaf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> tgamma(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_tgammad2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> tgamma(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_tgammaf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> tgamma(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_tgammad4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> tgamma(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_tgammaf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> tgamma(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_tgammad8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// lgamma
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> lgamma(const SIMDVector<float,simd_abi::sse> &a) {
return Sleef_lgammaf4_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> lgamma(const SIMDVector<double,simd_abi::sse> &a) {
return Sleef_lgammad2_u10(a.value);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> lgamma(const SIMDVector<float,simd_abi::avx> &a) {
return Sleef_lgammaf8_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> lgamma(const SIMDVector<double,simd_abi::avx> &a) {
return Sleef_lgammad4_u10(a.value);
}
#endif
#ifdef FASTOR_AVX512_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> lgamma(const SIMDVector<float,simd_abi::avx512> &a) {
return Sleef_lgammaf16_u10(a.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> lgamma(const SIMDVector<double,simd_abi::avx512> &a) {
return Sleef_lgammad8_u10(a.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// hypot - other variants are automatically taken care off through the generic overloads
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::sse> hypot(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
return Sleef_hypotf4_u05(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::sse> hypot(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
return Sleef_hypotd2_u05(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX2_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> hypot(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_hypotf8_u05avx2(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> hypot(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_hypotd4_u05avx2(a.value, b.value);
}
#elif defined(FASTOR_AVX_IMPL)
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx> hypot(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
return Sleef_hypotf8_u05avx(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx> hypot(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
return Sleef_hypotd4_u05avx(a.value, b.value);
}
#endif
#ifdef FASTOR_AVX512F_IMPL
template<>
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> hypot(const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
return Sleef_hypotf16_u05avx512f(a.value, b.value);
}
template<>
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> hypot(const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
return Sleef_hypotd8_u05avx512f(a.value, b.value);
}
#endif
//----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // FASTOR_USE_SLEEF_U35
#endif // FASTOR_USE_SLEEF_U35

16
noarch/include/Fastor/simd_vector/SIMDVector.h
View File

@@ -1,16 +0,0 @@
#ifndef SIMDVECTOR_H
#define SIMDVECTOR_H
#include "Fastor/simd_vector/simd_vector_base.h"
#include "Fastor/simd_vector/simd_vector_scalar.h"
#include "Fastor/simd_vector/simd_vector_float.h"
#include "Fastor/simd_vector/simd_vector_double.h"
#include "Fastor/simd_vector/simd_vector_int32.h"
#include "Fastor/simd_vector/simd_vector_int64.h"
#include "Fastor/simd_vector/simd_vector_complex_scalar.h"
#include "Fastor/simd_vector/simd_vector_complex_float.h"
#include "Fastor/simd_vector/simd_vector_complex_double.h"
#include "Fastor/simd_vector/simd_vector_common.h"
#endif // SIMDVECTOR_H

1255
noarch/include/Fastor/simd_vector/extintrin.h
View File

File diff suppressed because it is too large Load Diff

225
noarch/include/Fastor/simd_vector/simd_vector_abi.h
View File

@@ -1,225 +0,0 @@
#ifndef FASTOR_SIMD_VECTOR_ABI
#define FASTOR_SIMD_VECTOR_ABI
#include "Fastor/meta/meta.h"
#include "Fastor/config/config.h"
#include <complex>
#include <type_traits>
namespace Fastor {
namespace simd_abi {
struct scalar {};
struct sse {};
struct avx {};
struct avx512 {};
struct mic {};
template<size_t N> struct fixed_size {};
#ifndef FASTOR_DONT_VECTORISE
#if defined(FASTOR_AVX512_IMPL)
using native = simd_abi::avx512;
#elif defined(FASTOR_AVX_IMPL)
using native = simd_abi::avx;
#elif defined(FASTOR_SSE2_IMPL)
using native = simd_abi::sse;
#else
using native = simd_abi::scalar;
#endif
#else
using native = simd_abi::scalar;
#endif
}
// Definition of DEFAULT_ABI is here
//--------------------------------------------------------------------------------------------------------------//
#define DEFAULT_ABI simd_abi::native
//--------------------------------------------------------------------------------------------------------------//
//--------------------------------------------------------------------------------------------------------------//
namespace internal {
template<class __svec>
struct get_simd_vector_size;
template<template<typename, typename> class __svec, typename T, typename ABI>
struct get_simd_vector_size<__svec<T,ABI>> {
static constexpr size_t bitsize = std::is_same<ABI,simd_abi::avx512>::value
? FASTOR_AVX512_BITSIZE : (std::is_same<ABI,simd_abi::avx>::value
? FASTOR_AVX_BITSIZE : (std::is_same<ABI,simd_abi::sse>::value
? FASTOR_SSE_BITSIZE : sizeof(T)*8));
// Size should be at least 1UL
static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL;
};
// Specialisation for fixed size simd vectors
template<template<typename, typename> class __svec, typename T, size_t N>
struct get_simd_vector_size<__svec<T,simd_abi::fixed_size<N> > > {
static constexpr size_t bitsize = N*8UL;
static constexpr size_t value = N;
};
// Specialisation for complex simd vectors
template<template<typename, typename> class __svec, typename ABI>
struct get_simd_vector_size<__svec<std::complex<float>,ABI>> {
using T = float;
static constexpr size_t bitsize = std::is_same<ABI,simd_abi::avx512>::value
? FASTOR_AVX512_BITSIZE : (std::is_same<ABI,simd_abi::avx>::value
? FASTOR_AVX_BITSIZE : (std::is_same<ABI,simd_abi::sse>::value
? FASTOR_SSE_BITSIZE : sizeof(T)*8));
// Size should be at least 1UL
static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL;
};
template<template<typename, typename> class __svec, typename ABI>
struct get_simd_vector_size<__svec<std::complex<double>,ABI>> {
using T = double;
static constexpr size_t bitsize = std::is_same<ABI,simd_abi::avx512>::value
? FASTOR_AVX512_BITSIZE : (std::is_same<ABI,simd_abi::avx>::value
? FASTOR_AVX_BITSIZE : (std::is_same<ABI,simd_abi::sse>::value
? FASTOR_SSE_BITSIZE : sizeof(T)*8));
// Size should be at least 1UL
static constexpr size_t value = (bitsize / sizeof(T) / 8UL) != 0 ? (bitsize / sizeof(T) / 8UL) : 1UL;
};
template<class __svec>
struct get_half_simd_type;
template<template<typename, typename> class __svec, typename T, typename ABI>
struct get_half_simd_type<__svec<T,ABI>> {
// If not a half of simd we give back the actual incoming type
using actual_type = __svec<T,ABI>;
using type = typename std::conditional< std::is_same<ABI,simd_abi::avx512>::value, __svec<T,simd_abi::avx>,
typename std::conditional< std::is_same<ABI,simd_abi::avx>::value, __svec<T,simd_abi::sse>, actual_type>::type
>::type;
};
template<class __svec>
struct get_quarter_simd_type;
template<template<typename, typename> class __svec, typename T, typename ABI>
struct get_quarter_simd_type<__svec<T,ABI>> {
// If not a quarter of simd we give back the actual incoming type
using actual_type = __svec<T,ABI>;
using type = typename std::conditional< std::is_same<ABI,simd_abi::avx512>::value, __svec<T,simd_abi::sse>, actual_type>::type;
};
template<class __svec, size_t N>
struct is_exact_multiple_of_smaller_simd;
template<template<typename, typename> class __svec, typename T, typename ABI, size_t N>
struct is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N> {
// If not a fraction of simd we give back the actual incoming type
using actual_type = __svec<T,ABI>;
// if N is half simd which=2, if it is a 1/4th which=4, else which=1
static constexpr int which = get_simd_vector_size<__svec<T,ABI>>::value / N == 2UL ? 2UL
: (get_simd_vector_size<__svec<T,ABI>>::value / N == 4UL ? 4UL : 1UL);
static constexpr bool value = which != 1UL && !std::is_same<ABI,simd_abi::sse>::value ? true : false;
static constexpr bool is_half_of_avx512 = std::is_same<ABI,simd_abi::avx512>::value && which==2UL;
static constexpr bool is_half_of_avx = std::is_same<ABI,simd_abi::avx>::value && which==2UL;
static constexpr bool is_4th_of_avx512 = std::is_same<ABI,simd_abi::avx512>::value && which==4UL;
// static constexpr bool is_4th_of_avx = std::is_same<ABI,simd_abi::avx>::value && which==4UL;
static constexpr bool is_half_type = is_half_of_avx512 || is_half_of_avx;
static constexpr bool is_quarter_type = is_4th_of_avx512;
using half_type = typename std::conditional< is_half_of_avx512, __svec<T,simd_abi::avx>,
typename std::conditional< is_half_of_avx, __svec<T,simd_abi::sse>, actual_type>::type
>::type;
using quarter_type = typename std::conditional< is_4th_of_avx512, __svec<T,simd_abi::sse>, actual_type>::type;
using type = typename std::conditional<is_half_type, half_type,
typename std::conditional<is_quarter_type, quarter_type, actual_type>::type
>::type;
};
template<class __svec, size_t N>
struct choose_best_simd_type;
template<template<typename, typename> class __svec, typename TT, typename ABI, size_t N>
struct choose_best_simd_type<__svec<TT,ABI>,N> {
using T = remove_cv_ref_t<TT>;
using actual_type = __svec<T,ABI>;
static constexpr size_t _vec_size = get_simd_vector_size<__svec<T,ABI>>::value;
// For exact fractions simd gets proper speed up for instance for matmul
using exact_multiple_t = typename is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::type;
static constexpr bool is_exact_multiple = is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::value;
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
using size_based_type = typename std::conditional<is_exact_multiple, exact_multiple_t, actual_type>::type;
#else
using size_based_type = typename std::conditional<is_exact_multiple, exact_multiple_t,
typename std::conditional<is_less<N,_vec_size>::value, typename get_half_simd_type<__svec<T,ABI>>::type, actual_type
>::type
>::type;
#endif
// using size_based_type = typename std::conditional<is_exact_multiple, typename is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::type,
// typename std::conditional<is_greater<_vec_size,2UL*N>::value, typename get_quarter_simd_type<__svec<T,ABI>>::type,
// typename std::conditional<is_greater<_vec_size,N>::value, typename get_half_simd_type<__svec<T,ABI>>::type, actual_type
// >::type
// >::type
// >::type;
// // For other fractions masking might be a better idea than, hence this special logic for remainder using is_less.
// // For no special logic use the above case
// using size_based_type = typename std::conditional<is_exact_multiple, typename is_exact_multiple_of_smaller_simd<__svec<T,ABI>,N>::type,
// typename std::conditional<is_greater<_vec_size,2UL*N>::value && is_greater<_vec_size % N,1UL>::value,
// typename get_quarter_simd_type<__svec<T,ABI>>::type,
// typename std::conditional<is_greater<_vec_size,N>::value && is_greater<_vec_size % N,1UL>::value,
// typename get_half_simd_type<__svec<T,ABI>>::type, actual_type
// >::type
// >::type
// >::type;
using type = typename std::conditional< std::is_same<T,float>::value ||
std::is_same<T,double>::value ||
std::is_same<T,std::complex<float>>::value ||
std::is_same<T,std::complex<double>>::value ||
std::is_same<T,int32_t>::value ||
std::is_same<T,int64_t>::value,
size_based_type,
__svec<T,simd_abi::scalar>
>::type;
};
} // end of namesapce internal
// This is specifically for matmul and other backend tensor kernels that need
// to choose/switch between best simd types for utmost performance
template<class __svec, size_t N>
using choose_best_simd_t = typename internal::choose_best_simd_type<__svec,N>::type;
//--------------------------------------------------------------------------------------------------------------//
// For switching between complex and non-complex implementation of simd we need different return types
//--------------------------------------------------------------------------------------------------------------//
template<typename T, typename T2 = void>
struct get_simd_cmplx_value_type;
template<typename T>
struct get_simd_cmplx_value_type<T, enable_if_t_<!is_complex_v_<T> > > {
using type = T;
};
template<typename T>
struct get_simd_cmplx_value_type<T, enable_if_t_<is_complex_v_<T> > > {
using type = typename T::value_type;
};
template<class VecType>
struct simd_cmplx_value_type;
template<template<typename, typename> class __svec, typename T, typename ABI>
struct simd_cmplx_value_type< __svec<T,ABI> > {
using type = typename get_simd_cmplx_value_type<T>::type;
};
template<class VecType>
using simd_cmplx_value_t = typename simd_cmplx_value_type<VecType>::type;
//--------------------------------------------------------------------------------------------------------------//
} // end of namesapce Fastor
#endif // FASTOR_SIMD_VECTOR_ABI

391
noarch/include/Fastor/simd_vector/simd_vector_base.h
View File

@@ -1,391 +0,0 @@
#ifndef SIMD_VECTOR_BASE_H
#define SIMD_VECTOR_BASE_H
#include "Fastor/meta/meta.h"
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/simd_vector_abi.h"
#include<cmath>
#include<complex>
namespace Fastor {
/* The default SIMDVector class that falls back to scalar implementation
* if SIMD types are not available or if vectorisation is disallowed
*/
//--------------------------------------------------------------------------------------------------------------------//
template <typename CVT, typename ABI = simd_abi::native>
struct SIMDVector {
using T = remove_cv_ref_t<CVT>;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<T,ABI>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<T,ABI>>::value;}
using vector_type = SIMDVector<T,ABI>;
using value_type = T[Size];
using scalar_value_type = T;
using abi_type = ABI;
FASTOR_INLINE SIMDVector() : value{} {}
FASTOR_INLINE SIMDVector(T num) { std::fill(value, value+Size, num); }
FASTOR_INLINE SIMDVector(const SIMDVector<T,ABI> &a) { std::copy(a.value,a.value+a.Size,value); }
FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) { std::copy(data,data+Size,value); unused(Aligned); }
FASTOR_INLINE SIMDVector<T,ABI> operator=(T num) { std::fill(value, value+Size, num); return *this;}
FASTOR_INLINE SIMDVector<T,ABI> operator=(const SIMDVector<T,ABI> &a) { std::copy(a.value,a.value+a.Size,value); return *this; };
FASTOR_INLINE void load(const T *data, bool Aligned=true ) { std::copy(data,data+Size,value); unused(Aligned);}
FASTOR_INLINE void store(T *data, bool Aligned=true ) const { std::copy(value,value+Size,data); unused(Aligned);}
FASTOR_INLINE void aligned_load(const T *data) { std::copy(data,data+Size,value); }
FASTOR_INLINE void aligned_store(T *data) const { std::copy(value,value+Size,data);}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
std::fill(value, value+Size, 0);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
}
FASTOR_INLINE T operator[](FASTOR_INDEX i) const {return value[i];}
FASTOR_INLINE T operator()(FASTOR_INDEX i) const {return value[i];}
// For compatibility with complex simd vector
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> real() const {
simd_cmplx_value_t<vector_type> arr[Size];
for (FASTOR_INDEX i=0; i<Size; ++i) {
arr[i] = value[i].real();
}
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
return out;
}
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> imag() const {
simd_cmplx_value_t<vector_type> arr[Size];
for (FASTOR_INDEX i=0; i<Size; ++i) {
arr[i] = value[i].imag();
}
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
return out;
}
FASTOR_INLINE void set(T num) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] = num;
}
template<typename U, typename ... Args>
FASTOR_INLINE void set(U first, Args ... args) {
static_assert(sizeof...(args)+1==Size,"CANNOT SET VECTOR WITH SPECIFIED NUMBER OF VALUES DUE TO ABI CONSIDERATION");
T arr[Size] = {first,args...};
std::reverse_copy(arr, arr+Size, value);
}
FASTOR_INLINE void set_sequential(T num0) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] = num0+(T)i;
}
// In-place operators
FASTOR_INLINE void operator+=(T num) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] += num;
}
FASTOR_INLINE void operator+=(const SIMDVector<T,ABI> &a) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] += a.value[i];
}
FASTOR_INLINE void operator-=(T num) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] -= num;
}
FASTOR_INLINE void operator-=(const SIMDVector<T,ABI> &a) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] -= a.value[i];
}
FASTOR_INLINE void operator*=(T num) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] *= num;
}
FASTOR_INLINE void operator*=(const SIMDVector<T,ABI> &a) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] *= a.value[i];
}
FASTOR_INLINE void operator/=(T num) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] /= num;
}
FASTOR_INLINE void operator/=(const SIMDVector<T,ABI> &a) {
for (FASTOR_INDEX i=0; i<Size;++i)
value[i] /= a.value[i];
}
// end of in-place operators
FASTOR_INLINE SIMDVector<T,ABI> shift(FASTOR_INDEX i) {
SIMDVector<T,ABI> out;
std::fill(out.value,out.value+out.Size,static_cast<T>(0));
std::copy(value,value+Size, out.value+i);
return out;
}
FASTOR_INLINE T sum() {
T quan = 0;
for (FASTOR_INDEX i=0; i<Size;++i)
quan += value[i];
return quan;
}
FASTOR_INLINE T product() {
//! Don't use prod as that is the name of a meta-function
T quan = 1;
for (FASTOR_INDEX i=0; i<Size;++i)
quan *= value[i];
return quan;
}
FASTOR_INLINE SIMDVector<T,ABI> reverse() {
SIMDVector<T,ABI> out;
std::copy(value,value+Size,out.value);
std::reverse(out.value,out.value+Size);
return out;
}
// This is for comatibility with complex simd vectors
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> magnitude() {
simd_cmplx_value_t<vector_type> arr[Size];
for (FASTOR_INDEX i=0; i<Size; ++i) {
arr[i] = std::abs(value[i]);
}
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
return out;
}
template<typename U=T, enable_if_t_<is_complex_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<simd_cmplx_value_t<vector_type>,ABI> norm() {
simd_cmplx_value_t<vector_type> arr[Size];
for (FASTOR_INDEX i=0; i<Size; ++i) {
arr[i] = std::norm(value[i]);
}
SIMDVector<simd_cmplx_value_t<vector_type>,ABI> out(arr,false);
return out;
}
FASTOR_INLINE T minimum() {
T quan = 0;
for (FASTOR_INDEX i=0; i<Size;++i)
if (value[i]<quan)
quan = value[i];
return quan;
}
FASTOR_INLINE T maximum() {
T quan = 0;
for (FASTOR_INDEX i=0; i<Size;++i)
if (value[i]>quan)
quan = value[i];
return quan;
}
FASTOR_INLINE T dot(const SIMDVector<T,ABI> &other) {
T quan = 0;
for (FASTOR_INDEX i=0; i<Size;++i)
quan += value[i]*other.value[i];
return quan;
}
template<typename U>
FASTOR_INLINE SIMDVector<U,ABI> cast() {
SIMDVector<U,ABI> out;
for (FASTOR_INDEX i=0; i<Size;++i) {
out.value[i] = static_cast<U>(value[i]);
}
return out;
}
#ifdef FASTOR_ZERO_INITIALISE
FASTOR_ARCH_ALIGN T value[Size] = {};
#else
FASTOR_ARCH_ALIGN T value[Size];
#endif
};
template <typename T, typename ABI>
constexpr FASTOR_INDEX SIMDVector<T,ABI>::Size;
template<typename T, typename ABI>
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<T,ABI> a) {
os << "[";
for (FASTOR_INDEX i=0; i<a.size(); ++i)
os << a.value[i] << ' ';
os << "]";
return os;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator+(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] + b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator+(const SIMDVector<T,ABI> &a, T b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] + b;
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator+(T a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a + b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator+(const SIMDVector<T,ABI> &b) {
return b;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator-(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] - b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator-(const SIMDVector<T,ABI> &a, T b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] - b;
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator-(T a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a - b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator-(const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = -b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator*(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] * b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator*(const SIMDVector<T,ABI> &a, T b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] * b;
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator*(T a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a * b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator/(const SIMDVector<T,ABI> &a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] / b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator/(const SIMDVector<T,ABI> &a, T b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a.value[i] / b;
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> operator/(T a, const SIMDVector<T,ABI> &b) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<SIMDVector<T,ABI>::Size; ++i)
out.value[i] = a / b.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> rcp(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<a.Size; ++i)
out.value[i] = T(1.)/a.value[i];
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> sqrt(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<a.Size; ++i)
out.value[i] = std::sqrt(a.value[i]);
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> rsqrt(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<a.Size; ++i)
out.value[i] = T(1.)/std::sqrt(a.value[i]);
return out;
}
template<typename T, typename ABI>
FASTOR_INLINE SIMDVector<T,ABI> abs(const SIMDVector<T,ABI> &a) {
SIMDVector<T,ABI> out;
for (FASTOR_INDEX i=0; i<a.Size; ++i)
out.value[i] = std::abs(a.value[i]);
return out;
}
// For compatibility with complex simd vectors
template<typename T, typename ABI, enable_if_t_<is_complex_v_<T>,bool> = false>
FASTOR_INLINE SIMDVector<T,ABI> conj(const SIMDVector<T,ABI> &a) {
T arr[SIMDVector<T,ABI>::Size];
for (FASTOR_INDEX i=0UL; i<SIMDVector<T,ABI>::Size; ++i) {
arr[i] = std::conj(a[i]);
}
return SIMDVector<T,ABI>(arr,false);
}
template<typename T, typename ABI, enable_if_t_<is_complex_v_<T>,bool> = false>
FASTOR_INLINE SIMDVector<T,ABI> arg(const SIMDVector<T,ABI> &a) {
T arr[SIMDVector<T,ABI>::Size];
for (FASTOR_INDEX i=0UL; i<SIMDVector<T,ABI>::Size; ++i) {
arr[i] = std::arg(a[i]);
}
return SIMDVector<T,ABI>(arr,false);
}
} // end of namespace Fastor
#endif // SIMD_VECTOR_H

1177
noarch/include/Fastor/simd_vector/simd_vector_common.h
View File

File diff suppressed because it is too large Load Diff

1611
noarch/include/Fastor/simd_vector/simd_vector_complex_double.h
View File

File diff suppressed because it is too large Load Diff

1619
noarch/include/Fastor/simd_vector/simd_vector_complex_float.h
View File

File diff suppressed because it is too large Load Diff

378
noarch/include/Fastor/simd_vector/simd_vector_complex_scalar.h
View File

@@ -1,378 +0,0 @@
#ifndef SIMD_VECTOR_COMPLEX_SCALAR
#define SIMD_VECTOR_COMPLEX_SCALAR
#include "Fastor/util/extended_algorithms.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/simd_vector_base.h"
#include "Fastor/simd_vector/simd_vector_double.h"
#include <cmath>
#include <complex>
namespace Fastor {
// SCALAR IMPLEMENTATION OF SIMDVECTOR FOR COMPLEX<T>
//------------------------------------------------------------------------------------------------------------
template <typename T>
struct SIMDVector<std::complex<T>, simd_abi::scalar> {
using vector_type = SIMDVector<std::complex<T>, simd_abi::scalar>;
using value_type = T;
using scalar_value_type = std::complex<T>;
using abi_type = simd_abi::scalar;
static constexpr FASTOR_INDEX Size = 1;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;}
FASTOR_INLINE SIMDVector() : value_r(0), value_i(0) {}
FASTOR_INLINE SIMDVector(scalar_value_type num) {
value_r = num.real();
value_i = num.imag();
}
FASTOR_INLINE SIMDVector(value_type reg0, value_type reg1) : value_r(reg0), value_i(reg1) {}
FASTOR_INLINE SIMDVector(const scalar_value_type *data, bool Aligned=true) {
value_r = (*data).real();
value_i = (*data).imag();
}
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator=(std::complex<T> num) {
value_r = num.real();
value_i = num.imag();
return *this;
}
FASTOR_INLINE void load(const scalar_value_type *data, bool ) {
value_r = (*data).real();
value_i = (*data).imag();
}
FASTOR_INLINE void store(scalar_value_type *data, bool ) const {
data[0] = scalar_value_type(value_r,value_i);
}
FASTOR_INLINE void aligned_load(const T *data) {
value_r = (*data).real();
value_i = (*data).imag();
}
FASTOR_INLINE void aligned_store(T *data) const {
data[0] = scalar_value_type(value_r,value_i);
}
FASTOR_INLINE void mask_load(const scalar_value_type *data, uint8_t mask, bool ) {
if (mask != 0x0) {
value_r = (*data).real();
value_i = (*data).imag();
}
}
FASTOR_INLINE void mask_store(scalar_value_type *data, uint8_t mask, bool) const {
if (mask != 0x0) {
data[0] = scalar_value_type(value_r,value_i);
}
}
FASTOR_INLINE T operator[](FASTOR_INDEX) const {return scalar_value_type(value_r,value_i);}
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> real() const {
return value_r;
}
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> imag() const {
return value_i;
}
FASTOR_INLINE void set(scalar_value_type num) {
value_r = num.real();
value_i = num.imag();
}
FASTOR_INLINE void set_sequential(scalar_value_type num) {
value_r = num.real();
value_i = num.imag();
}
// In-place operators
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE void operator+=(U num) {
value_r += num;
}
FASTOR_INLINE void operator+=(scalar_value_type num) {
value_r += num.real();
value_i += num.imag();
}
FASTOR_INLINE void operator+=(const vector_type &a) {
value_r += a.value_r;
value_i += a.value_i;
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE void operator-=(U num) {
value_r -= num;
}
FASTOR_INLINE void operator-=(scalar_value_type num) {
value_r -= num.real();
value_i -= num.imag();
}
FASTOR_INLINE void operator-=(const vector_type &a) {
value_r -= a.value_r;
value_i -= a.value_i;
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE void operator*=(U num) {
value_r *= num;
value_i *= num;
}
FASTOR_INLINE void operator*=(scalar_value_type num) {
scalar_value_type tmp(value_r, value_i);
tmp *= num;
value_r = tmp.real();
value_i = tmp.imag();
}
FASTOR_INLINE void operator*=(const vector_type &a) {
scalar_value_type tmp0(value_r, value_i);
scalar_value_type tmp1(a.value_r, a.value_i);
tmp0 *= tmp1;
value_r = tmp0.real();
value_i = tmp0.imag();
}
template<typename U=T, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE void operator/=(U num) {
value_r /= num;
value_i /= num;
}
FASTOR_INLINE void operator/=(scalar_value_type num) {
scalar_value_type tmp(value_r, value_i);
tmp /= num;
value_r = tmp.real();
value_i = tmp.imag();
}
FASTOR_INLINE void operator/=(const vector_type &a) {
scalar_value_type tmp0(value_r, value_i);
scalar_value_type tmp1(a.value_r, a.value_i);
tmp0 /= tmp1;
value_r = tmp0.real();
value_i = tmp0.imag();
}
// end of in-place operators
FASTOR_INLINE scalar_value_type sum() const {return scalar_value_type(value_r, value_i);}
FASTOR_INLINE scalar_value_type product() const {return scalar_value_type(value_r, value_i);}
FASTOR_INLINE vector_type reverse() const { return scalar_value_type(value_r, value_i); }
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> magnitude() const { return std::abs(scalar_value_type(value_r, value_i));}
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> norm() const { return std::norm(scalar_value_type(value_r, value_i));}
FASTOR_INLINE scalar_value_type minimum() const {return scalar_value_type(value_r, value_i);}
FASTOR_INLINE scalar_value_type maximum() const {return scalar_value_type(value_r, value_i);}
FASTOR_INLINE scalar_value_type dot(const vector_type &other) const {
return vector_type(value_r, value_i)*vector_type(other.value_r, other.value_i);
}
value_type value_r;
value_type value_i;
};
template <typename T>
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<std::complex<T>,simd_abi::scalar> a) {
os << "[" << a.value_r << signum_string(a.value_i) << std::abs(a.value_i) << "j]\n";
return os;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out += b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out += b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator+(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out += b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out += b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator+(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
out += b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator+(const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
return b;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out -= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out -= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator-(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out -= b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out -= b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator-(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
out -= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator-(const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
return SIMDVector<std::complex<T>,simd_abi::scalar>(0,0) - b;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator*(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out *= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator*(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out *= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator*(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out *= b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator*(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out *= b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator*(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
out *= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator/(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out /= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator/(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, std::complex<T> b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out /= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator/(std::complex<T> a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out /= b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator/(const SIMDVector<std::complex<T>,simd_abi::scalar> &a, U b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(a);
out /= b;
return out;
}
template <typename T, typename U, enable_if_t_<is_arithmetic_v_<U>,bool> = false>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
operator/(U a, const SIMDVector<std::complex<T>,simd_abi::scalar> &b) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(a,0));
out /= b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
rcp(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
SIMDVector<std::complex<T>,simd_abi::scalar> out(std::complex<T>(1,0));
out /= a;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
sqrt(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
std::complex<T> out(a.value_r,a.value_i);
return std::sqrt(out);
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
rsqrt(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
return rcp(sqrt(a));
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
abs(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
SIMDVector<std::complex<T>,simd_abi::scalar> out;
out.value_r = a.magnitude().value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
conj(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
std::complex<T> out(a.value_r,a.value_i);
return std::conj(out);
}
template <typename T>
FASTOR_INLINE SIMDVector<std::complex<T>,simd_abi::scalar>
arg(const SIMDVector<std::complex<T>,simd_abi::scalar> &a) {
std::complex<T> out(a.value_r,a.value_i);
return SIMDVector<std::complex<T>,simd_abi::scalar>(std::arg(out),0);
}
//------------------------------------------------------------------------------------------------------------
} // end of namespace Fastor
#endif // SIMD_VECTOR_COMPLEX_SCALAR

914
noarch/include/Fastor/simd_vector/simd_vector_double.h
View File

@@ -1,914 +0,0 @@
#ifndef SIMD_VECTOR_DOUBLE_H
#define SIMD_VECTOR_DOUBLE_H
#include "Fastor/simd_vector/simd_vector_base.h"
namespace Fastor {
// AVX512 VERSION
//--------------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX512_IMPL
template <>
struct SIMDVector<double, simd_abi::avx512> {
using value_type = __m512d;
using scalar_value_type = double;
using abi_type = simd_abi::avx512;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx512>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx512>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm512_setzero_pd()) {}
FASTOR_INLINE SIMDVector(double num) : value(_mm512_set1_pd(num)) {}
FASTOR_INLINE SIMDVector(__m512d regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) {
if (Aligned)
value =_mm512_load_pd(data);
else
value = _mm512_loadu_pd(data);
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator=(double num) {
value = _mm512_set1_pd(num);
return *this;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator=(__m512d regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const double *data, bool Aligned=true) {
if (Aligned)
value =_mm512_load_pd(data);
else
value = _mm512_loadu_pd(data);
}
FASTOR_INLINE void store(double *data, bool Aligned=true) const {
if (Aligned)
_mm512_store_pd(data,value);
else
_mm512_storeu_pd(data,value);
}
FASTOR_INLINE void aligned_load(const double *data) {
value =_mm512_load_pd(data);
}
FASTOR_INLINE void aligned_store(double *data) const {
_mm512_store_pd(data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm512_mask_loadu_pd(value, mask, a);
else
value = _mm512_mask_load_pd(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm512_setzero_pd();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm512_mask_storeu_pd(a, mask, value);
else
_mm512_mask_store_pd(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE double operator[](FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
FASTOR_INLINE void set(double num) {
value = _mm512_set1_pd(num);
}
FASTOR_INLINE void set(double num0, double num1, double num2, double num3, double num4, double num5, double num6, double num7) {
value = _mm512_set_pd(num0,num1,num2,num3,num4,num5,num6,num7);
}
FASTOR_INLINE void set_sequential(double num0) {
value = _mm512_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0,num0+4.0,num0+5.0,num0+6.0,num0+7.0);
}
FASTOR_INLINE void broadcast(const double *data) {
// value = _mm512_broadcast_sd(data);
}
// In-place operators
FASTOR_INLINE void operator+=(double num) {
value = _mm512_add_pd(value,_mm512_set1_pd(num));
}
FASTOR_INLINE void operator+=(__m512d regi) {
value = _mm512_add_pd(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<double,simd_abi::avx512> &a) {
value = _mm512_add_pd(value,a.value);
}
FASTOR_INLINE void operator-=(double num) {
value = _mm512_sub_pd(value,_mm512_set1_pd(num));
}
FASTOR_INLINE void operator-=(__m512d regi) {
value = _mm512_sub_pd(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<double,simd_abi::avx512> &a) {
value = _mm512_sub_pd(value,a.value);
}
FASTOR_INLINE void operator*=(double num) {
value = _mm512_mul_pd(value,_mm512_set1_pd(num));
}
FASTOR_INLINE void operator*=(__m512d regi) {
value = _mm512_mul_pd(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<double,simd_abi::avx512> &a) {
value = _mm512_mul_pd(value,a.value);
}
FASTOR_INLINE void operator/=(double num) {
value = _mm512_div_pd(value,_mm512_set1_pd(num));
}
FASTOR_INLINE void operator/=(__m512d regi) {
value = _mm512_div_pd(value,regi);
}
FASTOR_INLINE void operator/=(const SIMDVector<double,simd_abi::avx512> &a) {
value = _mm512_div_pd(value,a.value);
}
// end of in-place operators
// FASTOR_INLINE SIMDVector<double,simd_abi::avx512> shift(FASTOR_INDEX i) {
// SIMDVector<double,simd_abi::avx512> out;
// if (i==1)
// out.value = _mm512_shift1_pd(value);
// else if (i==2)
// out.value = _mm512_shift2_pd(value);
// else if (i==3)
// out.value = _mm512_shift3_pd(value);
// return out;
// }
FASTOR_INLINE double sum() {
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
return _mm512_reduce_add_pd(value);
#else
__m256d low = _mm512_castpd512_pd256(value);
__m256d high = _mm512_extractf64x4_pd(value,0x1);
return _mm256_sum_pd(_mm256_add_pd(low,high));
#endif
}
FASTOR_INLINE double product() {
__m256d low = _mm512_castpd512_pd256(value);
__m256d high = _mm512_extractf64x4_pd(value,1);
return _mm256_prod_pd(_mm256_mul_pd(low,high));
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> reverse() {
return _mm512_reverse_pd(value);
}
// FASTOR_INLINE double minimum() {return _mm512_hmin_pd(value);}
// FASTOR_INLINE double maximum() {return _mm512_hmax_pd(value);}
FASTOR_INLINE double dot(const SIMDVector<double,simd_abi::avx512> &other) {
__m512d res = _mm512_mul_pd(value,other.value);
__m256d low = _mm512_castpd512_pd256(res);
__m256d high = _mm512_extractf64x4_pd(res,1);
return _mm256_sum_pd(_mm256_add_pd(low,high));
}
__m512d value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<double,simd_abi::avx512> a) {
// ICC crashes without a copy
const __m512d v = a.value;
const double* value = reinterpret_cast<const double*>(&v);
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3]
<< " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_add_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(const SIMDVector<double,simd_abi::avx512> &a, double b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_add_pd(a.value,_mm512_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(double a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_add_pd(_mm512_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator+(const SIMDVector<double,simd_abi::avx512> &b) {
return b;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_sub_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(const SIMDVector<double,simd_abi::avx512> &a, double b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_sub_pd(a.value,_mm512_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(double a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_sub_pd(_mm512_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator-(const SIMDVector<double,simd_abi::avx512> &b) {
return _mm512_neg_pd(b.value);
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator*(
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_mul_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator*(const SIMDVector<double,simd_abi::avx512> &a, double b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_mul_pd(a.value,_mm512_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator*(double a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_mul_pd(_mm512_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator/(
const SIMDVector<double,simd_abi::avx512> &a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_div_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator/(const SIMDVector<double,simd_abi::avx512> &a, double b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_div_pd(a.value,_mm512_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> operator/(double a, const SIMDVector<double,simd_abi::avx512> &b) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_div_pd(_mm512_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> rcp(const SIMDVector<double,simd_abi::avx512> &a) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_rcp14_pd(a.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> sqrt(const SIMDVector<double,simd_abi::avx512> &a) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_sqrt_pd(a.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> rsqrt(const SIMDVector<double,simd_abi::avx512> &a) {
SIMDVector<double,simd_abi::avx512> out;
out.value = _mm512_rsqrt14_pd(a.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx512> abs(const SIMDVector<double,simd_abi::avx512> &a) {
SIMDVector<double,simd_abi::avx512> out;
#ifdef FASTOR_HAS_AVX512_ABS
out.value = _mm512_abs_pd(a.value);
#else
for (FASTOR_INDEX i=0UL; i<8UL; ++i) {
((double*)&out.value)[i] = std::abs(((double*)&a.value)[i]);
}
#endif
return out;
}
#endif
// AVX VERSION
//--------------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX_IMPL
template <>
struct SIMDVector<double, simd_abi::avx> {
using value_type = __m256d;
using scalar_value_type = double;
using abi_type = simd_abi::avx;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<double,simd_abi::avx>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm256_setzero_pd()) {}
FASTOR_INLINE SIMDVector(double num) : value(_mm256_set1_pd(num)) {}
FASTOR_INLINE SIMDVector(__m256d regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) {
if (Aligned)
value =_mm256_load_pd(data);
else
value = _mm256_loadu_pd(data);
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator=(double num) {
value = _mm256_set1_pd(num);
return *this;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator=(__m256d regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const double *data, bool Aligned=true) {
if (Aligned)
value =_mm256_load_pd(data);
else
value = _mm256_loadu_pd(data);
}
FASTOR_INLINE void store(double *data, bool Aligned=true) const {
if (Aligned)
_mm256_store_pd(data,value);
else
_mm256_storeu_pd(data,value);
}
FASTOR_INLINE void aligned_load(const double *data) {
value =_mm256_load_pd(data);
}
FASTOR_INLINE void aligned_store(double *data) const {
_mm256_store_pd(data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm256_mask_loadu_pd(value, mask, a);
else
value = _mm256_mask_load_pd(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm256_setzero_pd();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm256_mask_storeu_pd(a, mask, value);
else
_mm256_mask_store_pd(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE double operator[](FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
FASTOR_INLINE void set(double num) {
value = _mm256_set1_pd(num);
}
FASTOR_INLINE void set(double num0, double num1, double num2, double num3) {
value = _mm256_set_pd(num0,num1,num2,num3);
}
FASTOR_INLINE void set_sequential(double num0) {
value = _mm256_setr_pd(num0,num0+1.0,num0+2.0,num0+3.0);
}
FASTOR_INLINE void broadcast(const double *data) {
value = _mm256_broadcast_sd(data);
}
// In-place operators
FASTOR_INLINE void operator+=(double num) {
value = _mm256_add_pd(value,_mm256_set1_pd(num));
}
FASTOR_INLINE void operator+=(__m256d regi) {
value = _mm256_add_pd(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<double,simd_abi::avx> &a) {
value = _mm256_add_pd(value,a.value);
}
FASTOR_INLINE void operator-=(double num) {
value = _mm256_sub_pd(value,_mm256_set1_pd(num));
}
FASTOR_INLINE void operator-=(__m256d regi) {
value = _mm256_sub_pd(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<double,simd_abi::avx> &a) {
value = _mm256_sub_pd(value,a.value);
}
FASTOR_INLINE void operator*=(double num) {
value = _mm256_mul_pd(value,_mm256_set1_pd(num));
}
FASTOR_INLINE void operator*=(__m256d regi) {
value = _mm256_mul_pd(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<double,simd_abi::avx> &a) {
value = _mm256_mul_pd(value,a.value);
}
FASTOR_INLINE void operator/=(double num) {
value = _mm256_div_pd(value,_mm256_set1_pd(num));
}
FASTOR_INLINE void operator/=(__m256d regi) {
value = _mm256_div_pd(value,regi);
}
FASTOR_INLINE void operator/=(const SIMDVector<double,simd_abi::avx> &a) {
value = _mm256_div_pd(value,a.value);
}
// end of in-place operators
FASTOR_INLINE SIMDVector<double,simd_abi::avx> shift(FASTOR_INDEX i) {
SIMDVector<double,simd_abi::avx> out;
if (i==1)
out.value = _mm256_shift1_pd(value);
else if (i==2)
out.value = _mm256_shift2_pd(value);
else if (i==3)
out.value = _mm256_shift3_pd(value);
return out;
}
FASTOR_INLINE double sum() {return _mm256_sum_pd(value);}
FASTOR_INLINE double product() {return _mm256_prod_pd(value);}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> reverse() {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_reverse_pd(value);
return out;
}
FASTOR_INLINE double minimum() {return _mm256_hmin_pd(value);}
FASTOR_INLINE double maximum() {return _mm256_hmax_pd(value);}
FASTOR_INLINE double dot(const SIMDVector<double,simd_abi::avx> &other) {
return _mm_cvtsd_f64(_mm256_dp_pd(value,other.value));
}
__m256d value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<double,simd_abi::avx> a) {
// ICC crashes without a copy
const __m256d v = a.value;
const double* value = reinterpret_cast<const double*>(&v);
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_add_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(const SIMDVector<double,simd_abi::avx> &a, double b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_add_pd(a.value,_mm256_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(double a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_add_pd(_mm256_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator+(const SIMDVector<double,simd_abi::avx> &b) {
return b;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_sub_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(const SIMDVector<double,simd_abi::avx> &a, double b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_sub_pd(a.value,_mm256_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(double a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_sub_pd(_mm256_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator-(const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_neg_pd(b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator*(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_mul_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator*(const SIMDVector<double,simd_abi::avx> &a, double b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_mul_pd(a.value,_mm256_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator*(double a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_mul_pd(_mm256_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator/(const SIMDVector<double,simd_abi::avx> &a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_div_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator/(const SIMDVector<double,simd_abi::avx> &a, double b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_div_pd(a.value,_mm256_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> operator/(double a, const SIMDVector<double,simd_abi::avx> &b) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_div_pd(_mm256_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> rcp(const SIMDVector<double,simd_abi::avx> &a) {
SIMDVector<double,simd_abi::avx> out;
// This is very inaccurate for double precision
out.value = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value)));
return out;
// // For making it more accurate using Newton Raphson use this
// __m128d xmm0 = _mm256_cvtps_pd(_mm_rcp_ps(_mm256_cvtpd_ps(a.value)));
// xmm0 = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0)));
// out.value = _mm256_mul_pd(xmm0,_mm256_sub_pd(VTWOPD,_mm256_mul_pd(x,xmm0)));
// return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> sqrt(const SIMDVector<double,simd_abi::avx> &a) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_sqrt_pd(a.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> rsqrt(const SIMDVector<double,simd_abi::avx> &a) {
SIMDVector<double,simd_abi::avx> out;
// This is very inaccurate for double precision
out.value = _mm256_cvtps_pd(_mm_rsqrt_ps(_mm256_cvtpd_ps(a.value)));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::avx> abs(const SIMDVector<double,simd_abi::avx> &a) {
SIMDVector<double,simd_abi::avx> out;
out.value = _mm256_abs_pd(a.value);
return out;
}
#endif
// SSE VERSION
//------------------------------------------------------------------------------------------------------------
#ifdef FASTOR_SSE2_IMPL
template <>
struct SIMDVector<double, simd_abi::sse> {
using value_type = __m128d;
using scalar_value_type = double;
using abi_type = simd_abi::sse;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<double,simd_abi::sse>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<double,simd_abi::sse>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm_setzero_pd()) {}
FASTOR_INLINE SIMDVector(double num) : value(_mm_set1_pd(num)) {}
FASTOR_INLINE SIMDVector(__m128d regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const double *data, bool Aligned=true) {
if (Aligned)
value =_mm_load_pd(data);
else
value = _mm_loadu_pd(data);
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator=(double num) {
value = _mm_set1_pd(num);
return *this;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator=(__m128d regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const double *data, bool Aligned=true) {
if (Aligned)
value =_mm_load_pd(data);
else
value = _mm_loadu_pd(data);
}
FASTOR_INLINE void store(double *data, bool Aligned=true) const {
if (Aligned)
_mm_store_pd(data,value);
else
_mm_storeu_pd(data,value);
}
FASTOR_INLINE void aligned_load(const double *data) {
value =_mm_load_pd(data);
}
FASTOR_INLINE void aligned_store(double *data) const {
_mm_store_pd(data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm_mask_loadu_pd(value, mask, a);
else
value = _mm_mask_load_pd(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm_setzero_pd();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm_mask_storeu_pd(a, mask, value);
else
_mm_mask_store_pd(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE double operator[](FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
FASTOR_INLINE double operator()(FASTOR_INDEX i) const {return reinterpret_cast<const double*>(&value)[i];}
FASTOR_INLINE void set(double num) {
value = _mm_set1_pd(num);
}
FASTOR_INLINE void set(double num0, double num1) {
value = _mm_set_pd(num0,num1);
}
FASTOR_INLINE void set_sequential(double num0) {
value = _mm_setr_pd(num0,num0+1.0);
}
FASTOR_INLINE void broadcast(const double *data) {
value = _mm_load1_pd(data);
}
// In-place operators
FASTOR_INLINE void operator+=(double num) {
value = _mm_add_pd(value,_mm_set1_pd(num));
}
FASTOR_INLINE void operator+=(__m128d regi) {
value = _mm_add_pd(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<double,simd_abi::sse> &a) {
value = _mm_add_pd(value,a.value);
}
FASTOR_INLINE void operator-=(double num) {
value = _mm_sub_pd(value,_mm_set1_pd(num));
}
FASTOR_INLINE void operator-=(__m128d regi) {
value = _mm_sub_pd(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<double,simd_abi::sse> &a) {
value = _mm_sub_pd(value,a.value);
}
FASTOR_INLINE void operator*=(double num) {
value = _mm_mul_pd(value,_mm_set1_pd(num));
}
FASTOR_INLINE void operator*=(__m128d regi) {
value = _mm_mul_pd(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<double,simd_abi::sse> &a) {
value = _mm_mul_pd(value,a.value);
}
FASTOR_INLINE void operator/=(double num) {
value = _mm_div_pd(value,_mm_set1_pd(num));
}
FASTOR_INLINE void operator/=(__m128d regi) {
value = _mm_div_pd(value,regi);
}
FASTOR_INLINE void operator/=(const SIMDVector<double,simd_abi::sse> &a) {
value = _mm_div_pd(value,a.value);
}
// end of in-place operators
FASTOR_INLINE SIMDVector<double,simd_abi::sse> shift(FASTOR_INDEX i) {
SIMDVector<double,simd_abi::sse> out;
FASTOR_ASSERT(i==1,"INCORRECT SHIFT INDEX");
out.value = _mm_shift1_pd(value);
return out;
}
FASTOR_INLINE double sum() {return _mm_sum_pd(value);}
FASTOR_INLINE double product() {return _mm_prod_pd(value);}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> reverse() {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_reverse_pd(value);
return out;
}
FASTOR_INLINE double minimum() {return _mm_hmin_pd(value);}
FASTOR_INLINE double maximum() {return _mm_hmax_pd(value);}
FASTOR_INLINE double dot(const SIMDVector<double,simd_abi::sse> &other) {
#ifdef FASTOR_SSE4_1_IMPL
return _mm_cvtsd_f64(_mm_dp_pd(value,other.value,0xff));
#else
return _mm_sum_pd(_mm_mul_pd(value,other.value));
#endif
}
__m128d value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<double,simd_abi::sse> a) {
// ICC crashes without a copy
const __m128d v = a.value;
const double* value = reinterpret_cast<const double*>(&v);
os << "[" << value[0] << " " << value[1] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_add_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(const SIMDVector<double,simd_abi::sse> &a, double b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_add_pd(a.value,_mm_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(double a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_add_pd(_mm_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator+(const SIMDVector<double,simd_abi::sse> &b) {
return b;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_sub_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(const SIMDVector<double,simd_abi::sse> &a, double b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_sub_pd(a.value,_mm_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(double a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_sub_pd(_mm_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator-(const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_neg_pd(b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator*(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_mul_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator*(const SIMDVector<double,simd_abi::sse> &a, double b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_mul_pd(a.value,_mm_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator*(double a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_mul_pd(_mm_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator/(const SIMDVector<double,simd_abi::sse> &a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_div_pd(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator/(const SIMDVector<double,simd_abi::sse> &a, double b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_div_pd(a.value,_mm_set1_pd(b));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> operator/(double a, const SIMDVector<double,simd_abi::sse> &b) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_div_pd(_mm_set1_pd(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> rcp(const SIMDVector<double,simd_abi::sse> &a) {
SIMDVector<double,simd_abi::sse> out;
// This is very inaccurate for double precision
out.value = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value)));
return out;
/*
// For making it more accurate using Newton Raphson use this
__m128d xmm0 = _mm_cvtps_pd(_mm_rcp_ps(_mm_cvtpd_ps(a.value)));
xmm0 = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0)));
out.value = _mm_mul_pd(xmm0,_mm_sub_pd(TWOPD,_mm_mul_pd(x,xmm0)));
return out;
*/
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> sqrt(const SIMDVector<double,simd_abi::sse> &a) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_sqrt_pd(a.value);
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> rsqrt(const SIMDVector<double,simd_abi::sse> &a) {
SIMDVector<double,simd_abi::sse> out;
// This is very inaccurate for double precision
out.value = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(a.value)));
return out;
}
FASTOR_INLINE SIMDVector<double,simd_abi::sse> abs(const SIMDVector<double,simd_abi::sse> &a) {
SIMDVector<double,simd_abi::sse> out;
out.value = _mm_abs_pd(a.value);
return out;
}
#endif
} // end of namespace Fastor
#endif // // SIMD_VECTOR_DOUBLE_H

918
noarch/include/Fastor/simd_vector/simd_vector_float.h
View File

@@ -1,918 +0,0 @@
#ifndef SIMD_VECTOR_FLOAT_H
#define SIMD_VECTOR_FLOAT_H
#include "Fastor/simd_vector/simd_vector_base.h"
namespace Fastor {
// AVX512 VERSION
//--------------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX512_IMPL
template <>
struct SIMDVector<float,simd_abi::avx512> {
using value_type = __m512;
using scalar_value_type = float;
using abi_type = simd_abi::avx512;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx512>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx512>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm512_setzero_ps()) {}
FASTOR_INLINE SIMDVector(float num) : value(_mm512_set1_ps(num)) {}
FASTOR_INLINE SIMDVector(__m512 regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) {
if (Aligned)
value =_mm512_load_ps(data);
else
value = _mm512_loadu_ps(data);
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator=(float num) {
value = _mm512_set1_ps(num);
return *this;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator=(__m512 regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const float *data, bool Aligned=true) {
if (Aligned)
value =_mm512_load_ps(data);
else
value = _mm512_loadu_ps(data);
}
FASTOR_INLINE void store(float *data, bool Aligned=true) const {
if (Aligned)
_mm512_store_ps(data,value);
else
_mm512_storeu_ps(data,value);
}
FASTOR_INLINE void aligned_load(const float *data) {
value =_mm512_load_ps(data);
}
FASTOR_INLINE void aligned_store(float *data) const {
_mm512_store_ps(data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint16_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm512_mask_loadu_ps(value, mask, a);
else
value = _mm512_mask_load_ps(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm512_setzero_ps();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint16_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm512_mask_storeu_ps(a, mask, value);
else
_mm512_mask_store_ps(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE float operator[](FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
FASTOR_INLINE void set(float num) {
value = _mm512_set1_ps(num);
}
FASTOR_INLINE void set(float num0, float num1, float num2, float num3,
float num4, float num5, float num6, float num7,
float num8, float num9, float num10, float num11,
float num12, float num13, float num14, float num15) {
value = _mm512_set_ps(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15);
}
FASTOR_INLINE void set_sequential(float num0) {
value = _mm512_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f,
num0+8.f,num0+9.f,num0+10.f,num0+11.f,num0+12.f,num0+13.f,num0+14.f,num0+15.f);
}
FASTOR_INLINE void broadcast(const float *data) {
// value = _mm512_broadcast_ss(data);
}
// In-place operators
FASTOR_INLINE void operator+=(float num) {
value = _mm512_add_ps(value,_mm512_set1_ps(num));
}
FASTOR_INLINE void operator+=(__m512 regi) {
value = _mm512_add_ps(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<float,simd_abi::avx512> &a) {
value = _mm512_add_ps(value,a.value);
}
FASTOR_INLINE void operator-=(float num) {
value = _mm512_sub_ps(value,_mm512_set1_ps(num));
}
FASTOR_INLINE void operator-=(__m512 regi) {
value = _mm512_sub_ps(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<float,simd_abi::avx512> &a) {
value = _mm512_sub_ps(value,a.value);
}
FASTOR_INLINE void operator*=(float num) {
value = _mm512_mul_ps(value,_mm512_set1_ps(num));
}
FASTOR_INLINE void operator*=(__m512 regi) {
value = _mm512_mul_ps(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<float,simd_abi::avx512> &a) {
value = _mm512_mul_ps(value,a.value);
}
FASTOR_INLINE void operator/=(float num) {
value = _mm512_div_ps(value,_mm512_set1_ps(num));
}
FASTOR_INLINE void operator/=(__m512 regi) {
value = _mm512_div_ps(value,regi);
}
FASTOR_INLINE void operator/=(const SIMDVector<float,simd_abi::avx512> &a) {
value = _mm512_div_ps(value,a.value);
}
// end of in-place operators
FASTOR_INLINE float sum() {
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
return _mm512_reduce_add_ps(value);
#else
__m256 low = _mm512_castps512_ps256(value);
__m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1));
return _mm256_sum_ps(_mm256_add_ps(low,high));
#endif
}
FASTOR_INLINE float product() {
__m256 low = _mm512_castps512_ps256(value);
__m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(value),1));
return _mm256_prod_ps(_mm256_mul_ps(low,high));
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> reverse() {
return _mm512_reverse_ps(value);
}
// FASTOR_INLINE float minimum() {return _mm512_hmin_ps(value);}
// FASTOR_INLINE float maximum() {return _mm512_hmax_ps(value);}
FASTOR_INLINE float dot(const SIMDVector<float,simd_abi::avx512> &other) {
__m512 res = _mm512_mul_ps(value,other.value);
__m256 low = _mm512_castps512_ps256(res);
__m256 high = _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(res),1));
return _mm256_sum_ps(_mm256_add_ps(low,high));
}
__m512 value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<float,simd_abi::avx512> a) {
// ICC crashes without a copy
const __m512 v = a.value;
const float* value = reinterpret_cast<const float*>(&v);
os << "["
<< value[0] << " " << value[1] << " "
<< value[2] << " " << value[3] << " "
<< value[4] << " " << value[5] << " "
<< value[6] << " " << value[7] << " "
<< value[8] << " " << value[9] << " "
<< value[10] << " " << value[11] << " "
<< value[12] << " " << value[13] << " "
<< value[14] << " " << value[15] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_add_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(const SIMDVector<float,simd_abi::avx512> &a, float b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_add_ps(a.value,_mm512_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(float a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_add_ps(_mm512_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator+(const SIMDVector<float,simd_abi::avx512> &b) {
return b;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_sub_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(const SIMDVector<float,simd_abi::avx512> &a, float b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_sub_ps(a.value,_mm512_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(float a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_sub_ps(_mm512_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator-(const SIMDVector<float,simd_abi::avx512> &b) {
return _mm512_neg_ps(b.value);
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator*(
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_mul_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator*(const SIMDVector<float,simd_abi::avx512> &a, float b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_mul_ps(a.value,_mm512_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator*(float a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_mul_ps(_mm512_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator/(
const SIMDVector<float,simd_abi::avx512> &a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_div_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator/(const SIMDVector<float,simd_abi::avx512> &a, float b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_div_ps(a.value,_mm512_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> operator/(float a, const SIMDVector<float,simd_abi::avx512> &b) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_div_ps(_mm512_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> rcp(const SIMDVector<float,simd_abi::avx512> &a) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_rcp14_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> sqrt(const SIMDVector<float,simd_abi::avx512> &a) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_sqrt_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> rsqrt(const SIMDVector<float,simd_abi::avx512> &a) {
SIMDVector<float,simd_abi::avx512> out;
out.value = _mm512_rsqrt14_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx512> abs(const SIMDVector<float,simd_abi::avx512> &a) {
SIMDVector<float,simd_abi::avx512> out;
#ifdef FASTOR_HAS_AVX512_ABS
out.value = _mm512_abs_ps(a.value);
#else
for (FASTOR_INDEX i=0UL; i<16UL; ++i) {
((float*)&out.value)[i] = std::abs(((float*)&a.value)[i]);
}
#endif
return out;
}
#endif
// AVX VERSION
//--------------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX_IMPL
template <>
struct SIMDVector<float,simd_abi::avx> {
using value_type = __m256;
using scalar_value_type = float;
using abi_type = simd_abi::avx;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<float,simd_abi::avx>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm256_setzero_ps()) {}
FASTOR_INLINE SIMDVector(float num) : value(_mm256_set1_ps(num)) {}
FASTOR_INLINE SIMDVector(__m256 regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) {
if (Aligned)
value =_mm256_load_ps(data);
else
value = _mm256_loadu_ps(data);
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator=(float num) {
value = _mm256_set1_ps(num);
return *this;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator=(__m256 regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const float *data, bool Aligned=true) {
if (Aligned)
value =_mm256_load_ps(data);
else
value = _mm256_loadu_ps(data);
}
FASTOR_INLINE void store(float *data, bool Aligned=true) const {
if (Aligned)
_mm256_store_ps(data,value);
else
_mm256_storeu_ps(data,value);
}
FASTOR_INLINE void aligned_load(const float *data) {
value =_mm256_load_ps(data);
}
FASTOR_INLINE void aligned_store(float *data) const {
_mm256_store_ps(data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm256_mask_loadu_ps(value, mask, a);
else
value = _mm256_mask_load_ps(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm256_setzero_ps();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm256_mask_storeu_ps(a, mask, value);
else
_mm256_mask_store_ps(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
#endif
}
FASTOR_INLINE float operator[](FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
FASTOR_INLINE void set(float num) {
value = _mm256_set1_ps(num);
}
FASTOR_INLINE void set(float num0, float num1, float num2, float num3,
float num4, float num5, float num6, float num7) {
value = _mm256_set_ps(num0,num1,num2,num3,num4,num5,num6,num7);
}
FASTOR_INLINE void set_sequential(float num0) {
value = _mm256_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f,num0+4.f,num0+5.f,num0+6.f,num0+7.f);
}
FASTOR_INLINE void broadcast(const float *data) {
value = _mm256_broadcast_ss(data);
}
// In-place operators
FASTOR_INLINE void operator+=(float num) {
value = _mm256_add_ps(value,_mm256_set1_ps(num));
}
FASTOR_INLINE void operator+=(__m256 regi) {
value = _mm256_add_ps(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<float,simd_abi::avx> &a) {
value = _mm256_add_ps(value,a.value);
}
FASTOR_INLINE void operator-=(float num) {
value = _mm256_sub_ps(value,_mm256_set1_ps(num));
}
FASTOR_INLINE void operator-=(__m256 regi) {
value = _mm256_sub_ps(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<float,simd_abi::avx> &a) {
value = _mm256_sub_ps(value,a.value);
}
FASTOR_INLINE void operator*=(float num) {
value = _mm256_mul_ps(value,_mm256_set1_ps(num));
}
FASTOR_INLINE void operator*=(__m256 regi) {
value = _mm256_mul_ps(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<float,simd_abi::avx> &a) {
value = _mm256_mul_ps(value,a.value);
}
FASTOR_INLINE void operator/=(float num) {
value = _mm256_div_ps(value,_mm256_set1_ps(num));
}
FASTOR_INLINE void operator/=(__m256 regi) {
value = _mm256_div_ps(value,regi);
}
FASTOR_INLINE void operator/=(const SIMDVector<float,simd_abi::avx> &a) {
value = _mm256_div_ps(value,a.value);
}
// end of in-place operators
FASTOR_INLINE SIMDVector<float,simd_abi::avx> shift(FASTOR_INDEX i) {
SIMDVector<float,simd_abi::avx> out;
if (i==1)
out.value = _mm256_shift1_ps(value);
else if (i==2)
out.value = _mm256_shift2_ps(value);
else if (i==3)
out.value = _mm256_shift3_ps(value);
else if (i==4)
out.value = _mm256_shift4_ps(value);
else if (i==5)
out.value = _mm256_shift5_ps(value);
else if (i==6)
out.value = _mm256_shift6_ps(value);
else if (i==7)
out.value = _mm256_shift7_ps(value);
return out;
}
FASTOR_INLINE float sum() {return _mm256_sum_ps(value);}
FASTOR_INLINE float product() {return _mm256_prod_ps(value);}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> reverse() {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_reverse_ps(value);
return out;
}
FASTOR_INLINE float minimum() {return _mm256_hmin_ps(value);}
FASTOR_INLINE float maximum() {return _mm256_hmax_ps(value);}
FASTOR_INLINE float dot(const SIMDVector<float,simd_abi::avx> &other) {
__m256 tmp = _mm256_dp_ps(value,other.value,0xff);
return _mm256_get0_ps(tmp)+_mm256_get4_ps(tmp);
}
__m256 value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<float,simd_abi::avx> a) {
// ICC crashes without a copy
const __m256 v = a.value;
const float* value = reinterpret_cast<const float*>(&v);
os << "[" << value[0] << " " << value[1] << " "
<< value[2] << " " << value[3] << " "
<< value[4] << " " << value[5] << " "
<< value[6] << " " << value[7] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_add_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(const SIMDVector<float,simd_abi::avx> &a, float b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_add_ps(a.value,_mm256_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(float a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_add_ps(_mm256_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator+(const SIMDVector<float,simd_abi::avx> &b) {
return b;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_sub_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(const SIMDVector<float,simd_abi::avx> &a, float b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_sub_ps(a.value,_mm256_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(float a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_sub_ps(_mm256_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator-(const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_neg_ps(b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator*(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_mul_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator*(const SIMDVector<float,simd_abi::avx> &a, float b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_mul_ps(a.value,_mm256_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator*(float a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_mul_ps(_mm256_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator/(const SIMDVector<float,simd_abi::avx> &a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_div_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator/(const SIMDVector<float,simd_abi::avx> &a, float b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_div_ps(a.value,_mm256_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> operator/(float a, const SIMDVector<float,simd_abi::avx> &b) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_div_ps(_mm256_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> rcp(const SIMDVector<float,simd_abi::avx> &a) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_rcp_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> sqrt(const SIMDVector<float,simd_abi::avx> &a) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_sqrt_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> rsqrt(const SIMDVector<float,simd_abi::avx> &a) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_rsqrt_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::avx> abs(const SIMDVector<float,simd_abi::avx> &a) {
SIMDVector<float,simd_abi::avx> out;
out.value = _mm256_abs_ps(a.value);
return out;
}
#endif
// SSE VERSION
//--------------------------------------------------------------------------------------------------
#ifdef FASTOR_SSE2_IMPL
template <>
struct SIMDVector<float,simd_abi::sse> {
using value_type = __m128;
using scalar_value_type = float;
using abi_type = simd_abi::sse;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<float,simd_abi::sse>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<float,simd_abi::sse>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm_setzero_ps()) {}
FASTOR_INLINE SIMDVector(float num) : value(_mm_set1_ps(num)) {}
FASTOR_INLINE SIMDVector(__m128 regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const float *data, bool Aligned=true) {
if (Aligned)
value =_mm_load_ps(data);
else
value = _mm_loadu_ps(data);
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator=(float num) {
value = _mm_set1_ps(num);
return *this;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator=(__m128 regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const float *data, bool Aligned=true) {
if (Aligned)
value =_mm_load_ps(data);
else
value = _mm_loadu_ps(data);
}
FASTOR_INLINE void store(float *data, bool Aligned=true) const {
if (Aligned)
_mm_store_ps(data,value);
else
_mm_storeu_ps(data,value);
}
FASTOR_INLINE void aligned_load(const float *data) {
value =_mm_load_ps(data);
}
FASTOR_INLINE void aligned_store(float *data) const {
_mm_store_ps(data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm_mask_loadu_ps(value, mask, a);
else
value = _mm_mask_load_ps(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm_setzero_ps();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm_mask_storeu_ps(a, mask, value);
else
_mm_mask_store_ps(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE float operator[](FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
FASTOR_INLINE float operator()(FASTOR_INDEX i) const {return reinterpret_cast<const float*>(&value)[i];}
FASTOR_INLINE void set(float num) {
value = _mm_set1_ps(num);
}
FASTOR_INLINE void set(float num0, float num1, float num2, float num3) {
value = _mm_set_ps(num0,num1,num2,num3);
}
FASTOR_INLINE void set_sequential(float num0) {
value = _mm_setr_ps(num0,num0+1.f,num0+2.f,num0+3.f);
}
FASTOR_INLINE void broadcast(const float *data) {
value = _mm_load1_ps(data);
}
// In-place operators
FASTOR_INLINE void operator+=(float num) {
value = _mm_add_ps(value,_mm_set1_ps(num));
}
FASTOR_INLINE void operator+=(__m128 regi) {
value = _mm_add_ps(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<float,simd_abi::sse> &a) {
value = _mm_add_ps(value,a.value);
}
FASTOR_INLINE void operator-=(float num) {
value = _mm_sub_ps(value,_mm_set1_ps(num));
}
FASTOR_INLINE void operator-=(__m128 regi) {
value = _mm_sub_ps(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<float,simd_abi::sse> &a) {
value = _mm_sub_ps(value,a.value);
}
FASTOR_INLINE void operator*=(float num) {
value = _mm_mul_ps(value,_mm_set1_ps(num));
}
FASTOR_INLINE void operator*=(__m128 regi) {
value = _mm_mul_ps(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<float,simd_abi::sse> &a) {
value = _mm_mul_ps(value,a.value);
}
FASTOR_INLINE void operator/=(float num) {
value = _mm_div_ps(value,_mm_set1_ps(num));
}
FASTOR_INLINE void operator/=(__m128 regi) {
value = _mm_div_ps(value,regi);
}
FASTOR_INLINE void operator/=(const SIMDVector<float,simd_abi::sse> &a) {
value = _mm_div_ps(value,a.value);
}
// end of in-place operators
FASTOR_INLINE SIMDVector<float,simd_abi::sse> shift(FASTOR_INDEX i) {
SIMDVector<float,simd_abi::sse> out;
if (i==1)
out.value = _mm_shift1_ps(value);
else if (i==2)
out.value = _mm_shift2_ps(value);
else if (i==3)
out.value = _mm_shift3_ps(value);
return out;
}
FASTOR_INLINE float sum() {return _mm_sum_ps(value);}
FASTOR_INLINE float product() {return _mm_prod_ps(value);}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> reverse() {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_reverse_ps(value);
return out;
}
FASTOR_INLINE float minimum() {return _mm_hmin_ps(value);}
FASTOR_INLINE float maximum() {return _mm_hmax_ps(value);}
FASTOR_INLINE float dot(const SIMDVector<float,simd_abi::sse> &other) {
#ifdef FASTOR_SSE4_1_IMPL
return _mm_cvtss_f32(_mm_dp_ps(value,other.value,0xff));
#else
return _mm_sum_ps(_mm_mul_ps(value,other.value));
#endif
}
__m128 value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<float,simd_abi::sse> a) {
// ICC crashes without a copy
const __m128 v = a.value;
const float* value = reinterpret_cast<const float*>(&v);
os << "[" << value[0] << " " << value[1] << " "
<< value[2] << " " << value[3] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_add_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(const SIMDVector<float,simd_abi::sse> &a, float b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_add_ps(a.value,_mm_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(float a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_add_ps(_mm_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator+(const SIMDVector<float,simd_abi::sse> &b) {
return b;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_sub_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(const SIMDVector<float,simd_abi::sse> &a, float b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_sub_ps(a.value,_mm_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(float a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_sub_ps(_mm_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator-(const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_neg_ps(b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator*(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_mul_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator*(const SIMDVector<float,simd_abi::sse> &a, float b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_mul_ps(a.value,_mm_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator*(float a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_mul_ps(_mm_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator/(const SIMDVector<float,simd_abi::sse> &a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_div_ps(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator/(const SIMDVector<float,simd_abi::sse> &a, float b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_div_ps(a.value,_mm_set1_ps(b));
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> operator/(float a, const SIMDVector<float,simd_abi::sse> &b) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_div_ps(_mm_set1_ps(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> rcp(const SIMDVector<float,simd_abi::sse> &a) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_rcp_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> sqrt(const SIMDVector<float,simd_abi::sse> &a) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_sqrt_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> rsqrt(const SIMDVector<float,simd_abi::sse> &a) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_rsqrt_ps(a.value);
return out;
}
FASTOR_INLINE SIMDVector<float,simd_abi::sse> abs(const SIMDVector<float,simd_abi::sse> &a) {
SIMDVector<float,simd_abi::sse> out;
out.value = _mm_abs_ps(a.value);
return out;
}
#endif
} // end of namespace Fastor
#endif // SIMD_VECTOR_FLOAT_H

988
noarch/include/Fastor/simd_vector/simd_vector_int32.h
View File

@@ -1,988 +0,0 @@
#ifndef SIMD_VECTOR_INT_H
#define SIMD_VECTOR_INT_H
#include "Fastor/simd_vector/simd_vector_base.h"
#include <cstdint>
namespace Fastor {
// AVX512 VERSION
//-----------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX512F_IMPL
template<>
struct SIMDVector<int32_t,simd_abi::avx512> {
using value_type = __m512i;
using scalar_value_type = int32_t;
using abi_type = simd_abi::avx512;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx512>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx512>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm512_setzero_si512()) {}
FASTOR_INLINE SIMDVector(int32_t num) : value(_mm512_set1_epi32(num)) {}
FASTOR_INLINE SIMDVector(__m512i regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) {
if (Aligned)
value =_mm512_load_si512((__m512i*)data);
else
value = _mm512_loadu_si512((__m512i*)data);
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator=(int32_t num) {
value = _mm512_set1_epi32(num);
return *this;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator=(__m512i regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) {
if (Aligned)
value =_mm512_load_si512((__m512i*)data);
else
value = _mm512_loadu_si512((__m512i*)data);
}
FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const {
if (Aligned)
_mm512_store_si512((__m512i*)data,value);
else
_mm512_storeu_si512((__m512i*)data,value);
}
FASTOR_INLINE void aligned_load(const int32_t *data) {
value =_mm512_load_si512((__m512i*)data);
}
FASTOR_INLINE void aligned_store(int32_t *data) const {
_mm512_store_si512((__m512i*)data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm512_mask_loadu_epi32(value, mask, a);
else
value = _mm512_mask_load_epi32(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm512_setzero_si512();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm512_mask_storeu_epi32(a, mask, value);
else
_mm512_mask_store_epi32(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE int32_t operator[](FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
FASTOR_INLINE void set(int32_t num) {
value = _mm512_set1_epi32(num);
}
FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7,
int32_t num8, int32_t num9, int32_t num10, int32_t num11, int32_t num12, int32_t num13, int32_t num14, int32_t num15) {
value = _mm512_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7,num8,num9,num10,num11,num12,num13,num14,num15);
}
FASTOR_INLINE void set_sequential(int32_t num0) {
value = _mm512_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7,
num0+8,num0+9,num0+10,num0+11,num0+12,num0+13,num0+14,num0+15);
}
// In-place operators
FASTOR_INLINE void operator+=(int32_t num) {
value = _mm512_add_epi32(value,_mm512_set1_epi32(num));
}
FASTOR_INLINE void operator+=(__m512i regi) {
value = _mm512_add_epi32(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
value = _mm512_add_epi32(value,a.value);
}
FASTOR_INLINE void operator-=(int32_t num) {
value = _mm512_sub_epi32(value,_mm512_set1_epi32(num));
}
FASTOR_INLINE void operator-=(__m512i regi) {
value = _mm512_sub_epi32(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
value = _mm512_sub_epi32(value,a.value);
}
FASTOR_INLINE void operator*=(int32_t num) {
value = _mm512_mullo_epi32(value,_mm512_set1_epi32(num));
}
FASTOR_INLINE void operator*=(__m512i regi) {
value = _mm512_mullo_epi32(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
value = _mm512_mullo_epi32(value,a.value);
}
FASTOR_INLINE void operator/=(int32_t num) {
#ifdef FASTOR_INTEL
value = _mm512_div_epi32(value,_mm512_set1_epi32(num));
#else
int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= num;
}
value = _mm512_loadu_si512((__m512i*)val);
#endif
}
FASTOR_INLINE void operator/=(__m512i regi) {
#ifdef FASTOR_INTEL
value = _mm512_div_epi32(value,regi);
#else
int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value);
int32_t val_num[Size]; _mm512_storeu_si512((__m512i*)val_num, regi);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= val_num[i];
}
value = _mm512_loadu_si512((__m512i*)val);
#endif
}
FASTOR_INLINE void operator/=(const SIMDVector<int32_t,simd_abi::avx512> &a) {
#ifdef FASTOR_INTEL
value = _mm512_div_epi32(value,a.value);
#else
int32_t val[Size]; _mm512_storeu_si512((__m512i*)val, value);
int32_t val_a[Size]; _mm512_storeu_si512((__m512i*)val_a, a.value);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= val_a[i];
}
value = _mm512_loadu_si512((__m512i*)val);
#endif
}
FASTOR_INLINE int32_t minimum() {
int32_t *vals = (int32_t*)&value;
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
if (vals[i]<quan)
quan = vals[i];
return quan;
}
FASTOR_INLINE int32_t maximum() {
int32_t *vals = (int32_t*)&value;
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
if (vals[i]>quan)
quan = vals[i];
return quan;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> reverse() {
return _mm512_reverse_epi32(value);
}
FASTOR_INLINE int32_t sum() {
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
return _mm512_reduce_add_epi32(value);
#else
int32_t vals[Size]; _mm512_storeu_si512((__m512i*)vals, value);
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
quan += vals[i];
return quan;
#endif
}
FASTOR_INLINE int32_t product() {
int32_t vals[Size]; _mm512_storeu_si512((__m512i*)vals, value);
int32_t quan = 1;
for (FASTOR_INDEX i=0; i<Size; ++i)
quan *= vals[i];
return quan;
}
FASTOR_INLINE int32_t dot(const SIMDVector<int32_t,simd_abi::avx512> &other) {
#ifdef FASTOR_HAS_AVX512_REDUCE_ADD
return _mm512_reduce_add_epi32(_mm512_mullo_epi32(value,other.value));
#else
return SIMDVector<int32_t,simd_abi::avx512>(_mm512_mullo_epi32(value,other.value)).sum();
#endif
}
__m512i value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<int32_t,simd_abi::avx512> a) {
const int32_t *value = (int32_t*) &a.value;
os << "["
<< value[0] << " " << value[1] << " "
<< value[2] << " " << value[3] << " "
<< value[4] << " " << value[5] << " "
<< value[6] << " " << value[7] << " "
<< value[8] << " " << value[9] << " "
<< value[10] << " " << value[11] << " "
<< value[12] << " " << value[13] << " "
<< value[14] << " " << value[15] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_add_epi32(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_add_epi32(a.value,_mm512_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_add_epi32(_mm512_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator+(const SIMDVector<int32_t,simd_abi::avx512> &b) {
return b;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_sub_epi32(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_sub_epi32(a.value,_mm512_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_sub_epi32(_mm512_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator-(const SIMDVector<int32_t,simd_abi::avx512> &b) {
return _mm512_castps_si512(_mm512_neg_ps(_mm512_castsi512_ps(b.value)));
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator*(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_mullo_epi32(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator*(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_mullo_epi32(a.value,_mm512_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator*(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
out.value = _mm512_mullo_epi32(_mm512_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator/(const SIMDVector<int32_t,simd_abi::avx512> &a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
#ifdef FASTOR_INTEL
out.value = _mm512_div_epi32(a.value,b.value);
#else
int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value);
int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value);
int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = val_a[i] / val_b[i];
}
out.value = _mm512_loadu_si512((__m512i*)val);
#endif
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator/(const SIMDVector<int32_t,simd_abi::avx512> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx512> out;
#ifdef FASTOR_INTEL
out.value = _mm512_div_epi32(a.value,_mm512_set1_epi32(b));
#else
int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value);
int32_t val_a[out.size()]; _mm512_storeu_si512((__m512i*)val_a, a.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = val_a[i] / b;
}
out.value = _mm512_loadu_si512((__m512i*)val);
#endif
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> operator/(int32_t a, const SIMDVector<int32_t,simd_abi::avx512> &b) {
SIMDVector<int32_t,simd_abi::avx512> out;
#ifdef FASTOR_INTEL
out.value = _mm512_div_epi32(_mm512_set1_epi32(a),b.value);
#else
int32_t val[out.size()]; _mm512_storeu_si512((__m512i*)val, out.value);
int32_t val_b[out.size()]; _mm512_storeu_si512((__m512i*)val_b, b.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = a / val_b[i];
}
out.value = _mm512_loadu_si512((__m512i*)val);
#endif
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx512> abs(const SIMDVector<int32_t,simd_abi::avx512> &a) {
SIMDVector<int32_t,simd_abi::avx512> out;
#ifdef FASTOR_HAS_AVX512_ABS
out.value = _mm512_abs_epi32(a.value);
#else
for (FASTOR_INDEX i=0UL; i<16UL; ++i) {
((int32_t*)&out.value)[i] = std::abs(((int32_t*)&a.value)[i]);
}
#endif
return out;
}
#endif
// AVX VERSION
//-----------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX2_IMPL
template<>
struct SIMDVector<int32_t,simd_abi::avx> {
using value_type = __m256i;
using scalar_value_type = int32_t;
using abi_type = simd_abi::avx;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::avx>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm256_setzero_si256()) {}
FASTOR_INLINE SIMDVector(int32_t num) : value(_mm256_set1_epi32(num)) {}
FASTOR_INLINE SIMDVector(__m256i regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) {
if (Aligned)
value =_mm256_load_si256((__m256i*)data);
else
value = _mm256_loadu_si256((__m256i*)data);
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator=(int32_t num) {
value = _mm256_set1_epi32(num);
return *this;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator=(__m256i regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) {
if (Aligned)
value =_mm256_load_si256((__m256i*)data);
else
value = _mm256_loadu_si256((__m256i*)data);
}
FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const {
if (Aligned)
_mm256_store_si256((__m256i*)data,value);
else
_mm256_storeu_si256((__m256i*)data,value);
}
FASTOR_INLINE void aligned_load(const int32_t *data) {
value =_mm256_load_si256((__m256i*)data);
}
FASTOR_INLINE void aligned_store(int32_t *data) const {
_mm256_store_si256((__m256i*)data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm256_mask_loadu_epi32(value, mask, a);
else
value = _mm256_mask_load_epi32(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm256_setzero_si256();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm256_mask_storeu_epi32(a, mask, value);
else
_mm256_mask_store_epi32(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE int32_t operator[](FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
FASTOR_INLINE void set(int32_t num) {
value = _mm256_set1_epi32(num);
}
FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3, int32_t num4, int32_t num5, int32_t num6, int32_t num7) {
value = _mm256_set_epi32(num0,num1,num2,num3,num4,num5,num6,num7);
}
FASTOR_INLINE void set_sequential(int32_t num0) {
value = _mm256_setr_epi32(num0,num0+1,num0+2,num0+3,num0+4,num0+5,num0+6,num0+7);
}
// In-place operators
FASTOR_INLINE void operator+=(int32_t num) {
value = _mm256_add_epi32x(value,_mm256_set1_epi32(num));
}
FASTOR_INLINE void operator+=(__m256i regi) {
value = _mm256_add_epi32x(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<int32_t,simd_abi::avx> &a) {
value = _mm256_add_epi32x(value,a.value);
}
FASTOR_INLINE void operator-=(int32_t num) {
value = _mm256_sub_epi32x(value,_mm256_set1_epi32(num));
}
FASTOR_INLINE void operator-=(__m256i regi) {
value = _mm256_sub_epi32x(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<int32_t,simd_abi::avx> &a) {
value = _mm256_sub_epi32x(value,a.value);
}
FASTOR_INLINE void operator*=(int32_t num) {
value = _mm256_mul_epi32x(value,_mm256_set1_epi32(num));
}
FASTOR_INLINE void operator*=(__m256i regi) {
value = _mm256_mul_epi32x(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<int32_t,simd_abi::avx> &a) {
value = _mm256_mul_epi32x(value,a.value);
}
FASTOR_INLINE void operator/=(int32_t num) {
int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= num;
}
value = _mm256_loadu_si256((__m256i*)val);
}
FASTOR_INLINE void operator/=(__m256i regi) {
int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value);
int32_t val_num[Size]; _mm256_storeu_si256((__m256i*)val_num, regi);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= val_num[i];
}
value = _mm256_loadu_si256((__m256i*)val);
}
FASTOR_INLINE void operator/=(const SIMDVector<int32_t,simd_abi::avx> &a) {
int32_t val[Size]; _mm256_storeu_si256((__m256i*)val, value);
int32_t val_a[Size]; _mm256_storeu_si256((__m256i*)val_a, a.value);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= val_a[i];
}
value = _mm256_loadu_si256((__m256i*)val);
}
FASTOR_INLINE int32_t minimum() {
int32_t *vals = (int32_t*)&value;
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
if (vals[i]<quan)
quan = vals[i];
return quan;
}
FASTOR_INLINE int32_t maximum() {
int32_t *vals = (int32_t*)&value;
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
if (vals[i]>quan)
quan = vals[i];
return quan;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> reverse() {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_reverse_epi32(value);
return out;
}
FASTOR_INLINE int32_t sum() {
int32_t vals[Size]; _mm256_storeu_si256((__m256i*)vals, value);
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
quan += vals[i];
return quan;
}
FASTOR_INLINE int32_t dot(const SIMDVector<int32_t,simd_abi::avx> &other) {
int32_t vals0[Size]; _mm256_storeu_si256((__m256i*)vals0, value);
int32_t vals1[Size]; _mm256_storeu_si256((__m256i*)vals1, other.value);
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
quan += vals0[i]*vals1[i];
return quan;
}
__m256i value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<int32_t,simd_abi::avx> a) {
const int32_t *value = (int32_t*) &a.value;
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3]
<< " " << value[4] << " " << value[5] << " " << value[6] << " " << value[7] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_add_epi32x(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_add_epi32x(a.value,_mm256_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_add_epi32x(_mm256_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator+(const SIMDVector<int32_t,simd_abi::avx> &b) {
return b;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_sub_epi32x(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_sub_epi32x(a.value,_mm256_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_sub_epi32x(_mm256_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator-(const SIMDVector<int32_t,simd_abi::avx> &b) {
return _mm256_castps_si256(_mm256_neg_ps(_mm256_castsi256_ps(b.value)));
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator*(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_mul_epi32x(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator*(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_mul_epi32x(a.value,_mm256_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator*(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
out.value = _mm256_mul_epi32x(_mm256_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator/(const SIMDVector<int32_t,simd_abi::avx> &a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value);
int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value);
int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = val_a[i] / val_b[i];
}
out.value = _mm256_loadu_si256((__m256i*)val);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator/(const SIMDVector<int32_t,simd_abi::avx> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::avx> out;
int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value);
int32_t val_a[out.size()]; _mm256_storeu_si256((__m256i*)val_a, a.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = val_a[i] / b;
}
out.value = _mm256_loadu_si256((__m256i*)val);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> operator/(int32_t a, const SIMDVector<int32_t,simd_abi::avx> &b) {
SIMDVector<int32_t,simd_abi::avx> out;
int32_t val[out.size()]; _mm256_storeu_si256((__m256i*)val, out.value);
int32_t val_b[out.size()]; _mm256_storeu_si256((__m256i*)val_b, b.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = a / val_b[i];
}
out.value = _mm256_loadu_si256((__m256i*)val);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::avx> abs(const SIMDVector<int32_t,simd_abi::avx> &a) {
SIMDVector<int32_t,simd_abi::avx> out;
#ifdef __AVX2__
out.value = _mm256_abs_epi32(a.value);
#else
// THIS IS ALSO AVX2 VERSION!
// __m128i lo = _mm_abs_epi32(_mm256_castsi256_si128(a.value));
// __m128i hi = _mm_abs_epi32(_mm256_extracti128_si256(a.value,0x1));
// out.value = _mm256_castsi128_si256(lo);
// out.value = _mm256_insertf128_si256(out.value,hi,0x1);
int32_t *value = (int32_t*) &a.value;
for (int32_t i=0; i<8; ++i) {
value[i] = std::abs(value[i]);
}
#endif
return out;
}
#endif
// SSE VERSION
//-----------------------------------------------------------------------------------------------
#ifdef FASTOR_SSE2_IMPL
template<>
struct SIMDVector<int32_t,simd_abi::sse> {
using value_type = __m128i;
using scalar_value_type = int32_t;
using abi_type = simd_abi::sse;
static constexpr FASTOR_INDEX Size = internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::sse>>::value;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return internal::get_simd_vector_size<SIMDVector<int32_t,simd_abi::sse>>::value;}
FASTOR_INLINE SIMDVector() : value(_mm_setzero_si128()) {}
FASTOR_INLINE SIMDVector(int32_t num) : value(_mm_set1_epi32(num)) {}
FASTOR_INLINE SIMDVector(__m128i regi) : value(regi) {}
FASTOR_INLINE SIMDVector(const int32_t *data, bool Aligned=true) {
if (Aligned)
value =_mm_load_si128((__m128i*)data);
else
value = _mm_loadu_si128((__m128i*)data);
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator=(int32_t num) {
value = _mm_set1_epi32(num);
return *this;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator=(__m128i regi) {
value = regi;
return *this;
}
FASTOR_INLINE void load(const int32_t *data, bool Aligned=true) {
if (Aligned)
value =_mm_load_si128((__m128i*)data);
else
value = _mm_loadu_si128((__m128i*)data);
}
FASTOR_INLINE void store(int32_t *data, bool Aligned=true) const {
if (Aligned)
_mm_store_si128((__m128i*)data,value);
else
_mm_storeu_si128((__m128i*)data,value);
}
FASTOR_INLINE void aligned_load(const int32_t *data) {
value =_mm_load_si128((__m128i*)data);
}
FASTOR_INLINE void aligned_store(int32_t *data) const {
_mm_store_si128((__m128i*)data,value);
}
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool Aligned=false) {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
value = _mm_mask_loadu_epi32(value, mask, a);
else
value = _mm_mask_load_epi32(value, mask, a);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
value = _mm_setzero_si128();
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
((scalar_value_type*)&value)[Size - i - 1] = a[Size - i - 1];
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool Aligned=false) const {
#ifdef FASTOR_HAS_AVX512_MASKS
if (!Aligned)
_mm_mask_storeu_epi32(a, mask, value);
else
_mm_mask_store_epi32(a, mask, value);
#else
// perhaps very inefficient but they never get used
int maska[Size];
mask_to_array(mask,maska);
for (FASTOR_INDEX i=0; i<Size; ++i) {
if (maska[i] == -1) {
a[Size - i - 1] = ((const scalar_value_type*)&value)[Size - i - 1];
}
else {
a[Size - i - 1] = 0;
}
}
unused(Aligned);
#endif
}
FASTOR_INLINE int32_t operator[](FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
FASTOR_INLINE int32_t operator()(FASTOR_INDEX i) const {return reinterpret_cast<const int32_t*>(&value)[i];}
FASTOR_INLINE void set(int32_t num) {
value = _mm_set1_epi32(num);
}
FASTOR_INLINE void set(int32_t num0, int32_t num1, int32_t num2, int32_t num3) {
value = _mm_set_epi32(num0,num1,num2,num3);
}
FASTOR_INLINE void set_sequential(int32_t num0) {
value = _mm_setr_epi32(num0,num0+1,num0+2,num0+3);
}
// In-place operators
FASTOR_INLINE void operator+=(int32_t num) {
value = _mm_add_epi32(value,_mm_set1_epi32(num));
}
FASTOR_INLINE void operator+=(__m128i regi) {
value = _mm_add_epi32(value,regi);
}
FASTOR_INLINE void operator+=(const SIMDVector<int32_t,simd_abi::sse> &a) {
value = _mm_add_epi32(value,a.value);
}
FASTOR_INLINE void operator-=(int32_t num) {
value = _mm_sub_epi32(value,_mm_set1_epi32(num));
}
FASTOR_INLINE void operator-=(__m128i regi) {
value = _mm_sub_epi32(value,regi);
}
FASTOR_INLINE void operator-=(const SIMDVector<int32_t,simd_abi::sse> &a) {
value = _mm_sub_epi32(value,a.value);
}
FASTOR_INLINE void operator*=(int32_t num) {
value = _mm_mul_epi32x(value,_mm_set1_epi32(num));
}
FASTOR_INLINE void operator*=(__m128i regi) {
value = _mm_mul_epi32x(value,regi);
}
FASTOR_INLINE void operator*=(const SIMDVector<int32_t,simd_abi::sse> &a) {
value = _mm_mul_epi32x(value,a.value);
}
FASTOR_INLINE void operator/=(int32_t num) {
int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= num;
}
value = _mm_loadu_si128((__m128i*)val);
}
FASTOR_INLINE void operator/=(__m128i regi) {
int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value);
int32_t val_num[Size]; _mm_storeu_si128((__m128i*)val_num, regi);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= val_num[i];
}
value = _mm_loadu_si128((__m128i*)val);
}
FASTOR_INLINE void operator/=(const SIMDVector<int32_t,simd_abi::sse> &a) {
int32_t val[Size]; _mm_storeu_si128((__m128i*)val, value);
int32_t val_a[Size]; _mm_storeu_si128((__m128i*)val_a, a.value);
for (FASTOR_INDEX i=0; i<Size; ++i) {
val[i] /= val_a[i];
}
value = _mm_loadu_si128((__m128i*)val);
}
FASTOR_INLINE int32_t minimum() {
int32_t *vals = (int32_t*)&value;
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
if (vals[i]<quan)
quan = vals[i];
return quan;
}
FASTOR_INLINE int32_t maximum() {
int32_t *vals = (int32_t*)&value;
int32_t quan = 0;
for (FASTOR_INDEX i=0; i<Size; ++i)
if (vals[i]>quan)
quan = vals[i];
return quan;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> reverse() {
return _mm_reverse_epi32(value);
}
FASTOR_INLINE int32_t sum() {return _mm_sum_epi32(value);}
FASTOR_INLINE int32_t product() {return _mm_prod_epi32(value);}
FASTOR_INLINE int32_t dot(const SIMDVector<int32_t,simd_abi::sse> &other) {
return _mm_sum_epi32(_mm_mul_epi32x(value,other.value));
}
__m128i value;
};
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<int32_t,simd_abi::sse> a) {
const int32_t *value = (int32_t*) &a.value;
os << "[" << value[0] << " " << value[1] << " " << value[2] << " " << value[3] << "]\n";
return os;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_add_epi32(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_add_epi32(a.value,_mm_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_add_epi32(_mm_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator+(const SIMDVector<int32_t,simd_abi::sse> &b) {
return b;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_sub_epi32(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_sub_epi32(a.value,_mm_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_sub_epi32(_mm_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator-(const SIMDVector<int32_t,simd_abi::sse> &b) {
return _mm_castps_si128(_mm_neg_ps(_mm_castsi128_ps(b.value)));
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator*(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_mul_epi32x(a.value,b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator*(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_mul_epi32x(a.value,_mm_set1_epi32(b));
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator*(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
out.value = _mm_mul_epi32x(_mm_set1_epi32(a),b.value);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator/(const SIMDVector<int32_t,simd_abi::sse> &a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value);
int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value);
int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = val_a[i] / val_b[i];
}
out.value = _mm_loadu_si128((__m128i*)val);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator/(const SIMDVector<int32_t,simd_abi::sse> &a, int32_t b) {
SIMDVector<int32_t,simd_abi::sse> out;
int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value);
int32_t val_a[out.size()]; _mm_storeu_si128((__m128i*)val_a, a.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = val_a[i] / b;
}
out.value = _mm_loadu_si128((__m128i*)val);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> operator/(int32_t a, const SIMDVector<int32_t,simd_abi::sse> &b) {
SIMDVector<int32_t,simd_abi::sse> out;
int32_t val[out.size()]; _mm_storeu_si128((__m128i*)val, out.value);
int32_t val_b[out.size()]; _mm_storeu_si128((__m128i*)val_b, b.value);
for (FASTOR_INDEX i=0; i<out.size(); ++i) {
val[i] = a / val_b[i];
}
out.value = _mm_loadu_si128((__m128i*)val);
return out;
}
FASTOR_INLINE SIMDVector<int32_t,simd_abi::sse> abs(const SIMDVector<int32_t,simd_abi::sse> &a) {
SIMDVector<int32_t,simd_abi::sse> out;
#ifdef FASTOR_SSSE3_IMPL
out.value = _mm_abs_epi32(a.value);
#else // SSE2
__m128i sign = _mm_srai_epi32(a.value, 31);
__m128i inv = _mm_xor_si128(a.value, sign);
out.value = _mm_sub_epi32(inv, sign);
#endif
return out;
}
#endif
} // end of namespace Fastor
#endif // SIMD_VECTOR_INT_H

1040
noarch/include/Fastor/simd_vector/simd_vector_int64.h
View File

File diff suppressed because it is too large Load Diff

216
noarch/include/Fastor/simd_vector/simd_vector_scalar.h
View File

@@ -1,216 +0,0 @@
#ifndef SIMD_VECTOR_T_SCALAR_H
#define SIMD_VECTOR_T_SCALAR_H
#include "Fastor/simd_vector/simd_vector_base.h"
namespace Fastor {
template <typename T>
struct SIMDVector<T, simd_abi::scalar> {
using value_type = T;
using scalar_value_type = T;
using abi_type = simd_abi::scalar;
static constexpr FASTOR_INDEX Size = 1;
static constexpr FASTOR_INLINE FASTOR_INDEX size() {return 1;}
FASTOR_INLINE SIMDVector() : value(0) {}
FASTOR_INLINE SIMDVector(T num) : value(num) {}
FASTOR_INLINE SIMDVector(const T *data, bool Aligned=true) : value(*data) {}
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator=(T num) {
value = num;
return *this;
}
FASTOR_INLINE void load(const T *data, bool Aligned=true) { value = *data; unused(Aligned); }
FASTOR_INLINE void store(T *data, bool Aligned=true) const { data[0] = value; unused(Aligned); }
FASTOR_INLINE void aligned_load(const T *data) { value = *data; }
FASTOR_INLINE void aligned_store(T *data) const { data[0] = value; }
FASTOR_INLINE void mask_load(const scalar_value_type *a, uint8_t mask, bool ) {
if (mask != 0x0) value = *a;
}
FASTOR_INLINE void mask_store(scalar_value_type *a, uint8_t mask, bool) const {
if (mask != 0x0) a[0] = value;
}
FASTOR_INLINE T operator[](FASTOR_INDEX) const {return value;}
FASTOR_INLINE T operator()(FASTOR_INDEX) const {return value;}
FASTOR_INLINE void set(T num) {
value = num;
}
FASTOR_INLINE void set_sequential(T num) {
value = num;
}
FASTOR_INLINE void broadcast(const T *data) {
value = *data;
}
// In-place operators
FASTOR_INLINE void operator+=(T num) {
value += num;
}
FASTOR_INLINE void operator+=(const SIMDVector<T,simd_abi::scalar> &a) {
value += a.value;
}
FASTOR_INLINE void operator-=(T num) {
value -= num;
}
FASTOR_INLINE void operator-=(const SIMDVector<T,simd_abi::scalar> &a) {
value -= a.value;
}
FASTOR_INLINE void operator*=(T num) {
value *= num;
}
FASTOR_INLINE void operator*=(const SIMDVector<T,simd_abi::scalar> &a) {
value *= a.value;
}
FASTOR_INLINE void operator/=(T num) {
value /= num;
}
FASTOR_INLINE void operator/=(const SIMDVector<T,simd_abi::scalar> &a) {
value /= a.value;
}
// end of in-place operators
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> shift(FASTOR_INDEX) {
return *this;
}
FASTOR_INLINE T sum() {return value;}
FASTOR_INLINE T product() {return value;}
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> reverse() {
return *this;
}
FASTOR_INLINE T minimum() {return value;}
FASTOR_INLINE T maximum() {return value;}
FASTOR_INLINE T dot(const SIMDVector<T,simd_abi::scalar> &other) {
return value*other.value;
}
T value;
};
template <typename T>
FASTOR_HINT_INLINE std::ostream& operator<<(std::ostream &os, SIMDVector<T,simd_abi::scalar> a) {
os << "[" << a.value << "]\n";
return os;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value+b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(const SIMDVector<T,simd_abi::scalar> &a, T b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value+b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(T a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a+b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator+(const SIMDVector<T,simd_abi::scalar> &b) {
return b;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value-b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(const SIMDVector<T,simd_abi::scalar> &a, T b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value-b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(T a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a-b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator-(const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = -b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator*(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value*b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator*(const SIMDVector<T,simd_abi::scalar> &a, T b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value*b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator*(T a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a*b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator/(const SIMDVector<T,simd_abi::scalar> &a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value/b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator/(const SIMDVector<T,simd_abi::scalar> &a, T b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a.value/b;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> operator/(T a, const SIMDVector<T,simd_abi::scalar> &b) {
SIMDVector<T,simd_abi::scalar> out;
out.value = a/b.value;
return out;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> rcp(const SIMDVector<T,simd_abi::scalar> &a) {
return T(1) / a.value;
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> sqrt(const SIMDVector<T,simd_abi::scalar> &a) {
return std::sqrt(a.value);
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> rsqrt(const SIMDVector<T,simd_abi::scalar> &a) {
return T(1) / std::sqrt(a.value);
}
template <typename T>
FASTOR_INLINE SIMDVector<T,simd_abi::scalar> abs(const SIMDVector<T,simd_abi::scalar> &a) {
return std::abs(a.value);
}
} // end of namespace Fastor
#endif // SIMD_VECTOR_T_SCALAR_H

33
noarch/include/Fastor/tensor/AbstractTensor.h
View File

@@ -1,33 +0,0 @@
#ifndef TENSORBASE_H
#define TENSORBASE_H
#include "Fastor/config/config.h"
#include "Fastor/meta/tensor_meta.h"
namespace Fastor {
template<typename T, size_t ... Rest>
class Tensor;
template<typename T, size_t ... Rest>
class TensorMap;
template<class Derived, FASTOR_INDEX Rank>
class AbstractTensor {
public:
constexpr FASTOR_INLINE AbstractTensor() = default;
constexpr FASTOR_INLINE const Derived& self() const {return *static_cast<const Derived*>(this);}
FASTOR_INLINE Derived& self() {return *static_cast<Derived*>(this);}
static constexpr FASTOR_INDEX Dimension = Rank;
#ifndef FASTOR_DYNAMIC_MODE
static constexpr FASTOR_INDEX size() {return Derived::Size;}
#else
FASTOR_INDEX size() const {return (*static_cast<const Derived*>(this)).size();}
#endif
};
}
#endif // TENSORBASE_H

393
noarch/include/Fastor/tensor/AbstractTensorFunctions.h
View File

@@ -1,393 +0,0 @@
#ifndef ABSTRACT_TENSOR_FUNCTIONS_H
#define ABSTRACT_TENSOR_FUNCTIONS_H
#include "Fastor/tensor/Tensor.h"
#include "Fastor/tensor/TensorTraits.h"
#include <limits>
namespace Fastor {
/* The implementation of the evaluate function that evaluates any expression in to a tensor
*/
//----------------------------------------------------------------------------------------------------------//
template<typename T, size_t ... Rest>
FASTOR_INLINE const Tensor<T,Rest...>& evaluate(const Tensor<T,Rest...> &src) {
return src;
}
template<class Derived, size_t DIMS>
FASTOR_INLINE typename Derived::result_type evaluate(const AbstractTensor<Derived,DIMS> &src) {
typename Derived::result_type out(src);
return out;
}
//----------------------------------------------------------------------------------------------------------//
/* IO for tensor expressions */
//----------------------------------------------------------------------------------------------------------//
template<class Expr, size_t DIM>
inline std::ostream& operator<<(std::ostream &os, const AbstractTensor<Expr,DIM> &src) {
using result_type = typename Expr::result_type;
result_type tmp(src);
print(tmp);
return os;
}
template<class Expr, size_t DIM>
inline void print(const AbstractTensor<Expr,DIM> &src) {
using result_type = typename Expr::result_type;
result_type tmp(src);
print(tmp);
}
//----------------------------------------------------------------------------------------------------------//
/* These are the set of functions that work on any expression that evaluate immediately
*/
/* Add all the elements of the tensor in a flattened sense
*/
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type sum(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out(src);
return out.sum();
}
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type sum(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using T = typename Derived::scalar_type;
using V = typename Derived::simd_vector_type;
FASTOR_INDEX i;
T _scal=0; V _vec(_scal);
for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) {
_vec += src.template eval<T>(i);
}
for (; i < src.size(); ++i) {
_scal += src.template eval_s<T>(i);
}
return _vec.sum() + _scal;
}
/* Multiply all the elements of the tensor in a flattened sense
*/
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type product(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out(src);
return out.product();
}
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type product(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using T = typename Derived::scalar_type;
using V = typename Derived::simd_vector_type;
FASTOR_INDEX i;
T _scal=1; V _vec(_scal);
for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) {
_vec *= src.template eval<T>(i);
}
for (; i < src.size(); ++i) {
_scal *= src.template eval_s<T>(i);
}
return _vec.product() * _scal;
}
/* Get minimum element of a tensor
*/
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type min(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out(src);
return min(out);
}
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type min(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using T = typename Derived::scalar_type;
using V = typename Derived::simd_vector_type;
FASTOR_INDEX i;
T _scal=std::numeric_limits<T>::max(); V _vec(_scal);
for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) {
_vec = min(src.template eval<T>(i),_vec);
}
for (; i < src.size(); ++i) {
_scal = std::min(src.template eval_s<T>(i),_scal);
}
return std::min(_vec.minimum(), _scal);
}
/* Get maximum element of a tensor
*/
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type max(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out(src);
return max(out);
}
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type max(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
using T = typename Derived::scalar_type;
using V = typename Derived::simd_vector_type;
FASTOR_INDEX i;
T _scal=std::numeric_limits<T>::min(); V _vec(_scal);
for (i = 0; i < ROUND_DOWN(src.size(),V::Size); i+=V::Size) {
_vec = max(src.template eval<T>(i),_vec);
}
for (; i < src.size(); ++i) {
_scal = std::max(src.template eval_s<T>(i),_scal);
}
return std::max(_vec.maximum(), _scal);
}
/* Get the lower triangular matrix from a 2D expression
*/
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type tril(const AbstractTensor<Derived,DIMS> &_src, int k = 0) {
static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIL");
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out(src);
return tril(out,k);
}
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::result_type tril(const AbstractTensor<Derived,DIMS> &_src, int k = 0) {
static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIL");
const Derived &src = _src.self();
using T = typename Derived::scalar_type;
typename Derived::result_type out(0);
int M = int(src.dimension(0));
int N = int(src.dimension(1));
for (int i = 0; i < M; ++i) {
int jcount = k + i < N ? k + i : N - 1;
for (int j = 0; j <= jcount; ++j) {
out(i,j) = src.template eval_s<T>(i,j);
}
}
return out;
}
/* Get the upper triangular matrix from a 2D expression
*/
template<class Derived, size_t DIMS, enable_if_t_<requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::scalar_type triu(const AbstractTensor<Derived,DIMS> &_src, int k = 0) {
static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIU");
const Derived &src = _src.self();
using result_type = typename Derived::result_type;
const result_type out(src);
return triu(out,k);
}
template<class Derived, size_t DIMS, enable_if_t_<!requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE typename Derived::result_type triu(const AbstractTensor<Derived,DIMS> &_src, int k = 0) {
static_assert(DIMS==2,"TENSOR HAS TO BE 2D FOR TRIU");
const Derived &src = _src.self();
using T = typename Derived::scalar_type;
typename Derived::result_type out(0);
int M = int(src.dimension(0));
int N = int(src.dimension(1));
for (int i = 0; i < M; ++i) {
int jcount = k + i < 0 ? 0 : k + i;
for (int j = jcount; j < N; ++j) {
out(i,j) = src.template eval_s<T>(i,j);
}
}
return out;
}
//----------------------------------------------------------------------------------------------------------//
// Boolean functions
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
template<class Derived, size_t DIMS, enable_if_t_<is_boolean_expression_v<Derived> && requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool all_of(const AbstractTensor<Derived,DIMS> &_src) {
using result_type = typename Derived::result_type;
const result_type tmp(_src.self());
return all_of(tmp);
}
template<class Derived, size_t DIMS, enable_if_t_<is_boolean_expression_v<Derived> && !requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool all_of(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
bool val = true;
for (FASTOR_INDEX i = 0; i < src.size(); ++i) {
if (src.template eval_s<bool>(i) == false) {
val = false;
break;
}
}
return val;
}
template<class Derived, size_t DIMS, enable_if_t_<is_boolean_expression_v<Derived> && requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool any_of(const AbstractTensor<Derived,DIMS> &_src) {
using result_type = typename Derived::result_type;
const result_type tmp(_src.self());
return any_of(tmp);
}
template<class Derived, size_t DIMS, enable_if_t_<is_boolean_expression_v<Derived> && !requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool any_of(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
bool val = false;
for (FASTOR_INDEX i = 0; i < src.size(); ++i) {
if (src.template eval_s<bool>(i) == true) {
val = true;
break;
}
}
return val;
}
template<class Derived, size_t DIMS, enable_if_t_<is_boolean_expression_v<Derived> && requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool none_of(const AbstractTensor<Derived,DIMS> &_src) {
using result_type = typename Derived::result_type;
const result_type tmp(_src.self());
return none_of(tmp);
}
template<class Derived, size_t DIMS, enable_if_t_<is_boolean_expression_v<Derived> && !requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool none_of(const AbstractTensor<Derived,DIMS> &_src) {
const Derived &src = _src.self();
bool val = false;
for (FASTOR_INDEX i = 0; i < src.size(); ++i) {
if (src.template eval_s<bool>(i) == true) {
val = true;
break;
}
}
return val;
}
/* Is a second order tensor expression a uniform
A tensor expression is uniform if it spans equally in all dimensions,
i.e. generalisation of square matrix to N-dimensions
*/
template<class Derived, size_t DIMS>
constexpr FASTOR_INLINE bool isuniform(const AbstractTensor<Derived,DIMS> &_src) {
return is_tensor_uniform_v<typename Derived::result_type>;
}
/* Is a second order tensor expression a square matrix
*/
template<class Derived, size_t DIMS, enable_if_t_<DIMS==2,bool> = false>
constexpr FASTOR_INLINE bool issquare(const AbstractTensor<Derived,DIMS> &_src) {
return is_tensor_uniform_v<typename Derived::result_type>;
}
/* Is a second order tensor expression orthogonal
*/
template<class Derived, size_t DIMS, enable_if_t_<DIMS==2,bool> = false>
FASTOR_INLINE bool isorthogonal(const AbstractTensor<Derived,DIMS> &_src, const double Tol=PRECI_TOL) {
typename Derived::result_type tmp1(_src.self());
typename Derived::result_type tmp2(matmul(transpose(tmp1),tmp1));
typename Derived::result_type I; I.eye2();
return isequal(tmp2,I,Tol);
}
/* Is a tensor expression symmetric - for higher order tensor two axes can defining a plane
provided to determine if a tensor expression is symmertric in that plane
*/
template<size_t axis0 = 0, size_t axis1 = 1, class Derived, size_t DIMS,
enable_if_t_<DIMS==2 && requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool issymmetric(const AbstractTensor<Derived,DIMS> &_src, const double Tol=PRECI_TOL) {
if (!issquare(_src.self())) return false;
return all_of( abs(evaluate(trans(_src.self()) - _src.self())) < Tol);
}
template<size_t axis0 = 0, size_t axis1 = 1, class Derived, size_t DIMS,
enable_if_t_<DIMS==2 && !requires_evaluation_v<Derived>,bool> = false>
FASTOR_INLINE bool issymmetric(const AbstractTensor<Derived,DIMS> &_src, const double Tol=PRECI_TOL) {
if (!issquare(_src.self())) return false;
// Avoid copies
const Derived& src = _src.self();
using T = typename Derived::scalar_type;
using result_type = typename Derived::result_type;
constexpr size_t M = get_tensor_dimension_v<0,result_type>;
constexpr size_t N = get_tensor_dimension_v<1,result_type>;
bool _issym = true;
for (size_t i=0; i<M; ++i) {
for (size_t j=0; j<N; ++j) {
if (std::abs( src.template eval_s<T>(i*N+j) - src.template eval_s<T>(j*N+i) ) > Tol ) {
_issym = false;
break;
}
}
}
return _issym;
}
/* Is a second order tensor expression deviatoric - a 2D tensor expression is deviatoric if it is
trace free
*/
template<class Derived, size_t DIMS, enable_if_t_<DIMS==2,bool> = false>
constexpr FASTOR_INLINE bool isdeviatoric(const AbstractTensor<Derived,DIMS> &_src, const double Tol=PRECI_TOL) {
return std::abs( trace(_src.self()) ) < Tol ? true : false;
}
/* Is a second order tensor expression volumetric - a 2D tensor expression is volumetric if 1/3*[A:I]*I = A
*/
template<class Derived, size_t DIMS, enable_if_t_<DIMS==2,bool> = false>
constexpr FASTOR_INLINE bool isvolumetric(const AbstractTensor<Derived,DIMS> &_src, const double Tol=PRECI_TOL) {
typename Derived::result_type I; I.eye2();
typename Derived::result_type tmp = trace(_src.self()) * I;
return all_of( abs(tmp - _src.self()) < Tol);
}
/* A second order tensor expression belongs to the special linear group if
it's determinant is +1
*/
template<class Derived, size_t DIMS, enable_if_t_<DIMS==2,bool> = false>
FASTOR_INLINE bool doesbelongtoSL3(const AbstractTensor<Derived,DIMS> &_src, const double Tol=PRECI_TOL) {
// Expression must be square
if ( !issquare(_src.self()) ) return false;
return std::abs(determinant(_src.self()) - 1) < Tol ? true : false;
}
/* A second order tensor/expression belongs to the special orthogonal group if
it is orthogonal and it's determinant is +1
*/
template<class Derived, size_t DIMS, enable_if_t_<DIMS==2,bool> = false>
FASTOR_INLINE bool doesbelongtoSO3(const AbstractTensor<Derived,DIMS> &_src, const double Tol=PRECI_TOL) {
// Expression must be square and orthogonal
return issquare(_src.self()) && isorthogonal(_src.self()) ? true : false;
}
/* Are two tensor expressions approximately equal
*/
template<class Derived0, size_t DIMS0, class Derived1, size_t DIMS1,
enable_if_t_<!requires_evaluation_v<Derived0> && !requires_evaluation_v<Derived1>,bool> = false>
FASTOR_INLINE bool isequal(
const AbstractTensor<Derived0,DIMS0> &_src0,
const AbstractTensor<Derived1,DIMS1> &_src1,
const double Tol=PRECI_TOL) {
if ( DIMS0 != DIMS1) return false;
if ( _src0.self().size() != _src1.self().size()) return false;
return all_of( abs(_src0.self() - _src1.self()) < Tol);
}
template<class Derived0, size_t DIMS0, class Derived1, size_t DIMS1,
enable_if_t_<requires_evaluation_v<Derived0> || requires_evaluation_v<Derived1>,bool> = false>
FASTOR_INLINE bool isequal(
const AbstractTensor<Derived0,DIMS0> &_src0,
const AbstractTensor<Derived1,DIMS1> &_src1,
const double Tol=PRECI_TOL) {
if ( DIMS0 != DIMS1) return false;
if ( _src0.self().size() != _src1.self().size()) return false;
return all_of( abs(evaluate(_src0.self() - _src1.self())) < Tol);
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
} // end of namespace Fastor
#endif // #ifndef TENSOR_FUNCTIONS_H

67
noarch/include/Fastor/tensor/Aliasing.h
View File

@@ -1,67 +0,0 @@
#ifndef ALIASING_H_
#define ALIASING_H_
#include "Fastor/tensor/ForwardDeclare.h"
#include "Fastor/tensor/Tensor.h"
namespace Fastor {
// template<typename T, size_t ...Rest0, size_t ... Rest1>
// FASTOR_INLINE bool does_alias(const Tensor<T,Rest0...> &dst, const Tensor<T,Rest1...> &src) {
// return dst.data() == src.data() ? true : false;
// }
template<typename Derived, size_t DIM, typename T, size_t ... Rest>
FASTOR_INLINE bool does_alias(const AbstractTensor<Derived,DIM> &dst, const Tensor<T,Rest...> &src) {
return dst.self().data() == src.data() ? true : false;
}
// template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM>
// FASTOR_INLINE bool does_alias(const AbstractTensor<Derived,DIM> &dst, const AbstractTensor<OtherDerived,OtherDIM> &src) {
// return does_alias(dst.self(),src.self());
// }
#define FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(NAME)\
template<typename Derived, size_t DIM, typename OtherDerived, size_t OtherDIM>\
FASTOR_INLINE bool does_alias(const AbstractTensor<Derived,DIM> &dst, const Unary ##NAME ## Op<OtherDerived,OtherDIM> &src) {\
return does_alias(dst.self(),src.expr().self());\
}\
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Add )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sub )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Abs )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sqrt)
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Exp )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Log )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sin )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Cos )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Tan )
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Asin)
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Acos)
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Atan)
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Sinh)
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Cosh)
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Tanh)
// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Det )
// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Norm)
// FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Trace)
FASTOR_MAKE_ALIAS_FUNC_UNARY_OPS(Trans)
#define FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(NAME)\
template<typename Derived, size_t DIM, typename TLhs, typename TRhs, size_t OtherDIM>\
FASTOR_INLINE bool does_alias(const AbstractTensor<Derived,DIM> &dst, const Binary ##NAME ## Op<TLhs,TRhs,OtherDIM> &src) {\
return does_alias(dst.self(),src.lhs().self()) || does_alias(dst.self(),src.rhs().self());\
}\
FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Add)
FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Sub)
FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Mul)
FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(Div)
FASTOR_MAKE_ALIAS_FUNC_BINARY_OPS(MatMul)
}
#endif // ALIASING_H_

502
noarch/include/Fastor/tensor/BlockIndexing.h
View File

@@ -1,502 +0,0 @@
#ifndef BLOCK_INDEXING_H
#define BLOCK_INDEXING_H
//----------------------------------------------------------------------------------------------------------//
// Block indexing
//----------------------------------------------------------------------------------------------------------//
// Calls scalar indexing so they are fully bounds checked.
template<size_t F, size_t L, size_t S>
FASTOR_INLINE Tensor<T,range_detector<F,L,S>::value> operator()(const iseq<F,L,S>& idx) {
static_assert(1==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<T,range_detector<F,L,S>::value> out;
FASTOR_INDEX counter = 0;
for (FASTOR_INDEX i=F; i<L; i+=S) {
out(counter) = this->operator()(i);
counter++;
}
return out;
}
template<size_t F0, size_t L0, size_t S0, size_t F1, size_t L1, size_t S1>
FASTOR_INLINE Tensor<T,range_detector<F0,L0,S0>::value,range_detector<F1,L1,S1>::value>
operator()(iseq<F0,L0,S0>, iseq<F1,L1,S1>) {
static_assert(2==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<T,range_detector<F0,L0,S0>::value,range_detector<F1,L1,S1>::value> out;
FASTOR_INDEX counter_i = 0;
for (FASTOR_INDEX i=F0; i<L0; i+=S0) {
FASTOR_INDEX counter_j = 0;
for (FASTOR_INDEX j=F1; j<L1; j+=S1) {
out(counter_i,counter_j) = this->operator()(i,j);
counter_j++;
}
counter_i++;
}
return out;
}
template<size_t F0, size_t L0, size_t S0, size_t F1, size_t L1, size_t S1, size_t F2, size_t L2, size_t S2>
FASTOR_INLINE Tensor<T,range_detector<F0,L0,S0>::value,range_detector<F1,L1,S1>::value,range_detector<F2,L2,S2>::value>
operator()(iseq<F0,L0,S0>, iseq<F1,L1,S1>, iseq<F2,L2,S2>) const {
static_assert(3==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<T,range_detector<F0,L0,S0>::value,
range_detector<F1,L1,S1>::value,
range_detector<F2,L2,S2>::value> out;
FASTOR_INDEX counter_i = 0;
for (FASTOR_INDEX i=F0; i<L0; i+=S0) {
FASTOR_INDEX counter_j = 0;
for (FASTOR_INDEX j=F1; j<L1; j+=S1) {
FASTOR_INDEX counter_k = 0;
for (FASTOR_INDEX k=F2; k<L2; k+=S2) {
out(counter_i,counter_j,counter_k) = this->operator()(i,j,k);
counter_k++;
}
counter_j++;
}
counter_i++;
}
return out;
}
template<size_t F0, size_t L0, size_t S0,
size_t F1, size_t L1, size_t S1,
size_t F2, size_t L2, size_t S2,
size_t F3, size_t L3, size_t S3>
FASTOR_INLINE Tensor<T,range_detector<F0,L0,S0>::value,
range_detector<F1,L1,S1>::value,
range_detector<F2,L2,S2>::value,
range_detector<F3,L3,S3>::value>
operator ()(iseq<F0,L0,S0>, iseq<F1,L1,S1>,
iseq<F2,L2,S2>, iseq<F3,L3,S3>) {
static_assert(4==dimension_t::value, "INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<T,range_detector<F0,L0,S0>::value,
range_detector<F1,L1,S1>::value,
range_detector<F2,L2,S2>::value,
range_detector<F3,L3,S3>::value> out;
FASTOR_INDEX counter_i = 0;
for (FASTOR_INDEX i=F0; i<L0; i+=S0) {
FASTOR_INDEX counter_j = 0;
for (FASTOR_INDEX j=F1; j<L1; j+=S1) {
FASTOR_INDEX counter_k = 0;
for (FASTOR_INDEX k=F2; k<L2; k+=S2) {
FASTOR_INDEX counter_l = 0;
for (FASTOR_INDEX l=F3; l<L3; l+=S3) {
out(counter_i,counter_j,counter_k,counter_l) = this->operator()(i,j,k,l);
counter_l++;
}
counter_k++;
}
counter_j++;
}
counter_i++;
}
return out;
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,1> operator()(seq _s) {
static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,1>(*this,_s);
}
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2> operator()(seq _s0, seq _s1) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,2>(*this,_s0,_s1);
}
template<int F0, int L0, int S0>
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2>
operator()(fseq<F0,L0,S0> _s0, seq _s1) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,2>(*this,_s0,_s1);
}
template<int F0, int L0, int S0>
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2>
operator()(seq _s0, fseq<F0,L0,S0> _s1) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,2>(*this,_s0,_s1);
}
template<typename Int, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2> operator()(seq _s0, Int num) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,2>(*this,_s0,seq(num));
}
template<typename Int, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2> operator()(Int num, seq _s1) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,2>(*this,seq(num),_s1);
}
template<typename ... Seq, enable_if_t_<!is_arithmetic_pack_v<Seq...> && !is_fixed_sequence_pack_v<Seq...>,bool> = false>
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,sizeof...(Seq)> operator()(Seq ... _seqs) {
static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,sizeof...(Seq)>(*this, {_seqs...});
}
template<typename ...Fseq, enable_if_t_<is_fixed_sequence_pack_v<Fseq...>,bool> = false>
FASTOR_INLINE TensorFixedViewExprnD<Tensor<T,Rest...>,Fseq...> operator()(Fseq... ) {
static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorFixedViewExprnD<Tensor<T,Rest...>,Fseq...>(*this);
}
// if fseq == fall - then just return a reference to the tensor
template<int F0, int L0, int S0,
typename std::enable_if<
internal::fseq_range_detector<typename to_positive<fseq<F0,L0,S0>,
get_value<1,Rest...>::value>::type>::value == get_value<1,Rest...>::value,
bool>::type =0>
FASTOR_INLINE Tensor<T,Rest...>&
operator()(fseq<F0,L0,S0>) {
static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return (*this);
}
// if fseq != fall - return a view
template<int F0, int L0, int S0,
typename std::enable_if<
internal::fseq_range_detector<typename to_positive<fseq<F0,L0,S0>,
get_value<1,Rest...>::value>::type>::value != get_value<1,Rest...>::value,
bool>::type =0>
FASTOR_INLINE TensorFixedViewExpr1D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,pack_prod<Rest...>::value>::type,1>
operator()(fseq<F0,L0,S0>) {
static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorFixedViewExpr1D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,pack_prod<Rest...>::value>::type,1>(*this);
}
// if fseq == fall - then just return a reference to the tensor
template<int F0, int L0, int S0, int F1, int L1, int S1,
typename std::enable_if<
internal::fseq_range_detector<typename to_positive<fseq<F0,L0,S0>,get_value<1,Rest...>::value>::type>::value == get_value<1,Rest...>::value &&
internal::fseq_range_detector<typename to_positive<fseq<F1,L1,S1>,get_value<2,Rest...>::value>::type>::value == get_value<2,Rest...>::value,
bool>::type =0>
FASTOR_INLINE Tensor<T,Rest...>&
operator()(fseq<F0,L0,S0>, fseq<F1,L1,S1>) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return (*this);
}
// if fseq != fall - return a view
template<int F0, int L0, int S0, int F1, int L1, int S1,
typename std::enable_if<
internal::fseq_range_detector<typename to_positive<fseq<F0,L0,S0>,get_value<1,Rest...>::value>::type>::value != get_value<1,Rest...>::value ||
internal::fseq_range_detector<typename to_positive<fseq<F1,L1,S1>,get_value<2,Rest...>::value>::type>::value != get_value<2,Rest...>::value,
bool>::type =0>
FASTOR_INLINE TensorFixedViewExpr2D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,get_value<1,Rest...>::value>::type,
typename to_positive<fseq<F1,L1,S1>,get_value<2,Rest...>::value>::type,2>
operator()(fseq<F0,L0,S0>, fseq<F1,L1,S1>) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorFixedViewExpr2D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,get_value<1,Rest...>::value>::type,
typename to_positive<fseq<F1,L1,S1>,get_value<2,Rest...>::value>::type,2>(*this);
}
template<int F0, int L0, int S0, typename Int, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2> operator()(fseq<F0,L0,S0> _s, Int num) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,2>(*this,seq(_s),seq(num));
}
template<int F0, int L0, int S0, typename Int,
typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2> operator()(Int num, fseq<F0,L0,S0> _s) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorViewExpr<Tensor<T,Rest...>,2>(*this,seq(num),seq(_s));
}
// Selecting a row and returning a TensorMap - does not seem to speed up the code
//----------------------------------------------------------------------------------------------------------//
// template<int F0, int L0, int S0, typename Int,
// typename std::enable_if<std::is_integral<Int>::value &&
// internal::fseq_range_detector<typename to_positive<fseq<F0,L0,S0>,
// get_value<2,Rest...>::value>::type>::value != get_value<2,Rest...>::value,bool>::type=0>
// FASTOR_INLINE TensorViewExpr<Tensor<T,Rest...>,2> operator()(Int num, fseq<F0,L0,S0> _s) {
// static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
// return TensorViewExpr<Tensor<T,Rest...>,2>(*this,seq(num),seq(_s));
// }
// // Selecting a row from a 2D tensor returns a TensorMap
// template<typename Int, int F0, int L0, int S0,
// typename std::enable_if<std::is_integral<Int>::value &&
// internal::fseq_range_detector<typename to_positive<fseq<F0,L0,S0>,
// get_value<2,Rest...>::value>::type>::value == get_value<2,Rest...>::value,bool>::type=0>
// FASTOR_INLINE TensorMap<T,get_value<2,Rest...>::value> operator()(Int num, fseq<F0,L0,S0> _s) {
// static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
// constexpr FASTOR_INDEX N = get_value<2,Rest...>::value;
// return TensorMap<T,N>(&_data[num*N]);
// }
//----------------------------------------------------------------------------------------------------------//
template<typename Int, size_t N, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,N>,1> operator()(const Tensor<Int,N> &_it) {
static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,N>,1>(*this,_it);
}
template<typename Int, size_t ... IterSizes, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,IterSizes...>,sizeof...(Rest)>
operator()(const Tensor<Int,IterSizes...> &_it) {
static_assert(dimension_t::value==sizeof...(IterSizes),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,IterSizes...>,sizeof...(Rest)>(*this,_it);
}
template<typename Int0, typename Int1, size_t M, size_t N,
typename std::enable_if<std::is_integral<Int0>::value && std::is_integral<Int1>::value,bool>::type=0>
FASTOR_INLINE TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,N>,2>
operator()(const Tensor<Int0,M> &_it0, const Tensor<Int1,N> &_it1) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<Int0,M,N> tmp_it;
constexpr int NCols = get_value<2,Rest...>::value;
for (FASTOR_INDEX i = 0; i<M; ++i) {
for (FASTOR_INDEX j=0; j<N; ++j) {
tmp_it(i,j) = _it0(i)*NCols + _it1(j);
}
}
return TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,N>,2>(*this,tmp_it);
}
template<typename Int0, typename Int1, size_t M,
typename std::enable_if<std::is_integral<Int0>::value && std::is_integral<Int1>::value,bool>::type=0>
FASTOR_INLINE TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>
operator()(const Tensor<Int0,M> &_it0, Int1 num) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<Int0,M,1> tmp_it;
constexpr int NCols = get_value<2,Rest...>::value;
for (FASTOR_INDEX i = 0; i<M; ++i) {
tmp_it(i,0) = _it0(i)*NCols + num;
}
return TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>(*this,tmp_it);
}
template<typename Int0, typename Int1, size_t M,
typename std::enable_if<std::is_integral<Int0>::value && std::is_integral<Int1>::value,bool>::type=0>
FASTOR_INLINE TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>
operator()(Int1 num, const Tensor<Int0,M> &_it0) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<Int0,M,1> tmp_it;
constexpr int NCols = get_value<2,Rest...>::value;
for (FASTOR_INDEX i = 0; i<M; ++i) {
tmp_it(i,0) = num*NCols + _it0(i);
}
return TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>(*this,tmp_it);
}
template<typename Int, size_t M, int F, int L, int S,
typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,M,
to_positive<fseq<F,L,S>,get_value<2,Rest...>::value>::type::Size>,2>
operator()(const Tensor<Int,M> &_it0, fseq<F,L,S>) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
constexpr int NCols = get_value<2,Rest...>::value;
using _seq = typename to_positive<fseq<F,L,S>,NCols>::type;
constexpr int ColSize = _seq::Size;
Tensor<Int,M,ColSize> tmp_it;
for (FASTOR_INDEX i = 0; i<M; ++i) {
for (FASTOR_INDEX j=0; j<ColSize; ++j) {
tmp_it(i,j) = _it0(i)*NCols + _seq::_step*j + _seq::_first;
}
}
return TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,M,
to_positive<fseq<F,L,S>,get_value<2,Rest...>::value>::type::Size>,2> (*this,tmp_it);
}
template<typename Int, size_t N, int F, int L, int S,
typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,
to_positive<fseq<F,L,S>,get_value<1,Rest...>::value>::type::Size,N>,2>
operator()(fseq<F,L,S>, const Tensor<Int,N> &_it0) {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
constexpr int NRows = get_value<1,Rest...>::value;
constexpr int NCols = get_value<2,Rest...>::value;
using _seq = typename to_positive<fseq<F,L,S>,NRows>::type;
constexpr int RowSize = _seq::Size;
Tensor<Int,RowSize,N> tmp_it;
for (FASTOR_INDEX i = 0; i<RowSize; ++i) {
for (FASTOR_INDEX j=0; j<N; ++j) {
tmp_it(i,j) = (_seq::_step*i + _seq::_first)*NCols + _it0(j);
}
}
return TensorRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,
to_positive<fseq<F,L,S>,get_value<1,Rest...>::value>::type::Size,N>,2> (*this,tmp_it);
}
//----------------------------------------------------------------------------------------------------------//
// Filter views
FASTOR_INLINE TensorFilterViewExpr<Tensor<T,Rest...>,Tensor<bool,Rest...>,sizeof...(Rest)>
operator()(const Tensor<bool,Rest...> &_fl) {
return TensorFilterViewExpr<Tensor<T,Rest...>,Tensor<bool,Rest...>,sizeof...(Rest)>(*this,_fl);
}
FASTOR_INLINE TensorFilterViewExpr<Tensor<T,Rest...>,TensorMap<bool,Rest...>,sizeof...(Rest)>
operator()(const TensorMap<bool,Rest...> &_fl) {
return TensorFilterViewExpr<Tensor<T,Rest...>,TensorMap<bool,Rest...>,sizeof...(Rest)>(*this,_fl);
}
//----------------------------------------------------------------------------------------------------------//
FASTOR_INLINE TensorConstViewExpr<Tensor<T,Rest...>,1> operator()(seq _s) const {
static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstViewExpr<Tensor<T,Rest...>,1>(*this,_s);
}
FASTOR_INLINE TensorConstViewExpr<Tensor<T,Rest...>,2> operator()(seq _s0, seq _s1) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstViewExpr<Tensor<T,Rest...>,2>(*this,_s0,_s1);
}
template<typename Int, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstViewExpr<Tensor<T,Rest...>,2> operator()(seq _s0, Int num) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstViewExpr<Tensor<T,Rest...>,2>(*this,_s0,seq(num));
}
template<typename Int, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstViewExpr<Tensor<T,Rest...>,2> operator()(Int num, seq _s1) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstViewExpr<Tensor<T,Rest...>,2>(*this,seq(num),_s1);
}
template<typename ... Seq, enable_if_t_<!is_arithmetic_pack_v<Seq...> && !is_fixed_sequence_pack_v<Seq...>,bool> = false>
FASTOR_INLINE TensorConstViewExpr<Tensor<T,Rest...>,sizeof...(Seq)> operator()(Seq ... _seqs) const {
static_assert(dimension_t::value==sizeof...(Seq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstViewExpr<Tensor<T,Rest...>,sizeof...(Seq)>(*this, {_seqs...});
}
template<typename ...Fseq, enable_if_t_<is_fixed_sequence_pack_v<Fseq...>,bool> = false>
FASTOR_INLINE TensorConstFixedViewExprnD<Tensor<T,Rest...>,Fseq...> operator()(Fseq... ) const {
static_assert(dimension_t::value==sizeof...(Fseq),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstFixedViewExprnD<Tensor<T,Rest...>,Fseq...>(*this);
}
template<int F0, int L0, int S0>
FASTOR_INLINE TensorConstFixedViewExpr1D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,pack_prod<Rest...>::value>::type,1> operator()(fseq<F0,L0,S0>) const {
static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstFixedViewExpr1D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,pack_prod<Rest...>::value>::type,1>(*this);
}
template<int F0, int L0, int S0, int F1, int L1, int S1>
FASTOR_INLINE TensorConstFixedViewExpr2D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,get_value<1,Rest...>::value>::type,
typename to_positive<fseq<F1,L1,S1>,get_value<2,Rest...>::value>::type,2>
operator()(fseq<F0,L0,S0>, fseq<F1,L1,S1>) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstFixedViewExpr2D<Tensor<T,Rest...>,
typename to_positive<fseq<F0,L0,S0>,get_value<1,Rest...>::value>::type,
typename to_positive<fseq<F1,L1,S1>,get_value<2,Rest...>::value>::type,2>(*this);
}
template<int F0, int L0, int S0, typename Int, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstViewExpr<Tensor<T,Rest...>,2> operator()(fseq<F0,L0,S0> _s, Int num) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstViewExpr<Tensor<T,Rest...>,2>(*this,seq(_s),seq(num));
}
template<int F0, int L0, int S0, typename Int, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstViewExpr<Tensor<T,Rest...>,2> operator()(Int num, fseq<F0,L0,S0> _s) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstViewExpr<Tensor<T,Rest...>,2>(*this,seq(num),seq(_s));
}
template<typename Int, size_t N, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,N>,1>
operator()(const Tensor<Int,N> &_it) const {
static_assert(dimension_t::value==1,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,N>,1>(*this,_it);
}
template<typename Int, size_t ... IterSizes, typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,IterSizes...>,sizeof...(Rest)>
operator()(const Tensor<Int,IterSizes...> &_it) const {
static_assert(dimension_t::value==sizeof...(IterSizes),"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
return TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,IterSizes...>,sizeof...(Rest)>(*this,_it);
}
template<typename Int0, typename Int1, size_t M, size_t N,
typename std::enable_if<std::is_integral<Int0>::value && std::is_integral<Int1>::value,bool>::type=0>
FASTOR_INLINE TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,N>,2>
operator()(const Tensor<Int0,M> &_it0, const Tensor<Int1,N> &_it1) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<Int0,M,N> tmp_it;
constexpr int NCols = get_value<2,Rest...>::value;
for (FASTOR_INDEX i = 0; i<M; ++i) {
for (FASTOR_INDEX j=0; j<N; ++j) {
tmp_it(i,j) = _it0(i)*NCols + _it1(j);
}
}
return TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,N>,2>(*this,tmp_it);
}
template<typename Int0, typename Int1, size_t M,
typename std::enable_if<std::is_integral<Int0>::value && std::is_integral<Int1>::value,bool>::type=0>
FASTOR_INLINE TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>
operator()(const Tensor<Int0,M> &_it0, Int1 num) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<Int0,M,1> tmp_it;
constexpr int NCols = get_value<2,Rest...>::value;
for (FASTOR_INDEX i = 0; i<M; ++i) {
tmp_it(i,0) = _it0(i)*NCols + num;
}
return TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>(*this,tmp_it);
}
template<typename Int0, typename Int1, size_t M,
typename std::enable_if<std::is_integral<Int0>::value && std::is_integral<Int1>::value,bool>::type=0>
FASTOR_INLINE TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>
operator()(Int1 num, const Tensor<Int0,M> &_it0) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
Tensor<Int0,M,1> tmp_it;
constexpr int NCols = get_value<2,Rest...>::value;
for (FASTOR_INDEX i = 0; i<M; ++i) {
tmp_it(i,0) = num*NCols + _it0(i);
}
return TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int0,M,1>,2>(*this,tmp_it);
}
template<typename Int, size_t M, int F, int L, int S,
typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,M,
to_positive<fseq<F,L,S>,get_value<2,Rest...>::value>::type::Size>,2>
operator()(const Tensor<Int,M> &_it0, fseq<F,L,S>) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
constexpr int NCols = get_value<2,Rest...>::value;
using _seq = typename to_positive<fseq<F,L,S>,NCols>::type;
constexpr int ColSize = _seq::Size;
Tensor<Int,M,ColSize> tmp_it;
for (FASTOR_INDEX i = 0; i<M; ++i) {
for (FASTOR_INDEX j=0; j<ColSize; ++j) {
tmp_it(i,j) = _it0(i)*NCols + _seq::_step*j + _seq::_first;
}
}
return TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,M,
to_positive<fseq<F,L,S>,get_value<2,Rest...>::value>::type::Size>,2> (*this,tmp_it);
}
template<typename Int, size_t N, int F, int L, int S,
typename std::enable_if<std::is_integral<Int>::value,bool>::type=0>
FASTOR_INLINE TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,
to_positive<fseq<F,L,S>,get_value<1,Rest...>::value>::type::Size,N>,2>
operator()(fseq<F,L,S>, const Tensor<Int,N> &_it0) const {
static_assert(dimension_t::value==2,"INDEXING TENSOR WITH INCORRECT NUMBER OF ARGUMENTS");
constexpr int NRows = get_value<1,Rest...>::value;
constexpr int NCols = get_value<2,Rest...>::value;
using _seq = typename to_positive<fseq<F,L,S>,NRows>::type;
constexpr int RowSize = _seq::Size;
Tensor<Int,RowSize,N> tmp_it;
for (FASTOR_INDEX i = 0; i<RowSize; ++i) {
for (FASTOR_INDEX j=0; j<N; ++j) {
tmp_it(i,j) = (_seq::_step*i + _seq::_first)*NCols + _it0(j);
}
}
return TensorConstRandomViewExpr<Tensor<T,Rest...>,Tensor<Int,
to_positive<fseq<F,L,S>,get_value<1,Rest...>::value>::type::Size,N>,2> (*this,tmp_it);
}
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
#endif // BLOCK_INDEXING_H

154
noarch/include/Fastor/tensor/ForwardDeclare.h
View File

@@ -1,154 +0,0 @@
#ifndef FORWARD_DECLARE_H
#define FORWARD_DECLARE_H
namespace Fastor {
// FORWARD DECLARATIONS
//----------------------------------------------------------------
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryAddOp;
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinarySubOp;
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryMulOp;
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryDivOp;
template<typename TLhs, typename TRhs, size_t DIM0>
struct BinaryMatMulOp;
template<typename Expr, size_t DIMS>
struct UnaryAddOp;
template<typename Expr, size_t DIMS>
struct UnarySubOp;
template<typename Expr, size_t DIMS>
struct UnaryAbsOp;
template<typename Expr, size_t DIMS>
struct UnarySqrtOp;
template<typename Expr, size_t DIMS>
struct UnaryExpOp;
template<typename Expr, size_t DIMS>
struct UnaryLogOp;
template<typename Expr, size_t DIMS>
struct UnarySinOp;
template<typename Expr, size_t DIMS>
struct UnaryCosOp;
template<typename Expr, size_t DIMS>
struct UnaryTanOp;
template<typename Expr, size_t DIMS>
struct UnaryAsinOp;
template<typename Expr, size_t DIMS>
struct UnaryAcosOp;
template<typename Expr, size_t DIMS>
struct UnaryAtanOp;
template<typename Expr, size_t DIMS>
struct UnarySinhOp;
template<typename Expr, size_t DIMS>
struct UnaryCoshOp;
template<typename Expr, size_t DIMS>
struct UnaryTanhOp;
template<typename Expr, size_t DIMS>
struct UnaryTransOp;
template<typename Expr, size_t DIMS>
struct TensorViewExpr;
template<typename Expr, size_t DIMS>
struct TensorConstViewExpr;
template<typename Expr, typename IterExpr, size_t DIMS>
struct TensorRandomViewExpr;
template<typename Expr, typename IterExpr, size_t DIMS>
struct TensorFilterViewExpr;
template<typename Expr, typename IterExpr, size_t DIMS>
struct TensorConstRandomViewExpr;
template<typename Expr, typename Seq0, size_t DIMS>
struct TensorFixedViewExpr1D;
template<typename Expr, typename Seq0, size_t DIMS>
struct TensorConstFixedViewExpr1D;
template<typename Expr, typename Seq0, typename Seq1, size_t DIMS>
struct TensorFixedViewExpr2D;
template<typename Expr, typename Seq0, typename Seq1, size_t DIMS>
struct TensorConstFixedViewExpr2D;
template<class TensorType, typename ... Fseqs>
struct TensorConstFixedViewExprnD;
template<class TensorType, typename ... Fseqs>
struct TensorFixedViewExprnD;
template<typename Expr, size_t DIM>
struct TensorDiagViewExpr;
template <FASTOR_INDEX ... All>
struct Index;
template<class Idx, class Seq>
struct nprods;
template<class Idx, class Seq>
struct nprods_views;
#define FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(NAME)\
template<typename Expr, size_t DIM0>\
struct Unary ##NAME ## Op;\
FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Not)
FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isinf)
FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isnan)
FASTOR_MAKE_UNARY_BOOL_OP_FORWARD_DECLARATION(Isfinite)
template<typename Derived>
struct is_unary_bool_op;
#define FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(NAME)\
template<typename TLhs, typename TRhs, size_t DIM0>\
struct BinaryCmpOp##NAME ;\
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(EQ)
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(NEQ)
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(LT)
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(GT)
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(LE)
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(GE)
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(AND)
FASTOR_MAKE_BINARY_CMP_OP_FORWARD_DECLARATION(OR)
template<typename Derived>
struct is_binary_cmp_op;
//----------------------------------------------------------------
}
#endif // FORWARD_DECLARE_H

120
noarch/include/Fastor/tensor/IndexRetriever.h
View File

@@ -1,120 +0,0 @@
#ifndef INDEX_RETRIEVER_H
#define INDEX_RETRIEVER_H
// Retrieving index
// Given a flat index get the flat index in to the tensor - note that for tensor class the incoming index is
// the out-going index but it may not be the same for special tensors for instance for SingleValueTensor and
// views and such the index may be different
//----------------------------------------------------------------------------------------------------------//
template<typename U>
FASTOR_INLINE U get_mem_index(U index) const {
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT((index>=0 && index < size()), "INDEX OUT OF BOUNDS");
#endif
return index;
}
// Retrieving index for nD tensors
// Given a multi-dimensional index get the flat index in to the tensor
//----------------------------------------------------------------------------------------------------------//
template<typename... Args, typename std::enable_if<sizeof...(Args)==1
&& sizeof...(Args)==dimension_t::value && is_arithmetic_pack<Args...>::value,bool>::type =0>
FASTOR_INLINE size_t get_flat_index(Args ... args) const {
constexpr size_t M = get_value<1,Rest...>::value;
const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...);
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT( ( (i>=0 && i<M)), "INDEX OUT OF BOUNDS");
#endif
return i;
}
template<typename... Args, typename std::enable_if<sizeof...(Args)==2
&& sizeof...(Args)==dimension_t::value && is_arithmetic_pack<Args...>::value,bool>::type =0>
FASTOR_INLINE size_t get_flat_index(Args ... args) const {
constexpr size_t M = get_value<1,Rest...>::value;
constexpr size_t N = get_value<2,Rest...>::value;
const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...);
const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...);
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT( ( (i>=0 && i<M) && (j>=0 && j<N)), "INDEX OUT OF BOUNDS");
#endif
return i*N+j;
}
template<typename... Args, typename std::enable_if<sizeof...(Args)==3
&& sizeof...(Args)==dimension_t::value && is_arithmetic_pack<Args...>::value,bool>::type =0>
FASTOR_INLINE size_t get_flat_index(Args ... args) const {
constexpr size_t M = get_value<1,Rest...>::value;
constexpr size_t N = get_value<2,Rest...>::value;
constexpr size_t P = get_value<3,Rest...>::value;
const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...);
const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...);
const size_t k = get_index<2>(args...) < 0 ? P + get_index<2>(args...) : get_index<2>(args...);
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT( ( (i>=0 && i<M) && (j>=0 && j<N) && (k>=0 && k<P)), "INDEX OUT OF BOUNDS");
#endif
return i*N*P+j*P+k;
}
template<typename... Args, typename std::enable_if<sizeof...(Args)==4
&& sizeof...(Args)==dimension_t::value && is_arithmetic_pack<Args...>::value,bool>::type =0>
FASTOR_INLINE size_t get_flat_index(Args ... args) const {
constexpr size_t M = get_value<1,Rest...>::value;
constexpr size_t N = get_value<2,Rest...>::value;
constexpr size_t P = get_value<3,Rest...>::value;
constexpr size_t Q = get_value<4,Rest...>::value;
const size_t i = get_index<0>(args...) < 0 ? M + get_index<0>(args...) : get_index<0>(args...);
const size_t j = get_index<1>(args...) < 0 ? N + get_index<1>(args...) : get_index<1>(args...);
const size_t k = get_index<2>(args...) < 0 ? P + get_index<2>(args...) : get_index<2>(args...);
const size_t l = get_index<3>(args...) < 0 ? Q + get_index<3>(args...) : get_index<3>(args...);
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT( ( (i>=0 && i<M) && (j>=0 && j<N)
&& (k>=0 && k<P) && (l>=0 && l<Q)), "INDEX OUT OF BOUNDS");
#endif
return i*N*P*Q+j*P*Q+k*Q+l;
}
template<typename... Args, typename std::enable_if<sizeof...(Args)>=5
&& sizeof...(Args)==dimension_t::value && is_arithmetic_pack<Args...>::value,bool>::type =0>
FASTOR_INLINE size_t get_flat_index(Args ... args) const {
/* The type of largs needs to be the type of incoming pack i.e.
whatever the tensor is indexed with
*/
get_nth_type<0,Args...> largs[sizeof...(Args)] = {args...};
constexpr std::array<size_t,dimension_t::value> products_ = nprods_views<Index<Rest...>,
typename std_ext::make_index_sequence<dimension_t::value>::type>::values;
constexpr size_t DimensionHolder[dimension_t::value] = {Rest...};
for (size_t i=0; i<dimension_t::value; ++i) {
if ( largs[i] < 0 ) largs[i] += DimensionHolder[i];
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT( (largs[i]>=0 && largs[i]<DimensionHolder[i]), "INDEX OUT OF BOUNDS");
#endif
}
size_t index = 0;
for (size_t i=0; i<dimension_t::value; ++i) {
index += products_[i]*largs[i];
}
return index;
}
//----------------------------------------------------------------------------------------------------------//
FASTOR_INLINE int get_flat_index(const std::array<int, dimension_t::value> &as) const {
constexpr std::array<size_t,dimension_t::value> products_ = nprods_views<Index<Rest...>,
typename std_ext::make_index_sequence<dimension_t::value>::type>::values;
size_t index = 0;
for (size_t i=0; i<dimension_t::value; ++i) {
index += products_[i]*as[i];
}
#if FASTOR_BOUNDS_CHECK
FASTOR_ASSERT((index>=0 && index<size()), "INDEX OUT OF BOUNDS");
#endif
return index;
}
//----------------------------------------------------------------------------------------------------------//
#endif // INDEX_RETRIEVER_H

111
noarch/include/Fastor/tensor/InitializerListConstructors.h
View File

@@ -1,111 +0,0 @@
#ifndef INITIALIZER_LIST_CONSTRUCTORS_H
#define INITIALIZER_LIST_CONSTRUCTORS_H
// Initialiser list constructors
//----------------------------------------------------------------------------------------------------------//
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false >
constexpr
FASTOR_INLINE Tensor(const std::initializer_list<U> &lst) {
static_assert(sizeof...(Rest)==1,"TENSOR RANK MISMATCH WITH LIST-INITIALISER");
#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE))
FASTOR_ASSERT(pack_prod<Rest...>::value==lst.size(), "TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
#endif
auto counter = 0;
for (const auto &i: lst) {_data[counter] = i; counter++;}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false >
constexpr
FASTOR_INLINE Tensor(const std::initializer_list<std::initializer_list<U>> &lst2d) {
static_assert(sizeof...(Rest)==2,"TENSOR RANK MISMATCH WITH LIST-INITIALISER");
#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE))
constexpr FASTOR_INDEX M = get_value<1,Rest...>::value;
constexpr FASTOR_INDEX N = get_value<2,Rest...>::value;
auto size_ = 0;
FASTOR_ASSERT(M==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
for (auto &lst: lst2d) {
auto curr_size = lst.size();
FASTOR_ASSERT(N==lst.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
size_ += curr_size;
}
FASTOR_ASSERT(pack_prod<Rest...>::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
#endif
auto counter = 0;
for (const auto &lst1d: lst2d) {
for (const auto &i: lst1d) {
_data[counter] = T(i);
counter++;
}
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false >
constexpr
FASTOR_INLINE Tensor(const std::initializer_list<std::initializer_list<std::initializer_list<U>>> &lst3d) {
static_assert(sizeof...(Rest)==3,"TENSOR RANK MISMATCH WITH LIST-INITIALISER");
#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE))
constexpr FASTOR_INDEX M = get_value<1,Rest...>::value;
constexpr FASTOR_INDEX N = get_value<2,Rest...>::value;
constexpr FASTOR_INDEX P = get_value<3,Rest...>::value;
auto size_ = 0;
FASTOR_ASSERT(M==lst3d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
for (const auto &lst2d: lst3d) {
FASTOR_ASSERT(N==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
for (const auto &lst: lst2d) {
const auto curr_size = lst.size();
FASTOR_ASSERT(P==lst.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
size_ += curr_size;
}
}
FASTOR_ASSERT(pack_prod<Rest...>::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
#endif
auto counter = 0;
for (const auto &lst2d: lst3d) {
for (const auto &lst1d: lst2d) {
for (const auto &i: lst1d) {
_data[counter] = i;
counter++;
}
}
}
}
template<typename U=T, enable_if_t_<is_primitive_v_<U>,bool> = false >
constexpr
FASTOR_INLINE Tensor(const std::initializer_list<std::initializer_list<std::initializer_list<std::initializer_list<U>>>> &lst4d) {
static_assert(sizeof...(Rest)==4,"TENSOR RANK MISMATCH WITH LIST-INITIALISER");
#if (!defined(NDEBUG) && !defined(FASTOR_ZERO_INITIALISE))
constexpr FASTOR_INDEX M = get_value<1,Rest...>::value;
constexpr FASTOR_INDEX N = get_value<2,Rest...>::value;
constexpr FASTOR_INDEX P = get_value<3,Rest...>::value;
constexpr FASTOR_INDEX Q = get_value<4,Rest...>::value;
auto size_ = 0;
FASTOR_ASSERT(M==lst4d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
for (const auto &lst3d: lst4d) {
FASTOR_ASSERT(N==lst3d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
for (const auto &lst2d: lst3d) {
FASTOR_ASSERT(P==lst2d.size(),"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
for (const auto &lst: lst2d) {
const auto curr_size = lst.size();
FASTOR_ASSERT(Q==curr_size,"TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
size_ += curr_size;
}
}
}
FASTOR_ASSERT(pack_prod<Rest...>::value==size_, "TENSOR SIZE MISMATCH WITH LIST-INITIALISER");
#endif
auto counter = 0;
for (const auto &lst3d: lst4d) {
for (const auto &lst2d: lst3d) {
for (const auto &lst1d: lst2d) {
for (const auto &i: lst1d) {
_data[counter] = i;
counter++;
}
}
}
}
}
//----------------------------------------------------------------------------------------------------------//
#endif // INITIALIZER_LIST_CONSTRUCTORS_H

24
noarch/include/Fastor/tensor/PODConverters.h
View File

@@ -1,24 +0,0 @@
#ifndef PODCONVERTERS_H
#define PODCONVERTERS_H
FASTOR_INLINE T toscalar() const {
//! Returns a scalar
static_assert(size()==1,"ONLY TENSORS OF SIZE 1 CAN BE CONVERTED TO A SCALAR");
return (*_data);
}
FASTOR_INLINE std::array<T,size()> toarray() const {
//! Returns std::array
std::array<T,size()> out;
std::copy(_data,_data+size(),out.begin());
return out;
}
FASTOR_INLINE std::vector<T> tovector() const {
//! Returns std::vector
std::vector<T> out(size());
std::copy(_data,_data+size(),out.begin());
return out;
}
#endif //PODCONVERTERS_H

Some files were not shown because too many files have changed in this diff Show More