bgirgis/cpp-thirdparty
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Fastor library

Browse Source
This commit is contained in:
Bassem Girgis
2025-03-22 01:17:52 -05:00
parent 5546e086f6
commit 4dd5939693
132 changed files with 55086 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
noarch/include/Fastor/backend/adjoint.h Normal file
View File

@@ -0,0 +1,105 @@
#ifndef ADJOINT_H
#define ADJOINT_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = *src;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
/* Compute adjoint: */
dst[0] = + src3;
dst[1] = - src1;
dst[2] = - src2;
dst[3] = + src0;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
T src4 = src[4];
T src5 = src[5];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute adjoint: */
dst[0] = + src4 * src8 - src5 * src7;
dst[1] = - src1 * src8 + src2 * src7;
dst[2] = + src1 * src5 - src2 * src4;
dst[3] = - src3 * src8 + src5 * src6;
dst[4] = + src0 * src8 - src2 * src6;
dst[5] = - src0 * src5 + src2 * src3;
dst[6] = + src3 * src7 - src4 * src6;
dst[7] = - src0 * src7 + src1 * src6;
dst[8] = + src0 * src4 - src1 * src3;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _adjoint(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
/* Compute adjoint: */
T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2];
T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1];
dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3;
dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3;
T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0];
dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5;
dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5;
t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0];
dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1;
dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1;
dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1;
dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1;
t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2];
t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1];
t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1];
dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3;
dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3;
t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0];
t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0];
dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5;
dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5;
t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0];
dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1;
dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1;
dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1;
dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1;
}
} // end of namespace Fastor
#endif // ADJOINT_H

24
noarch/include/Fastor/backend/backend.h Normal file
View File

@@ -0,0 +1,24 @@
#ifndef BACKEND_H
#define BACKEND_H
#include "Fastor/backend/adjoint.h"
#include "Fastor/backend/cofactor.h"
#include "Fastor/backend/cyclic_0.h"
#include "Fastor/backend/determinant.h"
#include "Fastor/backend/doublecontract.h"
#include "Fastor/backend/dyadic.h"
#include "Fastor/backend/inner.h"
#include "Fastor/backend/inverse.h"
#include "Fastor/backend/lufact.h"
#include "Fastor/backend/lut_inverse.h"
#include "Fastor/backend/matmul/matmul.h"
#include "Fastor/backend/matmul/tmatmul.h"
#include "Fastor/backend/norm.h"
#include "Fastor/backend/outer.h"
#include "Fastor/backend/tensor_cross.h"
#include "Fastor/backend/trace.h"
#include "Fastor/backend/transpose/transpose.h"
#endif // BACKEND_H

105
noarch/include/Fastor/backend/cofactor.h Normal file
View File

@@ -0,0 +1,105 @@
#ifndef COFACTOR_H
#define COFACTOR_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = *src;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
/* Compute cofactor: */
dst[0] = + src3;
dst[1] = - src2;
dst[2] = - src1;
dst[3] = + src0;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
T src4 = src[4];
T src5 = src[5];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute cofactor: */
dst[0] = + src4 * src8 - src5 * src7;
dst[1] = - src3 * src8 + src5 * src6;
dst[2] = + src3 * src7 - src4 * src6;
dst[3] = - src1 * src8 + src2 * src7;
dst[4] = + src0 * src8 - src2 * src6;
dst[5] = - src0 * src7 + src1 * src6;
dst[6] = + src1 * src5 - src2 * src4;
dst[7] = - src0 * src5 + src2 * src3;
dst[8] = + src0 * src4 - src1 * src3;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _cofactor(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
/* Compute cofactor: */
T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2];
T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1];
dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3;
dst[4] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3;
T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0];
dst[1] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5;
dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5;
t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0];
dst[2] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1;
dst[6] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1;
dst[3] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1;
dst[7] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1;
t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2];
t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1];
t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1];
dst[8] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3;
dst[12] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3;
t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0];
t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0];
dst[9] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5;
dst[13] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5;
t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0];
dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1;
dst[14] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1;
dst[11] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1;
dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1;
}
}
#endif // COFACTOR_H

172
noarch/include/Fastor/backend/cyclic_0.h Normal file
View File

@@ -0,0 +1,172 @@
#ifndef CYCLIC_0_H
#define CYCLIC_0_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
//! Version 0 of cyclic product of two second order tensors i.e. C_ijkl = A_ik * B_jl
template<typename T, size_t M0, size_t N0, size_t M1, size_t N1>
FASTOR_HINT_INLINE void _cyclic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
for (size_t i=0; i<M0; ++i) {
for (size_t j=0; j<N0; ++j) {
for (size_t k=0; k<M1; ++k) {
for (size_t l=0; l<N1; ++l) {
out[i*N1*M1*N0+j*M1*N0+k*N0+l] += a[i*N0+k]*b[j*N1+l];
}
}
}
}
}
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_HINT_INLINE void _cyclic<double,2,2,2,2>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
__m256d as = _mm256_load_pd(a);
__m256d bs = _mm256_load_pd(b);
// First 2x2 block
__m256d c0 = _mm256_mul_pd(as,bs);
_mm_store_sd(out,_mm256_castpd256_pd128(c0));
_mm_store_sd(out+1,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1));
__m128d c0_high = _mm256_extractf128_pd(c0,0x1);
_mm_store_sd(out+3,_mm_shuffle_pd(_mm256_castpd256_pd128(c0),_mm256_castpd256_pd128(c0),0x1));
_mm_store_sd(out+4,_mm_shuffle_pd(c0_high,c0_high,0x1));
// c_02
__m128d a0 = _mm256_castpd256_pd128(as);
__m128d b0 = _mm256_castpd256_pd128(bs);
b0 = _mm_shuffle_pd(b0,b0,0x1);
__m128d c1 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0)));
_mm_store_sd(out+2,c1);
_mm_store_sd(out+6,c1);
// c_12
__m128d a1 = _mm256_extractf128_pd(as,0x1);
__m128d b1 = _mm256_extractf128_pd(bs,0x1);
b1 = _mm_shuffle_pd(b1,b1,0x1);
__m128d c2 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1)));
_mm_store_sd(out+5,c2);
_mm_store_sd(out+7,c2);
// c_22
__m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b1)));
_mm_store_sd(out+8,c3);
}
template<>
FASTOR_HINT_INLINE void _cyclic<double,3,3,3,3>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// 34+ OPS
__m256d a_low = _mm256_load_pd(a);
__m256d a_high = _mm256_load_pd(a+4);
__m128d a_end = _mm_load_sd(a+8);
__m256d b_low = _mm256_load_pd(b);
__m256d b_high = _mm256_load_pd(b+4);
__m128d b_end = _mm_load_sd(b+8);
// The first 3x3 block
__m256d c0 = _mm256_mul_pd(a_low,b_low);
__m256d c1 = _mm256_mul_pd(a_high,b_high);
__m128d c2 = _mm_mul_sd(a_end,b_end);
_mm256_store_pd(out,c0);
_mm_store_sd(out+6,_mm_set_sd(_mm256_get1_pd(c0)));
_mm256_storeu_pd(out+7,c1);
// _mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0)));
_mm_store_sd(out+12,_mm256_extractf128_pd(c0,0x1));
_mm_store_sd(out+13,_mm_set_sd(_mm256_get1_pd(c1)));
_mm_store_sd(out+14,c2);
// rest
// c_03
__m128d a0 = _mm256_castpd256_pd128(a_low);
__m128d b0 = _mm256_castpd256_pd128(b_low);
b0 = _mm_shuffle_pd(b0,b0,0x1);
__m128d c3 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b0)));
_mm_store_sd(out+3,c3);
_mm_store_sd(out+18,c3);
// c_04
__m128d a1 = _mm_set_pd(_mm256_get0_pd(a_low),_mm256_get2_pd(a_low));
__m128d b1 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get0_pd(b_low));
__m128d c4 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b1)));
_mm_store_sd(out+4,c4);
_mm_store_sd(out+24,c4);
// c_05
__m128d a2 = _mm_set_pd(_mm256_get1_pd(a_low),_mm256_get2_pd(a_low));
__m128d b2 = _mm_set_pd(_mm256_get2_pd(b_low),_mm256_get1_pd(b_low));
__m128d c5 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b2)));
_mm_store_sd(out+5,c5);
_mm_store_sd(out+30,c5);
// c_13
__m128d a3 = _mm_setr_pd(_mm256_get3_pd(a_low),_mm256_get0_pd(a_high));
__m128d b3 = _mm_setr_pd(_mm256_get0_pd(b_high),_mm256_get3_pd(b_low));
__m128d c6 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b3)));
_mm_store_sd(out+9,c6);
_mm_store_sd(out+19,c6);
// c_14
__m128d a4 = _mm_set_pd(_mm256_get3_pd(a_low),_mm256_get1_pd(a_high));
__m128d b4 = _mm_set_pd(_mm256_get1_pd(b_high),_mm256_get3_pd(b_low));
__m128d c7 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b4)));
_mm_store_sd(out+10,c7);
_mm_store_sd(out+25,c7);
// c_15
__m128d a5 = _mm256_castpd256_pd128(a_high);
__m128d b5 = _mm256_castpd256_pd128(b_high);
b5 = _mm_shuffle_pd(b5,b5,0x1);
__m128d c8 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,b5)));
_mm_store_sd(out+11,c8);
_mm_store_sd(out+31,c8);
// c_23
__m128d a6 = _mm256_extractf128_pd(a_high,0x1);
__m128d b6 = _mm256_extractf128_pd(b_high,0x1);
b6 = _mm_shuffle_pd(b6,b6,0x1);
__m128d c9 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a6,b6)));
_mm_store_sd(out+15,c9);
_mm_store_sd(out+20,c9);
// c_24
__m128d a7 = _mm_set_pd(_mm256_get2_pd(a_high),_mm_get0_pd(a_end));
__m128d b7 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get2_pd(b_high));
__m128d c10 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a7,b7)));
_mm_store_sd(out+16,c10);
_mm_store_sd(out+26,c10);
// c_25
__m128d a8 = _mm_set_pd(_mm256_get3_pd(a_high),_mm_get0_pd(a_end));
__m128d b8 = _mm_set_pd(_mm_get0_pd(b_end),_mm256_get3_pd(b_high));
__m128d c11 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a8,b8)));
_mm_store_sd(out+17,c11);
_mm_store_sd(out+32,c11);
// c_33
__m128d c12 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b3)));
_mm_store_sd(out+21,c12);
// c_34
__m128d c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b4)));
_mm_store_sd(out+22,c13);
// c_43
c13 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a0,b6)));
_mm_store_sd(out+27,c13);
// c_35
__m128d c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,_mm_shuffle_pd(b5,b5,0x1))));
_mm_store_sd(out+23,c14);
// c_53
c14 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a3,b6)));
_mm_store_sd(out+33,c14);
// c_44
__m128d c15 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a1,b7)));
_mm_store_sd(out+28,c15);
// c_45
__m128d c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a2,b8)));
_mm_store_sd(out+29,c16);
c16 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a4,b7)));
_mm_store_sd(out+34,c16);
// c_55
__m128d c17 = _mm_mul_pd(HALFPD,_add_pd(_mm_mul_pd(a5,_mm_shuffle_pd(b8,b8,0x1))));
_mm_store_sd(out+35,c17);
}
#endif
}
#endif // CYCLIC_0_H

116
noarch/include/Fastor/backend/determinant.h Normal file
View File

@@ -0,0 +1,116 @@
#ifndef DETERMINANT_H
#define DETERMINANT_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/meta/tensor_meta.h"
namespace Fastor {
#ifndef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N, typename std::enable_if<M==2 && N==2, bool>::type=0>
#else
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==2 && N==2, bool>::type=0>
#endif
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
return a[0] * a[3] - a[1] * a[2];
}
#ifndef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N, typename std::enable_if<M==3 && N==3, bool>::type=0>
#else
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==3 && N==3, bool>::type=0>
#endif
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
return a[0]*a[4]*a[8] + a[1]*a[5]*a[6] + a[2]*a[3]*a[7] - a[2]*a[4]*a[6] - a[1]*a[3]*a[8] - a[0]*a[5]*a[7];
}
template<typename T, size_t M, size_t N, typename std::enable_if<M==4 && N==4, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT m) {
return m[12] * m[9] * m[6] * m[3] - m[8] * m[13] * m[6] * m[3] -
m[12] * m[5] * m[10] * m[3] + m[4] * m[13] * m[10] * m[3] +
m[8] * m[5] * m[14] * m[3] - m[4] * m[9] * m[14] * m[3] -
m[12] * m[9] * m[2] * m[7] + m[8] * m[13] * m[2] * m[7] +
m[12] * m[1] * m[10] * m[7] - m[0] * m[13] * m[10] * m[7] -
m[8] * m[1] * m[14] * m[7] + m[0] * m[9] * m[14] * m[7] +
m[12] * m[5] * m[2] * m[11] - m[4] * m[13] * m[2] * m[11] -
m[12] * m[1] * m[6] * m[11] + m[0] * m[13] * m[6] * m[11] +
m[4] * m[1] * m[14] * m[11] - m[0] * m[5] * m[14] * m[11] -
m[8] * m[5] * m[2] * m[15] + m[4] * m[9] * m[2] * m[15] +
m[8] * m[1] * m[6] * m[15] - m[0] * m[9] * m[6] * m[15] -
m[4] * m[1] * m[10] * m[15] + m[0] * m[5] * m[10] * m[15];
}
template<typename T, size_t M, size_t N, typename std::enable_if<is_greater<M,4>::value || is_greater<N,4>::value, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
static_assert(M==N, "2D TENSOR MUST BE SQUARE");
assert(false && "2D TENSOR MUST BE SQUARE");
}
#ifdef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
std::is_same<T,float>::value && M==2 && N==2, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// 10 OPS
__m128 a1 = _mm_load_ps(a);
__m128 a2 = _mm_shuffle_ps(a1,a1,_MM_SHUFFLE(0,1,2,3));
__m128 a3 = _mm_mul_ps(a1,a2);
return _mm_cvtss_f32(_mm_sub_ss(a3,_mm_shuffle_ps(a3,a3,_MM_SHUFFLE(0,0,0,1))));
}
template<typename T, size_t M, size_t N, typename std::enable_if<!std::is_same<T,double>::value &&
std::is_same<T,float>::value && M==3 && N==3, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// ?? OPS
__m128 r0 = {a[2],a[1],a[0],0.};
__m128 r1 = {a[3],a[5],a[4],0.};
__m128 r2 = {a[7],a[6],a[8],0.};
__m128 r3 = {a[6],a[7],a[8],0.};
__m128 r4 = {a[4],a[5],a[3],0.};
__m128 r5 = {a[2],a[0],a[1],0.};
__m128 out0 = _mm_mul_ps(r2,_mm_mul_ps(r0,r1));
__m128 out1 = _mm_mul_ps(r3,_mm_mul_ps(r4,r5));
return _mm_cvtss_f32(_mm_sub_ss(_add_ps(out0),_add_ps(out1)));
}
template<typename T, size_t M, size_t N, typename std::enable_if<std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==2 && N==2, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// 10 OPS
__m128d a1 = _mm_load_pd(a);
__m128d a2 = _mm_load_pd(a+2);
__m128d a3 = _mm_mul_pd(a1,_mm_shuffle_pd(a2,a2,1));
return _mm_cvtsd_f64(_mm_sub_pd(a3,_mm_shuffle_pd(a3,a3,0x1)));
}
template<typename T, size_t M, size_t N, typename std::enable_if<std::is_same<T,double>::value &&
!std::is_same<T,float>::value && M==3 && N==3, bool>::type=0>
FASTOR_INLINE T _det(const T* FASTOR_RESTRICT a) {
// ?? OPS
__m256d r0 = {a[2],a[1],a[0],0.};
__m256d r1 = {a[3],a[5],a[4],0.};
__m256d r2 = {a[7],a[6],a[8],0.};
__m256d r3 = {a[6],a[7],a[8],0.};
__m256d r4 = {a[4],a[5],a[3],0.};
__m256d r5 = {a[2],a[0],a[1],0.};
__m256d out0 = _mm256_mul_pd(r2,_mm256_mul_pd(r0,r1));
__m256d out1 = _mm256_mul_pd(r3,_mm256_mul_pd(r4,r5));
return _mm_cvtsd_f64(_mm_sub_sd(_add_pd(out0),_add_pd(out1)));
}
#endif
}
#endif // DETERMINANT_H

128
noarch/include/Fastor/backend/doublecontract.h Normal file
View File

@@ -0,0 +1,128 @@
#ifndef DOUBLECONTRACT_H
#define DOUBLECONTRACT_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
template<typename T, size_t M, size_t N,
enable_if_t_<is_greater_equal_v_<4*choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,M*N>::Size, M*N >, bool> = false>
FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
constexpr size_t Size = M*N;
using V = choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,Size>;
V omm0;
size_t i = 0;
for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) {
const V amm0(&a[i],false);
const V bmm0(&b[i],false);
omm0 = fmadd(amm0,bmm0,omm0);
}
T scalar = static_cast<T>(0);
for (; i < Size; ++i) {
scalar += a[i]*b[i];
}
return omm0.sum() + scalar;
}
template<typename T, size_t M, size_t N,
enable_if_t_<is_less_v_<4*choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,M*N>::Size, M*N >, bool> = false>
FASTOR_INLINE T _doublecontract(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
constexpr size_t Size = M*N;
using V = choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,Size>;
V omm0, omm1, omm2, omm3;
size_t i = 0;
for (; i< ROUND_DOWN(Size,4*V::Size); i+=4*V::Size) {
const V amm0(&a[i],false);
const V amm1(&a[i+V::Size],false);
const V amm2(&a[i+2*V::Size],false);
const V amm3(&a[i+3*V::Size],false);
const V bmm0(&b[i],false);
const V bmm1(&b[i+V::Size],false);
const V bmm2(&b[i+2*V::Size],false);
const V bmm3(&b[i+3*V::Size],false);
omm0 = fmadd(amm0,bmm0,omm0);
omm1 = fmadd(amm1,bmm1,omm1);
omm2 = fmadd(amm2,bmm2,omm2);
omm3 = fmadd(amm3,bmm3,omm3);
}
for (; i< ROUND_DOWN(Size,2*V::Size); i+=2*V::Size) {
const V amm0(&a[i],false);
const V amm1(&a[i+V::Size],false);
const V bmm0(&b[i],false);
const V bmm1(&b[i+V::Size],false);
omm0 = fmadd(amm0,bmm0,omm0);
omm1 = fmadd(amm1,bmm1,omm1);
}
for (; i< ROUND_DOWN(Size,V::Size); i+=V::Size) {
const V amm0(&a[i],false);
const V bmm0(&b[i],false);
omm0 = fmadd(amm0,bmm0,omm0);
}
T scalar = static_cast<T>(0);
for (; i < Size; ++i) {
scalar += a[i]*b[i];
}
return (omm0 + omm1 + omm2 + omm3).sum() + scalar;
}
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE float _doublecontract<float,2,2>(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) {
return _mm_sum_ps(_mm_mul_ps(_mm_loadu_ps(a),_mm_loadu_ps(b)));
}
template<>
FASTOR_INLINE float _doublecontract<float,3,3>(const float* FASTOR_RESTRICT a, const float* FASTOR_RESTRICT b) {
float r1 = _mm256_sum_ps(_mm256_mul_ps(_mm256_loadu_ps(a),_mm256_loadu_ps(b)));
float r2 = _mm_sum_ps(_mm_mul_ss(_mm_load_ss(a+8),_mm_load_ss(b+8)));
return r1+r2;
}
template<>
FASTOR_INLINE double _doublecontract<double,2,2>(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) {
return _mm256_sum_pd(_mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b)));
}
template<>
FASTOR_INLINE double _doublecontract<double,3,3>(const double* FASTOR_RESTRICT a, const double* FASTOR_RESTRICT b) {
__m256d r1 = _mm256_mul_pd(_mm256_loadu_pd(a),_mm256_loadu_pd(b));
__m256d r2 = _mm256_mul_pd(_mm256_loadu_pd(a+4),_mm256_loadu_pd(b+4));
__m128d r3 = _mm_mul_sd(_mm_load_sd(a+8),_mm_load_sd(b+8));
__m128d r4 = _add_pd(_mm256_add_pd(r1,r2));
__m128d summ = _mm_add_pd(_add_pd(r3),r4);
return _mm_cvtsd_f64(summ);
}
#endif
// doublecontract and transpose
template<typename T, size_t M, size_t N>
FASTOR_INLINE T _doublecontract_transpose(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
T dc = static_cast<T>(0);
for (FASTOR_INDEX i=0; i<M; ++i)
for (FASTOR_INDEX j=0; j<N; ++j)
dc += a[i*N+j]*b[j*M+i];
return dc;
}
}
#endif // DOUBLECONTRACT_H

212
noarch/include/Fastor/backend/dyadic.h Normal file
View File

@@ -0,0 +1,212 @@
#ifndef DYADIC_H
#define DYADIC_H
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
// The non-voigt version of outer product
//---------------------------------------------------------------------------------------------------
// dyadic template parameters are based on size
// of the two tensors and not the dimensions
//---------------------------------------------------------------------------------------------------
template<typename T, size_t SizeA, size_t SizeB>
FASTOR_INLINE
void _dyadic(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,SizeB>::type;
// constexpr size_t unrollOuterloop = 4UL;
// constexpr size_t M0 = SizeA / unrollOuterloop * unrollOuterloop;
// Unrolling the inner loop beyond 4 does not give any benefit neither on AVX
// nor on AVX512
size_t i = 0;
for (; i<SizeA; ++i) {
const V amm0(a[i ]);
size_t j=0;
for (; j<ROUND_DOWN(SizeB,4*V::Size); j+=4*V::Size) {
const V bmm0(&b[j],false);
const V bmm1(&b[j+V::Size],false);
const V bmm2(&b[j+2*V::Size],false);
const V bmm3(&b[j+3*V::Size],false);
V omm0(amm0*bmm0);
V omm1(amm0*bmm1);
V omm2(amm0*bmm2);
V omm3(amm0*bmm3);
omm0.store(&out[(i )*SizeB+j],false);
omm1.store(&out[(i )*SizeB+j+V::Size],false);
omm2.store(&out[(i )*SizeB+j+2*V::Size],false);
omm3.store(&out[(i )*SizeB+j+3*V::Size],false);
}
for (; j<ROUND_DOWN(SizeB,2*V::Size); j+=2*V::Size) {
const V bmm0(&b[j],false);
const V bmm1(&b[j+V::Size],false);
V omm0(amm0*bmm0);
V omm1(amm0*bmm1);
omm0.store(&out[(i )*SizeB+j],false);
omm1.store(&out[(i )*SizeB+j+V::Size],false);
}
for (; j<ROUND_DOWN(SizeB,V::Size); j+=V::Size) {
const V bmm0(&b[j],false);
V omm0(amm0*bmm0);
omm0.store(&out[(i )*SizeB+j],false);
}
for (; j<SizeB; ++j) {
const T bmm0(b[j]);
out[(i )*SizeB+j] = a[i ]*bmm0;
}
}
}
//---------------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------------
#ifdef FASTOR_AVX_IMPL
// Outer product (2x2) x (2x2)
template<>
FASTOR_INLINE
void _dyadic<float,4,4>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
__m128 vec_a = _mm_load_ps(a);
__m128 vec_b = _mm_load_ps(b);
__m128 a00 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0));
__m128 a01 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1));
__m128 a10 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2));
__m128 a11 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(3,3,3,3));
__m256 a0001 = _mm256_castps128_ps256(a00);
a0001 = _mm256_insertf128_ps(a0001,a01,0x1);
__m256 a1011 = _mm256_castps128_ps256(a10);
a1011 = _mm256_insertf128_ps(a1011,a11,0x1);
__m256 vec_b2 = _mm256_castps128_ps256(vec_b);
vec_b2 = _mm256_insertf128_ps(vec_b2,vec_b,0x1);
_mm256_store_ps(out,_mm256_mul_ps(a0001,vec_b2));
_mm256_store_ps(out+8,_mm256_mul_ps(a1011,vec_b2));
}
// Outer product (2x2) x (2x2)
template<>
FASTOR_INLINE
void _dyadic<double,4,4>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
__m256d vec_b = _mm256_loadu_pd(b);
__m256d a00 = _mm256_set1_pd(a[0]);
__m256d a01 = _mm256_set1_pd(a[1]);
__m256d a10 = _mm256_set1_pd(a[2]);
__m256d a11 = _mm256_set1_pd(a[3]);
_mm256_storeu_pd(out,_mm256_mul_pd(a00,vec_b));
_mm256_storeu_pd(out+4,_mm256_mul_pd(a01,vec_b));
_mm256_storeu_pd(out+8,_mm256_mul_pd(a10,vec_b));
_mm256_storeu_pd(out+12,_mm256_mul_pd(a11,vec_b));
}
// Outer product (1x2) x (1x2) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<float,2,2>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 7 OPS
__m128 vec_a = _mm_loadu_ps(a);
__m128 vec_b = _mm_loadu_ps(b);
vec_a = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,0,0));
vec_b = _mm_shuffle_ps(vec_b,vec_b,_MM_SHUFFLE(1,0,1,0));
_mm_storeu_ps(out,_mm_mul_ps(vec_a,vec_b));
}
// Outer product (1x2) x (1x2) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<double,2,2>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// IVY 9 OPS / HW 13 OPS
__m128d vec_a = _mm_loadu_pd(a);
__m128d vec_b = _mm_loadu_pd(b);
__m128d a0 = _mm_shuffle_pd(vec_a,vec_a,0x0);
__m128d a1 = _mm_shuffle_pd(vec_a,vec_a,0x3);
__m256d as = _mm256_castpd128_pd256(a0);
as = _mm256_insertf128_pd(as,a1,0x1);
__m256d bs = _mm256_castpd128_pd256(vec_b);
bs = _mm256_insertf128_pd(bs,vec_b,0x1);
_mm256_storeu_pd(out,_mm256_mul_pd(as,bs));
}
// Outer product (1x3) x (1x3) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<float,3,3>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 18 OPS
__m128 vec_a = _mm_loadu_ps(a);
__m128 vec_b = _mm_loadu_ps(b);
__m128 a0 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(0,0,0,0));
__m128 a1 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(1,1,1,1));
__m128 a2 = _mm_shuffle_ps(vec_a,vec_a,_MM_SHUFFLE(2,2,2,2));
_mm_storeu_ps(out,_mm_mul_ps(a0,vec_b));
_mm_storeu_ps(out+3,_mm_mul_ps(a1,vec_b));
_mm_storeu_ps(out+6,_mm_mul_ps(a2,vec_b));
}
// Outer product (1x3) x (1x3) [for vectors]
template<>
FASTOR_INLINE
void _dyadic<double,3,3>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// 15 OPS + set OPS
__m256d vec_b = _mm256_loadu_pd(b);
__m256d a0 = _mm256_set1_pd(a[0]);
__m256d a1 = _mm256_set1_pd(a[1]);
__m256d a2 = _mm256_set1_pd(a[2]);
_mm256_storeu_pd(out,_mm256_mul_pd(a0,vec_b));
_mm256_storeu_pd(out+3,_mm256_mul_pd(a1,vec_b));
_mm256_storeu_pd(out+6,_mm256_mul_pd(a2,vec_b));
}
#endif
//---------------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------------
// Outer product of scalars
template<>
FASTOR_INLINE
void _dyadic<double,1,1>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
out[0] = a[0]*b[0];
}
template<>
FASTOR_INLINE
void _dyadic<float,1,1>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
out[0] = a[0]*b[0];
}
//---------------------------------------------------------------------------------------------------
}
#endif // DYADIC_H

27
noarch/include/Fastor/backend/inner.h Normal file
View File

@@ -0,0 +1,27 @@
#ifndef INNER_H_
#define INNER_H_
#include "Fastor/meta/meta.h"
#include "Fastor/backend/doublecontract.h"
namespace Fastor {
/* The dependency on doublecontract here is on purpose
as it creates a necessary layer of indirection to avoid
the case where M == 0
*/
template<typename T, size_t M,
enable_if_t_<is_greater_v_<M,0>, bool> = false>
FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
return _doublecontract<T,M,1>(a,b);
}
template<typename T, size_t M,
enable_if_t_<M==0, bool> = false>
FASTOR_INLINE T _inner(const T* FASTOR_RESTRICT a, const T* FASTOR_RESTRICT b) {
return (*a)*(*b);
}
} // end of namespace Fastor
#endif // INNER_H_

450
noarch/include/Fastor/backend/inverse.h Normal file
View File

@@ -0,0 +1,450 @@
#ifndef INVERSE_H
#define INVERSE_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = T(1) / (*src);
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2> && !is_same_v_<T,float> && !is_same_v_<T,double>, bool> = false>
#else
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
#endif
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T det;
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
/* Compute adjoint: */
dst[0] = + src3;
dst[1] = - src1;
dst[2] = - src2;
dst[3] = + src0;
/* Compute determinant: */
det = src0 * dst[0] + src1 * dst[2];
/* Multiply adjoint with reciprocal of determinant: */
det = T(1.0) / det;
dst[0] *= det;
dst[1] *= det;
dst[2] *= det;
dst[3] *= det;
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2> && is_same_v_<T,float>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// This is much superior to the scalar code as
// gcc/clang can't auto-vectorise the scalar code
// 6 shuffles + 1 add + 1 mul + 1 div
// Sky 6 + 4 + 4 + 11 = 25
__m128 mat = _mm_loadu_ps(src);
// xor to swap off-diagonals sings
__m128 nmat = _mm_neg_ps(mat);
// two shuffles to get adjoint
__m128 adj = _mm_shuffle_ps(mat, nmat, 0x009C );
adj = _mm_shuffle_ps(adj, adj , 0x39 );
// compute determinat
__m128 tmp0 = _mm_shuffle_ps(mat , mat , 0x00D8);
tmp0 = _mm_mul_ps (adj , tmp0 );
__m128 tmp1 = _mm_shuffle_ps(tmp0, tmp0, 0x1 );
__m128 det = _mm_div_ss (ONEPS, _mm_add_ss(tmp0,tmp1));
// broadcast det to all elements of __m128
det = _mm_shuffle_ps(det, det, 0x0 );
// divide adjoint by determinant
__m128 inv = _mm_mul_ps (adj, det);
_mm_storeu_ps(dst, inv);
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2> && is_same_v_<T,double>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// This is much superior to the scalar code as
// gcc/clang can't auto-vectorise the scalar code
// 8 shuffles + 1 add + 3 mul + 1 div
// Sky 8 + 4 + 12 + 14 = 38
__m128d row0 = _mm_loadu_pd(src);
__m128d row1 = _mm_loadu_pd(src+2);
__m128d tmp = row0;
row0 = _mm_shuffle_pd(row0,_mm_neg_pd(row0),0x2);
row1 = _mm_shuffle_pd(_mm_neg_pd(row1),row1,0x2);
// these two registers hold the adjoint
__m128d irow0 = _mm_shuffle_pd(row1,row0,0x3);
__m128d irow1 = _mm_shuffle_pd(row1,row0,0x0);
// dot product to compute determinant
__m128d det = _mm_mul_pd(tmp,_mm_reverse_pd(row1));
det = _mm_add_pd(det,_mm_reverse_pd(det));
// one by determinant
__m128d invdet = _mm_div_pd(_mm_set1_pd(1.0),det);
// scale
irow0 = _mm_mul_pd(irow0,invdet);
irow1 = _mm_mul_pd(irow1,invdet);
_mm_storeu_pd(dst ,irow0);
_mm_storeu_pd(dst+2,irow1);
}
#endif
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T det;
T src0 = src[0];
T src1 = src[1];
T src2 = src[2];
T src3 = src[3];
T src4 = src[4];
T src5 = src[5];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute adjoint: */
dst[0] = + src4 * src8 - src5 * src7;
dst[1] = - src1 * src8 + src2 * src7;
dst[2] = + src1 * src5 - src2 * src4;
dst[3] = - src3 * src8 + src5 * src6;
dst[4] = + src0 * src8 - src2 * src6;
dst[5] = - src0 * src5 + src2 * src3;
dst[6] = + src3 * src7 - src4 * src6;
dst[7] = - src0 * src7 + src1 * src6;
dst[8] = + src0 * src4 - src1 * src3;
/* Compute determinant: */
det = src0 * dst[0] + src1 * dst[3] + src2 * dst[6];
/* Multiply adjoint with reciprocal of determinant: */
det = T(1.0) / det;
dst[0] *= det;
dst[1] *= det;
dst[2] *= det;
dst[3] *= det;
dst[4] *= det;
dst[5] *= det;
dst[6] *= det;
dst[7] *= det;
dst[8] *= det;
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4> && !is_same_v_<T,float> && !is_same_v_<T,double>, bool> = false>
#else
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
#endif
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T t1 = src[2*4+2]*src[3*4+3] - src[2*4+3]*src[3*4+2];
T t2 = src[2*4+1]*src[3*4+3] - src[2*4+3]*src[3*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[2*4+2]*src[3*4+1];
dst[0] = src[1*4+1]*t1 - src[1*4+2]*t2 + src[1*4+3]*t3;
dst[1] = src[0*4+2]*t2 - src[0*4+1]*t1 - src[0*4+3]*t3;
T t4 = src[2*4+0]*src[3*4+3] - src[2*4+3]*src[3*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[2*4+2]*src[3*4+0];
dst[4] = src[1*4+2]*t4 - src[1*4+0]*t1 - src[1*4+3]*t5;
dst[5] = src[0*4+0]*t1 - src[0*4+2]*t4 + src[0*4+3]*t5;
t1 = src[2*4+0]*src[3*4+1] - src[2*4+1]*src[3*4+0];
dst[8] = src[1*4+0]*t2 - src[1*4+1]*t4 + src[1*4+3]*t1;
dst[9] = src[0*4+1]*t4 - src[0*4+0]*t2 - src[0*4+3]*t1;
dst[12] = src[1*4+1]*t5 - src[1*4+0]*t3 - src[1*4+2]*t1;
dst[13] = src[0*4+0]*t3 - src[0*4+1]*t5 + src[0*4+2]*t1;
t1 = src[0*4+2]*src[1*4+3] - src[0*4+3]*src[1*4+2];
t2 = src[0*4+1]*src[1*4+3] - src[0*4+3]*src[1*4+1];
t3 = src[0*4+1]*src[1*4+2] - src[0*4+2]*src[1*4+1];
dst[2] = src[3*4+1]*t1 - src[3*4+2]*t2 + src[3*4+3]*t3;
dst[3] = src[2*4+2]*t2 - src[2*4+1]*t1 - src[2*4+3]*t3;
t4 = src[0*4+0]*src[1*4+3] - src[0*4+3]*src[1*4+0];
t5 = src[0*4+0]*src[1*4+2] - src[0*4+2]*src[1*4+0];
dst[6] = src[3*4+2]*t4 - src[3*4+0]*t1 - src[3*4+3]*t5;
dst[7] = src[2*4+0]*t1 - src[2*4+2]*t4 + src[2*4+3]*t5;
t1 = src[0*4+0]*src[1*4+1] - src[0*4+1]*src[1*4+0];
dst[10] = src[3*4+0]*t2 - src[3*4+1]*t4 + src[3*4+3]*t1;
dst[11] = src[2*4+1]*t4 - src[2*4+0]*t2 - src[2*4+3]*t1;
dst[14] = src[3*4+1]*t5 - src[3*4+0]*t3 - src[3*4+2]*t1;
dst[15] = src[2*4+0]*t3 - src[2*4+1]*t5 + src[2*4+2]*t1;
const T __det = src[0]*dst[0] + src[1]*dst[4] + src[2]*dst[8] + src[3]*dst[12];
const T __invdet = T(1)/__det;
for (int i=0; i<16; ++i)
dst[i] *= __invdet;
}
#ifdef FASTOR_SSE2_IMPL
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4> && is_same_v_<T,float>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// From Intel's SSE matrix library
// The inverse is calculated using "Divide and Conquer" technique. The
// original matrix is divide into four 2x2 sub-matrices. Since each
// register of the matrix holds two elements, the smaller matrices are
// consisted of two registers. Hence we get a better locality of the
// calculations.
const __m128 p4f_sign_PNNP = _mm_castsi128_ps(_mm_set_epi32(0x00000000, 0x80000000, 0x80000000, 0x00000000));
// Load the full matrix into registers
__m128 _L1 = _mm_loadu_ps(src + 0);
__m128 _L2 = _mm_loadu_ps(src + 4);
__m128 _L3 = _mm_loadu_ps(src + 8);
__m128 _L4 = _mm_loadu_ps(src + 12);
__m128 A, B, C, D; // the four sub-matrices
A = _mm_movelh_ps(_L1, _L2);
B = _mm_movehl_ps(_L2, _L1);
C = _mm_movelh_ps(_L3, _L4);
D = _mm_movehl_ps(_L4, _L3);
// partial inverse of the sub-matrices
__m128 iA, iB, iC, iD, DC, AB;
__m128 dA, dB, dC, dD; // determinant of the sub-matrices
__m128 det, d, d1, d2;
__m128 rd; // reciprocal of the determinant
// AB = A# * B
AB = _mm_mul_ps(_mm_shuffle_ps(A,A,0x0F), B);
AB = _mm_sub_ps(AB,_mm_mul_ps(_mm_shuffle_ps(A,A,0xA5), _mm_shuffle_ps(B,B,0x4E)));
// DC = D# * C
DC = _mm_mul_ps(_mm_shuffle_ps(D,D,0x0F), C);
DC = _mm_sub_ps(DC,_mm_mul_ps(_mm_shuffle_ps(D,D,0xA5), _mm_shuffle_ps(C,C,0x4E)));
// dA = |A|
dA = _mm_mul_ps(_mm_shuffle_ps(A, A, 0x5F),A);
dA = _mm_sub_ss(dA, _mm_movehl_ps(dA,dA));
// dB = |B|
dB = _mm_mul_ps(_mm_shuffle_ps(B, B, 0x5F),B);
dB = _mm_sub_ss(dB, _mm_movehl_ps(dB,dB));
// dC = |C|
dC = _mm_mul_ps(_mm_shuffle_ps(C, C, 0x5F),C);
dC = _mm_sub_ss(dC, _mm_movehl_ps(dC,dC));
// dD = |D|
dD = _mm_mul_ps(_mm_shuffle_ps(D, D, 0x5F),D);
dD = _mm_sub_ss(dD, _mm_movehl_ps(dD,dD));
// d = trace(AB*DC) = trace(A#*B*D#*C)
d = _mm_mul_ps(_mm_shuffle_ps(DC,DC,0xD8),AB);
// iD = C*A#*B
iD = _mm_mul_ps(_mm_shuffle_ps(C,C,0xA0), _mm_movelh_ps(AB,AB));
iD = _mm_add_ps(iD,_mm_mul_ps(_mm_shuffle_ps(C,C,0xF5), _mm_movehl_ps(AB,AB)));
// iA = B*D#*C
iA = _mm_mul_ps(_mm_shuffle_ps(B,B,0xA0), _mm_movelh_ps(DC,DC));
iA = _mm_add_ps(iA,_mm_mul_ps(_mm_shuffle_ps(B,B,0xF5), _mm_movehl_ps(DC,DC)));
// d = trace(AB*DC) = trace(A#*B*D#*C) [continue]
d = _mm_add_ps(d, _mm_movehl_ps(d, d));
d = _mm_add_ss(d, _mm_shuffle_ps(d, d, 1));
d1 = _mm_mul_ss(dA,dD);
d2 = _mm_mul_ss(dB,dC);
// iD = D*|A| - C*A#*B
iD = _mm_sub_ps(_mm_mul_ps(D,_mm_shuffle_ps(dA,dA,0)), iD);
// iA = A*|D| - B*D#*C;
iA = _mm_sub_ps(_mm_mul_ps(A,_mm_shuffle_ps(dD,dD,0)), iA);
// det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C)
det = _mm_sub_ss(_mm_add_ss(d1,d2),d);
rd = _mm_div_ss(_mm_set_ss(1.0f), det);
// iB = D * (A#B)# = D*B#*A
iB = _mm_mul_ps(D, _mm_shuffle_ps(AB,AB,0x33));
iB = _mm_sub_ps(iB, _mm_mul_ps(_mm_shuffle_ps(D,D,0xB1), _mm_shuffle_ps(AB,AB,0x66)));
// iC = A * (D#C)# = A*C#*D
iC = _mm_mul_ps(A, _mm_shuffle_ps(DC,DC,0x33));
iC = _mm_sub_ps(iC, _mm_mul_ps(_mm_shuffle_ps(A,A,0xB1), _mm_shuffle_ps(DC,DC,0x66)));
rd = _mm_shuffle_ps(rd,rd,0);
rd = _mm_xor_ps(rd, p4f_sign_PNNP);
// iB = C*|B| - D*B#*A
iB = _mm_sub_ps(_mm_mul_ps(C,_mm_shuffle_ps(dB,dB,0)), iB);
// iC = B*|C| - A*C#*D;
iC = _mm_sub_ps(_mm_mul_ps(B,_mm_shuffle_ps(dC,dC,0)), iC);
// iX = iX / det
iA = _mm_mul_ps(rd,iA);
iB = _mm_mul_ps(rd,iB);
iC = _mm_mul_ps(rd,iC);
iD = _mm_mul_ps(rd,iD);
_mm_storeu_ps(dst+0, _mm_shuffle_ps(iA,iB,0x77));
_mm_storeu_ps(dst+4, _mm_shuffle_ps(iA,iB,0x22));
_mm_storeu_ps(dst+8, _mm_shuffle_ps(iC,iD,0x77));
_mm_storeu_ps(dst+12, _mm_shuffle_ps(iC,iD,0x22));
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4> && is_same_v_<T,double>, bool> = false>
FASTOR_INLINE void _inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
// From Intel's SSE matrix library
// The inverse is calculated using "Divide and Conquer" technique. The
// original matrix is divide into four 2x2 sub-matrices. Since each
// register of the matrix holds two elements, the smaller matrices are
// consisted of two registers. Hence we get a better locality of the
// calculations.
const __m128d _Sign_NP = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0));
const __m128d _Sign_PN = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0));
// the four sub-matrices
__m128d A1, A2, B1, B2, C1, C2, D1, D2;
A1 = _mm_loadu_pd(src + 0); B1 = _mm_loadu_pd(src + 2);
A2 = _mm_loadu_pd(src + 4); B2 = _mm_loadu_pd(src + 6);
C1 = _mm_loadu_pd(src + 8); D1 = _mm_loadu_pd(src +10);
C2 = _mm_loadu_pd(src +12); D2 = _mm_loadu_pd(src +14);
// partial inverse of the sub-matrices
__m128d iA1, iA2, iB1, iB2, iC1, iC2, iD1, iD2, DC1, DC2, AB1, AB2;
__m128d dA, dB, dC, dD; // determinant of the sub-matrices
__m128d det, d1, d2, rd;
// dA = |A|
dA = _mm_shuffle_pd(A2, A2, 1);
dA = _mm_mul_pd(A1, dA);
dA = _mm_sub_sd(dA, _mm_shuffle_pd(dA,dA,3));
// dB = |B|
dB = _mm_shuffle_pd(B2, B2, 1);
dB = _mm_mul_pd(B1, dB);
dB = _mm_sub_sd(dB, _mm_shuffle_pd(dB,dB,3));
// AB = A# * B
AB1 = _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,3));
AB2 = _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,0));
AB1 = _mm_sub_pd(AB1, _mm_mul_pd(B2, _mm_shuffle_pd(A1,A1,3)));
AB2 = _mm_sub_pd(AB2, _mm_mul_pd(B1, _mm_shuffle_pd(A2,A2,0)));
// dC = |C|
dC = _mm_shuffle_pd(C2, C2, 1);
dC = _mm_mul_pd(C1, dC);
dC = _mm_sub_sd(dC, _mm_shuffle_pd(dC,dC,3));
// dD = |D|
dD = _mm_shuffle_pd(D2, D2, 1);
dD = _mm_mul_pd(D1, dD);
dD = _mm_sub_sd(dD, _mm_shuffle_pd(dD,dD,3));
// DC = D# * C
DC1 = _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,3));
DC2 = _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,0));
DC1 = _mm_sub_pd(DC1, _mm_mul_pd(C2, _mm_shuffle_pd(D1,D1,3)));
DC2 = _mm_sub_pd(DC2, _mm_mul_pd(C1, _mm_shuffle_pd(D2,D2,0)));
// rd = trace(AB*DC) = trace(A#*B*D#*C)
d1 = _mm_mul_pd(AB1, _mm_shuffle_pd(DC1, DC2, 0));
d2 = _mm_mul_pd(AB2, _mm_shuffle_pd(DC1, DC2, 3));
rd = _mm_add_pd(d1, d2);
rd = _mm_add_sd(rd, _mm_shuffle_pd(rd, rd,3));
// iD = C*A#*B
iD1 = _mm_mul_pd(AB1, _mm_shuffle_pd(C1,C1,0));
iD2 = _mm_mul_pd(AB1, _mm_shuffle_pd(C2,C2,0));
iD1 = _mm_add_pd(iD1, _mm_mul_pd(AB2, _mm_shuffle_pd(C1,C1,3)));
iD2 = _mm_add_pd(iD2, _mm_mul_pd(AB2, _mm_shuffle_pd(C2,C2,3)));
// iA = B*D#*C
iA1 = _mm_mul_pd(DC1, _mm_shuffle_pd(B1,B1,0));
iA2 = _mm_mul_pd(DC1, _mm_shuffle_pd(B2,B2,0));
iA1 = _mm_add_pd(iA1, _mm_mul_pd(DC2, _mm_shuffle_pd(B1,B1,3)));
iA2 = _mm_add_pd(iA2, _mm_mul_pd(DC2, _mm_shuffle_pd(B2,B2,3)));
// iD = D*|A| - C*A#*B
dA = _mm_shuffle_pd(dA,dA,0);
iD1 = _mm_sub_pd(_mm_mul_pd(D1, dA), iD1);
iD2 = _mm_sub_pd(_mm_mul_pd(D2, dA), iD2);
// iA = A*|D| - B*D#*C;
dD = _mm_shuffle_pd(dD,dD,0);
iA1 = _mm_sub_pd(_mm_mul_pd(A1, dD), iA1);
iA2 = _mm_sub_pd(_mm_mul_pd(A2, dD), iA2);
d1 = _mm_mul_sd(dA, dD);
d2 = _mm_mul_sd(dB, dC);
// iB = D * (A#B)# = D*B#*A
iB1 = _mm_mul_pd(D1, _mm_shuffle_pd(AB2,AB1,1));
iB2 = _mm_mul_pd(D2, _mm_shuffle_pd(AB2,AB1,1));
iB1 = _mm_sub_pd(iB1, _mm_mul_pd(_mm_shuffle_pd(D1,D1,1), _mm_shuffle_pd(AB2,AB1,2)));
iB2 = _mm_sub_pd(iB2, _mm_mul_pd(_mm_shuffle_pd(D2,D2,1), _mm_shuffle_pd(AB2,AB1,2)));
// det = |A|*|D| + |B|*|C| - trace(A#*B*D#*C)
det = _mm_add_sd(d1, d2);
det = _mm_sub_sd(det, rd);
// iC = A * (D#C)# = A*C#*D
iC1 = _mm_mul_pd(A1, _mm_shuffle_pd(DC2,DC1,1));
iC2 = _mm_mul_pd(A2, _mm_shuffle_pd(DC2,DC1,1));
iC1 = _mm_sub_pd(iC1, _mm_mul_pd(_mm_shuffle_pd(A1,A1,1), _mm_shuffle_pd(DC2,DC1,2)));
iC2 = _mm_sub_pd(iC2, _mm_mul_pd(_mm_shuffle_pd(A2,A2,1), _mm_shuffle_pd(DC2,DC1,2)));
rd = _mm_div_sd(_mm_set_sd(1.0), det);
rd = _mm_shuffle_pd(rd,rd,0);
// iB = C*|B| - D*B#*A
dB = _mm_shuffle_pd(dB,dB,0);
iB1 = _mm_sub_pd(_mm_mul_pd(C1, dB), iB1);
iB2 = _mm_sub_pd(_mm_mul_pd(C2, dB), iB2);
d1 = _mm_xor_pd(rd, _Sign_PN);
d2 = _mm_xor_pd(rd, _Sign_NP);
// iC = B*|C| - A*C#*D;
dC = _mm_shuffle_pd(dC,dC,0);
iC1 = _mm_sub_pd(_mm_mul_pd(B1, dC), iC1);
iC2 = _mm_sub_pd(_mm_mul_pd(B2, dC), iC2);
_mm_storeu_pd(dst+0, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 3), d1));
_mm_storeu_pd(dst+4, _mm_mul_pd(_mm_shuffle_pd(iA2, iA1, 0), d2));
_mm_storeu_pd(dst+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 3), d1));
_mm_storeu_pd(dst+4+2, _mm_mul_pd(_mm_shuffle_pd(iB2, iB1, 0), d2));
_mm_storeu_pd(dst+2*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 3), d1));
_mm_storeu_pd(dst+3*4, _mm_mul_pd(_mm_shuffle_pd(iC2, iC1, 0), d2));
_mm_storeu_pd(dst+2*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 3), d1));
_mm_storeu_pd(dst+3*4+2,_mm_mul_pd(_mm_shuffle_pd(iD2, iD1, 0), d2));
}
#endif
} // end of namespace Fastor
#endif // INVERSE_H

753
noarch/include/Fastor/backend/lufact.h Normal file
View File

@@ -0,0 +1,753 @@
#ifndef LUFACT_H
#define LUFACT_H
#include "Fastor/meta/meta.h"
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,8>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT a, T *FASTOR_RESTRICT l, T *FASTOR_RESTRICT u) {
*l = 1;
*u = *a;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
// [a11 a12] [1 0] [u11 u12]
// [a21 a22] [l21 1] [0 u22]
const T L21 = A[2]/A[0];
L[0] = 1;
L[1] = 0;
L[2] = L21;
L[3] = 1;
U[0] = A[0];
U[1] = A[1];
U[2] = 0;
U[3] = A[3] - L21 * A[1];
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0];
const T L21 = A[3]/A00;
const T L31 = A[6]/A00;
const T U22 = A[4] - L21 * A[1];
const T U23 = A[5] - L21 * A[2];
const T L32 = (A[7] - L31 * A[1]) / U22;
const T U33 = A[8] - L31 * A[2] - L32 * U23;
L[0] = 1;
L[1] = 0;
L[2] = 0;
L[3] = L21;
L[4] = 1;
L[5] = 0;
L[6] = L31;
L[7] = L32;
L[8] = 1;
U[0] = A00;
U[1] = A[1];
U[2] = A[2];
U[3] = 0;
U[4] = U22;
U[5] = U23;
U[6] = 0;
U[7] = 0;
U[8] = U33;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0];
const T L21 = A[4] /A00;
const T L31 = A[8] /A00;
const T L41 = A[12]/A00;
const T U22 = A[5] - L21 * A[1];
const T U23 = A[6] - L21 * A[2];
const T U24 = A[7] - L21 * A[3];
const T L32 = (A[9 ] - L31 * A[1]) / U22;
const T L42 = (A[13] - L41 * A[1]) / U22;
const T U33 = A[10] - L31 * A[2] - L32 * U23;
const T U34 = A[11] - L31 * A[3] - L32 * U24;
const T L43 = (A[14] - L41 * A[2] - L42 * U23) / U33;
const T U44 = A[15] - L41 * A[3] - L42 * U24 - L43 * U34;
L[0] = 1;
L[1] = 0;
L[2] = 0;
L[3] = 0;
L[4] = L21;
L[5] = 1;
L[6] = 0;
L[7] = 0;
L[8] = L31;
L[9] = L32;
L[10] = 1;
L[11] = 0;
L[12] = L41;
L[13] = L42;
L[14] = L43;
L[15] = 1;
U[0] = A00;
U[1] = A[1];
U[2] = A[2];
U[3] = A[3];
U[4] = 0;
U[5] = U22;
U[6] = U23;
U[7] = U24;
U[8] = 0;
U[9] = 0;
U[10] = U33;
U[11] = U34;
U[12] = 0;
U[13] = 0;
U[14] = 0;
U[15] = U44;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,5>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,6>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T L61 = A[5*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T U26 = A[1*N+5] - L21 * A[5];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T L62 = (A[5*N+1] - L61 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46;
const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55;
const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[0*N+5] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[1*N+5] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[2*N+5] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[3*N+5] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
L[4*N+5] = 0;
L[5*N+0] = L61;
L[5*N+1] = L62;
L[5*N+2] = L63;
L[5*N+3] = L64;
L[5*N+4] = L65;
L[5*N+5] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[0*N+5] = A[5];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[1*N+5] = U26;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[2*N+5] = U36;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[3*N+5] = U46;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
U[4*N+5] = U56;
U[5*N+0] = 0;
U[5*N+1] = 0;
U[5*N+2] = 0;
U[5*N+3] = 0;
U[5*N+4] = 0;
U[5*N+5] = U66;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,7>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T L61 = A[5*N] /A00;
const T L71 = A[6*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T U26 = A[1*N+5] - L21 * A[5];
const T U27 = A[1*N+6] - L21 * A[6];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T L62 = (A[5*N+1] - L61 * A[1]) / U22;
const T L72 = (A[6*N+1] - L71 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26;
const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33;
const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36;
const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44;
const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46;
const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47;
const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55;
const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55;
const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56;
const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57;
const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66;
const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[0*N+5] = 0;
L[0*N+6] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[1*N+5] = 0;
L[1*N+6] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[2*N+5] = 0;
L[2*N+6] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[3*N+5] = 0;
L[3*N+6] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
L[4*N+5] = 0;
L[4*N+6] = 0;
L[5*N+0] = L61;
L[5*N+1] = L62;
L[5*N+2] = L63;
L[5*N+3] = L64;
L[5*N+4] = L65;
L[5*N+5] = 1;
L[5*N+6] = 0;
L[6*N+0] = L71;
L[6*N+1] = L72;
L[6*N+2] = L73;
L[6*N+3] = L74;
L[6*N+4] = L75;
L[6*N+5] = L76;
L[6*N+6] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[0*N+5] = A[5];
U[0*N+6] = A[6];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[1*N+5] = U26;
U[1*N+6] = U27;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[2*N+5] = U36;
U[2*N+6] = U37;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[3*N+5] = U46;
U[3*N+6] = U47;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
U[4*N+5] = U56;
U[4*N+6] = U57;
U[5*N+0] = 0;
U[5*N+1] = 0;
U[5*N+2] = 0;
U[5*N+3] = 0;
U[5*N+4] = 0;
U[5*N+5] = U66;
U[5*N+6] = U67;
U[6*N+0] = 0;
U[6*N+1] = 0;
U[6*N+2] = 0;
U[6*N+3] = 0;
U[6*N+4] = 0;
U[6*N+5] = 0;
U[6*N+6] = U77;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,8>, bool> = false>
FASTOR_INLINE void _lufact(const T *FASTOR_RESTRICT A, T *FASTOR_RESTRICT L, T *FASTOR_RESTRICT U) {
const T A00 = A[0*N];
const T L21 = A[1*N] /A00;
const T L31 = A[2*N] /A00;
const T L41 = A[3*N] /A00;
const T L51 = A[4*N] /A00;
const T L61 = A[5*N] /A00;
const T L71 = A[6*N] /A00;
const T L81 = A[7*N] /A00;
const T U22 = A[1*N+1] - L21 * A[1];
const T U23 = A[1*N+2] - L21 * A[2];
const T U24 = A[1*N+3] - L21 * A[3];
const T U25 = A[1*N+4] - L21 * A[4];
const T U26 = A[1*N+5] - L21 * A[5];
const T U27 = A[1*N+6] - L21 * A[6];
const T U28 = A[1*N+7] - L21 * A[7];
const T L32 = (A[2*N+1] - L31 * A[1]) / U22;
const T L42 = (A[3*N+1] - L41 * A[1]) / U22;
const T L52 = (A[4*N+1] - L51 * A[1]) / U22;
const T L62 = (A[5*N+1] - L61 * A[1]) / U22;
const T L72 = (A[6*N+1] - L71 * A[1]) / U22;
const T L82 = (A[7*N+1] - L81 * A[1]) / U22;
const T U33 = A[2*N+2] - L31 * A[2] - L32 * U23;
const T U34 = A[2*N+3] - L31 * A[3] - L32 * U24;
const T U35 = A[2*N+4] - L31 * A[4] - L32 * U25;
const T U36 = A[2*N+5] - L31 * A[5] - L32 * U26;
const T U37 = A[2*N+6] - L31 * A[6] - L32 * U27;
const T U38 = A[2*N+7] - L31 * A[7] - L32 * U28;
const T L43 = (A[3*N+2] - L41 * A[2] - L42 * U23) / U33;
const T L53 = (A[4*N+2] - L51 * A[2] - L52 * U23) / U33;
const T L63 = (A[5*N+2] - L61 * A[2] - L62 * U23) / U33;
const T L73 = (A[6*N+2] - L71 * A[2] - L72 * U23) / U33;
const T L83 = (A[7*N+2] - L81 * A[2] - L82 * U23) / U33;
const T U44 = A[3*N+3] - L41 * A[3] - L42 * U24 - L43 * U34;
const T U45 = A[3*N+4] - L41 * A[4] - L42 * U25 - L43 * U35;
const T U46 = A[3*N+5] - L41 * A[5] - L42 * U26 - L43 * U36;
const T U47 = A[3*N+6] - L41 * A[6] - L42 * U27 - L43 * U37;
const T U48 = A[3*N+7] - L41 * A[7] - L42 * U28 - L43 * U38;
const T L54 = (A[4*N+3] - L51 * A[3] - L52 * U24 - L53 * U34) / U44;
const T L64 = (A[5*N+3] - L61 * A[3] - L62 * U24 - L63 * U34) / U44;
const T L74 = (A[6*N+3] - L71 * A[3] - L72 * U24 - L73 * U34) / U44;
const T L84 = (A[7*N+3] - L81 * A[3] - L82 * U24 - L83 * U34) / U44;
const T U55 = A[4*N+4] - L51 * A[4] - L52 * U25 - L53 * U35 - L54 * U45;
const T U56 = A[4*N+5] - L51 * A[5] - L52 * U26 - L53 * U36 - L54 * U46;
const T U57 = A[4*N+6] - L51 * A[6] - L52 * U27 - L53 * U37 - L54 * U47;
const T U58 = A[4*N+7] - L51 * A[7] - L52 * U28 - L53 * U38 - L54 * U48;
const T L65 = (A[5*N+4] - L61 * A[4] - L62 * U25 - L63 * U35 - L64 * U45) / U55;
const T L75 = (A[6*N+4] - L71 * A[4] - L72 * U25 - L73 * U35 - L74 * U45) / U55;
const T L85 = (A[7*N+4] - L81 * A[4] - L82 * U25 - L83 * U35 - L84 * U45) / U55;
const T U66 = A[5*N+5] - L61 * A[5] - L62 * U26 - L63 * U36 - L64 * U46 - L65 * U56;
const T U67 = A[5*N+6] - L61 * A[6] - L62 * U27 - L63 * U37 - L64 * U47 - L65 * U57;
const T U68 = A[5*N+7] - L61 * A[7] - L62 * U28 - L63 * U38 - L64 * U48 - L65 * U58;
const T L76 = (A[6*N+5] - L71 * A[5] - L72 * U26 - L73 * U36 - L74 * U46 - L75 * U56) / U66;
const T L86 = (A[7*N+5] - L81 * A[5] - L82 * U26 - L83 * U36 - L84 * U46 - L85 * U56) / U66;
const T U77 = A[6*N+6] - L71 * A[6] - L72 * U27 - L73 * U37 - L74 * U47 - L75 * U57 - L76 * U67;
const T U78 = A[6*N+7] - L71 * A[7] - L72 * U28 - L73 * U38 - L74 * U48 - L75 * U58 - L76 * U68;
const T L87 = (A[7*N+6] - L81 * A[6] - L82 * U27 - L83 * U37 - L84 * U47 - L85 * U57 - L86 * U67) / U77;
const T U88 = A[7*N+7] - L81 * A[7] - L82 * U28 - L83 * U38 - L84 * U48 - L85 * U58 - L86 * U68 - L87 * U78;
// L
L[0*N+0] = 1;
L[0*N+1] = 0;
L[0*N+2] = 0;
L[0*N+3] = 0;
L[0*N+4] = 0;
L[0*N+5] = 0;
L[0*N+6] = 0;
L[0*N+7] = 0;
L[1*N+0] = L21;
L[1*N+1] = 1;
L[1*N+2] = 0;
L[1*N+3] = 0;
L[1*N+4] = 0;
L[1*N+5] = 0;
L[1*N+6] = 0;
L[1*N+7] = 0;
L[2*N+0] = L31;
L[2*N+1] = L32;
L[2*N+2] = 1;
L[2*N+3] = 0;
L[2*N+4] = 0;
L[2*N+5] = 0;
L[2*N+6] = 0;
L[2*N+7] = 0;
L[3*N+0] = L41;
L[3*N+1] = L42;
L[3*N+2] = L43;
L[3*N+3] = 1;
L[3*N+4] = 0;
L[3*N+5] = 0;
L[3*N+6] = 0;
L[3*N+7] = 0;
L[4*N+0] = L51;
L[4*N+1] = L52;
L[4*N+2] = L53;
L[4*N+3] = L54;
L[4*N+4] = 1;
L[4*N+5] = 0;
L[4*N+6] = 0;
L[4*N+7] = 0;
L[5*N+0] = L61;
L[5*N+1] = L62;
L[5*N+2] = L63;
L[5*N+3] = L64;
L[5*N+4] = L65;
L[5*N+5] = 1;
L[5*N+6] = 0;
L[5*N+7] = 0;
L[6*N+0] = L71;
L[6*N+1] = L72;
L[6*N+2] = L73;
L[6*N+3] = L74;
L[6*N+4] = L75;
L[6*N+5] = L76;
L[6*N+6] = 1;
L[6*N+7] = 0;
L[7*N+0] = L81;
L[7*N+1] = L82;
L[7*N+2] = L83;
L[7*N+3] = L84;
L[7*N+4] = L85;
L[7*N+5] = L86;
L[7*N+6] = L87;
L[7*N+7] = 1;
// U
U[0*N+0] = A00;
U[0*N+1] = A[1];
U[0*N+2] = A[2];
U[0*N+3] = A[3];
U[0*N+4] = A[4];
U[0*N+5] = A[5];
U[0*N+6] = A[6];
U[0*N+7] = A[7];
U[1*N+0] = 0;
U[1*N+1] = U22;
U[1*N+2] = U23;
U[1*N+3] = U24;
U[1*N+4] = U25;
U[1*N+5] = U26;
U[1*N+6] = U27;
U[1*N+7] = U28;
U[2*N+0] = 0;
U[2*N+1] = 0;
U[2*N+2] = U33;
U[2*N+3] = U34;
U[2*N+4] = U35;
U[2*N+5] = U36;
U[2*N+6] = U37;
U[2*N+7] = U38;
U[3*N+0] = 0;
U[3*N+1] = 0;
U[3*N+2] = 0;
U[3*N+3] = U44;
U[3*N+4] = U45;
U[3*N+5] = U46;
U[3*N+6] = U47;
U[3*N+7] = U48;
U[4*N+0] = 0;
U[4*N+1] = 0;
U[4*N+2] = 0;
U[4*N+3] = 0;
U[4*N+4] = U55;
U[4*N+5] = U56;
U[4*N+6] = U57;
U[4*N+7] = U58;
U[5*N+0] = 0;
U[5*N+1] = 0;
U[5*N+2] = 0;
U[5*N+3] = 0;
U[5*N+4] = 0;
U[5*N+5] = U66;
U[5*N+6] = U67;
U[5*N+7] = U68;
U[6*N+0] = 0;
U[6*N+1] = 0;
U[6*N+2] = 0;
U[6*N+3] = 0;
U[6*N+4] = 0;
U[6*N+5] = 0;
U[6*N+6] = U77;
U[6*N+7] = U78;
U[7*N+0] = 0;
U[7*N+1] = 0;
U[7*N+2] = 0;
U[7*N+3] = 0;
U[7*N+4] = 0;
U[7*N+5] = 0;
U[7*N+6] = 0;
U[7*N+7] = U88;
}
} // end of namespace Fastor
#endif // LUFACT_H

80
noarch/include/Fastor/backend/lut_inverse.h Normal file
View File

@@ -0,0 +1,80 @@
#ifndef LOWUNITRI_INVERSE_H
#define LOWUNITRI_INVERSE_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
namespace Fastor {
template<typename T, size_t N, enable_if_t_<is_greater_v_<N,4>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst);
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,1>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst) {
*dst = T(1);
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,2>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
/* Compute adjoint: */
dst[0] = 1;
dst[1] = 0;
dst[2] = - src[2];
dst[3] = 1;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,3>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
T src3 = src[3];
T src6 = src[6];
T src7 = src[7];
T src8 = src[8];
/* Compute adjoint: */
dst[0] = 1;
dst[1] = 0;
dst[2] = 0;
dst[3] = - src3 * src8;
dst[4] = 1;
dst[5] = 0;
dst[6] = + src3 * src7 - src6;
dst[7] = - src7;
dst[8] = 1;
}
template<typename T, size_t N, enable_if_t_<is_equal_v_<N,4>, bool> = false>
FASTOR_INLINE void _lowunitri_inverse(const T *FASTOR_RESTRICT src, T *FASTOR_RESTRICT dst)
{
dst[0] = 1;
dst[1] = 0;
dst[2] = 0;
dst[3] = 0;
dst[4] = - src[1*4+0];
dst[5] = 1;
dst[6] = 0;
dst[7] = 0;
T t2 = src[2*4+1];
T t3 = src[2*4+1]*src[3*4+2] - src[3*4+1];
T t4 = src[2*4+0];
T t5 = src[2*4+0]*src[3*4+2] - src[3*4+0];
dst[8] = src[1*4+0]*t2 - t4;
dst[9] = - t2;
dst[10] = 1;
dst[11] = 0;
dst[12] = t5 - src[1*4+0]*t3;
dst[13] = t3;
dst[14] = - src[3*4+2];
dst[15] = 1;
}
} // end of namespace Fastor
#endif // LOWUNITRI_INVERSE_H

209
noarch/include/Fastor/backend/matmul/libxsmm_backend.h Normal file
View File

@@ -0,0 +1,209 @@
#ifndef LIBXSMM_BACKEND_H
#define LIBXSMM_BACKEND_H
#include <Fastor/tensor/Tensor.h>
#ifdef FASTOR_USE_LIBXSMM
#include <libxsmm.h>
namespace Fastor {
namespace blas {
// single
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulNN_libxsmm(
const float * FASTOR_RESTRICT a_data,
const float * FASTOR_RESTRICT b_data,
float * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr float alpha = 1.0;
constexpr float beta = 0.0;
constexpr char transa = 'N';
constexpr char transb = 'N';
libxsmm_sgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&NN /*lda*/,
a_data /*required*/,
&KK /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
// double
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulNN_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'N';
constexpr char transb = 'N';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&NN /*lda*/,
a_data /*required*/,
&KK /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulTN_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'N';
constexpr char transb = 'Y';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&NN /*lda*/,
a_data /*required*/,
&MM /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulNT_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'Y';
constexpr char transb = 'N';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&KK /*lda*/,
a_data /*required*/,
&KK /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<size_t M, size_t K, size_t N>
FASTOR_INLINE
void matmulTT_libxsmm(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
constexpr int MM= M;
constexpr int KK= K;
constexpr int NN= N;
constexpr double alpha = 1.0;
constexpr double beta = 0.0;
constexpr char transa = 'Y';
constexpr char transb = 'Y';
libxsmm_dgemm(
&transa /*transa*/,
&transb /*transb*/,
&NN /*required*/,
&MM /*required*/,
&KK /*required*/,
&alpha /*alpha*/,
b_data /*required*/,
&KK /*lda*/,
a_data /*required*/,
&MM /*ldb*/,
&beta /*beta*/,
out_data /*required*/,
&NN /*ldc*/
);
}
template<typename T, size_t M, size_t K, size_t N,
typename std::enable_if<std::is_same<T,double>::value,bool>::type=0>
FASTOR_INLINE
void matmul_libxsmm(
const T * FASTOR_RESTRICT a_data,
const T * FASTOR_RESTRICT b_data,
T * FASTOR_RESTRICT out_data) {
matmulNN_libxsmm<M,K,N>(a_data,b_data,out_data);
}
template<typename T, size_t M, size_t K, size_t N,
typename std::enable_if<std::is_same<T,float>::value,bool>::type=0>
FASTOR_INLINE
void matmul_libxsmm(
const T * FASTOR_RESTRICT a_data,
const T * FASTOR_RESTRICT b_data,
T * FASTOR_RESTRICT out_data) {
matmulNN_libxsmm<M,K,N>(a_data,b_data,out_data);
}
} // end of namespace blas
} // end of namespace Fastor
#endif // FASTOR_USE_LIBXSMM
#endif // LIBXSMM_BACKEND_H

150
noarch/include/Fastor/backend/matmul/matmul.h Normal file
View File

@@ -0,0 +1,150 @@
#ifndef MATMUL_H
#define MATMUL_H
#include "Fastor/meta/meta.h"
#include "Fastor/backend/matmul/matmul_kernels.h"
#ifdef FASTOR_USE_LIBXSMM
#include "Fastor/backend/matmul/libxsmm_backend.h"
#endif
#ifdef FASTOR_USE_MKL
#include "Fastor/backend/matmul/mkl_backend.h"
#endif
namespace Fastor {
// Forward declare
//-----------------------------------------------------------------------------------------------------------
namespace internal {
template<typename T, size_t M, size_t N>
FASTOR_INLINE
void _matvecmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out);
} // internal
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#if !defined(FASTOR_USE_LIBXSMM) && !defined(FASTOR_USE_MKL)
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) ),bool> = 0>
#else
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<
!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) )
&& is_less_equal<M*N*K/internal::meta_cube<FASTOR_BLAS_SWITCH_MATRIX_SIZE>::value,1>::value,
bool> = 0>
#endif
FASTOR_INLINE
void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
// Non-primitive types
FASTOR_IF_CONSTEXPR (!is_primitive_v_<T>) {
internal::_matmul_base_non_primitive<T,M,K,N>(a,b,out);
return;
}
// Matrix-vector specialisation
FASTOR_IF_CONSTEXPR (N==1UL) {
internal::_matvecmul<T,M,K>(a,b,out);
return;
}
using nativeV = SIMDVector<T,DEFAULT_ABI>;
using V = choose_best_simd_t<nativeV,N>;
// Use specialised kernels
FASTOR_IF_CONSTEXPR((N==V::Size || N==2*V::Size || N==3*V::Size || N==4*V::Size || N==5*V::Size) && V::Size!=1UL) {
internal::_matmul_mk_smalln<T,M,K,N>(a,b,out);
return;
}
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
FASTOR_IF_CONSTEXPR((N<5*V::Size && N!=1UL)) {
internal::_matmul_mk_smalln<T,M,K,N>(a,b,out);
return;
}
#endif
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size <= 1UL) {
internal::_matmul_base<T,M,K,N>(a,b,out);
return;
}
else FASTOR_IF_CONSTEXPR( M*N*K > 27UL && N % V::Size > 1UL) {
internal::_matmul_base_masked<T,M,K,N>(a,b,out);
return;
}
#else
FASTOR_IF_CONSTEXPR( M*N*K > 27UL ) {
internal::_matmul_base<T,M,K,N>(a,b,out);
return;
}
#endif
else
{
// For all other cases where M,N,K is too small
// this simple version is sufficient
constexpr int ROUND_ = ROUND_DOWN(N,V::Size);
for (size_t j=0; j<M; ++j) {
size_t k=0;
for (; k<ROUND_; k+=V::Size) {
V out_row;
for (size_t i=0; i<K; ++i) {
const V brow(&b[i*N+k],false);
const V vec_a(a[j*K+i]);
out_row = fmadd(vec_a,brow,out_row);
}
out_row.store(&out[k+N*j],false);
}
for (; k<N; k++) {
T out_row = 0.;
for (size_t i=0; i<K; ++i) {
out_row += a[j*K+i]*b[i*N+k];
}
out[N*j+k] = out_row;
}
}
}
}
#if defined(FASTOR_USE_LIBXSMM) && !defined(FASTOR_USE_MKL)
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<
!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) )
&& is_greater<M*N*K/internal::meta_cube<FASTOR_BLAS_SWITCH_MATRIX_SIZE>::value,1>::value,
bool> = 0>
FASTOR_INLINE
void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
blas::matmul_libxsmm<T,M,K,N>(a,b,c);
}
#endif
#if !defined(FASTOR_USE_LIBXSMM) && defined(FASTOR_USE_MKL)
template<typename T, size_t M, size_t K, size_t N,
enable_if_t_<
!(M!=K && M==N && (M==2UL || M==3UL || M==4UL || M==8UL) && (is_same_v_<T,float> || is_same_v_<T,double>) )
&& is_greater<M*N*K/internal::meta_cube<FASTOR_BLAS_SWITCH_MATRIX_SIZE>::value,1>::value,
bool> = 0>
FASTOR_INLINE
void _matmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
blas::matmul_mkl<T,M,K,N>(a,b,c);
}
#endif
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#include "Fastor/backend/matmul/matmul_specialisations_kernels.h"
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace
#endif // MATMUL_H

823
noarch/include/Fastor/backend/matmul/matmul_kernels.h Normal file
View File

@@ -0,0 +1,823 @@
#ifndef MATMUL_KERNELS_H
#define MATMUL_KERNELS_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/meta/tensor_meta.h"
namespace Fastor {
namespace internal {
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel)
// unroll the inner-most loop (on unrollOuterloop)
//-----------------------------------------------------------------------------------------------------------
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==2,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==3,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==4,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==5,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
const V bmm4(&b[k*N+j+4*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_scalar_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
T c_ij[unrollOuterloop*numSIMDCols] = {};
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const T bmm0(b[k*N+j]);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] += amm0*bmm0;
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n];
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const int (&maska)[V::Size]) {
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
const V bmm0(maskload<V>(&b[k*N+j],maska));
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]);
}
}
}
template<typename T, typename MaskT, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_matmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const MaskT mask) {
V bmm0;
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = 0; k < K; ++k) {
bmm0.mask_load(&b[k*N+j],mask,false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false);
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the base implementation of matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as gemm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in a scalar fashion
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; ++j) {
interior_block_matmul_scalar_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[unrollOuterloop];
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
T c_ij[unrollOuterloop] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * b[k*N+j];
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i + n)*N+j] = c_ij[n];
}
}
}
// Now treat the remaining M-M1 rows
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_matmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
T c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] += a[n*K+k] * b[k*N+j];
c[n*N+j] = c_ij[n-M1];
}
}
for (size_t n = M1; n < M; ++n) {
c[n*N+j] = c_ij[n-M1];
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the base implementation of matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as gemm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in vector mode with masked and conditional load/stores.
// Note that conditional load/store requires at least AVX intrinsics
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
int maska[V::Size];
std::fill(maska,&maska[V::Size], -1);
for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0;
#ifdef FASTOR_HAS_AVX512_MASKS
const auto mask = array_to_mask(maska);
#endif
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; j+= N-N1) {
#ifdef FASTOR_HAS_AVX512_MASKS
interior_block_matmul_mask_impl<T,decltype(mask),V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,mask);
#else
interior_block_matmul_mask_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,maska);
#endif
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_matmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[unrollOuterloop];
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
V c_ij[unrollOuterloop];
for (size_t k = 0; k < K; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[(i + n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n].mask_store(&c[(i+n)*N+j],mask,false);
#else
maskstore(&c[(i+n)*N+j],maska,c_ij[n]);
#endif
}
}
}
// Now treat the remaining M-M1 rows
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_matmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
V c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[n*K+k];
c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]);
}
}
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n-M1].mask_store(&c[n*N+j],mask,false);
#else
maskstore(&c[n*N+j],maska,c_ij[n-M1]);
#endif
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// matmul kernel for non-fundamental types
// The assumption here is that non-fundamental types are not SIMD vectorisable for instance
// Tensor<std::vector<T>,3,3> or Tensor<Tensor<...>,...> plus they cannot fuse [do fused-add-multiply]
// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in
// the inner most loops of matmul
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
// There is no SIMD here as V::Size == 1 anyway
// No outer loop unrolling otherwise the innermost loop
// will create unnecessary temporaries
for (size_t i=0; i<M; ++i) {
// V::Size == 1 so this loop can't be unrolled
for (size_t j=0; j<N; ++j) {
// This could potentially cost as opposed to directly writing in to c
T tmp {};
for (size_t k=0; k<K; ++k) {
tmp += a[i*K+k]*b[k*N+j];
}
c[i*N+j] = tmp;
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// Other variants and slightly older implementations
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the same implementation as the above case but does not unroll on block sizes and does not require
// the registers to be zeroed out but K must be !=1
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_mkn_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// Get 10 parallel independent chains of accumulators for bigger matrices
constexpr size_t unrollOuterloop = M >= 64 ? 10UL : (M % 8 == 0 ? 8UL : V::Size);
// The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore
// the last rows may need special treatment if M is not a multiple of unrollOuterloop.
// M0 is the number of rows that can safely be iterated with a stride of
// unrollOuterloop.
constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop;
for (size_t i = 0; i < M0; i += unrollOuterloop) {
// The iteration over the column index of b and c uses a stride of V::Size. This
// enables row-vector loads (from b) and stores (to c). The matrix storage is
// padded accordingly, ensuring correct bounds and alignment.
for (size_t j = 0; j < N; j += V::Size) {
// This temporary variables are used to accumulate the results of the products
// producing the new values for the c matrix. This variable is necessary
// because we need a V object for data-parallel accumulation. Storing to c
// directly stores to scalar objects and thus would drop the ability for
// data-parallel (SIMD) addition.
V c_ij[unrollOuterloop];
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = a[(i + n)*K]*V(&b[j]);
}
for (size_t k = 1; k < K - 1; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j]);
c_ij[n].store(&c[(i + n)*N+j]);
}
}
}
}
// This is the same implementation as the above case but does not unroll on block sizes and does not require
// the registers to be zeroed out but K must be !=1
template<typename T, size_t M, size_t K, size_t N>
FASTOR_INLINE
void _matmul_mkn_non_square(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
// This variant strictly cannot deal outer-product i.e. with K==1
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// Get 10 parallel independent chains of accumulators for bigger matrices
constexpr size_t unrollOuterloop = M < V::Size ? 1UL :
(( M >= 64 && K > 10 && N > V::Size ) ? 10UL : (M % 8 == 0 && N > V::Size ? 8UL : V::Size));
constexpr bool isPadded = N % V::Size == 0;
// The row index (for a and c) is unrolled using the unrollOuterloop stride. Therefore
// the last rows may need special treatment if M is not a multiple of unrollOuterloop.
// M0 is the number of rows that can safely be iterated with a stride of
// unrollOuterloop.
constexpr size_t M0 = M / unrollOuterloop * unrollOuterloop;
constexpr size_t N0 = N / V::Size * V::Size;
for (size_t i = 0; i < M0; i += unrollOuterloop) {
// The iteration over the column index of b and c uses a stride of V::size(). This
// enables row-vector loads (from b) and stores (to c). The matrix storage is
// padded accordingly, ensuring correct bounds and alignment.
size_t j = 0;
for (; j < N0; j += V::Size) {
// This temporary variables are used to accumulate the results of the products
// producing the new values for the c matrix. This variable is necessary
// because we need a V object for data-parallel accumulation. Storing to c
// directly stores to scalar objects and thus would drop the ability for
// data-parallel (SIMD) addition.
V c_ij[unrollOuterloop];
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] = a[(i + n)*K]*V(&b[j], isPadded);
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * V(&b[k*N+j], false);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] += a[(i + n)*K+(K - 1)] * V(&b[(K - 1)*N+j], false);
c_ij[n].store(&c[(i + n)*N+j], isPadded);
}
}
// Remainder N - N0 columns
for (; j < N; ++j) {
T c_ij[unrollOuterloop];
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] = a[(i + n)*K]*b[j];
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * b[k*N+j];
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) { // correct
c_ij[n] += a[(i + n)*K+(K - 1)] * b[(K - 1)*N+j];
c[(i + n)*N+j] = c_ij[n];
}
}
}
// This final loop treats the remaining M - M0 rows.
size_t j = 0;
for (; j < N0; j += V::Size) {
V c_ij[M-M0];
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] = a[n*K] * V(&b[j], isPadded);
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+k] * V(&b[k*N+j], false);
}
}
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+(K - 1)] * V(&b[(K - 1)*N+j], false);
c_ij[n - M0].store(&c[n*N+j], isPadded);
}
}
for (; j < N; ++j) {
T c_ij[M-M0];
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] = a[n*K] * b[j];
}
for (size_t k = 1; k < K - 1; ++k) { // correct
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+k] * b[k*N+j];
}
}
for (size_t n = M0; n < M; ++n) { // correct
c_ij[n - M0] += a[n*K+(K - 1)] * b[(K - 1)*N+j];
c[n*N+j] = c_ij[n - M0];
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace internal
} // end of namespace Fastor
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#include "Fastor/backend/matmul/matmul_mk_smalln.h"
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
#endif // MATMUL_KERNELS_H

3271
noarch/include/Fastor/backend/matmul/matmul_mk_smalln.h Normal file
View File

File diff suppressed because it is too large Load Diff

1187
noarch/include/Fastor/backend/matmul/matmul_specialisations_kernels.h Normal file
View File

File diff suppressed because it is too large Load Diff

67
noarch/include/Fastor/backend/matmul/mkl_backend.h Normal file
View File

@@ -0,0 +1,67 @@
#ifndef MKL_BACKEND_H
#define MKL_BACKEND_H
#include <Fastor/tensor/Tensor.h>
#ifdef FASTOR_USE_MKL
// Explicitly activate the jit
#ifndef MKL_DIRECT_CALL_SEQ_JIT
#define MKL_DIRECT_CALL_SEQ_JIT
#endif
#include <mkl.h>
namespace Fastor {
namespace blas {
// single
template<typename T, size_t M, size_t K, size_t N, enable_if_t_<is_same_v_<T,float>,bool> = false>
void matmul_mkl(
const float * FASTOR_RESTRICT a_data,
const float * FASTOR_RESTRICT b_data,
float * FASTOR_RESTRICT out_data) {
cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N);
}
// double
template<typename T, size_t M, size_t K, size_t N, enable_if_t_<is_same_v_<T,double>,bool> = false>
void matmul_mkl(
const double * FASTOR_RESTRICT a_data,
const double * FASTOR_RESTRICT b_data,
double * FASTOR_RESTRICT out_data) {
cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
M, N, K, 1.0, a_data, K, b_data, N, 0.0, out_data, N);
}
#if 0
// dedicated jit api, but the jit kernel has to be created before hand
template<typename T, size_t M, size_t K, size_t N>
Tensor<T,M,N> matmul_mkl_jit_api(const Tensor<T,M,K> &a, const Tensor<T,K,N> &b) {
Tensor<T,M,N> out;
void* jitter;
mkl_jit_status_t status = mkl_jit_create_dgemm(&jitter, MKL_ROW_MAJOR, MKL_NOTRANS, MKL_NOTRANS, M, N, K, 1.0, K, N, 0.0, N);
dgemm_jit_kernel_t _dgemm_kernel = mkl_jit_get_dgemm_ptr(jitter);
_dgemm_kernel(jitter, a.data(), b.data(), out.data());
mkl_jit_destroy(jitter);
return out;
}
#endif
} // end of namespace blas
} // end of namespace Fastor
#endif // FASTOR_USE_MKL
#endif // MKL_BACKEND_H

803
noarch/include/Fastor/backend/matmul/tmatmul.h Normal file
View File

@@ -0,0 +1,803 @@
#ifndef MATMUL_KERNELS2_H
#define MATMUL_KERNELS2_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
#include "Fastor/meta/meta.h"
#include "Fastor/meta/tensor_meta.h"
namespace Fastor {
namespace internal {
// TRMM implementation of Fastor - matrix-matrix multiplication when either or both operands are
// lower or upper triangular. The matrices do not need to be square and trapezoidal cases are also
// covered. For big matrices the speed-up is 2X or even better over matmul for when one operand is
// is triangular and nearly 4X for when both operands are triangular.
// For small matrices due to aggressive unrolling for SIMD the matrices cannot be exactly traversed
// within their triangular part(s) and a bit of the non-triangular part(s) need(s) to be loaded as well
// hence, the performance may not be exactly 2X/4X over the general matmul case
// The functions here are exact replica of those in matmul_kernels.h and will be eventually
// merged together as these variants have no associated overhead for the general case of matmul
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
/*
For triangular matmul only the iteration span of K is modified
using the following logic [lt = Lower, ut=Upper]
// if lhs == lt
const size_t kfirst = 0;
const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K);
// if lhs == ut
const size_t kfirst = i;
const size_t klast = K;
// if rhs == lt
const size_t kfirst = j;
const size_t klast = K;
// if rhs == ut
const size_t kfirst = 0;
const size_t klast = min(j+1,K); // or min(j+unrollOuterloop,K);
// both lower
const size_t kfirst = j;
const size_t klast = min(i+1,K); // or min(i+unrollOuterloop,K);
// if lhs == lt && rhs == ut
const size_t kfirst = 0;
const size_t klast = min(min(i+1,j+1),K); // or min(min(i+unrollOuterloop,j+unrollInnerloop),K);
// if lhs == ut && rhs == lt
const size_t kfirst = max(i,j);
const size_t klast = K;
// if both upper
const size_t kfirst = i;
const size_t klast = min(j+1,K); // or min(j+unrollInnerloop,K);
*/
template<typename T, T K, T unrollOuterloop=1,T unrollInnerloop=1, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
constexpr FASTOR_INLINE T find_kfirst(const T i, const T j) {
return is_same_v_<LhsType,UpLoType::Lower> || is_same_v_<LhsType,UpLoType::General> ?
( is_same_v_<RhsType,UpLoType::Lower> ? j : 0UL ) :
(is_same_v_<LhsType,UpLoType::Upper> ? ( is_same_v_<RhsType,UpLoType::Lower> ? internal::max_(i,j) : i ) : 0UL );
}
template<typename T, T K, T unrollOuterloop=1,T unrollInnerloop=1, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
constexpr FASTOR_INLINE T find_klast(const T i, const T j) {
return is_same_v_<LhsType,UpLoType::Lower> ?
( is_same_v_<RhsType,UpLoType::Upper> ? internal::min_(internal::min_(i+unrollOuterloop,j+unrollInnerloop),K) : internal::min_(i+unrollOuterloop,K) ) :
(is_same_v_<LhsType,UpLoType::Upper> || (is_same_v_<LhsType,UpLoType::General>) ?
( is_same_v_<RhsType,UpLoType::Upper> ? internal::min_(j+unrollInnerloop,K) : K ) : K );
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// A set of helper functions for the inner blocks of matmul. Almost all compilers (GCC/CLang/Intel)
// unroll the inner-most loop (on unrollOuterloop)
//-----------------------------------------------------------------------------------------------------------
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==2,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==3,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==4,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==5,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(&b[k*N+j],false);
const V bmm1(&b[k*N+j+V::Size],false);
const V bmm2(&b[k*N+j+2*V::Size],false);
const V bmm3(&b[k*N+j+3*V::Size],false);
const V bmm4(&b[k*N+j+4*V::Size],false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
c_ij[n+unrollOuterloop] = fmadd(amm0,bmm1,c_ij[n+unrollOuterloop]);
c_ij[n+2*unrollOuterloop] = fmadd(amm0,bmm2,c_ij[n+2*unrollOuterloop]);
c_ij[n+3*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+3*unrollOuterloop]);
c_ij[n+4*unrollOuterloop] = fmadd(amm0,bmm3,c_ij[n+4*unrollOuterloop]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i+ii*unrollOuterloop+n)*N+j],false);
c_ij[n+unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+V::Size],false);
c_ij[n+2*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+2*V::Size],false);
c_ij[n+3*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+3*V::Size],false);
c_ij[n+4*unrollOuterloop].store(&c[(i+ii*unrollOuterloop+n)*N+j+4*V::Size],false);
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_scalar_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
T c_ij[unrollOuterloop*numSIMDCols] = {};
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const T bmm0(b[k*N+j]);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const T amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] += amm0*bmm0;
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i+ii*unrollOuterloop+n)*N+j] = c_ij[n];
}
}
}
template<typename T, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const int (&maska)[V::Size]) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
const V bmm0(maskload<V>(&b[k*N+j],maska));
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
maskstore(&c[(i+ii*unrollOuterloop+n)*N+j],maska,c_ij[n]);
}
}
}
template<typename T, typename MaskT, typename V, size_t M, size_t K, size_t N, size_t unrollOuterloop, size_t numSIMDRows, size_t numSIMDCols,
typename LhsType = UpLoType::General, typename RhsType = UpLoType::General,
typename std::enable_if<numSIMDCols==1,bool>::type = false>
FASTOR_INLINE
void interior_block_tmatmul_mask_impl(
const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c,
const size_t i, const size_t j, const MaskT mask) {
V bmm0;
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop*numSIMDRows,numSIMDCols*V::Size,LhsType,RhsType>(i,j);
for (size_t ii = 0; ii < numSIMDRows; ++ii) {
V c_ij[unrollOuterloop*numSIMDCols];
// Loop over columns of a (rows of b)
for (size_t k = kfirst; k < klast; ++k) {
bmm0.mask_load(&b[k*N+j],mask,false);
for (size_t n = 0; n < unrollOuterloop; ++n) {
const V amm0 = a[(i+ii*unrollOuterloop+n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].mask_store(&c[(i+ii*unrollOuterloop+n)*N+j],mask,false);
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This is the base implementation of triangular matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as trmm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in a scalar fashion
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul_base(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = choose_best_simd_t<SIMDVector<T,DEFAULT_ABI>,N>;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols,LhsType,RhsType>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1,LhsType,RhsType>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; ++j) {
interior_block_tmatmul_scalar_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1,LhsType,RhsType>(a,b,c,i,j);
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols,LhsType,RhsType>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
V c_ij[unrollOuterloop];
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,1,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,1,LhsType,RhsType>(i,j);
T c_ij[unrollOuterloop] = {};
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] += a[(i + n)*K+k] * b[k*N+j];
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c[(i + n)*N+j] = c_ij[n];
}
}
}
// Now treat the remaining M-M1 rows - here the klast - kfirst range is not used
// so the implementation is exactly the same as matmul_base
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_tmatmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; ++j) {
T c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] += a[n*K+k] * b[k*N+j];
c[n*N+j] = c_ij[n-M1];
}
}
for (size_t n = M1; n < M; ++n) {
c[n*N+j] = c_ij[n-M1];
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// This the base implementation of triangular matrix-matrix multiplication for all 2D tensors and
// higher order tensor products that can be expressed as trmm
// The function uses two level unrolling one based on block sizes and one based on register widths
// with any remainder left treated in vector mode with masked and conditional load/stores.
// Note that conditional load/store requires at least AVX intrinsics
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul_base_masked(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
// This parameter can be adjusted and does not need to be 4UL/8UL etc
// constexpr size_t unrollOuterloop = M % 5UL == 0 ? 5UL : 4UL;
constexpr size_t unrollOuterloop = 4UL;
#ifndef FASTOR_MATMUL_OUTER_BLOCK_SIZE
// Unroll the rows of (a and c) (M) by [numSIMDRows * V::Size]
constexpr size_t numSIMDRows = M % (unrollOuterloop * 3UL) == 0 ? 3UL : (M < 2UL*V::Size ? 1UL : 2UL);
#else
constexpr size_t numSIMDRows = FASTOR_MATMUL_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_MATMUL_INNER_BLOCK_SIZE
// Unroll the columns of (b and c) (N) by [numSIMDCols * V::Size]
constexpr size_t numSIMDCols = (N % (V::Size * 3UL) == 0 && M % (V::Size * 3UL) == 0 && N > 24UL) ? 3UL : 2UL;
#else
constexpr size_t numSIMDCols = FASTOR_MATMUL_INNER_BLOCK_SIZE;
#endif
// The goal is to get 10 parallel independent chains of accumulators
// to saturate the pipeline by having a completely unrolled block of
// [(unrollOuterloop) * (numSIMDCols)] at a time. A minimum value of
// unrollOuterloop=4 ensures a minimum of 8 independent parallel chains
// while a maximum of 12 i.e. for numSIMDCols=2 and numSIMDCols=3 respectively.
// However, most recent X86/64 architectures can do 2 FMAs per load so
// so unrolling with numSIMDCols > 2 is not beneficial
constexpr size_t unrollOuterBlock = numSIMDRows*unrollOuterloop;
// Number of rows of c (M) that can be safely unrolled with this block size.
constexpr size_t M0 = M / unrollOuterBlock * unrollOuterBlock;
constexpr size_t unrollInnerBlock = numSIMDCols*V::Size;
// Number of columns of c (N) that can be safely unrolled with this block size
constexpr size_t N0 = N / unrollInnerBlock * unrollInnerBlock;
// Number of columns of c (N) that can be safely unrolled with V::Size
constexpr size_t N1 = N / V::Size * V::Size;
int maska[V::Size];
std::fill(maska,&maska[V::Size], -1);
for (size_t jj=0; jj < V::Size - (N-N1); ++jj) maska[jj] = 0;
#ifdef FASTOR_HAS_AVX512_MASKS
const auto mask = array_to_mask(maska);
#endif
size_t i = 0;
for (; i < M0; i += unrollOuterBlock) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j);
}
// Remaining N - N1 columns
for (; j < N; j+= N-N1) {
#ifdef FASTOR_HAS_AVX512_MASKS
interior_block_matmul_mask_impl<T,decltype(mask),V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,mask);
#else
interior_block_matmul_mask_impl<T,V,M,K,N,unrollOuterloop,numSIMDRows,1>(a,b,c,i,j,maska);
#endif
}
}
// The remaining M-M0 rows are now unrolled yet again by unrollOuterloop.
// This is necessary as for small sizes the earlier block loop may not be
// triggered if the size of the block is bigger than the number of rows of
// (a and c) i.e. M
constexpr size_t M1 = (M / unrollOuterloop * unrollOuterloop);
for (; i < M1; i += unrollOuterloop) {
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
interior_block_tmatmul_impl<T,V,M,K,N,unrollOuterloop,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
V c_ij[unrollOuterloop];
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n] = fmadd(V(a[(i + n)*K+k]), V(&b[k*N+j],false), c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
c_ij[n].store(&c[(i + n)*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
const size_t kfirst = find_kfirst<size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,unrollOuterloop,V::Size,LhsType,RhsType>(i,j);
V c_ij[unrollOuterloop];
for (size_t k = kfirst; k < klast; ++k) {
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[(i + n)*K+k];
c_ij[n] = fmadd(amm0,bmm0,c_ij[n]);
}
}
for (size_t n = 0; n < unrollOuterloop; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n].mask_store(&c[(i+n)*N+j],mask,false);
#else
maskstore(&c[(i+n)*N+j],maska,c_ij[n]);
#endif
}
}
}
// Now treat the remaining M-M1 rows - here the klast - kfirst range is not used
// so the implementation is exactly the same as matmul_base
FASTOR_IF_CONSTEXPR (M-M1 > 0) {
// Hack to get around zero length array issue
constexpr size_t MM1 = M-M1 != 0 ? M-M1 : 1;
size_t j = 0;
for (; j < N0; j += unrollInnerBlock) {
// If MM1==0 the function never gets invoked anyway
interior_block_tmatmul_impl<T,V,M,K,N,MM1,1,numSIMDCols>(a,b,c,i,j);
}
// Remaining N - N0 columns
for (; j < N1; j += V::Size) {
V c_ij[MM1];
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1] = fmadd(V(a[n*K+k]), V(&b[k*N+j],false), c_ij[n-M1]);
c_ij[n-M1].store(&c[n*N+j],false);
}
}
for (size_t n = M1; n < M; ++n) {
c_ij[n-M1].store(&c[n*N+j],false);
}
}
// Remaining N - N1 columns
for (; j < N; j+=N-N1) {
V c_ij[MM1] = {};
for (size_t k = 0; k < K; ++k) {
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
V bmm0; bmm0.mask_load(&b[k*N+j],mask);
#else
const V bmm0(maskload<V>(&b[k*N+j],maska));
#endif
const V amm0 = a[n*K+k];
c_ij[n-M1] = fmadd(amm0,bmm0,c_ij[n-M1]);
}
}
for (size_t n = M1; n < M; ++n) {
#ifdef FASTOR_HAS_AVX512_MASKS
c_ij[n-M1].mask_store(&c[n*N+j],mask,false);
#else
maskstore(&c[n*N+j],maska,c_ij[n-M1]);
#endif
}
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// tmatmul kernel for non-fundamental types
// The assumption here is that non-fundamental types are not SIMD vectorisable for instance
// Tensor<std::vector<T>,3,3> or Tensor<Tensor<...>,...> plus they cannot fuse [do fused-add-multiply]
// so operations like [c += a*b] or potentially [c = c + a*b] might introduce multiple copies in
// the inner most loops of matmul
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul_base_non_primitive(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT c) {
// There is no SIMD here as V::Size == 1 anyway
// No outer loop unrolling otherwise the innermost loop
// will create unnecessary temporaries
for (size_t i=0; i<M; ++i) {
// V::Size == 1 so this loop can't be unrolled
for (size_t j=0; j<N; ++j) {
const size_t kfirst = find_kfirst<size_t,K,1,1,LhsType,RhsType>(i,j);
const size_t klast = find_klast <size_t,K,1,1,LhsType,RhsType>(i,j);
// This could potentially cost as opposed to directly writing in to c
T tmp {};
for (size_t k=kfirst; k<klast; ++k) {
tmp += a[i*K+k]*b[k*N+j];
}
c[i*N+j] = tmp;
}
}
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace internal
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
// Backend tmatmul function
template<typename T, size_t M, size_t K, size_t N, typename LhsType = UpLoType::General, typename RhsType = UpLoType::General>
FASTOR_INLINE
void _tmatmul(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
// Non-primitive types
FASTOR_IF_CONSTEXPR (!is_primitive_v_<T>) {
internal::_tmatmul_base_non_primitive<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
}
// Use specialised kernels
#if defined(FASTOR_AVX2_IMPL) || defined(FASTOR_HAS_AVX512_MASKS)
using nativeV = SIMDVector<T,DEFAULT_ABI>;
using V = choose_best_simd_t<nativeV,N>;
FASTOR_IF_CONSTEXPR(N % V::Size <= 1UL) {
internal::_tmatmul_base<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
}
else {
internal::_tmatmul_base_masked<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
}
#else
internal::_tmatmul_base<T,M,K,N,LhsType,RhsType>(a,b,out);
return;
#endif
}
//-----------------------------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------------------------
} // end of namespace Fastor
#endif // MATMUL_KERNELS_H

149
noarch/include/Fastor/backend/norm.h Normal file
View File

@@ -0,0 +1,149 @@
#ifndef NORM_H
#define NORM_H
#include "Fastor/config/config.h"
#include "Fastor/meta/meta.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
template<typename T, size_t N,
#ifdef FASTOR_AVX512_IMPL
enable_if_t_<is_less_v_<8*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#else
enable_if_t_<is_less_v_<4*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#endif
FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
T _scal = 0;
#ifdef FASTOR_AVX512_IMPL
V omm0, omm1, omm2, omm3, omm4, omm5, omm6, omm7;
#else
V omm0, omm1, omm2, omm3;
#endif
FASTOR_INDEX i = 0;
// With AVX utilises all the 16 registers but hurts the performance
// due to spill if eval has created temporary registers so only
// activated for AVX512
#ifdef FASTOR_AVX512_IMPL
for (; i < ROUND_DOWN(N,8*V::Size); i+=8*V::Size) {
const V smm0(&a[i] , false);
const V smm1(&a[i+V::Size] , false);
const V smm2(&a[i+2*V::Size] , false);
const V smm3(&a[i+3*V::Size] , false);
const V smm4(&a[i+4*V::Size] , false);
const V smm5(&a[i+5*V::Size] , false);
const V smm6(&a[i+6*V::Size] , false);
const V smm7(&a[i+7*V::Size] , false);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
omm2 = fmadd(smm2,smm2,omm2);
omm3 = fmadd(smm3,smm3,omm3);
omm4 = fmadd(smm4,smm4,omm4);
omm5 = fmadd(smm5,smm5,omm5);
omm6 = fmadd(smm6,smm6,omm6);
omm7 = fmadd(smm7,smm7,omm7);
}
#endif
for (; i < ROUND_DOWN(N,4*V::Size); i+=4*V::Size) {
const V smm0(&a[i] , false);
const V smm1(&a[i+V::Size] , false);
const V smm2(&a[i+2*V::Size] , false);
const V smm3(&a[i+3*V::Size] , false);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
omm2 = fmadd(smm2,smm2,omm2);
omm3 = fmadd(smm3,smm3,omm3);
}
for (; i < ROUND_DOWN(N,2*V::Size); i+=2*V::Size) {
const V smm0(&a[i] , false);
const V smm1(&a[i+V::Size] , false);
omm0 = fmadd(smm0,smm0,omm0);
omm1 = fmadd(smm1,smm1,omm1);
}
for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) {
const V smm0(&a[i] , false);
omm0 = fmadd(smm0,smm0,omm0);
}
for (; i < N; ++i) {
const auto smm0(a[i]);
_scal += smm0*smm0;
}
#ifdef FASTOR_AVX512_IMPL
return sqrts( (omm0 + omm1 + omm2 + omm3 + omm4 + omm5 + omm6 + omm7).sum() + _scal);
#else
return sqrts( (omm0 + omm1 + omm2 + omm3).sum() + _scal);
#endif
}
template<typename T, size_t N,
#ifdef FASTOR_AVX512_IMPL
enable_if_t_<is_greater_equal_v_<8*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#else
enable_if_t_<is_greater_equal_v_<4*internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type::Size, N >, bool> = false>
#endif
FASTOR_INLINE T _norm(const T* FASTOR_RESTRICT a) {
using V = typename internal::choose_best_simd_type<SIMDVector<T,DEFAULT_ABI>,N>::type;
T _scal = 0;
V omm0;
FASTOR_INDEX i = 0;
for (; i < ROUND_DOWN(N,V::Size); i+=V::Size) {
const V smm0(&a[i] , false);
omm0 = fmadd(smm0,smm0,omm0);
}
for (; i < N; ++i) {
const auto smm0(a[i]);
_scal += smm0*smm0;
}
return sqrts( omm0.sum() + _scal);
}
#ifdef FASTOR_SSE4_2_IMPL
template<>
FASTOR_INLINE float _norm<float,4>(const float * FASTOR_RESTRICT a) {
// IVY 33 OPS / HW 31 OPS
__m128 a_reg = _mm_load_ps(a);
return _mm_cvtss_f32(_mm_sqrt_ps(_add_ps(_mm_mul_ps(a_reg,a_reg))));
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE float _norm<float,9>(const float * FASTOR_RESTRICT a) {
// IVY & HW 61 OPS
__m256 a_reg = _mm256_loadu_ps(a);
__m128 a_end = _mm_load_ss(a+8);
__m128 a0 = _add_ps(_mm256_mul_ps(a_reg,a_reg));
__m128 a1 = _add_ps(_mm_mul_ps(a_end,a_end));
return _mm_cvtss_f32(_mm_sqrt_ps(_mm_add_ss(a0,a1)));
}
template<>
FASTOR_INLINE double _norm<double,4>(const double * FASTOR_RESTRICT a) {
// IVY 34 OPS / HW 36 OPS
__m256d a_reg = _mm256_loadu_pd(a);
return _mm_cvtsd_f64(_mm_sqrt_pd(_add_pd(_mm256_mul_pd(a_reg,a_reg))));
}
template<>
FASTOR_INLINE double _norm<double,9>(const double * FASTOR_RESTRICT a) {
// IVY 63 OPS / HW 67 OPS
__m256d a_low = _mm256_loadu_pd(a);
__m256d a_high = _mm256_loadu_pd(a+4);
__m128d a_end = _mm_load_sd(a+8);
__m128d a0 = _add_pd(_mm256_mul_pd(a_low,a_low));
__m128d a1 = _add_pd(_mm256_mul_pd(a_high,a_high));
__m128d a2 = _add_pd(_mm_mul_pd(a_end,a_end));
return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_add_sd(a2,(_mm_add_sd(a0,a1)))));
}
#endif
}
#endif // NORM_H

292
noarch/include/Fastor/backend/outer.h Normal file
View File

@@ -0,0 +1,292 @@
#ifndef OUTER_H
#define OUTER_H
#include "Fastor/config/config.h"
namespace Fastor {
template<typename T, size_t M0, size_t N0, size_t M1, size_t N1>
FASTOR_HINT_INLINE void _outer(const T * FASTOR_RESTRICT a, const T * FASTOR_RESTRICT b, T * FASTOR_RESTRICT out) {
for (size_t i=0; i<M0; ++i) {
for (size_t j=0; j<N0; ++j) {
for (size_t k=0; k<M1; ++k) {
for (size_t l=0; l<N1; ++l) {
out[i*N1*M1*N0+j*M1*N0+k*N0+l] += a[i*N0+j]*b[k*N1+l];
}
}
}
}
}
#ifdef FASTOR_SSE4_2_IMPL
// The followings are Voigt overloads
template<>
FASTOR_HINT_INLINE void _outer<float,2,2,2,2>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 31 OPS
// Fetch a to L1-cache
// _mm_prefetch(a,_MM_HINT_T0);
__m128 a0 = _mm_set1_ps(a[0]);
__m128 a1 = _mm_set1_ps(a[1]);
// __m128 a2 = _mm_set1_ps(a[2]);
__m128 a3 = _mm_set1_ps(a[3]);
__m128 bs = _mm_load_ps(b);
__m128 r0 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(1,2,3,0));
__m128 r1 = _mm_shuffle_ps(bs,bs,_MM_SHUFFLE(3,1,2,0));
__m128 r2 = _mm_mul_ps(HALFPS,_mm_add_ps(r1,bs));
r2 = _mm_shuffle_ps(r2,r2,_MM_SHUFFLE(3,0,2,1));
// __m128 r3 = _mm_shift1_ps(_mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0)));
__m128 r3 = _mm_shuffle_ps(r0,r2,_MM_SHUFFLE(0,0,1,0));
// row0
__m128 row0 = _mm_mul_ps(a0,r3);
_mm_store_ps(out, row0);
// row1
__m128 row1 = _mm_mul_ps(a3,r3);
row1 = _mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+3,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,1)));
// _mm_storeu_ps(out+4,row1);
_mm_store_ps(out+4,row1);
// row2
__m128 row2 = _mm_mul_ps(a1,r3);
row2 = _mm_shuffle_ps(row2,row2,_MM_SHUFFLE(1,0,3,2));
_mm_store_ss(out+6,_mm_shuffle_ps(row0,row0,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+7,_mm_shuffle_ps(row1,row1,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+8, row2);
}
template<>
FASTOR_HINT_INLINE void _outer<float,3,3,3,3>(const float * FASTOR_RESTRICT a, const float * FASTOR_RESTRICT b, float * FASTOR_RESTRICT out) {
// 81 OPS
// Fetch a to L1-cache
// _mm_prefetch(a,_MM_HINT_T0);
__m128 a0 = _mm_set1_ps(a[0]);
__m128 a1 = _mm_set1_ps(a[1]);
__m128 a2 = _mm_set1_ps(a[2]);
// __m128 a3 = _mm_set1_ps(a[3]);
__m128 a4 = _mm_set1_ps(a[4]);
__m128 a5 = _mm_set1_ps(a[5]);
// __m128 a6 = _mm_set1_ps(a[6]);
// __m128 a7 = _mm_set1_ps(a[7]);
__m128 a8 = _mm_set1_ps(a[8]);
__m128 b_low = _mm_load_ps(b);
__m128 b_high =_mm_load_ps(b+4);
__m128 b_end = _mm_load_ss(b+8);
__m128 b_diag = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(1,0,1,0));
b_diag = _mm_shuffle_ps(b_diag,b_end,_MM_SHUFFLE(1,0,2,0));
__m128 ofdiag_str = _mm_shift1_ps(_mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,1,2,1)));
__m128 ofdiag_rev = _mm_shuffle_ps(b_low,b_high,_MM_SHUFFLE(3,2,2,3));
ofdiag_rev = _mm_shift1_ps(_mm_shuffle_ps(ofdiag_rev,ofdiag_rev,_MM_SHUFFLE(3,3,2,0)));
// Compute this only once
__m128 half_add_diag = _mm_mul_ps(HALFPS,_mm_add_ps(ofdiag_str,ofdiag_rev));
// row0
__m128 c0_d = _mm_mul_ps(b_diag,a0);
__m128 c0_off = _mm_mul_ps(a0,half_add_diag);
c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ps(out,c0_d);
_mm_storeu_ps(out+3,c0_off);
// row1
__m128 c1_d = _mm_mul_ps(b_diag,a4);
__m128 c1_off = _mm_mul_ps(a4,half_add_diag);
c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1));
c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1)));
_mm_storeu_ps(out+7,c1_d);
_mm_storeu_ps(out+9,c1_off);
// row2
__m128 c2_d = _mm_mul_ps(b_diag,a8);
__m128 c2_off = _mm_mul_ps(a8,half_add_diag);
c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2)));
_mm_storeu_ps(out+15,c2_off);
// row3
__m128 c3_off = _mm_mul_ps(a1,half_add_diag);
c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+18,c0_off);
_mm_store_ss(out+19,c1_off);
_mm_store_ss(out+20,c2_off);
_mm_storeu_ps(out+21,c3_off);
// row4
__m128 c4_off = _mm_mul_ps(a2,half_add_diag);
c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1)));
_mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1)));
_mm_storeu_ps(out+27,c4_off);
// row5
__m128 c5_off = _mm_mul_ps(a5,half_add_diag);
c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1));
_mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2)));
_mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2)));
_mm_storeu_ps(out+33,c5_off);
// row0
// __m128 c0_d = _mm_mul_ps(b_diag,a0);
// __m128 c_os = _mm_mul_ps(ofdiag_str,a0);
// __m128 c_or = _mm_mul_ps(ofdiag_rev,a0);
// __m128 c0_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c0_off = _mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ps(out,c0_d);
// _mm_storeu_ps(out+3,c0_off);
// // row1
// __m128 c1_d = _mm_mul_ps(b_diag,a4);
// c_os = _mm_mul_ps(ofdiag_str,a4);
// c_or = _mm_mul_ps(ofdiag_rev,a4);
// __m128 c1_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c1_off = _mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,3,2,1));
// c1_d = _mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+6,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1)));
// _mm_storeu_ps(out+7,c1_d);
// _mm_storeu_ps(out+9,c1_off);
// // row2
// __m128 c2_d = _mm_mul_ps(b_diag,a8);
// c_os = _mm_mul_ps(ofdiag_str,a8);
// c_or = _mm_mul_ps(ofdiag_rev,a8);
// __m128 c2_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c2_off = _mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+12,_mm_shuffle_ps(c0_d,c0_d,_MM_SHUFFLE(0,0,0,2)));
// _mm_store_ss(out+13,_mm_shuffle_ps(c1_d,c1_d,_MM_SHUFFLE(0,0,0,1)));
// _mm_store_ss(out+14,_mm_shuffle_ps(c2_d,c2_d,_MM_SHUFFLE(0,0,0,2)));
// _mm_storeu_ps(out+15,c2_off);
// // row3
// c_os = _mm_mul_ps(ofdiag_str,a1);
// c_or = _mm_mul_ps(ofdiag_rev,a1);
// __m128 c3_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c3_off = _mm_shuffle_ps(c3_off,c3_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+18,c0_off);
// _mm_store_ss(out+19,c1_off);
// _mm_store_ss(out+20,c2_off);
// _mm_storeu_ps(out+21,c3_off);
// // row4
// c_os = _mm_mul_ps(ofdiag_str,a2);
// c_or = _mm_mul_ps(ofdiag_rev,a2);
// __m128 c4_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c4_off = _mm_shuffle_ps(c4_off,c4_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+24,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,1)));
// _mm_store_ss(out+25,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,1)));
// _mm_store_ss(out+26,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,1)));
// _mm_storeu_ps(out+27,c4_off);
// // row5
// c_os = _mm_mul_ps(ofdiag_str,a5);
// c_or = _mm_mul_ps(ofdiag_rev,a5);
// __m128 c5_off = _mm_mul_ps(HALFPS,_mm_add_ps(c_os,c_or));
// c5_off = _mm_shuffle_ps(c5_off,c5_off,_MM_SHUFFLE(0,3,2,1));
// _mm_store_ss(out+30,_mm_shuffle_ps(c0_off,c0_off,_MM_SHUFFLE(0,0,0,2)));
// _mm_store_ss(out+31,_mm_shuffle_ps(c1_off,c1_off,_MM_SHUFFLE(0,0,0,2)));
// _mm_store_ss(out+32,_mm_shuffle_ps(c2_off,c2_off,_MM_SHUFFLE(0,0,0,2)));
// _mm_storeu_ps(out+33,c5_off);
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_HINT_INLINE void _outer<double,2,2,2,2>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// Fetch a to L1-cache
// _mm_prefetch(a,_MM_HINT_T0);
__m256d a0 = _mm256_set1_pd(a[0]);
__m256d a1 = _mm256_set1_pd(a[1]);
__m256d a3 = _mm256_set1_pd(a[3]);
__m256d bs = _mm256_load_pd(b);
__m128d r0 = _mm_setr_pd(_mm256_get2_pd(bs),_mm256_get1_pd(bs));
__m128d r1 = _mm_setr_pd(_mm256_get1_pd(bs),_mm256_get2_pd(bs));
__m128d r2 = _mm_mul_pd(HALFPD,_mm_add_pd(r0,r1));
__m256d r3 = _mm256_setr_pd(_mm256_get0_pd(bs),_mm256_get3_pd(bs),_mm_get0_pd(r2),0.0);
// row0
__m256d row0 = _mm256_mul_pd(a0,r3);
_mm256_store_pd(out, row0);
// row1
__m256d row1 = _mm256_mul_pd(a3,r3);
_mm_store_sd(out+3,_mm_set_sd(_mm256_get1_pd(row0)));
_mm_store_pd(out+4,_mm_setr_pd(_mm256_get1_pd(row1),_mm256_get2_pd(row1)));
// row2
__m256d row2 = _mm256_mul_pd(a1,r3);
_mm_store_sd(out+6,_mm256_extractf128_pd(row0,0x1));
_mm_store_sd(out+7,_mm256_extractf128_pd(row1,0x1));
_mm_store_sd(out+8,_mm256_extractf128_pd(row2,0x1));
}
template<>
FASTOR_HINT_INLINE void _outer<double,3,3,3,3>(const double * FASTOR_RESTRICT a, const double * FASTOR_RESTRICT b, double * FASTOR_RESTRICT out) {
// OPS
// _mm_prefetch(a,_MM_HINT_T0);
__m256d a0 = _mm256_set1_pd(a[0]);
__m256d a1 = _mm256_set1_pd(a[1]);
__m256d a2 = _mm256_set1_pd(a[2]);
__m256d a4 = _mm256_set1_pd(a[4]);
__m256d a5 = _mm256_set1_pd(a[5]);
__m256d a8 = _mm256_set1_pd(a[8]);
__m256d b_low = _mm256_load_pd(b);
__m256d b_high = _mm256_load_pd(b+4);
__m128d b_end = _mm_load_sd(b+8);
__m256d b_diag = _mm256_setr_pd(_mm_cvtsd_f64(_mm256_castpd256_pd128(b_low)),
_mm_cvtsd_f64(_mm256_castpd256_pd128(b_high)),
_mm_cvtsd_f64(b_end),0.0);
__m256d ofdiag_str = _mm256_setr_pd(b[1],b[2],b[5],0.0);
__m256d ofdiag_rev = _mm256_setr_pd(b[3],b[6],b[7],0.0);
// Compute this only once
__m256d half_add_diag = _mm256_mul_pd(VHALFPD,_mm256_add_pd(ofdiag_str,ofdiag_rev));
// row0
__m256d c0_d = _mm256_mul_pd(b_diag,a0);
__m256d c0_off = _mm256_mul_pd(a0,half_add_diag);
_mm256_store_pd(out,c0_d);
_mm256_storeu_pd(out+3,c0_off);
// row1
__m256d c1_d = _mm256_mul_pd(b_diag,a4);
__m256d c1_off = _mm256_mul_pd(a4,half_add_diag);
_mm256_storeu_pd(out+6,c1_d);
_mm_store_sd(out+6,_mm256_castpd256_pd128(_mm256_shuffle_pd(c0_d,c0_d,_MM_SHUFFLE(0,0,0,1))));
_mm256_storeu_pd(out+9,c1_off);
// row2
__m256d c2_d = _mm256_mul_pd(b_diag,a8);
__m256d c2_off = _mm256_mul_pd(a8,half_add_diag);
_mm_store_sd(out+12,_mm_set_sd(_mm256_get2_pd(c0_d)));
_mm_store_sd(out+13,_mm_set_sd(_mm256_get2_pd(c1_d)));
_mm_store_sd(out+14,_mm_set_sd(_mm256_get2_pd(c2_d)));
_mm256_storeu_pd(out+15,c2_off);
// row3
__m256d c3_off = _mm256_mul_pd(a1,half_add_diag);
_mm_store_sd(out+18,_mm256_castpd256_pd128(c0_off));
_mm_store_sd(out+19,_mm256_castpd256_pd128(c1_off));
_mm_store_sd(out+20,_mm256_castpd256_pd128(c2_off));
_mm256_storeu_pd(out+21,c3_off);
// row4
__m256d c4_off = _mm256_mul_pd(a2,half_add_diag);
_mm_store_sd(out+24,_mm_set_sd(_mm256_get1_pd(c0_off)));
_mm_store_sd(out+25,_mm_set_sd(_mm256_get1_pd(c1_off)));
_mm_store_sd(out+26,_mm_set_sd(_mm256_get1_pd(c2_off)));
_mm256_storeu_pd(out+27,c4_off);
// row4
__m256d c5_off = _mm256_mul_pd(a5,half_add_diag);
_mm_store_sd(out+30,_mm_set_sd(_mm256_get2_pd(c0_off)));
_mm_store_sd(out+31,_mm_set_sd(_mm256_get2_pd(c1_off)));
_mm_store_sd(out+32,_mm_set_sd(_mm256_get2_pd(c2_off)));
_mm256_storeu_pd(out+33,c5_off);
}
#endif
}
#endif // OUTER_H

1843
noarch/include/Fastor/backend/tensor_cross.h Normal file
View File

File diff suppressed because it is too large Load Diff

59
noarch/include/Fastor/backend/trace.h Normal file
View File

@@ -0,0 +1,59 @@
#ifndef TRACE_H
#define TRACE_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
template<typename T, size_t M, size_t N, typename std::enable_if<M==N,bool>::type=0>
FASTOR_INLINE T _trace(const T * FASTOR_RESTRICT a) {
T sum = static_cast<T>(0);
for (FASTOR_INDEX i=0; i<M; ++i)
sum +=a[i*N+i];
return sum;
}
#ifdef FASTOR_SSE4_2_IMPL
template<>
FASTOR_INLINE double _trace<double,2,2>(const double * FASTOR_RESTRICT a) {
// AVX VERSION
// IVY 5 OPS / HW 7 OPS
// __m256d a_reg = _mm256_load_pd(a);
// __m128d a_high = _mm256_extractf128_pd(a_reg,0x1);
// return _mm_cvtsd_f64(_mm_add_sd(_mm256_castpd256_pd128(a_reg),_mm_shuffle_pd(a_high,a_high,0x1)));
// SSE VERSION
// 3 OPS
__m128d a0 = _mm_load_sd(a);
__m128d a1 = _mm_load_sd(a+3);
return _mm_cvtsd_f64(_mm_add_pd(a0,a1));
}
template<>
FASTOR_INLINE double _trace<double,3,3>(const double * FASTOR_RESTRICT a) {
// No benefit in AVX
return _mm_cvtsd_f64(_mm_add_sd(_mm_load_sd(a),_mm_add_sd(_mm_load_sd(a+4),_mm_load_sd(a+8))));
}
template<>
FASTOR_INLINE float _trace<float,2,2>(const float * FASTOR_RESTRICT a) {
__m128 a_reg = _mm_load_ps(a);
return _mm_cvtss_f32(_mm_add_ss(a_reg,_mm_reverse_ps(a_reg)));
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE float _trace<float,3,3>(const float * FASTOR_RESTRICT a) {
__m256 a_reg = _mm256_load_ps(a);
__m128 sum_two = _mm_add_ps(_mm256_castps256_ps128(a_reg),_mm256_extractf128_ps(a_reg,0x1));
return _mm_cvtss_f32(_mm_add_ss(sum_two,_mm_load_ss(a+8)));
}
#endif
}
#endif // TRACE_H

448
noarch/include/Fastor/backend/transpose/transpose.h Normal file
View File

@@ -0,0 +1,448 @@
#ifndef TRANSPOSE_H
#define TRANSPOSE_H
#include "Fastor/config/config.h"
#include "Fastor/backend/transpose/transpose_kernels.h"
#include "Fastor/simd_vector/extintrin.h"
#include "Fastor/simd_vector/SIMDVector.h"
namespace Fastor {
// Forward declare
namespace internal {
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out);
} // internal
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_AVX_IMPL
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) {
using V = SIMDVector<T,DEFAULT_ABI>;
// Block sizes of 8x8 i.e. numSIMDRows=1
// numSIMDCols=1 and innerBlock=outerBlock=1
// give a much greater speed up, but causes
// significant slow-down for issue #42
#ifndef FASTOR_TRANS_OUTER_BLOCK_SIZE
constexpr size_t numSIMDRows = 1UL;
#else
constexpr size_t numSIMDRows = FASTOR_TRANS_OUTER_BLOCK_SIZE;
#endif
#ifndef FASTOR_TRANS_INNER_BLOCK_SIZE
constexpr size_t numSIMDCols = 1UL;
#else
constexpr size_t numSIMDCols = FASTOR_TRANS_INNER_BLOCK_SIZE;
#endif
constexpr size_t innerBlock = V::Size * numSIMDCols;
constexpr size_t outerBlock = V::Size * numSIMDRows;
FASTOR_ARCH_ALIGN T pack_a[outerBlock*innerBlock];
FASTOR_ARCH_ALIGN T pack_out[outerBlock*innerBlock];
constexpr size_t M0 = M / innerBlock * innerBlock;
constexpr size_t N0 = N / outerBlock * outerBlock;
V _vec;
// For row-major matrices we go over N
// and then M to get contiguous writes
size_t j=0;
for (; j<N0; j+=outerBlock) {
size_t i=0;
for (; i< M0; i+=innerBlock) {
// Pack A
for (size_t ii=0; ii<innerBlock; ++ii) {
_vec.load(&a[(i+ii)*N+(j)],false);
_vec.store(&pack_a[ii*outerBlock]);
}
// Perform transpose on pack_a and get the result
// on pack_out
internal::_transpose_dispatch<T,innerBlock,outerBlock>(pack_a,pack_out);
// Unpack pack_out to out
for (size_t jj=0; jj<outerBlock; ++jj) {
_vec.load(&pack_out[jj*innerBlock]);
_vec.store(&out[(j+jj)*M+(i)],false);
}
}
// Remainer M - M0 columns (of c)
for (; i< M; ++i) {
for (size_t jj=0; jj<outerBlock; ++jj) {
out[(j+jj)*M+(i)] = a[i*N+j+jj];
}
}
}
// Remainder N - N0 rows (of c)
for (; j<N; ++j) {
for (size_t i=0; i< M; ++i) {
out[(j)*M+(i)] = a[i*N+j];
}
}
}
#else
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) {
for (size_t j=0; j<N; ++j)
for (size_t i=0; i< M; ++i)
out[j*M+i] = a[i*N+j];
}
#endif
//----------------------------------------------------------------------------------------------------------//
// Specialisations - float
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE void _transpose<float,2,2>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
__m128 a_reg = _mm_loadu_ps(a);
_mm_storeu_ps(out,_mm_shuffle_ps(a_reg,a_reg,_MM_SHUFFLE(3,1,2,0)));
}
template<>
FASTOR_INLINE void _transpose<float,3,3>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
#ifndef FASTOR_AVX2_IMPL
// 5 OPS
__m128 row0 = _mm_loadu_ps(a);
__m128 row1 = _mm_loadu_ps(a+3);
__m128 row2 = _mm_loadu_ps(a+6);
__m128 T0 = _mm_unpacklo_ps(row0,row1);
__m128 T1 = _mm_unpackhi_ps(row0,row1);
row0 = _mm_movelh_ps ( T0,row2 );
row1 = _mm_shuffle_ps( T0,row2, _MM_SHUFFLE(3,1,3,2) );
row2 = _mm_shuffle_ps( T1,row2, _MM_SHUFFLE(3,2,1,0) );
_mm_storeu_ps(out,row0);
_mm_storeu_ps(out+3,row1);
_mm_storeu_ps(out+6,row2); // out of range for out[9]
#else
// 3 OPS
// gcc/clang emit vpermsps tht operate on (%rsp)
// less pressure on shuffle port perhaps
__m256 trans07 = _mm256_loadu_ps(a);
const __m256i trans_mask = _mm256_setr_epi32(
0,3,6,
1,4,7,
2,5);
// does not shuffle across 256 lanes, only 128 lanes
// __m256 _res = _mm256_permutevar_ps(trans07, trans_mask);
// this one shuffles correctly
__m256 _res = _mm256_permutevar8x32_ps(trans07, trans_mask);
_mm256_storeu_ps(out,_res);
// out[8] = a[8];
_mm_store_ss(out+8,_mm_load_ss(a+8));
#endif
}
template<>
FASTOR_INLINE void _transpose<float,4,4>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
#ifdef FASTOR_AVX512F_IMPL
__m512 amm = _mm512_loadu_ps(a);
__m512i idx = _mm512_setr_epi32( 0, 4, 8, 12,
1, 5, 9, 13,
2, 6, 10, 14,
3, 7, 11, 15);
__m512 omm = _mm512_permutexvar_ps(idx, amm);
_mm512_storeu_ps(out, omm);
#else
__m128 row1 = _mm_loadu_ps(a);
__m128 row2 = _mm_loadu_ps(a+4);
__m128 row3 = _mm_loadu_ps(a+8);
__m128 row4 = _mm_loadu_ps(a+12);
_MM_TRANSPOSE4_PS(row1, row2, row3, row4);
_mm_storeu_ps(out , row1);
_mm_storeu_ps(out+4 , row2);
_mm_storeu_ps(out+8 , row3);
_mm_storeu_ps(out+12, row4);
#endif
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE void _transpose<float,8,8>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
__m256 row1 = _mm256_loadu_ps(a);
__m256 row2 = _mm256_loadu_ps(a+8);
__m256 row3 = _mm256_loadu_ps(a+16);
__m256 row4 = _mm256_loadu_ps(a+24);
__m256 row5 = _mm256_loadu_ps(a+32);
__m256 row6 = _mm256_loadu_ps(a+40);
__m256 row7 = _mm256_loadu_ps(a+48);
__m256 row8 = _mm256_loadu_ps(a+56);
internal::_MM_TRANSPOSE8_PS(row1, row2, row3, row4, row5, row6, row7, row8);
_mm256_storeu_ps(out, row1);
_mm256_storeu_ps(out+8, row2);
_mm256_storeu_ps(out+16, row3);
_mm256_storeu_ps(out+24, row4);
_mm256_storeu_ps(out+32, row5);
_mm256_storeu_ps(out+40, row6);
_mm256_storeu_ps(out+48, row7);
_mm256_storeu_ps(out+56, row8);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL)
template<>
FASTOR_INLINE void _transpose<float,16,16>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
internal::_MM_TRANSPOSE16_PS(a,out);
}
#endif
//----------------------------------------------------------------------------------------------------------//
// Specialisations - double
//----------------------------------------------------------------------------------------------------------//
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE void _transpose<double,2,2>(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) {
/*-------------------------------------------------------*/
// 2 OPS
__m128d row0 = _mm_loadu_pd(a);
__m128d row1 = _mm_loadu_pd(a+2);
__m128d tmp = row0;
row0 = _mm_shuffle_pd(row0,row1,0x0);
row1 = _mm_shuffle_pd(tmp ,row1,0x3);
_mm_storeu_pd(out ,row0);
_mm_storeu_pd(out+2,row1);
/*-------------------------------------------------------*/
/*-------------------------------------------------------*/
// // AVX VERSION
// // IVY 4 OPS / HW 8 OPS
// __m256d a1 = _mm256_loadu_pd(a);
// __m128d a2 = _mm256_castpd256_pd128(a1);
// __m128d a3 = _mm256_extractf128_pd(a1,0x1);
// __m128d a4 = _mm_shuffle_pd(a2,a3,0x0);
// a3 = _mm_shuffle_pd(a2,a3,0x3);
// a1 = _mm256_castpd128_pd256(a4);
// a1 = _mm256_insertf128_pd(a1,a3,0x1);
// _mm256_storeu_pd(out,a1);
/*-------------------------------------------------------*/
}
#endif
#ifdef FASTOR_SSE2_IMPL
template<>
FASTOR_INLINE void _transpose<double,3,3>(const double* FASTOR_RESTRICT a, double* FASTOR_RESTRICT out) {
// AVX512 is the fastest & AVX version despite more instructions is faster than SSE
#if defined(FASTOR_AVX512F_IMPL)
// AVX512
/*-------------------------------------------------------*/
__m512d a07 = _mm512_loadu_pd(a);
const __m512i trans_mask = _mm512_setr_epi64(
0,3,6,
1,4,7,
2,5);
__m512d trans07 = _mm512_permutexvar_pd(trans_mask, a07);
_mm512_storeu_pd(out,trans07);
_mm_store_sd(out+8,_mm_load_sd(a+8));
/*-------------------------------------------------------*/
#elif defined(FASTOR_AVX_IMPL)
// AVX
/*-------------------------------------------------------*/
__m256d row1 = _mm256_loadu_pd(a);
__m256d row2 = _mm256_loadu_pd(a+4);
__m128d a11 = _mm256_castpd256_pd128(row1);
__m128d a12 = _mm256_extractf128_pd(row1,0x1);
__m128d a21 = _mm256_castpd256_pd128(row2);
__m128d a22 = _mm256_extractf128_pd(row2,0x1);
row1 = _mm256_castpd128_pd256(_mm_shuffle_pd(a11,a12,0x2));
row1 = _mm256_insertf128_pd(row1,_mm_shuffle_pd(a22,a11,0x2),0x1);
row2 = _mm256_castpd128_pd256(_mm_shuffle_pd(a21,a22,0x2));
row2 = _mm256_insertf128_pd(row2,_mm_shuffle_pd(a12,a21,0x2),0x1);
_mm256_storeu_pd(out,row1);
_mm256_storeu_pd(out+4,row2);
_mm_store_sd(out+8,_mm_load_sd(a+8));
/*-------------------------------------------------------*/
#else
// SSE
/*-------------------------------------------------------*/
__m128d a11 = _mm_loadu_pd(a);
__m128d a12 = _mm_loadu_pd(a+2);
__m128d a21 = _mm_loadu_pd(a+4);
__m128d a22 = _mm_loadu_pd(a+6);
_mm_storeu_pd(out ,_mm_shuffle_pd(a11,a12,0x2));
_mm_storeu_pd(out+2,_mm_shuffle_pd(a22,a11,0x2));
_mm_storeu_pd(out+4,_mm_shuffle_pd(a21,a22,0x2));
_mm_storeu_pd(out+6,_mm_shuffle_pd(a12,a21,0x2));
_mm_store_sd (out+8,_mm_load_sd(a+8));
/*-------------------------------------------------------*/
#endif
}
#endif
#ifdef FASTOR_AVX_IMPL
template<>
FASTOR_INLINE void _transpose<double,4,4>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
#ifdef FASTOR_AVX512F_IMPL
__m512d amm0 = _mm512_loadu_pd(a);
__m512d amm1 = _mm512_loadu_pd(a+8);
__m512i idx0 = _mm512_setr_epi64(0, 4, 8, 12, 1, 5, 9, 13);
__m512i idx1 = _mm512_setr_epi64(2, 6, 10, 14, 3, 7, 11, 15);
__m512d omm0 = _mm512_permutex2var_pd(amm0, idx0, amm1);
__m512d omm1 = _mm512_permutex2var_pd(amm0, idx1, amm1);
_mm512_storeu_pd(out , omm0);
_mm512_storeu_pd(out+8, omm1);
#else
__m256d row1 = _mm256_loadu_pd(a);
__m256d row2 = _mm256_loadu_pd(a+4);
__m256d row3 = _mm256_loadu_pd(a+8);
__m256d row4 = _mm256_loadu_pd(a+12);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(out, row1);
_mm256_storeu_pd(out+4, row2);
_mm256_storeu_pd(out+8, row3);
_mm256_storeu_pd(out+12, row4);
#endif
}
#endif
#if defined(FASTOR_AVX512F_IMPL)
template<>
FASTOR_INLINE void _transpose<double,8,8>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
__m512d row0 = _mm512_loadu_pd(a);
__m512d row1 = _mm512_loadu_pd(a+8);
__m512d row2 = _mm512_loadu_pd(a+16);
__m512d row3 = _mm512_loadu_pd(a+24);
__m512d row4 = _mm512_loadu_pd(a+32);
__m512d row5 = _mm512_loadu_pd(a+40);
__m512d row6 = _mm512_loadu_pd(a+48);
__m512d row7 = _mm512_loadu_pd(a+56);
internal::_MM_TRANSPOSE8_PD(row0,row1,row2,row3,row4,row5,row6,row7);
_mm512_storeu_pd(out , row0);
_mm512_storeu_pd(out+8 , row1);
_mm512_storeu_pd(out+16, row2);
_mm512_storeu_pd(out+24, row3);
_mm512_storeu_pd(out+32, row4);
_mm512_storeu_pd(out+40, row5);
_mm512_storeu_pd(out+48, row6);
_mm512_storeu_pd(out+56, row7);
}
#elif defined(FASTOR_AVX_IMPL)
template<>
FASTOR_INLINE void _transpose<double,8,8>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
{
__m256d row1 = _mm256_loadu_pd(&a[0]);
__m256d row2 = _mm256_loadu_pd(&a[8]);
__m256d row3 = _mm256_loadu_pd(&a[16]);
__m256d row4 = _mm256_loadu_pd(&a[24]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[0], row1);
_mm256_storeu_pd(&out[8], row2);
_mm256_storeu_pd(&out[16], row3);
_mm256_storeu_pd(&out[24], row4);
}
{
__m256d row1 = _mm256_loadu_pd(&a[32]);
__m256d row2 = _mm256_loadu_pd(&a[40]);
__m256d row3 = _mm256_loadu_pd(&a[48]);
__m256d row4 = _mm256_loadu_pd(&a[56]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[4], row1);
_mm256_storeu_pd(&out[12], row2);
_mm256_storeu_pd(&out[20], row3);
_mm256_storeu_pd(&out[28], row4);
}
{
__m256d row1 = _mm256_loadu_pd(&a[4]);
__m256d row2 = _mm256_loadu_pd(&a[12]);
__m256d row3 = _mm256_loadu_pd(&a[20]);
__m256d row4 = _mm256_loadu_pd(&a[28]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[32], row1);
_mm256_storeu_pd(&out[40], row2);
_mm256_storeu_pd(&out[48], row3);
_mm256_storeu_pd(&out[56], row4);
}
{
__m256d row1 = _mm256_loadu_pd(&a[36]);
__m256d row2 = _mm256_loadu_pd(&a[44]);
__m256d row3 = _mm256_loadu_pd(&a[52]);
__m256d row4 = _mm256_loadu_pd(&a[60]);
internal::_MM_TRANSPOSE4_PD(row1, row2, row3, row4);
_mm256_storeu_pd(&out[36], row1);
_mm256_storeu_pd(&out[44], row2);
_mm256_storeu_pd(&out[52], row3);
_mm256_storeu_pd(&out[60], row4);
}
}
#endif
//----------------------------------------------------------------------------------------------------------//
//----------------------------------------------------------------------------------------------------------//
namespace internal {
// To get around compilers recusive inlining depth issue
template<typename T, size_t M, size_t N>
FASTOR_INLINE void _transpose_dispatch(const T * FASTOR_RESTRICT a, T * FASTOR_RESTRICT out) {
for (size_t j=0; j<N; ++j)
for (size_t i=0; i< M; ++i)
out[j*M+i] = a[i*N+j];
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,2,2>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,2,2>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,3,3>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,3,3>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,4,4>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,4,4>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,8,8>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,8,8>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<float,16,16>(const float * FASTOR_RESTRICT a, float * FASTOR_RESTRICT out) {
_transpose<float,16,16>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,2,2>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,2,2>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,3,3>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,3,3>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,4,4>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,4,4>(a,out);
}
template<>
FASTOR_INLINE void _transpose_dispatch<double,8,8>(const double * FASTOR_RESTRICT a, double * FASTOR_RESTRICT out) {
_transpose<double,8,8>(a,out);
}
} // internal
//----------------------------------------------------------------------------------------------------------//
}
#endif // TRANSPOSE_H

233
noarch/include/Fastor/backend/transpose/transpose_kernels.h Normal file
View File

@@ -0,0 +1,233 @@
#ifndef TRANSPOSE_KERNELS_H
#define TRANSPOSE_KERNELS_H
#include "Fastor/config/config.h"
#include "Fastor/simd_vector/extintrin.h"
namespace Fastor {
namespace internal {
#ifdef FASTOR_SSE_IMPL
// 4x4 PS - defined
// _MM_TRANSPOSE4_PS
#endif
#ifdef FASTOR_AVX_IMPL
// 4x4 PD
FASTOR_INLINE void _MM_TRANSPOSE4_PD(__m256d &row0, __m256d &row1, __m256d &row2, __m256d &row3)
{
__m256d tmp3, tmp2, tmp1, tmp0;
tmp0 = _mm256_shuffle_pd((row0),(row1), 0x0);
tmp2 = _mm256_shuffle_pd((row0),(row1), 0xF);
tmp1 = _mm256_shuffle_pd((row2),(row3), 0x0);
tmp3 = _mm256_shuffle_pd((row2),(row3), 0xF);
row0 = _mm256_permute2f128_pd(tmp0, tmp1, 0x20);
row1 = _mm256_permute2f128_pd(tmp2, tmp3, 0x20);
row2 = _mm256_permute2f128_pd(tmp0, tmp1, 0x31);
row3 = _mm256_permute2f128_pd(tmp2, tmp3, 0x31);
}
// 8x8 PS
FASTOR_INLINE void _MM_TRANSPOSE8_PS(__m256 &row0, __m256 &row1, __m256 &row2, __m256 &row3,
__m256 &row4, __m256 &row5, __m256 &row6, __m256 &row7)
{
__m256 __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7;
__m256 __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7;
__t0 = _mm256_unpacklo_ps(row0, row1);
__t1 = _mm256_unpackhi_ps(row0, row1);
__t2 = _mm256_unpacklo_ps(row2, row3);
__t3 = _mm256_unpackhi_ps(row2, row3);
__t4 = _mm256_unpacklo_ps(row4, row5);
__t5 = _mm256_unpackhi_ps(row4, row5);
__t6 = _mm256_unpacklo_ps(row6, row7);
__t7 = _mm256_unpackhi_ps(row6, row7);
__tt0 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(1,0,1,0));
__tt1 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(3,2,3,2));
__tt2 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(1,0,1,0));
__tt3 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(3,2,3,2));
__tt4 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(1,0,1,0));
__tt5 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(3,2,3,2));
__tt6 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(1,0,1,0));
__tt7 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(3,2,3,2));
row0 = _mm256_permute2f128_ps(__tt0, __tt4, 0x20);
row1 = _mm256_permute2f128_ps(__tt1, __tt5, 0x20);
row2 = _mm256_permute2f128_ps(__tt2, __tt6, 0x20);
row3 = _mm256_permute2f128_ps(__tt3, __tt7, 0x20);
row4 = _mm256_permute2f128_ps(__tt0, __tt4, 0x31);
row5 = _mm256_permute2f128_ps(__tt1, __tt5, 0x31);
row6 = _mm256_permute2f128_ps(__tt2, __tt6, 0x31);
row7 = _mm256_permute2f128_ps(__tt3, __tt7, 0x31);
}
#endif
#ifdef FASTOR_AVX512F_IMPL
// 8x8 PD
inline void _MM_TRANSPOSE8_PD(__m512d &row0, __m512d &row1, __m512d &row2, __m512d &row3,
__m512d &row4, __m512d &row5, __m512d &row6, __m512d &row7)
{
__m512d __t0, __t1, __t2, __t3, __t4, __t5, __t6, __t7;
__m512d __tt0, __tt1, __tt2, __tt3, __tt4, __tt5, __tt6, __tt7;
FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {0, 8 , 1 , 9 , 4 , 12, 5 , 13};
FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {2, 10, 3 , 11, 6 , 14, 7 , 15};
FASTOR_ARCH_ALIGN constexpr int64_t idx3[8] = {0, 1 , 8 , 9 , 4 , 5 , 12, 13};
FASTOR_ARCH_ALIGN constexpr int64_t idx4[8] = {2, 3 , 10, 11, 6 , 7 , 14, 15};
FASTOR_ARCH_ALIGN constexpr int64_t idx5[8] = {4, 5 , 6 , 7 , 12, 13, 14, 15};
__m512i vidx1 = _mm512_load_epi64(idx1);
__m512i vidx2 = _mm512_load_epi64(idx2);
__m512i vidx3 = _mm512_load_epi64(idx3);
__m512i vidx4 = _mm512_load_epi64(idx4);
__m512i vidx5 = _mm512_load_epi64(idx5);
__t0 = _mm512_permutex2var_pd(row0, vidx1, row1);
__t1 = _mm512_permutex2var_pd(row0, vidx2, row1);
__t2 = _mm512_permutex2var_pd(row2, vidx1, row3);
__t3 = _mm512_permutex2var_pd(row2, vidx2, row3);
__t4 = _mm512_permutex2var_pd(row4, vidx1, row5);
__t5 = _mm512_permutex2var_pd(row4, vidx2, row5);
__t6 = _mm512_permutex2var_pd(row6, vidx1, row7);
__t7 = _mm512_permutex2var_pd(row6, vidx2, row7);
__tt0 = _mm512_permutex2var_pd(__t0, vidx3, __t2);
__tt1 = _mm512_permutex2var_pd(__t0, vidx4, __t2);
__tt2 = _mm512_permutex2var_pd(__t1, vidx3, __t3);
__tt3 = _mm512_permutex2var_pd(__t1, vidx4, __t3);
__tt4 = _mm512_permutex2var_pd(__t4, vidx3, __t6);
__tt5 = _mm512_permutex2var_pd(__t4, vidx4, __t6);
__tt6 = _mm512_permutex2var_pd(__t5, vidx3, __t7);
__tt7 = _mm512_permutex2var_pd(__t5, vidx4, __t7);
row0 = _mm512_insertf64x4(__tt0,_mm512_castpd512_pd256(__tt4),0x1);
row1 = _mm512_insertf64x4(__tt1,_mm512_castpd512_pd256(__tt5),0x1);
row2 = _mm512_insertf64x4(__tt2,_mm512_castpd512_pd256(__tt6),0x1);
row3 = _mm512_insertf64x4(__tt3,_mm512_castpd512_pd256(__tt7),0x1);
row4 = _mm512_permutex2var_pd(__tt0, vidx5, __tt4);
row5 = _mm512_permutex2var_pd(__tt1, vidx5, __tt5);
row6 = _mm512_permutex2var_pd(__tt2, vidx5, __tt6);
row7 = _mm512_permutex2var_pd(__tt3, vidx5, __tt7);
}
#endif
#if defined(FASTOR_AVX512F_IMPL) && defined(FASTOR_AVX512DQ_IMPL)
// 16x16
FASTOR_INLINE void _MM_TRANSPOSE16_PS(const float * FASTOR_RESTRICT mat, float * FASTOR_RESTRICT matT)
{
__m512 t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, ta, tb, tc, td, te, tf;
__m512 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, ra, rb, rc, rd, re, rf;
int mask;
FASTOR_ARCH_ALIGN constexpr int64_t idx1[8] = {2, 3, 0, 1, 6, 7, 4, 5};
FASTOR_ARCH_ALIGN constexpr int64_t idx2[8] = {1, 0, 3, 2, 5, 4, 7, 6};
FASTOR_ARCH_ALIGN constexpr int32_t idx3[16] = {1, 0, 3, 2, 5 ,4 ,7 ,6 ,9 ,8 , 11, 10, 13, 12 ,15, 14};
__m512i vidx1 = _mm512_load_epi64(idx1);
__m512i vidx2 = _mm512_load_epi64(idx2);
__m512i vidx3 = _mm512_load_epi32(idx3);
t0 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+0])), _mm256_loadu_ps(&mat[ 8*16+0]), 1);
t1 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+0])), _mm256_loadu_ps(&mat[ 9*16+0]), 1);
t2 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+0])), _mm256_loadu_ps(&mat[10*16+0]), 1);
t3 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+0])), _mm256_loadu_ps(&mat[11*16+0]), 1);
t4 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+0])), _mm256_loadu_ps(&mat[12*16+0]), 1);
t5 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+0])), _mm256_loadu_ps(&mat[13*16+0]), 1);
t6 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+0])), _mm256_loadu_ps(&mat[14*16+0]), 1);
t7 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+0])), _mm256_loadu_ps(&mat[15*16+0]), 1);
t8 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 0*16+8])), _mm256_loadu_ps(&mat[ 8*16+8]), 1);
t9 = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 1*16+8])), _mm256_loadu_ps(&mat[ 9*16+8]), 1);
ta = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 2*16+8])), _mm256_loadu_ps(&mat[10*16+8]), 1);
tb = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 3*16+8])), _mm256_loadu_ps(&mat[11*16+8]), 1);
tc = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 4*16+8])), _mm256_loadu_ps(&mat[12*16+8]), 1);
td = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 5*16+8])), _mm256_loadu_ps(&mat[13*16+8]), 1);
te = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 6*16+8])), _mm256_loadu_ps(&mat[14*16+8]), 1);
tf = _mm512_insertf32x8(_mm512_castps256_ps512(_mm256_loadu_ps(&mat[ 7*16+8])), _mm256_loadu_ps(&mat[15*16+8]), 1);
mask= 0xcc;
r0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t0), (__mmask8)mask, vidx1, _mm512_castps_pd(t4)));
r1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t1), (__mmask8)mask, vidx1, _mm512_castps_pd(t5)));
r2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t2), (__mmask8)mask, vidx1, _mm512_castps_pd(t6)));
r3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t3), (__mmask8)mask, vidx1, _mm512_castps_pd(t7)));
r8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t8), (__mmask8)mask, vidx1, _mm512_castps_pd(tc)));
r9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t9), (__mmask8)mask, vidx1, _mm512_castps_pd(td)));
ra = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ta), (__mmask8)mask, vidx1, _mm512_castps_pd(te)));
rb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tb), (__mmask8)mask, vidx1, _mm512_castps_pd(tf)));
mask= 0x33;
r4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t4), (__mmask8)mask, vidx1, _mm512_castps_pd(t0)));
r5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t5), (__mmask8)mask, vidx1, _mm512_castps_pd(t1)));
r6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t6), (__mmask8)mask, vidx1, _mm512_castps_pd(t2)));
r7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(t7), (__mmask8)mask, vidx1, _mm512_castps_pd(t3)));
rc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tc), (__mmask8)mask, vidx1, _mm512_castps_pd(t8)));
rd = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(td), (__mmask8)mask, vidx1, _mm512_castps_pd(t9)));
re = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(te), (__mmask8)mask, vidx1, _mm512_castps_pd(ta)));
rf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(tf), (__mmask8)mask, vidx1, _mm512_castps_pd(tb)));
mask = 0xaa;
t0 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r0), (__mmask8)mask, vidx2, _mm512_castps_pd(r2)));
t1 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r1), (__mmask8)mask, vidx2, _mm512_castps_pd(r3)));
t4 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r4), (__mmask8)mask, vidx2, _mm512_castps_pd(r6)));
t5 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r5), (__mmask8)mask, vidx2, _mm512_castps_pd(r7)));
t8 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r8), (__mmask8)mask, vidx2, _mm512_castps_pd(ra)));
t9 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r9), (__mmask8)mask, vidx2, _mm512_castps_pd(rb)));
tc = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rc), (__mmask8)mask, vidx2, _mm512_castps_pd(re)));
td = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rd), (__mmask8)mask, vidx2, _mm512_castps_pd(rf)));
mask = 0x55;
t2 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r2), (__mmask8)mask, vidx2, _mm512_castps_pd(r0)));
t3 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r3), (__mmask8)mask, vidx2, _mm512_castps_pd(r1)));
t6 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r6), (__mmask8)mask, vidx2, _mm512_castps_pd(r4)));
t7 = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(r7), (__mmask8)mask, vidx2, _mm512_castps_pd(r5)));
ta = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(ra), (__mmask8)mask, vidx2, _mm512_castps_pd(r8)));
tb = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rb), (__mmask8)mask, vidx2, _mm512_castps_pd(r9)));
te = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(re), (__mmask8)mask, vidx2, _mm512_castps_pd(rc)));
tf = _mm512_castpd_ps(_mm512_mask_permutexvar_pd(_mm512_castps_pd(rf), (__mmask8)mask, vidx2, _mm512_castps_pd(rd)));
mask = 0xaaaa;
r0 = _mm512_mask_permutexvar_ps(t0, (__mmask16)mask, vidx3, t1);
r2 = _mm512_mask_permutexvar_ps(t2, (__mmask16)mask, vidx3, t3);
r4 = _mm512_mask_permutexvar_ps(t4, (__mmask16)mask, vidx3, t5);
r6 = _mm512_mask_permutexvar_ps(t6, (__mmask16)mask, vidx3, t7);
r8 = _mm512_mask_permutexvar_ps(t8, (__mmask16)mask, vidx3, t9);
ra = _mm512_mask_permutexvar_ps(ta, (__mmask16)mask, vidx3, tb);
rc = _mm512_mask_permutexvar_ps(tc, (__mmask16)mask, vidx3, td);
re = _mm512_mask_permutexvar_ps(te, (__mmask16)mask, vidx3, tf);
mask = 0x5555;
r1 = _mm512_mask_permutexvar_ps(t1, (__mmask16)mask, vidx3, t0);
r3 = _mm512_mask_permutexvar_ps(t3, (__mmask16)mask, vidx3, t2);
r5 = _mm512_mask_permutexvar_ps(t5, (__mmask16)mask, vidx3, t4);
r7 = _mm512_mask_permutexvar_ps(t7, (__mmask16)mask, vidx3, t6);
r9 = _mm512_mask_permutexvar_ps(t9, (__mmask16)mask, vidx3, t8);
rb = _mm512_mask_permutexvar_ps(tb, (__mmask16)mask, vidx3, ta);
rd = _mm512_mask_permutexvar_ps(td, (__mmask16)mask, vidx3, tc);
rf = _mm512_mask_permutexvar_ps(tf, (__mmask16)mask, vidx3, te);
_mm512_storeu_ps(&matT[ 0*16], r0);
_mm512_storeu_ps(&matT[ 1*16], r1);
_mm512_storeu_ps(&matT[ 2*16], r2);
_mm512_storeu_ps(&matT[ 3*16], r3);
_mm512_storeu_ps(&matT[ 4*16], r4);
_mm512_storeu_ps(&matT[ 5*16], r5);
_mm512_storeu_ps(&matT[ 6*16], r6);
_mm512_storeu_ps(&matT[ 7*16], r7);
_mm512_storeu_ps(&matT[ 8*16], r8);
_mm512_storeu_ps(&matT[ 9*16], r9);
_mm512_storeu_ps(&matT[10*16], ra);
_mm512_storeu_ps(&matT[11*16], rb);
_mm512_storeu_ps(&matT[12*16], rc);
_mm512_storeu_ps(&matT[13*16], rd);
_mm512_storeu_ps(&matT[14*16], re);
_mm512_storeu_ps(&matT[15*16], rf);
}
#endif
} // end of namespace internal
} // end of namespace Fastor
#endif // TRANSPOSE_KERNELS_H

182
noarch/include/Fastor/backend/voigt.h Normal file
View File

@@ -0,0 +1,182 @@
#ifndef VOIGT_H
#define VOIGT_H
#include "Fastor/tensor/Tensor.h"
// Conversion of tensors to symmetrised Voigt forms
namespace Fastor {
template<typename T, size_t ... Rest>
struct VoigtType;
template<typename T>
struct VoigtType<T,2,2,2,2> {
using return_type = Tensor<T,3,3>;
};
template<typename T>
struct VoigtType<T,3,3,3,3> {
using return_type = Tensor<T,6,6>;
};
template<typename T>
struct VoigtType<T,2,2,2> {
using return_type = Tensor<T,3,2>;
};
template<typename T>
struct VoigtType<T,3,3,3> {
using return_type = Tensor<T,6,3>;
};
template<typename T>
struct VoigtType<T,2,2> {
using return_type = Tensor<T,3>;
};
template<typename T>
struct VoigtType<T,3,3> {
using return_type = Tensor<T,6>;
};
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==4 && pack_prod<Rest...>::value == 16
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = a_data[0];
VoigtA[1] = a_data[3];
VoigtA[2] = 0.5*(a_data[1]+a_data[2]);
VoigtA[3] = VoigtA[1];
VoigtA[4] = a_data[15];
VoigtA[5] = 0.5*(a_data[13]+a_data[14]);
VoigtA[6] = VoigtA[2];
VoigtA[7] = VoigtA[5];
VoigtA[8] = 0.5*(a_data[5]+a_data[6]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==4 && pack_prod<Rest...>::value == 81
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT a_data, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = a_data[0];
VoigtA[1] = a_data[4];
VoigtA[2] = a_data[8];
VoigtA[3] = 0.5*(a_data[1]+a_data[3]);
VoigtA[4] = 0.5*(a_data[2]+a_data[6]);
VoigtA[5] = 0.5*(a_data[5]+a_data[7]);
VoigtA[6] = VoigtA[1];
VoigtA[7] = a_data[40];
VoigtA[8] = a_data[44];
VoigtA[9] = 0.5*(a_data[37]+a_data[39]);
VoigtA[10] = 0.5*(a_data[38]+a_data[42]);
VoigtA[11] = 0.5*(a_data[41]+a_data[43]);
VoigtA[12] = VoigtA[2];
VoigtA[13] = VoigtA[8];
VoigtA[14] = a_data[80];
VoigtA[15] = 0.5*(a_data[73]+a_data[75]);
VoigtA[16] = 0.5*(a_data[74]+a_data[78]);
VoigtA[17] = 0.5*(a_data[77]+a_data[79]);
VoigtA[18] = VoigtA[3];
VoigtA[19] = VoigtA[9];
VoigtA[20] = VoigtA[15];
VoigtA[21] = 0.5*(a_data[10]+a_data[12]);
VoigtA[22] = 0.5*(a_data[11]+a_data[15]);
VoigtA[23] = 0.5*(a_data[14]+a_data[16]);
VoigtA[24] = VoigtA[4];
VoigtA[25] = VoigtA[10];
VoigtA[26] = VoigtA[16];
VoigtA[27] = VoigtA[22];
VoigtA[28] = 0.5*(a_data[20]+a_data[24]);
VoigtA[29] = 0.5*(a_data[23]+a_data[25]);
VoigtA[30] = VoigtA[5];
VoigtA[31] = VoigtA[11];
VoigtA[32] = VoigtA[17];
VoigtA[33] = VoigtA[23];
VoigtA[34] = VoigtA[29];
VoigtA[35] = 0.5*(a_data[50]+a_data[52]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==3 && pack_prod<Rest...>::value == 8
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
// 3rd order tensor to 2nd order tensor
// 3rd order tensor should be symmetric in the first two indices
VoigtA[0] = e[0];
VoigtA[1] = e[1];
VoigtA[2] = e[6];
VoigtA[3] = e[7];
VoigtA[4] = 0.5*(e[2]+e[4]);
VoigtA[5] = 0.5*(e[3]+e[5]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==3 && pack_prod<Rest...>::value == 27
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
// 3rd order tensor to 2nd order tensor
// 3rd order tensor should be symmetric in the first two indices
VoigtA[0] = e[0];
VoigtA[1] = e[1];
VoigtA[2] = e[2];
VoigtA[3] = e[12];
VoigtA[4] = e[13];
VoigtA[5] = e[14];
VoigtA[6] = e[24];
VoigtA[7] = e[25];
VoigtA[8] = e[26];
VoigtA[9] = 0.5*(e[3]+e[9]);
VoigtA[10] = 0.5*(e[4]+e[10]);
VoigtA[11] = 0.5*(e[5]+e[11]);
VoigtA[12] = 0.5*(e[6]+e[18]);
VoigtA[13] = 0.5*(e[7]+e[19]);
VoigtA[14] = 0.5*(e[8]+e[20]);
VoigtA[15] = 0.5*(e[15]+e[21]);
VoigtA[16] = 0.5*(e[16]+e[22]);
VoigtA[17] = 0.5*(e[17]+e[23]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==2 && pack_prod<Rest...>::value == 4
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = e[0];
VoigtA[1] = e[3];
VoigtA[2] = 0.5*(e[1]+e[2]);
}
template<typename T, size_t ... Rest,
typename std::enable_if<sizeof...(Rest)==2 && pack_prod<Rest...>::value == 9
,bool>::type=0>
FASTOR_INLINE void _voigt(const T * FASTOR_RESTRICT e, T * FASTOR_RESTRICT VoigtA) {
VoigtA[0] = e[0];
VoigtA[1] = e[4];
VoigtA[2] = e[8];
VoigtA[3] = 0.5*(e[1]+e[3]);
VoigtA[4] = 0.5*(e[2]+e[6]);
VoigtA[5] = 0.5*(e[5]+e[7]);
}
template<typename T, size_t ... Rest>
FASTOR_INLINE auto voigt(const Tensor<T,Rest...> &a)
-> typename VoigtType<T,Rest...>::return_type {
T *a_data = a.data();
using ret_type = typename VoigtType<T,Rest...>::return_type;
ret_type voigt_a;
T *VoigtA = voigt_a.data();
_voigt<T,Rest...>(a_data,VoigtA);
return voigt_a;
}
}
#endif // VOIGT_H