#ifndef CONTRACTION_H #define CONTRACTION_H #include "Fastor/backend/dyadic.h" #include "Fastor/tensor/Tensor.h" #include "Fastor/tensor_algebra/indicial.h" namespace Fastor { //using namespace details; //namespace details { // Define this if fastest (complete meta-engine based) tensor contraction is required. // Note that this blows up memory consumption and compilation time exponentially //#define CONTRACT_OPT 2 // Define this if faster (partial meta-engine based) tensor contraction is required. // Note that this blows up memory consumption and compilation time but not as much //#define CONTRACT_OPT 1 template struct RecursiveCartesian; template struct RecursiveCartesian, Index, Tensor, Tensor, First, Lasts...> { static constexpr int out_dim = no_of_unique::value; static FASTOR_INLINE void Do(const T *a_data, const T *b_data, T *out_data, std::array &as, std::array &idx) { for (size_t i=0; i, Index, Tensor, Tensor,Lasts...>::Do(a_data, b_data, out_data, as, idx); } } }; template struct RecursiveCartesian, Index, Tensor, Tensor,Last> { using _contraction_impl = contraction_impl, Tensor, typename std_ext::make_index_sequence::type>; using resulting_tensor = typename _contraction_impl::type; using resulting_index = typename _contraction_impl::indices; static constexpr bool _is_reduction = resulting_tensor::dimension_t::value == 0; static constexpr int a_dim = sizeof...(Rest0); static constexpr int b_dim = sizeof...(Rest1); static constexpr int out_dim = no_of_unique::value; static constexpr auto& idx_a = IndexTensors< Index, Tensor, Index,Tensor, typename std_ext::make_index_sequence::type>::indices; static constexpr auto& idx_b = IndexTensors< Index, Tensor, Index,Tensor, typename std_ext::make_index_sequence::type>::indices; static constexpr auto& idx_out = IndexTensors< Index, Tensor, resulting_index,resulting_tensor, typename std_ext::make_index_sequence::type>::indices; using nloops = loop_setter< Index, Tensor, typename std_ext::make_index_sequence::type>; static constexpr auto& maxes_out = nloops::dims; static constexpr int total = nloops::value; static constexpr std::array products_a = nprods, typename std_ext::make_index_sequence::type>::values; static constexpr std::array products_b = nprods, typename std_ext::make_index_sequence::type>::values; #ifndef FASTOR_DONT_VECTORISE using vectorisability = is_vectorisable,Index,Tensor>; static constexpr int stride = vectorisability::stride; using V = typename vectorisability::type; #else static constexpr int stride = 1; using V = SIMDVector; #endif static FASTOR_INLINE void Do(const T *a_data, const T *b_data, T *out_data, std::array &as, std::array &idx) { FASTOR_IF_CONSTEXPR(!_is_reduction) { using Index_with_dims = typename put_dims_in_Index::type; constexpr std::array products_out = \ nprods::type>::values; V _vec_a; for (size_t i=0; i(resulting_tensor::Dimension); it++) { index_out += products_out[it]*as[idx_out[it]]; } // out_data[index_out] += a_data[index_a]*b_data[index_b]; // _vec_a.set(*(a_data+index_a)); _vec_a.set(a_data[index_a]); // V _vec_out = _vec_a*V(b_data+index_b) + V(out_data+index_out); V _vec_out = fmadd(_vec_a,V(&b_data[index_b]), V(&out_data[index_out])); _vec_out.store(out_data+index_out,false); // _vec_out.aligned_store(&out_data[index_out]); } } else { using Index_with_dims = typename put_dims_in_Index::type; constexpr std::array products_out = \ nprods::type>::values; V _vec_a; for (size_t i=0; i // struct RecursiveCartesian, Index, Tensor, Tensor> // { // using OutTensor = typename contraction_impl, Tensor, // typename std_ext::make_index_sequence::type>::type; // using OutIndices = typename contraction_impl, Tensor, // typename std_ext::make_index_sequence::type>::indices; // static constexpr int a_dim = sizeof...(Rest0); // static constexpr int b_dim = sizeof...(Rest1); // static constexpr int out_dim = no_of_unique::value; // static constexpr auto& idx_a = IndexTensors< // Index, // Tensor, // Index,Tensor, // typename std_ext::make_index_sequence::type>::indices; // static constexpr auto& idx_b = IndexTensors< // Index, // Tensor, // Index,Tensor, // typename std_ext::make_index_sequence::type>::indices; // static constexpr auto& idx_out = IndexTensors< // Index, // Tensor, // OutIndices,OutTensor, // typename std_ext::make_index_sequence::type>::indices; // using nloops = loop_setter< // Index, // Tensor, // typename std_ext::make_index_sequence::type>; // static constexpr auto& maxes_out = nloops::dims; // static constexpr int total = nloops::value; // static constexpr std::array products_a = nprods,typename std_ext::make_index_sequence::type>::values; // static constexpr std::array products_b = nprods,typename std_ext::make_index_sequence::type>::values; // using Index_with_dims = typename put_dims_in_Index::type; // static constexpr std::array products_out = // nprods::type>::values; // #ifndef FASTOR_DONT_VECTORISE // using vectorisability = is_vectorisable,Index,Tensor>; // static constexpr int stride = vectorisability::stride; // using V = typename vectorisability::type; // #else // static constexpr int stride = 1; // using V = SIMDVector; // #endif // static void Do(const T *a_data, const T *b_data, T *out_data, std::array &as, std::array &idx) // { // std::reverse_copy(idx.begin(),idx.end(),as.begin()); // int index_a = as[idx_a[a_dim-1]]; // for(int it = 0; it< a_dim; it++) { // index_a += products_a[it]*as[idx_a[it]]; // } // int index_b = as[idx_b[b_dim-1]]; // for(int it = 0; it< b_dim; it++) { // index_b += products_b[it]*as[idx_b[it]]; // } // int index_out = as[idx_out[OutTensor::Dimension-1]]; // for(int it = 0; it< static_cast(OutTensor::Dimension); it++) { // index_out += products_out[it]*as[idx_out[it]]; // } // out_data[index_out] += a_data[index_a]*b_data[index_b]; // return; // } // }; template constexpr std::array RecursiveCartesian, Index, Tensor, Tensor,Last>::products_a; template constexpr std::array RecursiveCartesian, Index, Tensor, Tensor,Last>::products_b; // template // constexpr std::array, Index, // Tensor, Tensor,Last>::OutTensor::Dimension> RecursiveCartesian, Index, // Tensor, Tensor,Last>::products_out; template struct RecursiveCartesianDispatcher; template struct RecursiveCartesianDispatcher, Index, Tensor, Tensor, Index > { static constexpr int out_dim = no_of_unique::value; static FASTOR_INLINE void Do(const T *a_data, const T *b_data, T *out_data, std::array &as, std::array &idx) { return RecursiveCartesian, Index, Tensor, Tensor, Args...>::Do(a_data, b_data, out_data, as, idx); } }; template struct extractor_contract_2 {}; template struct extractor_contract_2, Index, typename std::enable_if::value!=sizeof...(Idx0)+sizeof...(Idx1)>::type> { template static FASTOR_INLINE typename contraction_impl, Tensor, typename std_ext::make_index_sequence::type>::type contract_impl(const Tensor &a, const Tensor &b) { using _contraction_impl = contraction_impl, Tensor, typename std_ext::make_index_sequence::type>; using resulting_tensor = typename _contraction_impl::type; // is reduction including permuted reduction constexpr bool _is_reduction = resulting_tensor::dimension_t::value == 0; #if CONTRACT_OPT==-3 static_assert(!_is_reduction,"THIS VARIANT OF EINSUM CANNOT DEAL WITH REDUCTION CASES"); constexpr int total = no_of_loops_to_set,Index,Tensor,Tensor, typename std_ext::make_index_sequence::value>::type>::value; using index_generator = contract_meta_engine,Index,Tensor,Tensor, typename std_ext::make_index_sequence::type>; using OutTensor = typename index_generator::OutTensor; constexpr auto& index_a = index_generator::index_a; constexpr auto& index_b = index_generator::index_b; constexpr auto& index_out = index_generator::index_out; OutTensor out; out.zeros(); const T *a_data = a.data(); const T *b_data = b.data(); T *out_data = out.data(); #ifndef FASTOR_DONT_VECTORISE using vectorisability = is_vectorisable,Index,Tensor>; constexpr int stride = vectorisability::stride; using V = typename vectorisability::type; #else constexpr int stride = 1; using V = SIMDVector; #endif V _vec_a; for (int i = 0; i < total; i+=stride) { // out_data[index_out[i]] += a_data[index_a[i]]*b_data[index_b[i]]; _vec_a.set(*(a_data+index_a[i])); V _vec_out = _vec_a*V(b_data+index_b[i]) + V(out_data+index_out[i]); // _vec_out.store(out_data+index_out[i]); _vec_out.aligned_store(out_data+index_out[i]); } return out; } #elif CONTRACT_OPT==-2 static_assert(!_is_reduction,"THIS VARIANT OF EINSUM CANNOT DEAL WITH REDUCTION CASES"); resulting_tensor out; out.zeros(); const T *a_data = a.data(); const T *b_data = b.data(); T *out_data = out.data(); constexpr int a_dim = sizeof...(Rest0); constexpr int b_dim = sizeof...(Rest1); constexpr auto& idx_a = IndexFirstTensor,Index, Tensor,Tensor, typename std_ext::make_index_sequence::type>::indices; constexpr auto& idx_b = IndexSecondTensor,Index, Tensor,Tensor, typename std_ext::make_index_sequence::type>::indices; constexpr auto& idx_out = IndexResultingTensor,Index, Tensor,Tensor, typename std_ext::make_index_sequence::type>::indices; constexpr int total = no_of_loops_to_set,Index,Tensor,Tensor, typename std_ext::make_index_sequence::value>::type>::value; using maxes_out_type = typename no_of_loops_to_set,Index,Tensor,Tensor, typename std_ext::make_index_sequence::value>::type>::type; constexpr auto& as_all = cartesian_product::type>::values; // constexpr auto as_all = cartesian_product_2(maxes_out); constexpr std::array products_a = nprods,typename std_ext::make_index_sequence::type>::values; constexpr std::array products_b = nprods,typename std_ext::make_index_sequence::type>::values; using Index_with_dims = typename put_dims_in_Index::type; constexpr std::array products_out = nprods::type>::values; #ifndef FASTOR_DONT_VECTORISE using vectorisability = is_vectorisable,Index,Tensor>; constexpr int stride = vectorisability::stride; using V = typename vectorisability::type; #else constexpr int stride = 1; using V = SIMDVector; #endif int it; V _vec_a; for (int i = 0; i < total; i+=stride) { int index_a = as_all[i][idx_a[a_dim-1]]; for(it = 0; it< a_dim; it++) { index_a += products_a[it]*as_all[i][idx_a[it]]; } int index_b = as_all[i][idx_b[b_dim-1]]; for(it = 0; it< b_dim; it++) { index_b += products_b[it]*as_all[i][idx_b[it]]; } int index_out = as_all[i][idx_out[idx_out.size()-1]]; for(it = 0; it< idx_out.size(); it++) { index_out += products_out[it]*as_all[i][idx_out[it]]; } _vec_a.set(*(a_data+index_a)); V _vec_out = _vec_a*V(b_data+index_b) + V(out_data+index_out); // _vec_out.store(out_data+index_out); _vec_out.aligned_store(out_data+index_out); } return out; } #elif CONTRACT_OPT==-1 using resulting_index = typename _contraction_impl::indices; resulting_tensor out; out.zeros(); const T *a_data = a.data(); const T *b_data = b.data(); T *out_data = out.data(); constexpr int a_dim = sizeof...(Rest0); constexpr int b_dim = sizeof...(Rest1); constexpr int out_dim = no_of_unique::value; constexpr auto& idx_a = IndexTensors< Index, Tensor, Index,Tensor, typename std_ext::make_index_sequence::type>::indices; constexpr auto& idx_b = IndexTensors< Index, Tensor, Index,Tensor, typename std_ext::make_index_sequence::type>::indices; constexpr auto& idx_out = IndexTensors< Index, Tensor, resulting_index,resulting_tensor, typename std_ext::make_index_sequence::type>::indices; using nloops = loop_setter< Index, Tensor, typename std_ext::make_index_sequence::type>; constexpr auto& maxes_out = nloops::dims; constexpr int total = nloops::value; constexpr std::array products_a = nprods,typename std_ext::make_index_sequence::type>::values; constexpr std::array products_b = nprods,typename std_ext::make_index_sequence::type>::values; #ifndef FASTOR_DONT_VECTORISE using vectorisability = is_vectorisable,Index,Tensor>; constexpr int stride = vectorisability::stride; using V = typename vectorisability::type; #else constexpr int stride = 1; using V = SIMDVector; #endif std::array as = {}; int it; V _vec_a; #if FASTOR_CXX_VERSION >= 2017 constexpr std::array remainings = find_remaining(maxes_out, total); #endif FASTOR_IF_CONSTEXPR(!_is_reduction) { using Index_with_dims = typename put_dims_in_Index::type; constexpr std::array products_out = \ nprods::type>::values; for (int i = 0; i < total; i+=stride) { #if FASTOR_CXX_VERSION >= 2017 for (int n = 0; n < out_dim; ++n) { as[n] = ( i / remainings[n] ) % (int)maxes_out[n]; } #else int remaining = total; for (int n = 0; n < out_dim; ++n) { remaining /= maxes_out[n]; as[n] = ( i / remaining ) % maxes_out[n]; } #endif int index_a = as[idx_a[a_dim-1]]; for(it = 0; it< a_dim; it++) { index_a += products_a[it]*as[idx_a[it]]; } int index_b = as[idx_b[b_dim-1]]; for(it = 0; it< b_dim; it++) { index_b += products_b[it]*as[idx_b[it]]; } int index_out = as[idx_out[resulting_tensor::Dimension-1]]; for(it = 0; it< static_cast(resulting_tensor::Dimension); it++) { index_out += products_out[it]*as[idx_out[it]]; } // println(index_out,index_a,index_b,"\n"); _vec_a.set(*(a_data+index_a)); // _vec_a.broadcast(&a_data[index_a]); V _vec_out = _vec_a*V(b_data+index_b) + V(out_data+index_out); // V _vec_out = fmadd(_vec_a,V(b_data+index_b), V(out_data+index_out)); // _vec_out.store(out_data+index_out); _vec_out.aligned_store(out_data+index_out); } // ACTUALLY MUCH SLOWER // constexpr auto as_all = cartesian_product_2(maxes_out); // for (int i = 0; i < total; i+=stride) { // int index_a = as_all[i][idx_a[a_dim-1]]; // for(it = 0; it< a_dim; it++) { // index_a += products_a[it]*as_all[i][idx_a[it]]; // } // int index_b = as_all[i][idx_b[b_dim-1]]; // for(it = 0; it< b_dim; it++) { // index_b += products_b[it]*as_all[i][idx_b[it]]; // } // int index_out = as_all[i][idx_out[idx_out.size()-1]]; // for(it = 0; it< idx_out.size(); it++) { // index_out += products_out[it]*as_all[i][idx_out[it]]; // } // _vec_a.set(*(a_data+index_a)); // // V _vec_out = _vec_a*V(b_data+index_b) + V(out_data+index_out); // V _vec_out = fmadd(_vec_a,V(b_data+index_b), V(out_data+index_out)); // _vec_out.aligned_store(out_data+index_out); // } } else { for (int i = 0; i < total; i+=stride) { #if FASTOR_CXX_VERSION >= 2017 for (int n = 0; n < out_dim; ++n) { as[n] = ( i / remainings[n] ) % (int)maxes_out[n]; } #else int remaining = total; for (int n = 0; n < out_dim; ++n) { remaining /= maxes_out[n]; as[n] = ( i / remaining ) % maxes_out[n]; } #endif int index_a = as[idx_a[a_dim-1]]; for(it = 0; it< a_dim; it++) { index_a += products_a[it]*as[idx_a[it]]; } int index_b = as[idx_b[b_dim-1]]; for(it = 0; it< b_dim; it++) { index_b += products_b[it]*as[idx_b[it]]; } out_data[0] += a_data[index_a]*b_data[index_b]; } } return out; } #elif CONTRACT_OPT==1 using resulting_index = typename _contraction_impl::indices; resulting_tensor out; out.zeros(); const T *a_data = a.data(); const T *b_data = b.data(); T *out_data = out.data(); constexpr int a_dim = sizeof...(Rest0); constexpr int b_dim = sizeof...(Rest1); constexpr int out_dim = no_of_unique::value; constexpr auto& idx_a = IndexTensors< Index, Tensor, Index,Tensor, typename std_ext::make_index_sequence::type>::indices; constexpr auto& idx_b = IndexTensors< Index, Tensor, Index,Tensor, typename std_ext::make_index_sequence::type>::indices; constexpr auto& idx_out = IndexTensors< Index, Tensor, resulting_index,resulting_tensor, typename std_ext::make_index_sequence::type>::indices; using nloops = loop_setter< Index, Tensor, typename std_ext::make_index_sequence::type>; constexpr auto& maxes_out = nloops::dims; constexpr int total = nloops::value; constexpr std::array products_a = nprods,typename std_ext::make_index_sequence::type>::values; constexpr std::array products_b = nprods,typename std_ext::make_index_sequence::type>::values; #ifndef FASTOR_DONT_VECTORISE using vectorisability = is_vectorisable,Index,Tensor>; constexpr int stride = vectorisability::stride; using V = typename vectorisability::type; #else constexpr int stride = 1; using V = SIMDVector; #endif std::array as = {}; int it, jt, counter = 0; V _vec_a; FASTOR_IF_CONSTEXPR(!_is_reduction) { using Index_with_dims = typename put_dims_in_Index::type; constexpr std::array products_out = \ nprods::type>::values; while(true) { int index_a = as[idx_a[a_dim-1]]; for(it = 0; it< a_dim; it++) { index_a += products_a[it]*as[idx_a[it]]; } int index_b = as[idx_b[b_dim-1]]; for(it = 0; it< b_dim; it++) { index_b += products_b[it]*as[idx_b[it]]; } int index_out = as[idx_out[resulting_tensor::Dimension-1]]; for(it = 0; it< static_cast(resulting_tensor::Dimension); it++) { index_out += products_out[it]*as[idx_out[it]]; } _vec_a.set(*(a_data+index_a)); // V _vec_out = _vec_a*V(b_data+index_b) + V(out_data+index_out); V _vec_out = fmadd(_vec_a,V(b_data+index_b), V(out_data+index_out)); _vec_out.store(out_data+index_out,false); // _vec_out.aligned_store(out_data+index_out); counter++; for(jt = maxes_out.size()-1; jt>=0 ; jt--) { if (jt == maxes_out.size()-1) as[jt]+=stride; else as[jt] +=1; if(as[jt]=0 ; jt--) { if (jt == maxes_out.size()-1) as[jt]+=1; else as[jt] +=1; if(as[jt]::value; std::array as = {}; std::array idx = {}; using nloops = loop_setter< Index, Tensor, typename std_ext::make_index_sequence::type>; using dims_type = typename nloops::dims_type; RecursiveCartesianDispatcher,Index, Tensor,Tensor,dims_type>::Do(a_data,b_data,out_data,as,idx); return out; } #endif }; // Specialisation for outer product case template struct extractor_contract_2, Index, typename std::enable_if::value==sizeof...(Idx0)+sizeof...(Idx1)>::type> { template static Tensor FASTOR_INLINE contract_impl(const Tensor &a, const Tensor &b) { Tensor out; _dyadic::value, pack_prod::value>(a.data(),b.data(),out.data()); return out; } }; template FASTOR_INLINE auto contraction(const Tensor &a, const Tensor &b) -> decltype(extractor_contract_2::contract_impl(a,b)) { return extractor_contract_2::contract_impl(a,b); } } #endif // CONTRACTION_H