Initial commit

linx64/mpi/openmpi/bin/aggregate_profile.pl

@@ -0,0 +1,71 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright (c) 2013-2015 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2013-2015 Inria.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+#
+# Author Emmanuel Jeannot <emmanuel.jeannot@inria.fr>
+#
+# This script aggregates the profiles generated by the flush_monitoring function.
+# The files need to be in in given format: name_<phase_id>_<process_id>
+# They are then aggregated by phases.
+# If one needs the profile of all the phases he can concatenate the different files,
+# or use the output of the monitoring system done at MPI_Finalize
+# in the example it should be call as:
+# ./aggregate_profile.pl prof/phase to generate
+# prof/phase_1.prof
+# prof/phase_2.prof
+#
+# ensure that this script as the executable right: chmod +x ...
+#
+
+die "$0 <name of the  profile>\n\tProfile files should be of the form  \"name_phaseid_processesid.prof\"\n\tFor instance if you saved the monitoring into phase_0.0.prof, phase_0.1.prof, ..., phase_1.0.prof etc you should call: $0 phase\n" if ($#ARGV!=0);
+
+$name = $ARGV[0];
+
+@files = glob ($name."*");
+
+%phaseid = ();
+
+
+# Detect the different phases
+foreach $file (@files) {
+  ($id)=($file =~ m/$name\_(\d+)\.\d+/);
+  $phaseid{$id} = 1 if ($id);
+}
+
+# for each phases aggregate the files
+foreach $id (sort {$a <=> $b} keys %phaseid) {
+  aggregate($name."_".$id);
+}
+
+
+
+
+sub aggregate{
+  $phase = $_[0];
+
+  print "Building $phase.prof\n";
+
+  open OUT,">$phase.prof";
+
+  @files = glob ($phase."*");
+
+  foreach $file ( @files) {
+    open IN,$file;
+    while (<IN>) {
+      print OUT;
+    }
+    close IN;
+  }
+  close OUT;
+}

linx64/mpi/openmpi/bin/mpic++

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/mpicc

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/mpicxx

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/mpiexec

@@ -0,0 +1 @@
+orterun

linx64/mpi/openmpi/bin/mpif77

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/mpif90

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/mpifort

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/mpirun

@@ -0,0 +1 @@
+orterun

linx64/mpi/openmpi/bin/ompi-clean

@@ -0,0 +1 @@
+orte-clean

linx64/mpi/openmpi/bin/ompi-dvm

@@ -0,0 +1 @@
+orte-dvm

linx64/mpi/openmpi/bin/ompi-ps

@@ -0,0 +1 @@
+orte-ps

linx64/mpi/openmpi/bin/ompi-server

@@ -0,0 +1 @@
+orte-server

linx64/mpi/openmpi/bin/ompi-top

@@ -0,0 +1 @@
+orte-top

linx64/mpi/openmpi/bin/ortecc

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/oshc++

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/oshcc

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/oshcxx

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/oshfort

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/oshrun

@@ -0,0 +1 @@
+mpirun

linx64/mpi/openmpi/bin/profile2mat.pl

@@ -0,0 +1,125 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright (c) 2013-2015 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2013-2016 Inria.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+#
+# Author Emmanuel Jeannot <emmanuel.jeannot@inria.fr>
+#
+# Take a profile file and aggregates all the recorded communicaton into matrices.
+# It generated a matrices for teh number of messages, (msg),
+# for the total bytes transmitted (size) and
+# the average nulber of bytes per messages (avg)
+#
+# The output matix is symetric
+#
+# If possible it creates file with "internal" tags (collexctive and eta data),
+# "external" tags (point to point messages)  and "all" (every messgaes).
+#
+# ensure that this script as the executable right: chmod +x ...
+#
+
+
+if($#ARGV < 0){
+   die("Usage: $0 <\".prof\" filename>\n");
+}else{
+   $filename=$ARGV[0];
+}
+
+profile($filename,"I|E|S|R|C","all");
+if ( profile($filename,"E","external") ){
+    profile($filename,"I","internal");
+    profile($filename,"S|R","osc");
+    profile($filename,"C","coll");
+}
+
+sub profile{
+   my $filename= $_[0];
+   my $filter= $_[1];
+   my $suffix= $_[2];
+   my $done = 0;
+
+   $outfile=$filename;
+   $outfile=~s/\.prof$/_size_$suffix\.mat/;
+
+
+   open IN,"<$filename";
+   $n=0;
+   @mat1=();
+   @mat2=();
+   @mat3=();
+   $i=0;
+   while (<IN>) {
+      $i++;
+      if (($f,$p1,$p2,$s,$m)=/^($filter)\s+(\d+)\s+(\d+)\s+(\d+)\D+(\d+)/){
+	 $done = 1;
+	 $f++;
+	 #print "$p1 | $p2 | $s | $m\n";
+	 $mat1[$p1][$p2]+=$s;
+	 $mat1[$p2][$p1]+=$s;
+	 $mat2[$p1][$p2]+=$m;
+	 $mat2[$p2][$p1]+=$m;
+	 $n=$p1 if ($p1>$n);
+	 $n=$p2 if ($p2>$n);
+      }else {
+	 # print("file $filename line $i: $_\n");
+      }
+   }
+   close IN;
+
+   #print "$done\n";
+
+   foreach $i (0..$n) {
+      foreach $j (0..$n) {
+	 $mat1[$i][$j]+=0;
+	 $mat2[$i][$j]+=0;
+	 $mat1[$i][$j]/=2;
+	 $mat2[$i][$j]/=2;
+	 if ($mat2[$i][$j]){
+	    $mat3[$i][$j]=$mat1[$i][$j]/$mat2[$i][$j] ;
+	    #printf"%f\t%f\t%f\n",$mat1[$i][$j],$mat2[$i][$j],$mat3[$i][$j];
+	 }else{
+	    $mat3[$i][$j]=0;
+	 }
+      }
+   }
+
+
+   if ($done) {
+      print "$filename -> $suffix\n";
+      save_file($outfile,$n,\@mat1);
+      $outfile=~s/_size/_msg/;
+      save_file($outfile,$n,\@mat2);
+      $outfile=~s/_msg/_avg/;
+      save_file($outfile,$n,\@mat3);
+      print"\n";
+   }
+   return $done;
+}
+
+
+sub save_file{
+   my $outfile=$_[0];
+   my $n=$_[1];
+   my @mat=@{$_[2]};
+   $s=$n+1;
+   print "$outfile\n";
+   open OUT,">$outfile";
+   foreach $i (0..$n) {
+      foreach $j (0..$n) {
+	 printf OUT "%.0f ",$mat[$i][$j];
+      }
+      print OUT "\n";
+   }
+ #  print"\n------------\n\n";
+   close OUT;
+}

linx64/mpi/openmpi/bin/shmemc++

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/shmemcc

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/shmemcxx

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/shmemfort

@@ -0,0 +1 @@
+opal_wrapper

linx64/mpi/openmpi/bin/shmemrun

@@ -0,0 +1 @@
+mpirun

linx64/mpi/openmpi/etc/openmpi-default-hostfile

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the default hostfile for Open MPI.  Notice that it does not
+# contain any hosts (not even localhost).  This file should only
+# contain hosts if a system administrator wants users to always have
+# the same set of default hosts, and is not using a batch scheduler
+# (such as SLURM, PBS, etc.).
+#
+# Note that this file is *not* used when running in "managed"
+# environments (e.g., running in a job under a job scheduler, such as
+# SLURM or PBS / Torque).
+#
+# If you are primarily interested in running Open MPI on one node, you
+# should *not* simply list "localhost" in here (contrary to prior MPI
+# implementations, such as LAM/MPI).  A localhost-only node list is
+# created by the RAS component named "localhost" if no other RAS
+# components were able to find any hosts to run on (this behavior can
+# be disabled by excluding the localhost RAS component by specifying
+# the value "^localhost" [without the quotes] to the "ras" MCA
+# parameter).
+

linx64/mpi/openmpi/etc/openmpi-mca-params.conf

@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2017 Cisco Systems, Inc.  All rights reserved
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This is the default system-wide MCA parameters defaults file.
+# Specifically, the MCA parameter "mca_param_files" defaults to a
+# value of
+# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf"
+# (this file is the latter of the two).  So if the default value of
+# mca_param_files is not changed, this file is used to set system-wide
+# MCA parameters.  This file can therefore be used to set system-wide
+# default MCA parameters for all users.  Of course, users can override
+# these values if they want, but this file is an excellent location
+# for setting system-specific MCA parameters for those users who don't
+# know / care enough to investigate the proper values for them.
+
+# Note that this file is only applicable where it is visible (in a
+# filesystem sense).  Specifically, MPI processes each read this file
+# during their startup to determine what default values for MCA
+# parameters should be used.  mpirun does not bundle up the values in
+# this file from the node where it was run and send them to all nodes;
+# the default value decisions are effectively distributed.  Hence,
+# these values are only applicable on nodes that "see" this file.  If
+# $sysconf is a directory on a local disk, it is likely that changes
+# to this file will need to be propagated to other nodes.  If $sysconf
+# is a directory that is shared via a networked filesystem, changes to
+# this file will be visible to all nodes that share this $sysconf.
+
+# The format is straightforward: one per line, mca_param_name =
+# rvalue.  Quoting is ignored (so if you use quotes or escape
+# characters, they'll be included as part of the value).  For example:
+
+# Disable run-time MPI parameter checking
+#   mpi_param_check = 0
+
+# Note that the value "~/" will be expanded to the current user's home
+# directory.  For example:
+
+# Change component loading path
+#   component_path = /usr/local/lib/openmpi:~/my_openmpi_components
+
+# See "ompi_info --param all all --level 9" for a full listing of Open
+# MPI MCA parameters available and their default values.

linx64/mpi/openmpi/etc/openmpi-totalview.tcl

@@ -0,0 +1,38 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Check if the newly loaded image is one of the MPI starter programs
+# and start it immediately if it is.
+
+proc mpi_auto_run_starter {loaded_id} {
+    set starter_programs {mpirun mpiexec orterun}
+    set executable_name [TV::symbol get $loaded_id full_pathname]
+    set file_component [file tail $executable_name]
+
+    if {[lsearch -exact $starter_programs $file_component] != -1} {
+        puts "**************************************"
+        puts "Automatically starting $file_component"
+        puts "**************************************"
+        dgo
+    }
+}
+
+# Append this function to TotalView's image load callbacks so that
+# TotalView run this program automatically.
+
+dlappend TV::image_load_callbacks mpi_auto_run_starter

linx64/mpi/openmpi/etc/pmix-mca-params.conf

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2017 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This is the default system-wide MCA parameters defaults file.
+# Specifically, the MCA parameter "mca_param_files" defaults to a
+# value of
+# "$HOME/.pmix/mca-params.conf:$sysconf/pmix-mca-params.conf"
+# (this file is the latter of the two).  So if the default value of
+# mca_param_files is not changed, this file is used to set system-wide
+# MCA parameters.  This file can therefore be used to set system-wide
+# default MCA parameters for all users.  Of course, users can override
+# these values if they want, but this file is an excellent location
+# for setting system-specific MCA parameters for those users who don't
+# know / care enough to investigate the proper values for them.
+
+# Note that this file is only applicable where it is visible (in a
+# filesystem sense).  Specifically, processes each read this file
+# during their call to PMIx_Init to determine what default values for MCA
+# parameters should be used.  Launchers generally do not bundle up the values in
+# this file from the node where they are run and send them to all nodes;
+# the default value decisions are effectively distributed.  Hence,
+# these values are typically only applicable on nodes that "see" this file.  If
+# $sysconf is a directory on a local disk, it is likely that changes
+# to this file will need to be propagated to other nodes.  If $sysconf
+# is a directory that is shared via a networked filesystem, changes to
+# this file will be visible to all nodes that share this $sysconf.
+
+# The format is straightforward: one per line, mca_param_name =
+# rvalue.  Quoting is ignored (so if you use quotes or escape
+# characters, they'll be included as part of the value).  For example:
+
+# Set debugging verbosity on the PTL framework
+#   ptl_base_verbose = 10
+
+# Note that the value "~/" will be expanded to the current user's home
+# directory.  For example:
+
+# Change component loading path
+#   mca_base_component_path = /usr/local/lib/pmix:~/my_pmix_components
+
+# See "pinfo --param all all --level 9" for a full listing of PMIx
+# MCA parameters available and their default values.

linx64/mpi/openmpi/include/mpi-ext.h

@@ -0,0 +1,28 @@
+/*
+ * $HEADER$
+ */
+
+#ifndef OMPI_MPI_EXT_H
+#define OMPI_MPI_EXT_H 1
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+#define OMPI_HAVE_MPI_EXT 1
+
+/* Enabled Extension: affinity */
+#define OMPI_HAVE_MPI_EXT_AFFINITY 1
+#include "openmpi/ompi/mpiext/affinity/c/mpiext_affinity_c.h"
+
+/* Enabled Extension: cuda */
+#define OMPI_HAVE_MPI_EXT_CUDA 1
+#include "openmpi/ompi/mpiext/cuda/c/mpiext_cuda_c.h"
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif /* OMPI_MPI_EXT_H */
+

linx64/mpi/openmpi/include/mpi_portable_platform.h

@@ -0,0 +1,401 @@
+/*
+ * Header file with preprocessor magic to figure out, which compiler the user has been calling!
+ *
+ * This code is adapted from the file other/portable_platform.h of GASnet-1.14.0:
+ *  - Ripping out the required parts.
+ *  - Get rid of brackets as it messes up autoconf
+ *  - Delete version tests for older PGI versions (#include "omp.h" not acceptabe)
+ *  - Indent ('#' should be in column 0)
+ *
+ * External packages (i.e., romio) depend on top_build_dir/ompi/include, therefore
+ * although this is not changed in the configure process, this has to be set as
+ * a .in file...
+ * ---------------------------------------------------------------------------
+ */
+#ifndef OPAL_PORTABLE_PLATFORM_H
+#define OPAL_PORTABLE_PLATFORM_H
+
+/* All files in this directory and all sub-directories (except where otherwise noted)
+ * are subject to the following licensing terms:
+ *
+ * ---------------------------------------------------------------------------
+ * "Copyright (c) 2000-2003 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without written agreement is
+ * hereby granted, provided that the above copyright notice and the following
+ * two paragraphs appear in all copies of this software.
+ *
+ * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT
+ * OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF
+ * CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATION TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS."
+ * ---------------------------------------------------------------------------
+ *
+ * Please see the license.txt files within the gm-conduit, lapi-conduit and
+ * vapi-conduit directories for the licensing terms governing those
+ * contributed components.
+ *
+ * The authors/contributors of GASNet include:
+ *
+ *  Dan Bonachea <bonachea@cs.berkeley.edu>:
+ *    General infrastructure & documentation
+ *    mpi-conduit
+ *    elan-conduit
+ *    smp-conduit
+ *    udp-conduit
+ *    extended-ref
+ *    template-conduit
+ *  Christian Bell <csbell@cs.berkeley.edu>: gm-conduit, shmem-conduit
+ *  Mike Welcome <mlwelcome@lbl.gov>: lapi-conduit, portals-conduit
+ *  Paul H. Hargrove <phhargrove@lbl.gov>: vapi-conduit, ibv-conduit
+ *  Rajesh Nishtala <rajeshn@cs.berkeley.edu>: collectives, dcmf-conduit
+ *  Parry Husbands (PJRHusbands@lbl.gov): lapi-conduit
+ *
+ * For more information about GASNet, visit our home page at:
+ *  http://gasnet.cs.berkeley.edu/
+ * Or send email to:
+ *  <upc@lbl.gov>
+ *
+ * Source code contributions (fixes, patches, extensions etc.) should be
+ * sent to <upc@lbl.gov> to be reviewed for acceptance into the primary
+ * distribution. Contributions are most likely to be accepted if they
+ * are provided as public domain, or under a BSD-style license such as
+ * the one above.
+ *
+ */
+#ifndef _STRINGIFY
+#define _STRINGIFY_HELPER(x) #x
+#define _STRINGIFY(x) _STRINGIFY_HELPER(x)
+#endif
+
+#if defined(__INTEL_COMPILER)
+#    define PLATFORM_COMPILER_FAMILYNAME INTEL
+#    define PLATFORM_COMPILER_FAMILYID 2
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_INTEL_CXX  1
+#    else
+#        define PLATFORM_COMPILER_INTEL_C  1
+#    endif
+#    define _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE 19700000 /* year 1970: predates most intel products :) */
+#    ifdef __INTEL_COMPILER_BUILD_DATE
+#        define _PLATFORM_INTEL_COMPILER_BUILD_DATE __INTEL_COMPILER_BUILD_DATE
+#    else
+#        define _PLATFORM_INTEL_COMPILER_BUILD_DATE _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE
+#    endif
+     /* patch number is a decimal build date: YYYYMMDD */
+#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat)         \
+         (((((maj) * 10) | (min)) << 20) |                      \
+            ((pat) < _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE ?   \
+             _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE : ((pat)-_PLATFORM_COMPILER_INTEL_MIN_BUILDDATE)))
+#    define PLATFORM_COMPILER_VERSION \
+         PLATFORM_COMPILER_VERSION_INT(__INTEL_COMPILER/10, __INTEL_COMPILER/100, _PLATFORM_INTEL_COMPILER_BUILD_DATE)
+#    define PLATFORM_COMPILER_VERSION_STR \
+         _STRINGIFY(__INTEL_COMPILER) "." _STRINGIFY(_PLATFORM_INTEL_COMPILER_BUILD_DATE)
+
+#elif defined(__PATHSCALE__)
+#    define PLATFORM_COMPILER_PATHSCALE  1
+#    define PLATFORM_COMPILER_FAMILYNAME PATHSCALE
+#    define PLATFORM_COMPILER_FAMILYID 3
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_PATHSCALE_CXX  1
+#    else
+#        define PLATFORM_COMPILER_PATHSCALE_C  1
+#    endif
+#    define PLATFORM_COMPILER_VERSION \
+         PLATFORM_COMPILER_VERSION_INT(__PATHCC__,__PATHCC_MINOR__,__PATHCC_PATCHLEVEL__)
+#    define PLATFORM_COMPILER_VERSION_STR __PATHSCALE__
+
+#elif defined(__PGI)
+#    define PLATFORM_COMPILER_PGI  1
+#    define PLATFORM_COMPILER_FAMILYNAME PGI
+#    define PLATFORM_COMPILER_FAMILYID 4
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_PGI_CXX  1
+#    else
+#        define PLATFORM_COMPILER_PGI_C  1
+#    endif
+#    if __PGIC__ == 99
+      /* bug 2230: PGI versioning was broken for some platforms in 7.0
+         no way to know exact version, but provide something slightly more accurate */
+#      define PLATFORM_COMPILER_VERSION 0x070000
+#      define PLATFORM_COMPILER_VERSION_STR "7.?-?"
+#    elif defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__)
+#      define PLATFORM_COMPILER_VERSION \
+            PLATFORM_COMPILER_VERSION_INT(__PGIC__,__PGIC_MINOR__,__PGIC_PATCHLEVEL__)
+#      define PLATFORM_COMPILER_VERSION_STR \
+            _STRINGIFY(__PGIC__) "." _STRINGIFY(__PGIC_MINOR__) "-" _STRINGIFY(__PGIC_PATCHLEVEL__)
+#    else
+       /* PGI before 6.1-4 lacks any version ID preprocessor macros - so use this filthy hack */
+       /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+        * We cannot do these within mpi.h.in, as we should not include ompi.h
+        * Hopefully, compilers with integrated preprocessors will not analyse code within the #if 0-block
+        * XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+        */
+#if 0
+#        ifdef PLATFORM_PGI_IS_ANCIENT
+         /* Include below might fail for ancient versions lacking this header, but testing shows it
+            works back to at least 5.1-3 (Nov 2003), and based on docs probably back to 3.2 (Sep 2000) */
+#            define PLATFORM_COMPILER_VERSION 0
+#        elif defined(__x86_64__) /* bug 1753 - 64-bit omp.h upgrade happenned in <6.0-8,6.1-1) */
+#            include "omp.h"
+#            if defined(_PGOMP_H)
+                 /* 6.1.1 or newer */
+#                define PLATFORM_COMPILER_VERSION 0x060101
+#                define PLATFORM_COMPILER_VERSION_STR ">=6.1-1"
+#            else
+                 /* 6.0.8 or older */
+#                define PLATFORM_COMPILER_VERSION 0
+#                define PLATFORM_COMPILER_VERSION_STR "<=6.0-8"
+#            endif
+#        else /* 32-bit omp.h upgrade happenned in <5.2-4,6.0-8 */
+#            include "omp.h"
+#            if defined(_PGOMP_H)
+                 /* 6.0-8 or newer */
+#                define PLATFORM_COMPILER_VERSION 0x060008
+#                define PLATFORM_COMPILER_VERSION_STR ">=6.0-8"
+#            else
+                 /* 5.2-4 or older */
+#                define PLATFORM_COMPILER_VERSION 0
+#                define PLATFORM_COMPILER_VERSION_STR "<=5.2-4"
+#            endif
+#        endif
+#endif /* 0 */
+       /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */
+#    endif
+
+#elif defined(__xlC__)
+#    define PLATFORM_COMPILER_XLC  1
+#    define PLATFORM_COMPILER_FAMILYNAME XLC
+#    define PLATFORM_COMPILER_FAMILYID 5
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_XLC_CXX  1
+#    else
+#        define PLATFORM_COMPILER_XLC_C  1
+#    endif
+#    define PLATFORM_COMPILER_VERSION __xlC__
+#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+         ( ((maj) << 8) | ((min) << 4) | (pat) )
+
+#elif defined(__DECC) || defined(__DECCXX)
+#    define PLATFORM_COMPILER_COMPAQ  1
+#    define PLATFORM_COMPILER_FAMILYNAME COMPAQ
+#    define PLATFORM_COMPILER_FAMILYID 6
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_COMPAQ_CXX  1
+#    else
+#        define PLATFORM_COMPILER_COMPAQ_C  1
+#    endif
+#    if defined(__DECC_VER)
+#        define PLATFORM_COMPILER_VERSION __DECC_VER
+#    elif defined(__DECCXX_VER)
+#        define PLATFORM_COMPILER_VERSION __DECCXX_VER
+#    endif
+
+#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+         ( ((maj) * 10000000) + ((min) * 100000) + (90000) + (pat) )
+         /* 90000 = official ver, 80000 = customer special ver, 60000 = field test ver */
+
+#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC)
+#    define PLATFORM_COMPILER_SUN  1
+#    define PLATFORM_COMPILER_FAMILYNAME SUN
+#    define PLATFORM_COMPILER_FAMILYID 7
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_SUN_CXX  1
+#    else
+#        define PLATFORM_COMPILER_SUN_C  1
+#    endif
+#    if defined(__SUNPRO_C) && __SUNPRO_C > 0
+#        define PLATFORM_COMPILER_VERSION __SUNPRO_C
+#    elif defined(__SUNPRO_CC) && __SUNPRO_CC > 0
+#        define PLATFORM_COMPILER_VERSION __SUNPRO_CC
+#    endif
+#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+         ( ((maj) << 8) | ((min) << 4) | (pat) )
+
+#elif defined(__HP_cc) || defined(__HP_aCC)
+#    define PLATFORM_COMPILER_HP  1
+#    define PLATFORM_COMPILER_FAMILYNAME HP
+#    define PLATFORM_COMPILER_FAMILYID 8
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_HP_CXX  1
+#    else
+#        define PLATFORM_COMPILER_HP_C  1
+#    endif
+#    if defined(__HP_cc) && __HP_cc > 0
+#        define PLATFORM_COMPILER_VERSION __HP_cc
+#    elif defined(__HP_aCC) && __HP_aCC > 0
+#        define PLATFORM_COMPILER_VERSION __HP_aCC
+#    endif
+#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+         ( ((maj) << 16) | ((min) << 8) | (pat) )
+
+#elif defined(_SGI_COMPILER_VERSION) || \
+   (defined(_COMPILER_VERSION) && defined(__sgi) && !defined(__GNUC__)) /* 7.3.0 and earlier lack _SGI_COMPILER_VERSION */
+#    define PLATFORM_COMPILER_SGI  1
+#    define PLATFORM_COMPILER_FAMILYNAME SGI
+#    define PLATFORM_COMPILER_FAMILYID 9
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_SGI_CXX  1
+#    else
+#        define PLATFORM_COMPILER_SGI_C  1
+#    endif
+#    if defined(_SGI_COMPILER_VERSION) && _SGI_COMPILER_VERSION > 0
+#        define PLATFORM_COMPILER_VERSION _SGI_COMPILER_VERSION
+#    elif defined(_COMPILER_VERSION) && _COMPILER_VERSION > 0
+#        define PLATFORM_COMPILER_VERSION _COMPILER_VERSION
+#    endif
+#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+         ( ((maj) << 8) | ((min) << 4) | (pat) )
+
+#elif defined(_CRAYC)
+#    define PLATFORM_COMPILER_CRAY  1
+#    define PLATFORM_COMPILER_FAMILYNAME CRAY
+#    define PLATFORM_COMPILER_FAMILYID 10
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_CRAY_CXX  1
+#    else
+#        define PLATFORM_COMPILER_CRAY_C  1
+#    endif
+#    if defined(_RELEASE) && defined(_RELEASE_MINOR) /* X1 and XT */
+#        define PLATFORM_COMPILER_VERSION \
+             PLATFORM_COMPILER_VERSION_INT(_RELEASE,_RELEASE_MINOR,0)
+#     elif defined(_RELEASE) /* T3E */
+#        define PLATFORM_COMPILER_VERSION \
+             PLATFORM_COMPILER_VERSION_INT(_RELEASE,0,0)
+#     endif
+#     ifdef _RELEASE_STRING /* X1 and XT */
+#        define PLATFORM_COMPILER_VERSION_STR _RELEASE_STRING
+#     endif
+
+#elif defined(__KCC)
+#    define PLATFORM_COMPILER_KAI  1
+#    define PLATFORM_COMPILER_FAMILYNAME KAI
+#    define PLATFORM_COMPILER_FAMILYID 11
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_KAI_CXX  1
+#    else
+#        define PLATFORM_COMPILER_KAI_C  1
+#    endif
+
+#elif defined(__MTA__)
+#    define PLATFORM_COMPILER_MTA  1
+#    define PLATFORM_COMPILER_FAMILYNAME MTA
+#    define PLATFORM_COMPILER_FAMILYID 12
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_MTA_CXX  1
+#    else
+#        define PLATFORM_COMPILER_MTA_C  1
+#    endif
+
+#elif defined(_SX)
+#    define PLATFORM_COMPILER_NECSX  1
+#    define PLATFORM_COMPILER_FAMILYNAME NECSX
+#    define PLATFORM_COMPILER_FAMILYID 13
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_NECSX_CXX  1
+#    else
+#        define PLATFORM_COMPILER_NECSX_C  1
+#    endif
+
+#elif defined(_MSC_VER)
+#    define PLATFORM_COMPILER_MICROSOFT  1
+#    define PLATFORM_COMPILER_FAMILYNAME MICROSOFT
+#    define PLATFORM_COMPILER_FAMILYID 14
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_MICROSOFT_CXX  1
+#    else
+#        define PLATFORM_COMPILER_MICROSOFT_C  1
+#    endif
+#    define PLATFORM_COMPILER_VERSION _MSC_VER
+
+#elif defined(__TINYC__)
+#    define PLATFORM_COMPILER_TINY  1
+#    define PLATFORM_COMPILER_FAMILYNAME TINY
+#    define PLATFORM_COMPILER_FAMILYID 15
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_TINY_CXX  1
+#    else
+#        define PLATFORM_COMPILER_TINY_C  1
+#    endif
+
+#elif defined(__LCC__)
+#    define PLATFORM_COMPILER_LCC 1
+#    define PLATFORM_COMPILER_FAMILYNAME LCC
+#    define PLATFORM_COMPILER_FAMILYID 16
+#    ifdef __cplusplus
+#        define PLATFORM_COMPILER_LCC_CXX  1
+#    else
+#        define PLATFORM_COMPILER_LCC_C  1
+#    endif
+
+#else /* unknown compiler */
+#     define PLATFORM_COMPILER_UNKNOWN  1
+#endif
+
+/* this stanza comes last, because many vendor compilers lie and claim
+   to be GNU C for compatibility reasons and/or because they share a frontend */
+#if defined(__GNUC__)
+#    undef PLATFORM_COMPILER_UNKNOWN
+#    ifndef PLATFORM_COMPILER_FAMILYID
+#        define PLATFORM_COMPILER_GNU  1
+#        define PLATFORM_COMPILER_FAMILYNAME GNU
+#        define PLATFORM_COMPILER_FAMILYID 1
+#        ifdef __cplusplus
+#            define PLATFORM_COMPILER_GNU_CXX  1
+#        else
+#            define PLATFORM_COMPILER_GNU_C  1
+#        endif
+#        if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+#            define PLATFORM_COMPILER_VERSION \
+                 PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__)
+#        elif defined(__GNUC_MINOR__) /* older versions of egcs lack __GNUC_PATCHLEVEL__ */
+#            define PLATFORM_COMPILER_VERSION \
+                 PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,0)
+#        else
+#            define PLATFORM_COMPILER_VERSION \
+                 PLATFORM_COMPILER_VERSION_INT(__GNUC__,0,0)
+#        endif
+#        define PLATFORM_COMPILER_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
+#    else
+#        define _PLATFORM_COMPILER_GNU_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
+#    endif
+     /* gather any advertised GNU version number info, even for non-gcc compilers */
+#    if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+#        define __PLATFORM_COMPILER_GNU_VERSION_STR \
+             _STRINGIFY(__GNUC__) "." _STRINGIFY(__GNUC_MINOR__) "." _STRINGIFY(__GNUC_PATCHLEVEL__)
+#    elif defined(__GNUC_MINOR__)
+#        define __PLATFORM_COMPILER_GNU_VERSION_STR \
+             _STRINGIFY(__GNUC__) "." _STRINGIFY(__GNUC_MINOR__) ".?"
+#    else
+#        define __PLATFORM_COMPILER_GNU_VERSION_STR \
+             _STRINGIFY(__GNUC__) ".?.?"
+#    endif
+#elif defined(PLATFORM_COMPILER_UNKNOWN) /* unknown compiler */
+#    define PLATFORM_COMPILER_FAMILYNAME UNKNOWN
+#    define PLATFORM_COMPILER_FAMILYID 0
+#endif
+
+/* Default Values */
+#ifndef PLATFORM_COMPILER_VERSION
+#    define PLATFORM_COMPILER_VERSION 0 /* don't know */
+#endif
+
+#ifndef PLATFORM_COMPILER_VERSION_STR
+#    define PLATFORM_COMPILER_VERSION_STR _STRINGIFY(PLATFORM_COMPILER_VERSION)
+#endif
+
+#ifndef PLATFORM_COMPILER_VERSION_INT
+#    define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
+        (((maj) << 16) | ((min) << 8) | (pat))
+#endif
+
+
+#endif /* OPAL_PORTABLE_PLATFORM_H */

linx64/mpi/openmpi/include/mpif-ext.h

@@ -0,0 +1,24 @@
+! -*- fortran -*-
+! $HEADER$
+!
+! *** THIS FILE IS AUTOMATICALLY GENERATED!
+! *** Any manual edits will be lost!
+!
+      integer OMPI_HAVE_MPI_EXT
+      parameter (OMPI_HAVE_MPI_EXT=1)
+!
+!
+!     Enabled Extension: affinity
+!     No mpif.h bindings available
+!
+      integer OMPI_HAVE_MPI_EXT_AFFINITY
+      parameter (OMPI_HAVE_MPI_EXT_AFFINITY=0)
+
+!
+!     Enabled Extension: cuda
+!     No mpif.h bindings available
+!
+      integer OMPI_HAVE_MPI_EXT_CUDA
+      parameter (OMPI_HAVE_MPI_EXT_CUDA=0)
+
+!

linx64/mpi/openmpi/include/mpif-sizeof.h

@@ -0,0 +1,30 @@
+! -*- f90 -*-
+! WARNING: This is a generated file!  Edits will be lost!
+!
+! Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+! $COPYRIGHT$
+!
+! This file was generated by gen-mpi-sizeof.pl for all the MPI_SIZEOF
+! interface possibilities for intrinsic types.  Once TS 29113 is
+! supported in all compilers, we can simply have *one* procedure for
+! each type and use dimension(..) to indicate scalars+all array ranks.
+! But until more compilers support this, we simply generate a
+! procedure for scalars and all possible ranks in an attempt to
+! support lots of Fortran compilers.
+
+! *** ATTENTION!
+!
+! Sad panda.
+!
+! This compiler does not support the Right Stuff to enable MPI_SIZEOF.
+! Specifically: we need support for the INTERFACE keyword,
+! ISO_FORTRAN_ENV, and the STORAGE_SIZE() intrinsic on all types.
+! Apparently, this compiler does not support both of those things, so
+! this file will be (effecitvely) blank (i.e., we didn't bother
+! generating the necessary stuff for MPI_SIZEOF because the compiler
+! doesn't support
+! it).
+!
+! If you want support for MPI_SIZEOF, please use a different Fortran
+! compiler to build Open MPI.
+

linx64/mpi/openmpi/include/mpif.h

@@ -0,0 +1,63 @@
+! -*- fortran -*-
+!
+! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+!                         University Research and Technology
+!                         Corporation.  All rights reserved.
+! Copyright (c) 2004-2005 The University of Tennessee and The University
+!                         of Tennessee Research Foundation.  All rights
+!                         reserved.
+! Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+!                         University of Stuttgart.  All rights reserved.
+! Copyright (c) 2004-2005 The Regents of the University of California.
+!                         All rights reserved.
+! Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
+! Copyright (c) 2017      Research Organization for Information Science
+!                         and Technology (RIST). All rights reserved.
+! $COPYRIGHT$
+!
+! Additional copyrights may follow
+!
+! $HEADER$
+!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!
+! Do ***not*** copy this file to the directory where your Fortran
+! fortran application is compiled unless it is absolutely necessary!  Most
+! modern Fortran compilers now support the -I command line flag, which
+! tells the compiler where to find .h files (specifically, this one).  For
+! example:
+!
+!      shell$ mpifort foo.f -o foo -I$OMPI_HOME/include
+!
+! will probably do the trick (assuming that you have set OMPI_HOME
+! properly).
+!
+! That being said, OMPI's "mpifort" wrapper compiler should
+! automatically include the -I option for you.  The following command
+! should be equivalent to the command listed above:
+!
+!      shell$ mpifort foo.f -o foo
+!
+! You should not copy this file to your local directory because it is
+! possible that this file will be changed between versions of Open MPI.
+! Indeed, this mpif.h is incompatible with the mpif.f of other
+! implementations of MPI.  Using this mpif.h with other implementations
+! of MPI, or with other versions of Open MPI will result in undefined
+! behavior (to include incorrect results, segmentation faults,
+! unexplainable "hanging" in your application, etc.).  Always use the
+! -I command line option instead (or let mpifort do it for you).
+!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+      include 'mpif-config.h'
+      include 'mpif-constants.h'
+      include 'mpif-handles.h'
+      include 'mpif-io-constants.h'
+      include 'mpif-io-handles.h'
+      include 'mpif-externals.h'
+      include 'mpif-sentinels.h'
+      include 'mpif-sizeof.h'

linx64/mpi/openmpi/include/mpp/shmem.fh

@@ -0,0 +1 @@
+../shmem.fh

linx64/mpi/openmpi/include/mpp/shmem.h

@@ -0,0 +1 @@
+../shmem.h

linx64/mpi/openmpi/include/openmpi/ompi/mpiext/affinity/c/mpiext_affinity_c.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2004-2009 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#define OMPI_AFFINITY_STRING_MAX 1024
+
+typedef enum ompi_affinity_fmt {
+    OMPI_AFFINITY_RSRC_STRING_FMT,
+    OMPI_AFFINITY_LAYOUT_FMT
+} ompi_affinity_fmt_t;
+
+OMPI_DECLSPEC int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type,
+				    char ompi_bound[OMPI_AFFINITY_STRING_MAX],
+                                    char current_binding[OMPI_AFFINITY_STRING_MAX],
+                                    char exists[OMPI_AFFINITY_STRING_MAX]);

linx64/mpi/openmpi/include/openmpi/ompi/mpiext/cuda/c/mpiext_cuda_c.h

@@ -0,0 +1,17 @@
+/* ompi/mpiext/cuda/c/mpiext_cuda_c.h.  Generated from mpiext_cuda_c.h.in by configure.  */
+/*
+ * Copyright (c) 2004-2009 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015      NVIDIA, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#define MPIX_CUDA_AWARE_SUPPORT 0
+OMPI_DECLSPEC int MPIX_Query_cuda_support(void);

linx64/mpi/openmpi/include/openshmem/oshmem/constants.h

@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2013      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OSHMEM_CONSTANTS_H
+#define OSHMEM_CONSTANTS_H
+
+#include "orte/constants.h"
+#include "oshmem/include/shmem.h"
+
+
+#define OSHMEM_ERR_BASE   ORTE_ERR_MAX
+
+/* error codes */
+enum {
+    /* Error codes inherited from ORTE/OPAL.  Still enum values so
+       that we might get nice debugger help */
+    OSHMEM_SUCCESS                  = ORTE_SUCCESS,
+
+    OSHMEM_ERROR                    = ORTE_ERROR,
+    OSHMEM_ERR_OUT_OF_RESOURCE      = ORTE_ERR_OUT_OF_RESOURCE,
+    OSHMEM_ERR_TEMP_OUT_OF_RESOURCE = ORTE_ERR_TEMP_OUT_OF_RESOURCE,
+    OSHMEM_ERR_RESOURCE_BUSY        = ORTE_ERR_RESOURCE_BUSY,
+    OSHMEM_ERR_BAD_PARAM            = ORTE_ERR_BAD_PARAM,
+    OSHMEM_ERR_FATAL                = ORTE_ERR_FATAL,
+    OSHMEM_ERR_NOT_IMPLEMENTED      = ORTE_ERR_NOT_IMPLEMENTED,
+    OSHMEM_ERR_NOT_SUPPORTED        = ORTE_ERR_NOT_SUPPORTED,
+    OSHMEM_ERR_INTERUPTED           = ORTE_ERR_INTERUPTED,
+    OSHMEM_ERR_WOULD_BLOCK          = ORTE_ERR_WOULD_BLOCK,
+    OSHMEM_ERR_IN_ERRNO             = ORTE_ERR_IN_ERRNO,
+    OSHMEM_ERR_UNREACH              = ORTE_ERR_UNREACH,
+    OSHMEM_ERR_NOT_FOUND            = ORTE_ERR_NOT_FOUND,
+    OSHMEM_EXISTS                   = ORTE_EXISTS, /* indicates that the specified object already exists */
+    OSHMEM_ERR_TIMEOUT              = ORTE_ERR_TIMEOUT,
+    OSHMEM_ERR_NOT_AVAILABLE        = ORTE_ERR_NOT_AVAILABLE,
+    OSHMEM_ERR_PERM                 = ORTE_ERR_PERM,
+    OSHMEM_ERR_VALUE_OUT_OF_BOUNDS  = ORTE_ERR_VALUE_OUT_OF_BOUNDS,
+    OSHMEM_ERR_FILE_READ_FAILURE    = ORTE_ERR_FILE_READ_FAILURE,
+    OSHMEM_ERR_FILE_WRITE_FAILURE   = ORTE_ERR_FILE_WRITE_FAILURE,
+    OSHMEM_ERR_FILE_OPEN_FAILURE    = ORTE_ERR_FILE_OPEN_FAILURE,
+
+    OSHMEM_ERR_RECV_LESS_THAN_POSTED      = ORTE_ERR_RECV_LESS_THAN_POSTED,
+    OSHMEM_ERR_RECV_MORE_THAN_POSTED      = ORTE_ERR_RECV_MORE_THAN_POSTED,
+    OSHMEM_ERR_NO_MATCH_YET               = ORTE_ERR_NO_MATCH_YET,
+    OSHMEM_ERR_BUFFER                     = ORTE_ERR_BUFFER,
+    OSHMEM_ERR_REQUEST                    = ORTE_ERR_REQUEST,
+    OSHMEM_ERR_NO_CONNECTION_ALLOWED      = ORTE_ERR_NO_CONNECTION_ALLOWED,
+    OSHMEM_ERR_CONNECTION_REFUSED         = ORTE_ERR_CONNECTION_REFUSED   ,
+    OSHMEM_ERR_CONNECTION_FAILED          = ORTE_ERR_CONNECTION_FAILED,
+    OSHMEM_PACK_MISMATCH                  = ORTE_ERR_PACK_MISMATCH,
+    OSHMEM_ERR_PACK_FAILURE               = ORTE_ERR_PACK_FAILURE,
+    OSHMEM_ERR_UNPACK_FAILURE             = ORTE_ERR_UNPACK_FAILURE,
+    OSHMEM_ERR_COMM_FAILURE               = ORTE_ERR_COMM_FAILURE,
+    OSHMEM_UNPACK_INADEQUATE_SPACE        = ORTE_ERR_UNPACK_INADEQUATE_SPACE,
+    OSHMEM_UNPACK_READ_PAST_END_OF_BUFFER = ORTE_ERR_UNPACK_READ_PAST_END_OF_BUFFER,
+    OSHMEM_ERR_TYPE_MISMATCH              = ORTE_ERR_TYPE_MISMATCH,
+    OSHMEM_ERR_COMPARE_FAILURE            = ORTE_ERR_COMPARE_FAILURE,
+    OSHMEM_ERR_COPY_FAILURE               = ORTE_ERR_COPY_FAILURE,
+    OSHMEM_ERR_UNKNOWN_DATA_TYPE          = ORTE_ERR_UNKNOWN_DATA_TYPE,
+    OSHMEM_ERR_DATA_TYPE_REDEF            = ORTE_ERR_DATA_TYPE_REDEF,
+    OSHMEM_ERR_DATA_OVERWRITE_ATTEMPT     = ORTE_ERR_DATA_OVERWRITE_ATTEMPT
+};
+
+
+/* C datatypes */
+/*
+ * SHMEM_Init_thread constants
+ * Do not change the order of these without also modifying mpif.h.in.
+ */
+enum {
+  SHMEM_NULL	= 0,
+  SHMEM_CHAR,
+  SHMEM_UCHAR,
+  SHMEM_SHORT,
+  SHMEM_USHORT,
+  SHMEM_INT,
+  SHMEM_UINT,
+  SHMEM_LONG,
+  SHMEM_ULONG,
+  SHMEM_LLONG,
+  SHMEM_INT32_T,
+  SHMEM_INT64_T,
+  SHMEM_ULLONG,
+  SHMEM_FLOAT,
+  SHMEM_DOUBLE,
+  SHMEM_LDOUBLE,
+
+  SHMEM_FINT,
+  SHMEM_FINT4,
+  SHMEM_FINT8
+};
+
+
+/*
+ * Miscellaneous constants
+ */
+#define SHMEM_ANY_SOURCE         -1                      /* match any source rank */
+#define SHMEM_PROC_NULL          -2                      /* rank of null process */
+#define SHMEM_UNDEFINED          -32766                  /* undefined stuff */
+
+
+#ifndef UNREFERENCED_PARAMETER
+#define UNREFERENCED_PARAMETER(P) ((void)P)
+#endif
+
+#define OSHMEM_PREDEFINED_GLOBAL(type, global) ((type) ((void *) &(global)))
+
+#if OPAL_WANT_MEMCHECKER
+#define MEMCHECKER(x) do {       \
+                x;                     \
+       } while(0)
+#else
+#define MEMCHECKER(x)
+#endif /* OPAL_WANT_MEMCHECKER */
+
+#endif /* OSHMEM_CONSTANTS_H */
+

linx64/mpi/openmpi/include/openshmem/oshmem/frameworks.h

@@ -0,0 +1,25 @@
+/*
+ * This file is autogenerated by autogen.pl. Do not edit this file by hand.
+ */
+#ifndef OSHMEM_FRAMEWORKS_H
+#define OSHMEM_FRAMEWORKS_H
+
+#include <opal/mca/base/mca_base_framework.h>
+
+extern mca_base_framework_t oshmem_atomic_base_framework;
+extern mca_base_framework_t oshmem_memheap_base_framework;
+extern mca_base_framework_t oshmem_scoll_base_framework;
+extern mca_base_framework_t oshmem_spml_base_framework;
+extern mca_base_framework_t oshmem_sshmem_base_framework;
+
+static mca_base_framework_t *oshmem_frameworks[] = {
+    &oshmem_atomic_base_framework,
+    &oshmem_memheap_base_framework,
+    &oshmem_scoll_base_framework,
+    &oshmem_spml_base_framework,
+    &oshmem_sshmem_base_framework,
+    NULL
+};
+
+#endif /* OSHMEM_FRAMEWORKS_H */
+

linx64/mpi/openmpi/include/openshmem/oshmem/types.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2013      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#ifndef SHMEM_TYPES_H
+#define SHMEM_TYPES_H
+
+
+/*
+ * Predefine some internal types so we dont need all the include
+ * dependencies.
+ */
+
+   struct ompi_proc_t;
+   struct oshmem_group_t;
+   struct oshmem_op_t;
+
+#endif

linx64/mpi/openmpi/include/openshmem/oshmem/version.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2013      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * This file should be included by any file that needs full
+ * version information for the OSHMEM project
+ */
+
+#ifndef OSHMEM_VERSIONS_H
+#define OSHMEM_VERSIONS_H
+
+#define OSHMEM_MAJOR_VERSION 3
+#define OSHMEM_MINOR_VERSION 1
+#define OSHMEM_RELEASE_VERSION 2
+#define OSHMEM_GREEK_VERSION ""
+#define OSHMEM_WANT_REPO_REV @OSHMEM_WANT_REPO_REV@
+#define OSHMEM_REPO_REV "v3.1.2"
+#ifdef OSHMEM_VERSION
+/* If we included version.h, we want the real version, not the
+   stripped (no-r number) version */
+#undef OSHMEM_VERSION
+#endif
+#define OSHMEM_VERSION "3.1.2"
+
+#endif

linx64/mpi/openmpi/include/openshmem/oshmem_config.h

@@ -0,0 +1,67 @@
+/* -*- c -*-
+ *
+ * Copyright (c) 2013      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * Function: - OS, CPU and compiler dependent configuration
+ */
+
+#ifndef OSHMEM_CONFIG_H
+#define OSHMEM_CONFIG_H
+
+/* Need to include a bunch of infrastructure from the OMPI layer */
+#include "ompi_config.h"
+
+#define OSHMEM_IDENT_STRING OPAL_IDENT_STRING
+
+#if defined(__WINDOWS__)
+
+#  if defined(_USRDLL)    /* building shared libraries (.DLL) */
+#    if defined(OSHMEM_EXPORTS)
+#      define OSHMEM_DECLSPEC        __declspec(dllexport)
+#      define OSHMEM_MODULE_DECLSPEC
+#    else
+#      define OSHMEM_DECLSPEC        __declspec(dllimport)
+#      if defined(OSHMEM_MODULE_EXPORTS)
+#        define OSHMEM_MODULE_DECLSPEC __declspec(dllexport)
+#      else
+#        define OSHMEM_MODULE_DECLSPEC __declspec(dllimport)
+#      endif  /* defined(OSHMEM_MODULE_EXPORTS) */
+#    endif  /* defined(OSHMEM_EXPORTS) */
+#  else          /* building static library */
+#    if defined(OSHMEM_IMPORTS)
+#      define OSHMEM_DECLSPEC        __declspec(dllimport)
+#    else
+#      define OSHMEM_DECLSPEC
+#    endif  /* defined(OSHMEM_IMPORTS) */
+#    define OSHMEM_MODULE_DECLSPEC
+#  endif  /* defined(_USRDLL) */
+
+#else
+
+#  if OPAL_C_HAVE_VISIBILITY
+#    ifndef OSHMEM_DECLSPEC
+#      define OSHMEM_DECLSPEC            __opal_attribute_visibility__("default")
+#    endif
+#    ifndef OSHMEM_MODULE_DECLSPEC
+#      define OSHMEM_MODULE_DECLSPEC     __opal_attribute_visibility__("default")
+#    endif
+#  else
+#    ifndef OSHMEM_DECLSPEC
+#      define OSHMEM_DECLSPEC
+#    endif
+#    ifndef OSHMEM_MODULE_DECLSPEC
+#      define OSHMEM_MODULE_DECLSPEC
+#    endif
+#  endif
+#endif  /* defined(__WINDOWS__) */
+
+#endif

linx64/mpi/openmpi/include/pshmem.h

@@ -0,0 +1,546 @@
+/*
+ * Copyright (c) 2014-2016 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Intel, Inc. All rights reserved
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PSHMEM_SHMEM_H
+#define PSHMEM_SHMEM_H
+
+
+#include <stddef.h>     /* include for ptrdiff_t */
+#include <stdint.h>     /* include for fixed width types */
+#if defined(c_plusplus) || defined(__cplusplus)
+#    include <complex>
+#    define OSHMEM_COMPLEX_TYPE(type)    std::complex<type>
+#else
+#    include <complex.h>
+#    define OSHMEM_COMPLEX_TYPE(type)    type complex
+#endif
+
+
+#ifndef OSHMEM_DECLSPEC
+#  if defined(OPAL_C_HAVE_VISIBILITY) && (OPAL_C_HAVE_VISIBILITY == 1)
+#     define OSHMEM_DECLSPEC __attribute__((visibility("default")))
+#  else
+#     define OSHMEM_DECLSPEC
+#  endif
+#endif
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
+#define OSHMEMP_HAVE_C11 1
+#else
+#define OSHMEMP_HAVE_C11 0
+#endif
+
+#include <pshmemx.h>
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * Profiling API
+ */
+
+/*
+ * Initialization routines
+ */
+OSHMEM_DECLSPEC  void pshmem_init(void);
+OSHMEM_DECLSPEC  void pshmem_global_exit(int status);
+
+/*
+ * Finalization routines
+ */
+OSHMEM_DECLSPEC  void pshmem_finalize(void);
+
+/*
+ * Query routines
+ */
+OSHMEM_DECLSPEC  int pshmem_n_pes(void);
+OSHMEM_DECLSPEC  int pshmem_my_pe(void);
+
+
+/*
+ * Accessability routines
+ */
+OSHMEM_DECLSPEC int pshmem_pe_accessible(int pe);
+OSHMEM_DECLSPEC int pshmem_addr_accessible(const void *addr, int pe);
+
+/*
+ * Symmetric heap routines
+ */
+OSHMEM_DECLSPEC  void* pshmem_malloc(size_t size);
+OSHMEM_DECLSPEC  void* pshmem_align(size_t align, size_t size);
+OSHMEM_DECLSPEC  void* pshmem_realloc(void *ptr, size_t size);
+OSHMEM_DECLSPEC  void pshmem_free(void* ptr);
+
+/*
+ * Remote pointer operations
+ */
+OSHMEM_DECLSPEC  void *pshmem_ptr(const void *ptr, int pe);
+
+/*
+ * Elemental put routines
+ */
+OSHMEM_DECLSPEC  void pshmem_char_p(char* addr, char value, int pe);
+OSHMEM_DECLSPEC  void pshmem_short_p(short* addr, short value, int pe);
+OSHMEM_DECLSPEC  void pshmem_int_p(int* addr, int value, int pe);
+OSHMEM_DECLSPEC  void pshmem_long_p(long* addr, long value, int pe);
+OSHMEM_DECLSPEC  void pshmem_float_p(float* addr, float value, int pe);
+OSHMEM_DECLSPEC  void pshmem_double_p(double* addr, double value, int pe);
+OSHMEM_DECLSPEC  void pshmem_longlong_p(long long* addr, long long value, int pe);
+OSHMEM_DECLSPEC  void pshmem_longdouble_p(long double* addr, long double value, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_p(dst, val, pe)                                \
+    _Generic(&*(dst),                                         \
+            char*:        pshmem_char_p,                      \
+            short*:       pshmem_short_p,                     \
+            int*:         pshmem_int_p,                       \
+            long*:        pshmem_long_p,                      \
+            long long*:   pshmem_longlong_p,                  \
+            float*:       pshmem_float_p,                     \
+            double*:      pshmem_double_p,                    \
+            long double*: pshmem_longdouble_p)(dst, val, pe)
+#endif
+
+/*
+ * Block data put routines
+ */
+OSHMEM_DECLSPEC  void pshmem_char_put(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_short_put(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_int_put(int* target, const int* source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_long_put(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_float_put(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_double_put(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longlong_put(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longdouble_put(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_put(dst, src, len, pe)               \
+    _Generic(&*(dst),                               \
+            char*:        pshmem_char_put,          \
+            short*:       pshmem_short_put,         \
+            int*:         pshmem_int_put,           \
+            long*:        pshmem_long_put,          \
+            long long*:   pshmem_longlong_put,      \
+            float*:       pshmem_float_put,         \
+            double*:      pshmem_double_put,        \
+            long double*: pshmem_longdouble_put)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void pshmem_put8(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put16(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put32(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put64(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put128(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_putmem(void *target, const void *source, size_t len, int pe);
+
+/*
+ * Strided put routines
+ */
+OSHMEM_DECLSPEC void pshmem_char_iput(char* target, const char* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_short_iput(short* target, const short* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_int_iput(int* target, const int* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_long_iput(long* target, const long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_float_iput(float* target, const float* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_double_iput(double* target, const double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_iput(long long* target, const long long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_longdouble_iput(long double* target, const long double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_iput(dst, src, tst, sst, len, pe)     \
+    _Generic(&*(dst),                                \
+            char*:        pshmem_char_iput,          \
+            short*:       pshmem_short_iput,         \
+            int*:         pshmem_int_iput,           \
+            long*:        pshmem_long_iput,          \
+            long long*:   pshmem_longlong_iput,      \
+            float*:       pshmem_float_iput,         \
+            double*:      pshmem_double_iput,        \
+            long double*: pshmem_longdouble_iput)(dst, src, tst, sst, len, pe)
+#endif
+
+OSHMEM_DECLSPEC void pshmem_iput8(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iput16(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iput32(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iput64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iput128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+
+/*
+ * Nonblocking put routines
+ */
+OSHMEM_DECLSPEC  void pshmem_putmem_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_char_put_nbi(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_short_put_nbi(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_int_put_nbi(int* target, const int* source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_long_put_nbi(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longlong_put_nbi(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_float_put_nbi(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_double_put_nbi(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longdouble_put_nbi(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_put_nbi(dst, src, len, pe)               \
+    _Generic(&*(dst),                                   \
+            char*:        pshmem_char_put_nbi,          \
+            short*:       pshmem_short_put_nbi,         \
+            int*:         pshmem_int_put_nbi,           \
+            long*:        pshmem_long_put_nbi,          \
+            long long*:   pshmem_longlong_put_nbi,      \
+            float*:       pshmem_float_put_nbi,         \
+            double*:      pshmem_double_put_nbi,        \
+            long double*: pshmem_longdouble_put_nbi)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void pshmem_put8_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put16_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put32_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put64_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_put128_nbi(void *target, const void *source, size_t len, int pe);
+
+/*
+ * Elemental get routines
+ */
+OSHMEM_DECLSPEC  char pshmem_char_g(const char* addr, int pe);
+OSHMEM_DECLSPEC  short pshmem_short_g(const short* addr, int pe);
+OSHMEM_DECLSPEC  int pshmem_int_g(const int* addr, int pe);
+OSHMEM_DECLSPEC  long pshmem_long_g(const long* addr, int pe);
+OSHMEM_DECLSPEC  float pshmem_float_g(const float* addr, int pe);
+OSHMEM_DECLSPEC  double pshmem_double_g(const double* addr, int pe);
+OSHMEM_DECLSPEC  long long pshmem_longlong_g(const long long* addr, int pe);
+OSHMEM_DECLSPEC  long double pshmem_longdouble_g(const long double* addr, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_g(addr, pe)                                    \
+    _Generic(&*(dst),                                         \
+            char*:        pshmem_char_g,                      \
+            short*:       pshmem_short_g,                     \
+            int*:         pshmem_int_g,                       \
+            long*:        pshmem_long_g,                      \
+            long long*:   pshmem_longlong_g,                  \
+            float*:       pshmem_float_g,                     \
+            double*:      pshmem_double_g,                    \
+            long double*: pshmem_longdouble_g)(addr, pe)
+#endif
+
+/*
+ * Block data get routines
+ */
+OSHMEM_DECLSPEC  void pshmem_char_get(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_short_get(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_int_get(int *target, const int *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_long_get(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_float_get(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_double_get(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longlong_get(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longdouble_get(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_get(dst, src, len, pe)               \
+    _Generic(&*(dst),                              \
+            char*:        pshmem_char_get,          \
+            short*:       pshmem_short_get,         \
+            int*:         pshmem_int_get,           \
+            long*:        pshmem_long_get,          \
+            long long*:   pshmem_longlong_get,      \
+            float*:       pshmem_float_get,         \
+            double*:      pshmem_double_get,        \
+            long double*: pshmem_longdouble_get)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void pshmem_get8(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get16(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get32(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get64(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get128(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_getmem(void *target, const void *source, size_t len, int pe);
+
+/*
+ * Strided get routines
+ */
+OSHMEM_DECLSPEC void pshmem_char_iget(char* target, const char* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_short_iget(short* target, const short* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_int_iget(int* target, const int* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_float_iget(float* target, const float* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_double_iget(double* target, const double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_iget(long long* target, const long long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_longdouble_iget(long double* target, const long double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_long_iget(long* target, const long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_iget(dst, src, tst, sst, len, pe)     \
+    _Generic(&*(dst),                               \
+            char*:        pshmem_char_iget,          \
+            short*:       pshmem_short_iget,         \
+            int*:         pshmem_int_iget,           \
+            long*:        pshmem_long_iget,          \
+            long long*:   pshmem_longlong_iget,      \
+            float*:       pshmem_float_iget,         \
+            double*:      pshmem_double_iget,        \
+            long double*: pshmem_longdouble_iget)(dst, src, tst, sst, len, pe)
+#endif
+
+OSHMEM_DECLSPEC void pshmem_iget8(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iget16(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iget32(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iget64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void pshmem_iget128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+
+/*
+ * Nonblocking data get routines
+ */
+OSHMEM_DECLSPEC  void pshmem_getmem_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_char_get_nbi(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_short_get_nbi(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_int_get_nbi(int *target, const int *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_long_get_nbi(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longlong_get_nbi(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_float_get_nbi(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_double_get_nbi(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_longdouble_get_nbi(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_get_nbi(dst, src, len, pe)               \
+    _Generic(&*(dst),                                  \
+            char*:        pshmem_char_get_nbi,          \
+            short*:       pshmem_short_get_nbi,         \
+            int*:         pshmem_int_get_nbi,           \
+            long*:        pshmem_long_get_nbi,          \
+            long long*:   pshmem_longlong_get_nbi,      \
+            float*:       pshmem_float_get_nbi,         \
+            double*:      pshmem_double_get_nbi,        \
+            long double*: pshmem_longdouble_get_nbi)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void pshmem_get8_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get16_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get32_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get64_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_get128_nbi(void *target, const void *source, size_t len, int pe);
+
+/*
+ * Atomic operations
+ */
+/* Atomic swap */
+OSHMEM_DECLSPEC int pshmem_int_swap(int *target, int value, int pe);
+OSHMEM_DECLSPEC long pshmem_long_swap(long *target, long value, int pe);
+OSHMEM_DECLSPEC long long pshmem_longlong_swap(long long*target, long long value, int pe);
+OSHMEM_DECLSPEC float pshmem_float_swap(float *target, float value, int pe);
+OSHMEM_DECLSPEC double pshmem_double_swap(double *target, double value, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_swap(dst, val, pe)                             \
+    _Generic(&*(dst),                                        \
+            int*:         pshmem_int_swap,                    \
+            long*:        pshmem_long_swap,                   \
+            long long*:   pshmem_longlong_swap,               \
+            float*:       pshmem_float_swap,               \
+            double*:      pshmem_double_swap)(dst, val, pe)
+#endif
+
+/* Atomic set */
+OSHMEM_DECLSPEC void pshmem_int_set(int *target, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_long_set(long *target, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_set(long long*target, long long value, int pe);
+OSHMEM_DECLSPEC void pshmem_float_set(float *target, float value, int pe);
+OSHMEM_DECLSPEC void pshmem_double_set(double *target, double value, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_set(dst, val, pe)                             \
+    _Generic(&*(dst),                                        \
+            int*:         pshmem_int_set,                    \
+            long*:        pshmem_long_set,                   \
+            long long*:   pshmem_longlong_set,               \
+            float*:       pshmem_float_set,                  \
+            double*:      pshmem_double_set)(dst, val, pe)
+#endif
+
+/* Atomic conditional swap */
+OSHMEM_DECLSPEC int pshmem_int_cswap(int *target, int cond, int value, int pe);
+OSHMEM_DECLSPEC long pshmem_long_cswap(long *target, long cond, long value, int pe);
+OSHMEM_DECLSPEC long long pshmem_longlong_cswap(long long *target, long long cond, long long value, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_cswap(dst, cond, val, pe)                       \
+    _Generic(&*(dst),                                          \
+            int*:         pshmem_int_cswap,                    \
+            long*:        pshmem_long_cswap,                   \
+            long long*:   pshmem_longlong_cswap)(dst, cond, val, pe)
+#endif
+
+/* Atomic Fetch&Add */
+OSHMEM_DECLSPEC int pshmem_int_fadd(int *target, int value, int pe);
+OSHMEM_DECLSPEC long pshmem_long_fadd(long *target, long value, int pe);
+OSHMEM_DECLSPEC long long pshmem_longlong_fadd(long long *target, long long value, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_fadd(dst, val, pe)                             \
+    _Generic(&*(dst),                                         \
+            int*:         pshmem_int_fadd,                    \
+            long*:        pshmem_long_fadd,                   \
+            long long*:   pshmem_longlong_fadd)(dst, val, pe)
+#endif
+
+/* Atomic Fetch */
+OSHMEM_DECLSPEC int pshmem_int_fetch(const int *target, int pe);
+OSHMEM_DECLSPEC long pshmem_long_fetch(const long *target, int pe);
+OSHMEM_DECLSPEC long long pshmem_longlong_fetch(const long long *target, int pe);
+OSHMEM_DECLSPEC float pshmem_float_fetch(const float *target, int pe);
+OSHMEM_DECLSPEC double pshmem_double_fetch(const double *target, int pe);
+#if OSHMEM_HAVE_C11
+#define pshmem_fetch(dst, pe)                             \
+    _Generic(&*(dst),                                     \
+            int*:         pshmem_int_fetch,               \
+            long*:        pshmem_long_fetch,              \
+            long long*:   pshmem_longlong_fetch,          \
+            float*:       pshmem_float_fetch,             \
+            double*:      pshmem_double_fetch)(dst, pe)
+#endif
+
+/* Atomic Fetch&Inc */
+OSHMEM_DECLSPEC int pshmem_int_finc(int *target, int pe);
+OSHMEM_DECLSPEC long pshmem_long_finc(long *target, int pe);
+OSHMEM_DECLSPEC long long pshmem_longlong_finc(long long *target, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_finc(dst, val, pe)                             \
+    _Generic(&*(dst),                                         \
+            int*:         pshmem_int_finc,                    \
+            long*:        pshmem_long_finc,                   \
+            long long*:   pshmem_longlong_finc)(dst, val, pe)
+#endif
+
+/* Atomic Add*/
+OSHMEM_DECLSPEC void pshmem_int_add(int *target, int value, int pe);
+OSHMEM_DECLSPEC void pshmem_long_add(long *target, long value, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_add(long long *target, long long value, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_add(dst, val, pe)                             \
+    _Generic(&*(dst),                                        \
+            int*:         pshmem_int_add,                    \
+            long*:        pshmem_long_add,                   \
+            long long*:   pshmem_longlong_add)(dst, val, pe)
+#endif
+
+/* Atomic Inc */
+OSHMEM_DECLSPEC void pshmem_int_inc(int *target, int pe);
+OSHMEM_DECLSPEC void pshmem_long_inc(long *target, int pe);
+OSHMEM_DECLSPEC void pshmem_longlong_inc(long long *target, int pe);
+#if OSHMEMP_HAVE_C11
+#define pshmem_inc(dst, pe)                            \
+    _Generic(&*(dst),                                  \
+            int*:         pshmem_int_inc,              \
+            long*:        pshmem_long_inc,             \
+            long long*:   pshmem_longlong_inc)(dst, pe)
+#endif
+/*
+ * Lock functions
+ */
+OSHMEM_DECLSPEC void pshmem_set_lock(volatile long *lock);
+OSHMEM_DECLSPEC void pshmem_clear_lock(volatile long *lock);
+OSHMEM_DECLSPEC int pshmem_test_lock(volatile long *lock);
+
+/*
+ * P2P sync routines
+ */
+OSHMEM_DECLSPEC  void pshmem_short_wait(volatile short *addr, short value);
+OSHMEM_DECLSPEC  void pshmem_int_wait(volatile int *addr, int value);
+OSHMEM_DECLSPEC  void pshmem_long_wait(volatile long *addr, long value);
+OSHMEM_DECLSPEC  void pshmem_longlong_wait(volatile long long *addr, long long value);
+OSHMEM_DECLSPEC  void pshmem_wait(volatile long *addr, long value);
+
+OSHMEM_DECLSPEC  void pshmem_short_wait_until(volatile short *addr, int cmp, short value);
+OSHMEM_DECLSPEC  void pshmem_int_wait_until(volatile int *addr, int cmp, int value);
+OSHMEM_DECLSPEC  void pshmem_long_wait_until(volatile long *addr, int cmp, long value);
+OSHMEM_DECLSPEC  void pshmem_longlong_wait_until(volatile long long *addr, int cmp, long long value);
+OSHMEM_DECLSPEC  void pshmem_wait_until(volatile long *addr, int cmp, long value);
+
+/*
+ * Barrier sync routines
+ */
+OSHMEM_DECLSPEC  void pshmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC  void pshmem_barrier_all(void);
+OSHMEM_DECLSPEC  void pshmem_fence(void);
+OSHMEM_DECLSPEC  void pshmem_quiet(void);
+
+/*
+ * Collective routines
+ */
+OSHMEM_DECLSPEC void pshmem_broadcast32(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_broadcast64(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_broadcast(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_collect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_collect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_fcollect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_fcollect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_alltoall32(void *target, const void *source, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_alltoall64(void *target, const void *source, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_alltoalls32(void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void pshmem_alltoalls64(void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+
+/*
+ * Reduction routines
+ */
+OSHMEM_DECLSPEC void pshmem_short_and_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_int_and_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_long_and_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longlong_and_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmem_short_or_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_int_or_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_long_or_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longlong_or_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmem_short_xor_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_int_xor_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_long_xor_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longlong_xor_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmem_short_max_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_int_max_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_long_max_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longlong_max_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_float_max_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_double_max_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longdouble_max_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmem_short_min_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_int_min_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_long_min_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longlong_min_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_float_min_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_double_min_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longdouble_min_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmem_short_sum_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_int_sum_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_long_sum_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longlong_sum_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_float_sum_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_double_sum_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longdouble_sum_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_complexf_sum_to_all(OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(float) *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_complexd_sum_to_all(OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(double) *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmem_short_prod_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_int_prod_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_long_prod_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longlong_prod_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_float_prod_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_double_prod_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_longdouble_prod_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_complexf_prod_to_all(OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(float) *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmem_complexd_prod_to_all(OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(double) *pWrk, long *pSync);
+
+/*
+ * Platform specific cache management routines
+ */
+OSHMEM_DECLSPEC void pshmem_udcflush(void);
+OSHMEM_DECLSPEC void pshmem_udcflush_line(void* target);
+OSHMEM_DECLSPEC void pshmem_set_cache_inv(void);
+OSHMEM_DECLSPEC void pshmem_set_cache_line_inv(void* target);
+OSHMEM_DECLSPEC void pshmem_clear_cache_inv(void);
+OSHMEM_DECLSPEC void pshmem_clear_cache_line_inv(void* target);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+
+#endif /* PSHMEM_SHMEM_H */

linx64/mpi/openmpi/include/pshmemx.h

@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2014      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Intel, Inc. All rights reserved
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PSHMEM_SHMEMX_H
+#define PSHMEM_SHMEMX_H
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/*
+ * Legacy API
+ * old init/destruct functions - not in the open shmem spec but still supported
+ */
+OSHMEM_DECLSPEC  void pstart_pes(int npes);
+
+OSHMEM_DECLSPEC  int p_num_pes(void);
+OSHMEM_DECLSPEC  int p_my_pe(void);
+
+OSHMEM_DECLSPEC  void* pshmalloc(size_t size);
+OSHMEM_DECLSPEC  void* pshmemalign(size_t align, size_t size);
+OSHMEM_DECLSPEC  void* pshrealloc(void *ptr, size_t size);
+OSHMEM_DECLSPEC  void pshfree(void* ptr);
+
+OSHMEM_DECLSPEC  void pshmem_char_put(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void pshmem_char_get(char *target, const char *source, size_t len, int pe);
+
+
+/*
+ * All OpenSHMEM extension APIs that are not part of this specification must be defined in the shmemx.h include
+ * file. These extensions shall use the shmemx_ prefix for all routine, variable, and constant names.
+ */
+
+/*
+ * Elemental put routines
+ */
+OSHMEM_DECLSPEC  void pshmemx_int16_p(int16_t* addr, int16_t value, int pe);
+OSHMEM_DECLSPEC  void pshmemx_int32_p(int32_t* addr, int32_t value, int pe);
+OSHMEM_DECLSPEC  void pshmemx_int64_p(int64_t* addr, int64_t value, int pe);
+
+/*
+ * Block data put routines
+ */
+
+/*
+ * Strided put routines
+ */
+
+/*
+ * Elemental get routines
+ */
+OSHMEM_DECLSPEC  int16_t pshmemx_int16_g(const int16_t* addr, int pe);
+OSHMEM_DECLSPEC  int32_t pshmemx_int32_g(const int32_t* addr, int pe);
+OSHMEM_DECLSPEC  int64_t pshmemx_int64_g(const int64_t* addr, int pe);
+
+/*
+ * Block data get routines
+ */
+
+/*
+ * Strided get routines
+ */
+
+/*
+ * Atomic operations
+ */
+/* Atomic swap */
+OSHMEM_DECLSPEC int32_t pshmemx_int32_swap(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmemx_int64_swap(int64_t *target, int64_t value, int pe);
+
+/*Atomic set */
+OSHMEM_DECLSPEC void pshmemx_int32_set(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmemx_int64_set(int64_t *target, int64_t value, int pe);
+
+/* Atomic conditional swap */
+OSHMEM_DECLSPEC int32_t pshmemx_int32_cswap(int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmemx_int64_cswap(int64_t *target, int64_t cond, int64_t value, int pe);
+
+/* Atomic Fetch&Add */
+OSHMEM_DECLSPEC int32_t pshmemx_int32_fadd(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t pshmemx_int64_fadd(int64_t *target, int64_t value, int pe);
+
+/* Atomic Fetch */
+OSHMEM_DECLSPEC int32_t pshmemx_int32_fetch(const int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t pshmemx_int64_fetch(const int64_t *target, int pe);
+
+/* Atomic Fetch&Inc */
+OSHMEM_DECLSPEC int32_t pshmemx_int32_finc(int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t pshmemx_int64_finc(int64_t *target, int pe);
+
+/* Atomic Add*/
+OSHMEM_DECLSPEC void pshmemx_int32_add(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void pshmemx_int64_add(int64_t *target, int64_t value, int pe);
+
+/* Atomic Inc */
+OSHMEM_DECLSPEC void pshmemx_int32_inc(int32_t *target, int pe);
+OSHMEM_DECLSPEC void pshmemx_int64_inc(int64_t *target, int pe);
+
+/*
+ * P2P sync routines
+ */
+OSHMEM_DECLSPEC  void pshmemx_int32_wait(int32_t *addr, int32_t value);
+OSHMEM_DECLSPEC  void pshmemx_int64_wait(int64_t *addr, int64_t value);
+
+OSHMEM_DECLSPEC  void pshmemx_int32_wait_until(int32_t *addr, int cmp, int32_t value);
+OSHMEM_DECLSPEC  void pshmemx_int64_wait_until(int64_t *addr, int cmp, int64_t value);
+
+/*
+ * Reduction routines
+ */
+OSHMEM_DECLSPEC void pshmemx_int16_and_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int32_and_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int64_and_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmemx_int16_or_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int32_or_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int64_or_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmemx_int16_xor_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int32_xor_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int64_xor_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmemx_int16_max_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int32_max_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int64_max_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmemx_int16_min_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int32_min_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int64_min_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmemx_int16_sum_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int32_sum_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int64_sum_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void pshmemx_int16_prod_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int32_prod_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void pshmemx_int64_prod_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+/*
+ * Backward compatibility section
+ */
+#define pshmem_int16_p               pshmemx_int16_p
+#define pshmem_int32_p               pshmemx_int32_p
+#define pshmem_int64_p               pshmemx_int64_p
+
+#define pshmem_int16_g               pshmemx_int16_g
+#define pshmem_int32_g               pshmemx_int32_g
+#define pshmem_int64_g               pshmemx_int64_g
+
+#define pshmem_int32_swap            pshmemx_int32_swap
+#define pshmem_int64_swap            pshmemx_int64_swap
+
+#define pshmem_int32_set             pshmemx_int32_set
+#define pshmem_int64_set             pshmemx_int64_set
+
+#define pshmem_int32_cswap           pshmemx_int32_cswap
+#define pshmem_int64_cswap           pshmemx_int64_cswap
+
+#define pshmem_int32_fadd            pshmemx_int32_fadd
+#define pshmem_int64_fadd            pshmemx_int64_fadd
+
+#define pshmem_int32_fetch           pshmemx_int32_fetch
+#define pshmem_int64_fetch           pshmemx_int64_fetch
+
+#define pshmem_int32_finc            pshmemx_int32_finc
+#define pshmem_int64_finc            pshmemx_int64_finc
+
+#define pshmem_int32_add             pshmemx_int32_add
+#define pshmem_int64_add             pshmemx_int64_add
+
+#define pshmem_int32_inc             pshmemx_int32_inc
+#define pshmem_int64_inc             pshmemx_int64_inc
+
+#define pshmem_int32_wait            pshmemx_int32_wait
+#define pshmem_int64_wait            pshmemx_int64_wait
+#define pshmem_int32_wait_until      pshmemx_int32_wait_until
+#define pshmem_int64_wait_until      pshmemx_int64_wait_until
+
+#define pshmem_int16_and_to_all      pshmemx_int16_and_to_all
+#define pshmem_int32_and_to_all      pshmemx_int32_and_to_all
+#define pshmem_int64_and_to_all      pshmemx_int64_and_to_all
+
+#define pshmem_int16_or_to_all       pshmemx_int16_or_to_all
+#define pshmem_int32_or_to_all       pshmemx_int32_or_to_all
+#define pshmem_int64_or_to_all       pshmemx_int64_or_to_all
+
+#define pshmem_int16_xor_to_all      pshmemx_int16_xor_to_all
+#define pshmem_int32_xor_to_all      pshmemx_int32_xor_to_all
+#define pshmem_int64_xor_to_all      pshmemx_int64_xor_to_all
+
+#define pshmem_int16_max_to_all      pshmemx_int16_max_to_all
+#define pshmem_int32_max_to_all      pshmemx_int32_max_to_all
+#define pshmem_int64_max_to_all      pshmemx_int64_max_to_all
+
+#define pshmem_int16_min_to_all      pshmemx_int16_min_to_all
+#define pshmem_int32_min_to_all      pshmemx_int32_min_to_all
+#define pshmem_int64_min_to_all      pshmemx_int64_min_to_all
+
+#define pshmem_int16_sum_to_all      pshmemx_int16_sum_to_all
+#define pshmem_int32_sum_to_all      pshmemx_int32_sum_to_all
+#define pshmem_int64_sum_to_all      pshmemx_int64_sum_to_all
+
+#define pshmem_int16_prod_to_all     pshmemx_int16_prod_to_all
+#define pshmem_int32_prod_to_all     pshmemx_int32_prod_to_all
+#define pshmem_int64_prod_to_all     pshmemx_int64_prod_to_all
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif /* PSHMEM_SHMEMX_H */

linx64/mpi/openmpi/include/shmem-compat.h

@@ -0,0 +1,47 @@
+/* oshmem/include/shmem-compat.h. This file contains OpenSHMEM lagacy API  */
+/*
+ * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OSHMEM_SHMEM_COMPAT_H
+#define OSHMEM_SHMEM_COMPAT_H
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * Legacy API
+ * old init/destruct functions - not in the open shmem spec but still supported
+ */
+OSHMEM_DECLSPEC  int num_pes(void);
+OSHMEM_DECLSPEC  int my_pe(void);
+
+OSHMEM_DECLSPEC  void start_pes(int npes);
+
+OSHMEM_DECLSPEC  int _num_pes(void);
+OSHMEM_DECLSPEC  int _my_pe(void);
+
+OSHMEM_DECLSPEC  void* shmalloc(size_t size);
+OSHMEM_DECLSPEC  void* shmemalign(size_t align, size_t size);
+OSHMEM_DECLSPEC  void* shrealloc(void *ptr, size_t size);
+OSHMEM_DECLSPEC  void shfree(void* ptr);
+
+OSHMEM_DECLSPEC  void shmem_char_put(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_char_get(char *target, const char *source, size_t len, int pe);
+
+OSHMEM_DECLSPEC  void shmem_put(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void globalexit(int status);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif /* OSHMEM_SHMEM_COMPAT_H */

linx64/mpi/openmpi/include/shmem.fh

@@ -0,0 +1,65 @@
+! Emacs: -*- mode: fortran; -*-
+!
+!   Copyright (c) 2013      Mellanox Technologies, Inc.
+!                           All rights reserved.
+!   Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+!   $COPYRIGHT$
+!
+!   Additional copyrights may follow
+!
+!   $HEADER$
+!
+
+!
+! TODO: exact values should be found during configuration
+!
+      integer SHMEM_MAJOR_VERSION
+      parameter ( SHMEM_MAJOR_VERSION = 1 )
+
+      integer SHMEM_MINOR_VERSION
+      parameter ( SHMEM_MINOR_VERSION = 3 )
+
+      integer SHMEM_MAX_NAME_LEN
+      parameter ( SHMEM_MAX_NAME_LEN = 256-1 )
+
+      character(LEN = SHMEM_MAX_NAME_LEN) SHMEM_VENDOR_STRING
+      parameter ( SHMEM_VENDOR_STRING = "http://www.open-mpi.org/" )
+
+      integer SHMEM_BARRIER_SYNC_SIZE
+      parameter ( SHMEM_BARRIER_SYNC_SIZE = 4 )
+
+      integer SHMEM_BCAST_SYNC_SIZE
+      parameter ( SHMEM_BCAST_SYNC_SIZE = 8 )
+
+      integer SHMEM_COLLECT_SYNC_SIZE
+      parameter ( SHMEM_COLLECT_SYNC_SIZE = 8 )
+
+      integer SHMEM_REDUCE_SYNC_SIZE
+      parameter ( SHMEM_REDUCE_SYNC_SIZE = 8 )
+
+      integer SHMEM_SYNC_VALUE
+      parameter ( SHMEM_SYNC_VALUE = -1 )
+
+      integer SHMEM_REDUCE_MIN_WRKDATA_SIZE
+      parameter ( SHMEM_REDUCE_MIN_WRKDATA_SIZE = 8 )
+
+!
+!     waits
+!
+      integer SHMEM_CMP_EQ
+      parameter ( SHMEM_CMP_EQ = 0 )
+      integer SHMEM_CMP_NE
+      parameter ( SHMEM_CMP_NE = 1 )
+      integer SHMEM_CMP_GT
+      parameter ( SHMEM_CMP_GT = 2 )
+      integer SHMEM_CMP_LE
+      parameter ( SHMEM_CMP_LE = 3 )
+      integer SHMEM_CMP_LT
+      parameter ( SHMEM_CMP_LT = 4 )
+      integer SHMEM_CMP_GE
+      parameter ( SHMEM_CMP_GE = 5 )
+
+      logical shmem_pe_accessible
+      logical shmem_addr_accessible
+
+      integer*8 shmem_ptr

linx64/mpi/openmpi/include/shmem.h

@@ -0,0 +1,635 @@
+/* oshmem/include/shmem.h.  Generated from shmem.h.in by configure.  */
+/*
+ * Copyright (c) 2014-2016 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014      Intel, Inc. All rights reserved
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OSHMEM_SHMEM_H
+#define OSHMEM_SHMEM_H
+
+
+#include <stddef.h>     /* include for ptrdiff_t */
+#include <stdint.h>     /* include for fixed width types */
+#if defined(c_plusplus) || defined(__cplusplus)
+#    include <complex>
+#    define OSHMEM_COMPLEX_TYPE(type)    std::complex<type>
+#else
+#    include <complex.h>
+#    define OSHMEM_COMPLEX_TYPE(type)    type complex
+#endif
+
+/*
+ * SHMEM version
+ */
+#define OSHMEM_MAJOR_VERSION 3
+#define OSHMEM_MINOR_VERSION 1
+#define OSHMEM_RELEASE_VERSION 2
+
+
+#ifndef OSHMEM_DECLSPEC
+#  if defined(OPAL_C_HAVE_VISIBILITY) && (OPAL_C_HAVE_VISIBILITY == 1)
+#     define OSHMEM_DECLSPEC __attribute__((visibility("default")))
+#  else
+#     define OSHMEM_DECLSPEC
+#  endif
+#endif
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
+#define OSHMEM_HAVE_C11 1
+#else
+#define OSHMEM_HAVE_C11 0
+#endif
+
+#include <shmem-compat.h>
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/*
+ * OpenSHMEM API (www.openshmem.org)
+ */
+
+/*
+ * Environment variables
+ */
+
+/* Following environment variables are Mellanox extension */
+
+/* size of symmetric heap in bytes.
+ * Can be qualified with the letter 'K', 'M', 'G' or 'T'
+ */
+#define SHMEM_HEAP_SIZE "SHMEM_SYMMETRIC_HEAP_SIZE"
+
+/*
+ * Type of allocator used by symmetric heap
+ */
+#define SHMEM_HEAP_TYPE "SHMEM_SYMMETRIC_HEAP_ALLOCATOR"
+
+/*
+ * Constants and definitions
+ */
+#define SHMEM_MAJOR_VERSION             1
+#define SHMEM_MINOR_VERSION             3 
+#define SHMEM_VENDOR_STRING             "http://www.open-mpi.org/"
+#define SHMEM_MAX_NAME_LEN              256
+
+/*
+ * Deprecated (but still valid) names
+ */
+#define _SHMEM_MAJOR_VERSION            SHMEM_MAJOR_VERSION
+#define _SHMEM_MINOR_VERSION            SHMEM_MINOR_VERSION
+#define _SHMEM_MAX_NAME_LEN             SHMEM_MAX_NAME_LEN
+
+#ifndef OSHMEM_SPEC_VERSION
+#define OSHMEM_SPEC_VERSION (SHMEM_MAJOR_VERSION * 10000 + SHMEM_MINOR_VERSION * 100)
+#endif
+
+enum shmem_wait_ops {
+    SHMEM_CMP_EQ,
+    SHMEM_CMP_NE,
+    SHMEM_CMP_GT,
+    SHMEM_CMP_LE,
+    SHMEM_CMP_LT,
+    SHMEM_CMP_GE
+};
+
+/*
+ * Deprecated (but still valid) names
+ */
+#define _SHMEM_CMP_EQ                   SHMEM_CMP_EQ
+#define _SHMEM_CMP_NE                   SHMEM_CMP_NE
+#define _SHMEM_CMP_GT                   SHMEM_CMP_GT
+#define _SHMEM_CMP_LE                   SHMEM_CMP_LE
+#define _SHMEM_CMP_LT                   SHMEM_CMP_LT
+#define _SHMEM_CMP_GE                   SHMEM_CMP_GE
+
+#define _SHMEM_BARRIER_SYNC_SIZE        (1)
+#define _SHMEM_BCAST_SYNC_SIZE          (1 + _SHMEM_BARRIER_SYNC_SIZE)
+#define _SHMEM_COLLECT_SYNC_SIZE        (1 + _SHMEM_BCAST_SYNC_SIZE)
+#define _SHMEM_REDUCE_SYNC_SIZE         (1 + _SHMEM_BCAST_SYNC_SIZE)
+#define _SHMEM_ALLTOALL_SYNC_SIZE       (_SHMEM_BARRIER_SYNC_SIZE)
+#define _SHMEM_ALLTOALLS_SYNC_SIZE      (_SHMEM_BARRIER_SYNC_SIZE)
+#define _SHMEM_REDUCE_MIN_WRKDATA_SIZE  (1)
+#define _SHMEM_SYNC_VALUE               (-1)
+
+#define SHMEM_BARRIER_SYNC_SIZE        _SHMEM_BARRIER_SYNC_SIZE
+#define SHMEM_BCAST_SYNC_SIZE          _SHMEM_BCAST_SYNC_SIZE
+#define SHMEM_COLLECT_SYNC_SIZE        _SHMEM_COLLECT_SYNC_SIZE
+#define SHMEM_REDUCE_SYNC_SIZE         _SHMEM_REDUCE_SYNC_SIZE
+#define SHMEM_ALLTOALL_SYNC_SIZE       _SHMEM_ALLTOALL_SYNC_SIZE
+#define SHMEM_ALLTOALLS_SYNC_SIZE      _SHMEM_ALLTOALLS_SYNC_SIZE
+#define SHMEM_REDUCE_MIN_WRKDATA_SIZE  _SHMEM_REDUCE_MIN_WRKDATA_SIZE
+#define SHMEM_SYNC_VALUE               _SHMEM_SYNC_VALUE
+
+
+/*
+ * Initialization routines
+ */
+OSHMEM_DECLSPEC  void shmem_init(void);
+OSHMEM_DECLSPEC  void shmem_finalize(void);
+OSHMEM_DECLSPEC  void shmem_global_exit(int status);
+
+/*
+ * Query routines
+ */
+OSHMEM_DECLSPEC  int shmem_n_pes(void);
+OSHMEM_DECLSPEC  int shmem_my_pe(void);
+
+/*
+ * Info routines
+ */
+OSHMEM_DECLSPEC void shmem_info_get_version(int *major, int *minor);
+OSHMEM_DECLSPEC void shmem_info_get_name(char *name);
+
+/*
+ * Accessability routines
+ */
+OSHMEM_DECLSPEC int shmem_pe_accessible(int pe);
+OSHMEM_DECLSPEC int shmem_addr_accessible(const void *addr, int pe);
+
+/*
+ * Symmetric heap routines
+ */
+OSHMEM_DECLSPEC  void* shmem_malloc(size_t size);
+OSHMEM_DECLSPEC  void* shmem_align(size_t align, size_t size);
+OSHMEM_DECLSPEC  void* shmem_realloc(void *ptr, size_t size);
+OSHMEM_DECLSPEC  void shmem_free(void* ptr);
+
+/*
+ * Remote pointer operations
+ */
+OSHMEM_DECLSPEC  void *shmem_ptr(const void *ptr, int pe);
+
+/*
+ * Elemental put routines
+ */
+OSHMEM_DECLSPEC  void shmem_char_p(char* addr, char value, int pe);
+OSHMEM_DECLSPEC  void shmem_short_p(short* addr, short value, int pe);
+OSHMEM_DECLSPEC  void shmem_int_p(int* addr, int value, int pe);
+OSHMEM_DECLSPEC  void shmem_long_p(long* addr, long value, int pe);
+OSHMEM_DECLSPEC  void shmem_float_p(float* addr, float value, int pe);
+OSHMEM_DECLSPEC  void shmem_double_p(double* addr, double value, int pe);
+OSHMEM_DECLSPEC  void shmem_longlong_p(long long* addr, long long value, int pe);
+OSHMEM_DECLSPEC  void shmem_longdouble_p(long double* addr, long double value, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_p(dst, val, pe)                                \
+    _Generic(&*(dst),                                        \
+            char*:        shmem_char_p,                      \
+            short*:       shmem_short_p,                     \
+            int*:         shmem_int_p,                       \
+            long*:        shmem_long_p,                      \
+            long long*:   shmem_longlong_p,                  \
+            float*:       shmem_float_p,                     \
+            double*:      shmem_double_p,                    \
+            long double*: shmem_longdouble_p)(dst, val, pe)
+#endif
+
+/*
+ * Block data put routines
+ */
+OSHMEM_DECLSPEC  void shmem_char_put(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_short_put(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_int_put(int* target, const int* source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_long_put(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_float_put(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_double_put(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longlong_put(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longdouble_put(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_put(dst, src, len, pe)               \
+    _Generic(&*(dst),                              \
+            char*:        shmem_char_put,          \
+            short*:       shmem_short_put,         \
+            int*:         shmem_int_put,           \
+            long*:        shmem_long_put,          \
+            long long*:   shmem_longlong_put,      \
+            float*:       shmem_float_put,         \
+            double*:      shmem_double_put,        \
+            long double*: shmem_longdouble_put)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void shmem_put8(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put16(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put32(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put64(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put128(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_putmem(void *target, const void *source, size_t len, int pe);
+
+
+/*
+ * Strided put routines
+ */
+OSHMEM_DECLSPEC void shmem_char_iput(char* target, const char* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_short_iput(short* target, const short* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_int_iput(int* target, const int* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_long_iput(long* target, const long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_float_iput(float* target, const float* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_double_iput(double* target, const double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_iput(long long* target, const long long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_longdouble_iput(long double* target, const long double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_iput(dst, src, tst, sst, len, pe)     \
+    _Generic(&*(dst),                               \
+            char*:        shmem_char_iput,          \
+            short*:       shmem_short_iput,         \
+            int*:         shmem_int_iput,           \
+            long*:        shmem_long_iput,          \
+            long long*:   shmem_longlong_iput,      \
+            float*:       shmem_float_iput,         \
+            double*:      shmem_double_iput,        \
+            long double*: shmem_longdouble_iput)(dst, src, tst, sst, len, pe)
+#endif
+
+OSHMEM_DECLSPEC void shmem_iput8(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iput16(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iput32(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iput64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iput128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+
+/*
+ * Nonblocking put routines
+ */
+OSHMEM_DECLSPEC  void shmem_char_put_nbi(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_short_put_nbi(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_int_put_nbi(int* target, const int* source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_long_put_nbi(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longlong_put_nbi(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_float_put_nbi(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_double_put_nbi(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longdouble_put_nbi(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_put_nbi(dst, src, len, pe)               \
+    _Generic(&*(dst),                                  \
+            char*:        shmem_char_put_nbi,          \
+            short*:       shmem_short_put_nbi,         \
+            int*:         shmem_int_put_nbi,           \
+            long*:        shmem_long_put_nbi,          \
+            long long*:   shmem_longlong_put_nbi,      \
+            float*:       shmem_float_put_nbi,         \
+            double*:      shmem_double_put_nbi,        \
+            long double*: shmem_longdouble_put_nbi)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void shmem_put8_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put16_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put32_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put64_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_put128_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_putmem_nbi(void *target, const void *source, size_t len, int pe);
+
+/*
+ * Elemental get routines
+ */
+OSHMEM_DECLSPEC  char shmem_char_g(const char* addr, int pe);
+OSHMEM_DECLSPEC  short shmem_short_g(const short* addr, int pe);
+OSHMEM_DECLSPEC  int shmem_int_g(const int* addr, int pe);
+OSHMEM_DECLSPEC  long shmem_long_g(const long* addr, int pe);
+OSHMEM_DECLSPEC  float shmem_float_g(const float* addr, int pe);
+OSHMEM_DECLSPEC  double shmem_double_g(const double* addr, int pe);
+OSHMEM_DECLSPEC  long long shmem_longlong_g(const long long* addr, int pe);
+OSHMEM_DECLSPEC  long double shmem_longdouble_g(const long double* addr, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_g(addr, pe)                                    \
+    _Generic(&*(addr),                                        \
+            char*:        shmem_char_g,                      \
+            short*:       shmem_short_g,                     \
+            int*:         shmem_int_g,                       \
+            long*:        shmem_long_g,                      \
+            long long*:   shmem_longlong_g,                  \
+            float*:       shmem_float_g,                     \
+            double*:      shmem_double_g,                    \
+            long double*: shmem_longdouble_g)(addr, pe)
+#endif
+
+/*
+ * Block data get routines
+ */
+OSHMEM_DECLSPEC  void shmem_char_get(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_short_get(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_int_get(int *target, const int *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_long_get(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_float_get(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_double_get(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longlong_get(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longdouble_get(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_get(dst, src, len, pe)               \
+    _Generic(&*(dst),                              \
+            char*:        shmem_char_get,          \
+            short*:       shmem_short_get,         \
+            int*:         shmem_int_get,           \
+            long*:        shmem_long_get,          \
+            long long*:   shmem_longlong_get,      \
+            float*:       shmem_float_get,         \
+            double*:      shmem_double_get,        \
+            long double*: shmem_longdouble_get)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void shmem_get8(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get16(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get32(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get64(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get128(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_getmem(void *target, const void *source, size_t len, int pe);
+
+/*
+ * Strided get routines
+ */
+OSHMEM_DECLSPEC void shmem_char_iget(char* target, const char* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_short_iget(short* target, const short* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_int_iget(int* target, const int* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_float_iget(float* target, const float* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_double_iget(double* target, const double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_iget(long long* target, const long long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_longdouble_iget(long double* target, const long double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_long_iget(long* target, const long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_iget(dst, src, tst, sst, len, pe)     \
+    _Generic(&*(dst),                               \
+            char*:        shmem_char_iget,          \
+            short*:       shmem_short_iget,         \
+            int*:         shmem_int_iget,           \
+            long*:        shmem_long_iget,          \
+            long long*:   shmem_longlong_iget,      \
+            float*:       shmem_float_iget,         \
+            double*:      shmem_double_iget,        \
+            long double*: shmem_longdouble_iget)(dst, src, tst, sst, len, pe)
+#endif
+
+OSHMEM_DECLSPEC void shmem_iget8(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iget16(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iget32(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iget64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+OSHMEM_DECLSPEC void shmem_iget128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
+
+/*
+ * Nonblocking data get routines
+ */
+OSHMEM_DECLSPEC  void shmem_getmem_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_char_get_nbi(char *target, const char *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_short_get_nbi(short *target, const short *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_int_get_nbi(int *target, const int *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_long_get_nbi(long *target, const long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longlong_get_nbi(long long *target, const long long *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_float_get_nbi(float *target, const float *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_double_get_nbi(double *target, const double *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_longdouble_get_nbi(long double *target, const long double *source, size_t len, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_get_nbi(dst, src, len, pe)               \
+    _Generic(&*(dst),                                  \
+            char*:        shmem_char_get_nbi,          \
+            short*:       shmem_short_get_nbi,         \
+            int*:         shmem_int_get_nbi,           \
+            long*:        shmem_long_get_nbi,          \
+            long long*:   shmem_longlong_get_nbi,      \
+            float*:       shmem_float_get_nbi,         \
+            double*:      shmem_double_get_nbi,        \
+            long double*: shmem_longdouble_get_nbi)(dst, src, len, pe)
+#endif
+
+OSHMEM_DECLSPEC  void shmem_get8_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get16_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get32_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get64_nbi(void *target, const void *source, size_t len, int pe);
+OSHMEM_DECLSPEC  void shmem_get128_nbi(void *target, const void *source, size_t len, int pe);
+
+/*
+ * Atomic operations
+ */
+/* Atomic swap */
+OSHMEM_DECLSPEC int shmem_int_swap(int *target, int value, int pe);
+OSHMEM_DECLSPEC long shmem_long_swap(long *target, long value, int pe);
+OSHMEM_DECLSPEC long long shmem_longlong_swap(long long*target, long long value, int pe);
+OSHMEM_DECLSPEC float shmem_float_swap(float *target, float value, int pe);
+OSHMEM_DECLSPEC double shmem_double_swap(double *target, double value, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_swap(dst, val, pe)                             \
+    _Generic(&*(dst),                                        \
+            int*:         shmem_int_swap,                    \
+            long*:        shmem_long_swap,                   \
+            long long*:   shmem_longlong_swap,               \
+            float*:       shmem_float_swap,                  \
+            double*:      shmem_double_swap)(dst, val, pe)
+#endif
+
+/* Atomic set */
+OSHMEM_DECLSPEC void shmem_int_set(int *target, int value, int pe);
+OSHMEM_DECLSPEC void shmem_long_set(long *target, long value, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_set(long long*target, long long value, int pe);
+OSHMEM_DECLSPEC void shmem_float_set(float *target, float value, int pe);
+OSHMEM_DECLSPEC void shmem_double_set(double *target, double value, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_set(dst, val, pe)                             \
+    _Generic(&*(dst),                                       \
+            int*:         shmem_int_set,                    \
+            long*:        shmem_long_set,                   \
+            long long*:   shmem_longlong_set,               \
+            float*:       shmem_float_set,                  \
+            double*:      shmem_double_set)(dst, val, pe)
+#endif
+
+/* Atomic conditional swap */
+OSHMEM_DECLSPEC int shmem_int_cswap(int *target, int cond, int value, int pe);
+OSHMEM_DECLSPEC long shmem_long_cswap(long *target, long cond, long value, int pe);
+OSHMEM_DECLSPEC long long shmem_longlong_cswap(long long *target, long long cond, long long value, int pe);
+
+#if OSHMEM_HAVE_C11
+#define shmem_cswap(dst, cond, val, pe)                       \
+    _Generic(&*(dst),                                         \
+            int*:         shmem_int_cswap,                    \
+            long*:        shmem_long_cswap,                   \
+            long long*:   shmem_longlong_cswap)(dst, cond, val, pe)
+#endif
+
+/* Atomic Fetch&Add */
+OSHMEM_DECLSPEC int shmem_int_fadd(int *target, int value, int pe);
+OSHMEM_DECLSPEC long shmem_long_fadd(long *target, long value, int pe);
+OSHMEM_DECLSPEC long long shmem_longlong_fadd(long long *target, long long value, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_fadd(dst, val, pe)                             \
+    _Generic(&*(dst),                                        \
+            int*:         shmem_int_fadd,                    \
+            long*:        shmem_long_fadd,                   \
+            long long*:   shmem_longlong_fadd)(dst, val, pe)
+#endif
+
+/* Atomic Fetch */
+OSHMEM_DECLSPEC int shmem_int_fetch(const int *target, int pe);
+OSHMEM_DECLSPEC long shmem_long_fetch(const long *target, int pe);
+OSHMEM_DECLSPEC long long shmem_longlong_fetch(const long long *target, int pe);
+OSHMEM_DECLSPEC float shmem_float_fetch(const float *target, int pe);
+OSHMEM_DECLSPEC double shmem_double_fetch(const double *target, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_fetch(dst, pe)                             \
+    _Generic(&*(dst),                                    \
+            int*:         shmem_int_fetch,               \
+            long*:        shmem_long_fetch,              \
+            long long*:   shmem_longlong_fetch,          \
+            float*:       shmem_float_fetch,             \
+            double*:      shmem_double_fetch)(dst, pe)
+#endif
+
+/* Atomic Fetch&Inc */
+OSHMEM_DECLSPEC int shmem_int_finc(int *target, int pe);
+OSHMEM_DECLSPEC long shmem_long_finc(long *target, int pe);
+OSHMEM_DECLSPEC long long shmem_longlong_finc(long long *target, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_finc(dst, pe)                            \
+    _Generic(&*(dst),                                  \
+            int*:         shmem_int_finc,              \
+            long*:        shmem_long_finc,             \
+            long long*:   shmem_longlong_finc)(dst, pe)
+#endif
+
+/* Atomic Add*/
+OSHMEM_DECLSPEC void shmem_int_add(int *target, int value, int pe);
+OSHMEM_DECLSPEC void shmem_long_add(long *target, long value, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_add(long long *target, long long value, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_add(dst, val, pe)                             \
+    _Generic(&*(dst),                                       \
+            int*:         shmem_int_add,                    \
+            long*:        shmem_long_add,                   \
+            long long*:   shmem_longlong_add)(dst, val, pe)
+#endif
+
+/* Atomic Inc */
+OSHMEM_DECLSPEC void shmem_int_inc(int *target, int pe);
+OSHMEM_DECLSPEC void shmem_long_inc(long *target, int pe);
+OSHMEM_DECLSPEC void shmem_longlong_inc(long long *target, int pe);
+#if OSHMEM_HAVE_C11
+#define shmem_inc(dst, pe)                            \
+    _Generic(&*(dst),                                 \
+            int*:         shmem_int_inc,              \
+            long*:        shmem_long_inc,             \
+            long long*:   shmem_longlong_inc)(dst, pe)
+#endif
+
+
+/*
+ * Lock functions
+ */
+OSHMEM_DECLSPEC void shmem_set_lock(volatile long *lock);
+OSHMEM_DECLSPEC void shmem_clear_lock(volatile long *lock);
+OSHMEM_DECLSPEC int shmem_test_lock(volatile long *lock);
+
+/*
+ * P2P sync routines
+ */
+OSHMEM_DECLSPEC  void shmem_short_wait(volatile short *addr, short value);
+OSHMEM_DECLSPEC  void shmem_int_wait(volatile int *addr, int value);
+OSHMEM_DECLSPEC  void shmem_long_wait(volatile long *addr, long value);
+OSHMEM_DECLSPEC  void shmem_longlong_wait(volatile long long *addr, long long value);
+OSHMEM_DECLSPEC  void shmem_wait(volatile long *addr, long value);
+
+OSHMEM_DECLSPEC  void shmem_short_wait_until(volatile short *addr, int cmp, short value);
+OSHMEM_DECLSPEC  void shmem_int_wait_until(volatile int *addr, int cmp, int value);
+OSHMEM_DECLSPEC  void shmem_long_wait_until(volatile long *addr, int cmp, long value);
+OSHMEM_DECLSPEC  void shmem_longlong_wait_until(volatile long long *addr, int cmp, long long value);
+OSHMEM_DECLSPEC  void shmem_wait_until(volatile long *addr, int cmp, long value);
+
+/*
+ * Barrier sync routines
+ */
+OSHMEM_DECLSPEC  void shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC  void shmem_barrier_all(void);
+OSHMEM_DECLSPEC  void shmem_fence(void);
+OSHMEM_DECLSPEC  void shmem_quiet(void);
+
+/*
+ * Collective routines
+ */
+OSHMEM_DECLSPEC void shmem_broadcast32(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_broadcast64(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_broadcast(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_collect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_collect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_fcollect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_fcollect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_alltoall32(void *target, const void *source, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_alltoall64(void *target, const void *source, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_alltoalls32(void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+OSHMEM_DECLSPEC void shmem_alltoalls64(void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync);
+
+
+/*
+ * Reduction routines
+ */
+OSHMEM_DECLSPEC void shmem_short_and_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_int_and_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_long_and_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longlong_and_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmem_short_or_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_int_or_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_long_or_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longlong_or_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmem_short_xor_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_int_xor_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_long_xor_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longlong_xor_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmem_short_max_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_int_max_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_long_max_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longlong_max_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_float_max_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_double_max_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longdouble_max_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmem_short_min_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_int_min_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_long_min_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longlong_min_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_float_min_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_double_min_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longdouble_min_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmem_short_sum_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_int_sum_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_long_sum_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longlong_sum_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_float_sum_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_double_sum_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longdouble_sum_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_complexf_sum_to_all(OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(float) *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_complexd_sum_to_all(OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(double) *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmem_short_prod_to_all(short *target, const short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_int_prod_to_all(int *target, const int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_long_prod_to_all(long *target, const long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longlong_prod_to_all(long long *target, const long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_float_prod_to_all(float *target, const float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_double_prod_to_all(double *target, const double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_longdouble_prod_to_all(long double *target, const long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_complexf_prod_to_all(OSHMEM_COMPLEX_TYPE(float) *target, const OSHMEM_COMPLEX_TYPE(float) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(float) *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmem_complexd_prod_to_all(OSHMEM_COMPLEX_TYPE(double) *target, const OSHMEM_COMPLEX_TYPE(double) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(double) *pWrk, long *pSync);
+
+/*
+ * Platform specific cache management routines
+ */
+OSHMEM_DECLSPEC void shmem_udcflush(void);
+OSHMEM_DECLSPEC void shmem_udcflush_line(void* target);
+OSHMEM_DECLSPEC void shmem_set_cache_inv(void);
+OSHMEM_DECLSPEC void shmem_set_cache_line_inv(void* target);
+OSHMEM_DECLSPEC void shmem_clear_cache_inv(void);
+OSHMEM_DECLSPEC void shmem_clear_cache_line_inv(void* target);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+
+#endif /* OSHMEM_SHMEM_H */

linx64/mpi/openmpi/include/shmemx.h

@@ -0,0 +1,206 @@
+/* oshmem/include/shmemx.h. This file contains vendor extension functions  */
+/*
+ * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OSHMEM_SHMEMX_H
+#define OSHMEM_SHMEMX_H
+
+#include <shmem.h>
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * All OpenSHMEM extension APIs that are not part of this specification must be defined in the shmemx.h include
+ * file. These extensions shall use the shmemx_ prefix for all routine, variable, and constant names.
+ */
+
+/*
+ * Elemental put routines
+ */
+OSHMEM_DECLSPEC  void shmemx_int16_p(int16_t* addr, int16_t value, int pe);
+OSHMEM_DECLSPEC  void shmemx_int32_p(int32_t* addr, int32_t value, int pe);
+OSHMEM_DECLSPEC  void shmemx_int64_p(int64_t* addr, int64_t value, int pe);
+
+/*
+ * Elemental put routines
+ */
+
+/*
+ * Block data put routines
+ */
+
+/*
+ * Strided put routines
+ */
+
+/*
+ * Elemental get routines
+ */
+OSHMEM_DECLSPEC  int16_t shmemx_int16_g(const int16_t* addr, int pe);
+OSHMEM_DECLSPEC  int32_t shmemx_int32_g(const int32_t* addr, int pe);
+OSHMEM_DECLSPEC  int64_t shmemx_int64_g(const int64_t* addr, int pe);
+
+/*
+ * Block data get routines
+ */
+
+/*
+ * Strided get routines
+ */
+
+/*
+ * Atomic operations
+ */
+/* Atomic swap */
+OSHMEM_DECLSPEC int32_t shmemx_int32_swap(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmemx_int64_swap(int64_t *target, int64_t value, int pe);
+
+/* Atomic set */
+OSHMEM_DECLSPEC void shmemx_int32_set(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmemx_int64_set(int64_t *target, int64_t value, int pe);
+
+/* Atomic conditional swap */
+OSHMEM_DECLSPEC int32_t shmemx_int32_cswap(int32_t *target, int32_t cond, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmemx_int64_cswap(int64_t *target, int64_t cond, int64_t value, int pe);
+
+/* Atomic Fetch&Add */
+OSHMEM_DECLSPEC int32_t shmemx_int32_fadd(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC int64_t shmemx_int64_fadd(int64_t *target, int64_t value, int pe);
+
+/* Atomic Fetch */
+OSHMEM_DECLSPEC int32_t shmemx_int32_fetch(const int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t shmemx_int64_fetch(const int64_t *target, int pe);
+
+/* Atomic Fetch&Inc */
+OSHMEM_DECLSPEC int32_t shmemx_int32_finc(int32_t *target, int pe);
+OSHMEM_DECLSPEC int64_t shmemx_int64_finc(int64_t *target, int pe);
+
+/* Atomic Add*/
+OSHMEM_DECLSPEC void shmemx_int32_add(int32_t *target, int32_t value, int pe);
+OSHMEM_DECLSPEC void shmemx_int64_add(int64_t *target, int64_t value, int pe);
+
+/* Atomic Inc */
+OSHMEM_DECLSPEC void shmemx_int32_inc(int32_t *target, int pe);
+OSHMEM_DECLSPEC void shmemx_int64_inc(int64_t *target, int pe);
+
+/*
+ * P2P sync routines
+ */
+OSHMEM_DECLSPEC  void shmemx_int32_wait(int32_t *addr, int32_t value);
+OSHMEM_DECLSPEC  void shmemx_int64_wait(int64_t *addr, int64_t value);
+
+OSHMEM_DECLSPEC  void shmemx_int32_wait_until(int32_t *addr, int cmp, int32_t value);
+OSHMEM_DECLSPEC  void shmemx_int64_wait_until(int64_t *addr, int cmp, int64_t value);
+
+/*
+ * Reduction routines
+ */
+OSHMEM_DECLSPEC void shmemx_int16_and_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int32_and_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int64_and_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmemx_int16_or_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int32_or_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int64_or_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmemx_int16_xor_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int32_xor_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int64_xor_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmemx_int16_max_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int32_max_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int64_max_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmemx_int16_min_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int32_min_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int64_min_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmemx_int16_sum_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int32_sum_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int64_sum_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+OSHMEM_DECLSPEC void shmemx_int16_prod_to_all(int16_t *target, const int16_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int16_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int32_prod_to_all(int32_t *target, const int32_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int32_t *pWrk, long *pSync);
+OSHMEM_DECLSPEC void shmemx_int64_prod_to_all(int64_t *target, const int64_t *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int64_t *pWrk, long *pSync);
+
+/*
+ * Backward compatibility section
+ */
+#define shmem_int16_p               shmemx_int16_p
+#define shmem_int32_p               shmemx_int32_p
+#define shmem_int64_p               shmemx_int64_p
+
+#define shmem_int16_g               shmemx_int16_g
+#define shmem_int32_g               shmemx_int32_g
+#define shmem_int64_g               shmemx_int64_g
+
+#define shmem_int32_swap            shmemx_int32_swap
+#define shmem_int64_swap            shmemx_int64_swap
+
+#define shmem_int32_set             shmemx_int32_set
+#define shmem_int64_set             shmemx_int64_set
+
+#define shmem_int32_cswap           shmemx_int32_cswap
+#define shmem_int64_cswap           shmemx_int64_cswap
+
+#define shmem_int32_fadd            shmemx_int32_fadd
+#define shmem_int64_fadd            shmemx_int64_fadd
+
+#define shmem_int32_fetch           shmemx_int32_fetch
+#define shmem_int64_fetch           shmemx_int64_fetch
+
+#define shmem_int32_finc            shmemx_int32_finc
+#define shmem_int64_finc            shmemx_int64_finc
+
+#define shmem_int32_add             shmemx_int32_add
+#define shmem_int64_add             shmemx_int64_add
+#define shmem_int32_inc             shmemx_int32_inc
+#define shmem_int64_inc             shmemx_int64_inc
+
+#define shmem_int32_wait            shmemx_int32_wait
+#define shmem_int64_wait            shmemx_int64_wait
+#define shmem_int32_wait_until      shmemx_int32_wait_until
+#define shmem_int64_wait_until      shmemx_int64_wait_until
+
+#define shmem_int16_and_to_all      shmemx_int16_and_to_all
+#define shmem_int32_and_to_all      shmemx_int32_and_to_all
+#define shmem_int64_and_to_all      shmemx_int64_and_to_all
+
+#define shmem_int16_or_to_all       shmemx_int16_or_to_all
+#define shmem_int32_or_to_all       shmemx_int32_or_to_all
+#define shmem_int64_or_to_all       shmemx_int64_or_to_all
+
+#define shmem_int16_xor_to_all      shmemx_int16_xor_to_all
+#define shmem_int32_xor_to_all      shmemx_int32_xor_to_all
+#define shmem_int64_xor_to_all      shmemx_int64_xor_to_all
+
+#define shmem_int16_max_to_all      shmemx_int16_max_to_all
+#define shmem_int32_max_to_all      shmemx_int32_max_to_all
+#define shmem_int64_max_to_all      shmemx_int64_max_to_all
+
+#define shmem_int16_min_to_all      shmemx_int16_min_to_all
+#define shmem_int32_min_to_all      shmemx_int32_min_to_all
+#define shmem_int64_min_to_all      shmemx_int64_min_to_all
+
+#define shmem_int16_sum_to_all      shmemx_int16_sum_to_all
+#define shmem_int32_sum_to_all      shmemx_int32_sum_to_all
+#define shmem_int64_sum_to_all      shmemx_int64_sum_to_all
+
+#define shmem_int16_prod_to_all     shmemx_int16_prod_to_all
+#define shmem_int32_prod_to_all     shmemx_int32_prod_to_all
+#define shmem_int64_prod_to_all     shmemx_int64_prod_to_all
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif /* OSHMEM_SHMEMX_H */

linx64/mpi/openmpi/lib64/libmca_common_monitoring.la

@@ -0,0 +1,41 @@
+# libmca_common_monitoring.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libmca_common_monitoring.so.50'
+
+# Names of this library.
+library_names='libmca_common_monitoring.so.50.0.0 libmca_common_monitoring.so.50 libmca_common_monitoring.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libmca_common_monitoring.
+current=50
+age=0
+revision=0
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libmca_common_monitoring.so

@@ -0,0 +1 @@
+libmca_common_monitoring.so.50.0.0

linx64/mpi/openmpi/lib64/libmca_common_monitoring.so.50

@@ -0,0 +1 @@
+libmca_common_monitoring.so.50.0.0

linx64/mpi/openmpi/lib64/libmca_common_ompio.la

@@ -0,0 +1,41 @@
+# libmca_common_ompio.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libmca_common_ompio.so.41'
+
+# Names of this library.
+library_names='libmca_common_ompio.so.41.9.0 libmca_common_ompio.so.41 libmca_common_ompio.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libmca_common_ompio.
+current=50
+age=9
+revision=0
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libmca_common_ompio.so

@@ -0,0 +1 @@
+libmca_common_ompio.so.41.9.0

linx64/mpi/openmpi/lib64/libmca_common_ompio.so.41

@@ -0,0 +1 @@
+libmca_common_ompio.so.41.9.0

linx64/mpi/openmpi/lib64/libmca_common_sm.la

@@ -0,0 +1,41 @@
+# libmca_common_sm.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libmca_common_sm.so.40'
+
+# Names of this library.
+library_names='libmca_common_sm.so.40.10.0 libmca_common_sm.so.40 libmca_common_sm.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libmca_common_sm.
+current=50
+age=10
+revision=0
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libmca_common_sm.so

@@ -0,0 +1 @@
+libmca_common_sm.so.40.10.0

linx64/mpi/openmpi/lib64/libmca_common_sm.so.40

@@ -0,0 +1 @@
+libmca_common_sm.so.40.10.0

linx64/mpi/openmpi/lib64/libmca_common_verbs.la

@@ -0,0 +1,41 @@
+# libmca_common_verbs.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libmca_common_verbs.so.40'
+
+# Names of this library.
+library_names='libmca_common_verbs.so.40.10.0 libmca_common_verbs.so.40 libmca_common_verbs.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' -libverbs -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libmca_common_verbs.
+current=50
+age=10
+revision=0
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libmca_common_verbs.so

@@ -0,0 +1 @@
+libmca_common_verbs.so.40.10.0

linx64/mpi/openmpi/lib64/libmca_common_verbs.so.40

@@ -0,0 +1 @@
+libmca_common_verbs.so.40.10.0

linx64/mpi/openmpi/lib64/libmpi.la

@@ -0,0 +1,41 @@
+# libmpi.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libmpi.so.40'
+
+# Names of this library.
+library_names='libmpi.so.40.10.2 libmpi.so.40 libmpi.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' /home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64/libopen-rte.la /home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64/libopen-pal.la -ldl -ludev -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libmpi.
+current=50
+age=10
+revision=2
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libmpi.so

@@ -0,0 +1 @@
+libmpi.so.40.10.2

linx64/mpi/openmpi/lib64/libmpi.so.40

@@ -0,0 +1 @@
+libmpi.so.40.10.2

linx64/mpi/openmpi/lib64/libompitrace.la

@@ -0,0 +1,41 @@
+# libompitrace.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libompitrace.so.40'
+
+# Names of this library.
+library_names='libompitrace.so.40.10.0 libompitrace.so.40 libompitrace.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libompitrace.
+current=50
+age=10
+revision=0
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libompitrace.so

@@ -0,0 +1 @@
+libompitrace.so.40.10.0

linx64/mpi/openmpi/lib64/libompitrace.so.40

@@ -0,0 +1 @@
+libompitrace.so.40.10.0

linx64/mpi/openmpi/lib64/libopen-pal.la

@@ -0,0 +1,41 @@
+# libopen-pal.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libopen-pal.so.40'
+
+# Names of this library.
+library_names='libopen-pal.so.40.10.2 libopen-pal.so.40 libopen-pal.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' -ldl -ludev -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libopen-pal.
+current=50
+age=10
+revision=2
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libopen-pal.so

@@ -0,0 +1 @@
+libopen-pal.so.40.10.2

linx64/mpi/openmpi/lib64/libopen-pal.so.40

@@ -0,0 +1 @@
+libopen-pal.so.40.10.2

linx64/mpi/openmpi/lib64/libopen-rte.la

@@ -0,0 +1,41 @@
+# libopen-rte.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='libopen-rte.so.40'
+
+# Names of this library.
+library_names='libopen-rte.so.40.10.2 libopen-rte.so.40 libopen-rte.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' /home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64/libopen-pal.la -ldl -ludev -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for libopen-rte.
+current=50
+age=10
+revision=2
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/libopen-rte.so

@@ -0,0 +1 @@
+libopen-rte.so.40.10.2

linx64/mpi/openmpi/lib64/libopen-rte.so.40

@@ -0,0 +1 @@
+libopen-rte.so.40.10.2

linx64/mpi/openmpi/lib64/liboshmem.la

@@ -0,0 +1,41 @@
+# liboshmem.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='liboshmem.so.40'
+
+# Names of this library.
+library_names='liboshmem.so.40.10.1 liboshmem.so.40 liboshmem.so'
+
+# The name of the static archive.
+old_library=''
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=' -pthread'
+
+# Libraries that this one depends upon.
+dependency_libs=' /home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64/libmpi.la /home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64/libopen-rte.la /home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64/libopen-pal.la -ldl -ludev -lrt -lm -lutil -lz'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for liboshmem.
+current=50
+age=10
+revision=1
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=no
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/home/brgirgis/Downloads/openmpi/3.1.2/build/../release_install/lib64'

linx64/mpi/openmpi/lib64/liboshmem.so

@@ -0,0 +1 @@
+liboshmem.so.40.10.1

linx64/mpi/openmpi/lib64/liboshmem.so.40

@@ -0,0 +1 @@
+liboshmem.so.40.10.1