Upgrade MPIs on linux

linx64/mpi/mpich/bin/mpic++

@@ -0,0 +1 @@
+mpicxx

linx64/mpi/mpich/bin/mpicc

@@ -1,9 +1,9 @@
 #! /usr/bin/bash
-# 
-# (C) 2006 by Argonne National Laboratory.
-#     See COPYRIGHT in top-level directory.
-#
-# mpicc
+##
+## Copyright (C) by Argonne National Laboratory
+##     See COPYRIGHT in top-level directory
+##
+
 # Simple script to compile and/or link MPI programs.
 # This script knows the default flags and libraries, and can handle
 # alternative C compilers and the associated flags and libraries.
@@ -27,19 +27,18 @@
 #
 # Directory locations: Fixed for any MPI implementation.
 # Set from the directory arguments to configure (e.g., --prefix=/usr/local)
-prefix=/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1
-exec_prefix=/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1
-sysconfdir=/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1/etc
-includedir=/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1/include
-libdir=/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1/lib64
+prefix=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0
+exec_prefix=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0
+sysconfdir=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/etc
+includedir=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/include
+libdir=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/lib
 
 # Default settings for compiler, flags, and libraries.
 # Determined by a combination of environment variables and tests within
-# configure (e.g., determining whehter -lsocket is needee)
+# configure (e.g., determining whether -lsocket is needed)
 CC="gcc"
-MPICH_VERSION="3.3.1"
+MPICH_VERSION="4.3.0"
 
-enable_wrapper_rpath="yes"
 
 # How to pass a linker flag through the compiler.
 wl="-Wl,"
@@ -58,7 +57,11 @@ library_names_spec="\$libname\$shrext"
 
 # Flag to hardcode $libdir into a binary during linking.
 # This must work even if $libdir does not exist.
-hardcode_libdir_flag_spec="\${wl}-rpath \${wl}\$libdir -Wl,--enable-new-dtags"
+hardcode_libdir_flag_spec="\${wl}-rpath \${wl}\$libdir"
+
+# Flag to add dtags to allow using runpath instead of rpath
+enable_dtags_flag="-Wl,--enable-new-dtags"
+disable_dtags_flag="-Wl,--disable-new-dtags"
 
 # Whether we need a single -rpath flag with a separated argument.
 hardcode_libdir_separator=""
@@ -72,6 +75,18 @@ hardcode_direct="no"
 hardcode_minus_L="no"
 
 
+# Attempt to construct dynamic loading info, based on the user
+# preference of rpath, runpath or none and on the detected libdir
+# flags.
+with_wrapper_dl_type=runpath
+if test "X${with_wrapper_dl_type}" = "Xrunpath" ; then
+    eval wrapper_dl_type_flags=\"${hardcode_libdir_flag_spec} ${enable_dtags_flag}\"
+elif test "X${with_wrapper_dl_type}" = "Xrpath" ; then
+    eval wrapper_dl_type_flags=\"${hardcode_libdir_flag_spec} ${disable_dtags_flag}\"
+else
+    wrapper_dl_type_flags=""
+fi
+
 # Internal variables
 # Show is set to echo to cause the compilation command to be echoed instead 
 # of executed.
@@ -79,6 +94,7 @@ Show=
 #
 # End of initialization of variables
 #---------------------------------------------------------------------
+
 # Environment Variables.
 # The environment variables MPICH_CC may be used to override the 
 # default choices.
@@ -87,8 +103,8 @@ Show=
 # (e.g., "cc -64" becomes "cc--64", that file is sources, allowing other
 # changes to the compilation environment.  See the variables used by the 
 # script (defined above)
-# Added MPICH_CC_OLD, MPICH_CC can be used to prefix CC with external utility,
-# e.g. setenv MPICH_CC 'eval linkcache $MPICH_CC_OLD'
+# Added MPICH_CC_OLD, MPICH_CC can be used to prefix CC
+# with external utility, e.g. setenv MPICH_CC 'eval linkcache $MPICH_CC_OLD'
 if [ -n "$MPICH_CC" ] ; then
     MPICH_CC_OLD="$CC"
     CC="$MPICH_CC"
@@ -106,7 +122,7 @@ fi
 # Argument processing.
 # This is somewhat awkward because of the handling of arguments within
 # the shell.  We want to handle arguments that include spaces without 
-# loosing the spacing (an alternative would be to use a more powerful
+# losing the spacing (an alternative would be to use a more powerful
 # scripting language that would allow us to retain the array of values, 
 # which the basic (rather than enhanced) Bourne shell does not.  
 #
@@ -118,6 +134,9 @@ allargs=("$@")
 argno=0
 interlib_deps=yes
 static_mpi=no
+showinfo=""
+delay_libs=
+mpi_abi=no
 for arg in "$@" ; do
     # Set addarg to no if this arg should be ignored by the C compiler
     addarg=yes
@@ -151,6 +170,16 @@ for arg in "$@" ; do
     addarg=no
     Show=echo
     ;;
+    -show-compile-info)
+    addarg=no
+    Show=echo
+    show_info=compile
+    ;;
+    -show-link-info)
+    addarg=no
+    Show=echo
+    show_info=link
+    ;;
     -config=*)
     addarg=no
     CCname=`echo A$arg | sed -e 's/A-config=//g'`
@@ -191,6 +220,14 @@ for arg in "$@" ; do
     nativelinking=yes
     addarg=no
     ;;
+    -lcuda|-lcudart|-lze_loader)
+    delay_libs="$delay_libs $arg"
+    addarg=no
+    ;;
+    -mpi-abi|-mpi_abi)
+    mpi_abi=yes
+    addarg=no
+    ;;
     -help)
     NC=`echo "$CC" | sed 's%\/% %g' | awk '{print $NF}' -`
     if [ -f "$sysconfdir/mpixxx_opts.conf" ] ; then
@@ -221,6 +258,16 @@ if [ $# -eq 0 ] ; then
     exit 1
 fi
 
+# Check recursive situations
+# User mistakes, e.g. setting MPICH_CC=mpicc, may end up recursively running
+# this script. A simple check to bail if that's the case.
+if [ -n "$MPICH_RECURSION_CHECK" ] ; then
+    echo "This script ($0) is being called recursively, check that MPICH_CC does not refer to mpicc."
+    exit 1
+fi
+MPICH_RECURSION_CHECK=1
+export MPICH_RECURSION_CHECK
+
 # -----------------------------------------------------------------------
 # Derived variables.  These are assembled from variables set from the
 # default, environment, configuration file (if any) and command-line
@@ -250,13 +297,25 @@ final_ldflags=" "
 final_libs=""
 if test "yes" = "no" -o "${interlib_deps}" = "no" ; then
     final_ldflags="${final_ldflags} "
-    final_libs="${final_libs}  -lm   -lpthread  -lrt "
+    final_libs="${final_libs}  -L/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/lib -lm -ludev -latomic -lpthread -ldl"
+fi
+
+if [ "${0%_abi}" != "$0" ] ; then
+    mpi_abi=yes
+fi
+if [ "$mpi_abi" = no ] ; then
+    mpi_libs_shared="-lmpi "
+    mpi_libs_static="$libdir/libmpi.a "
+else
+    final_cppflags="-DMPI_ABI ${final_cppflags}"
+    mpi_libs_shared="-lmpi_abi "
+    mpi_libs_static="$libdir/libmpi_abi.a "
 fi
 
 # -----------------------------------------------------------------------
 #
 # A temporary statement to invoke the compiler
-# Place the -L before any args incase there are any mpi libraries in there.
+# Place the -L before any args in case there are any mpi libraries in there.
 # Eventually, we'll want to move this after any non-MPI implementation 
 # libraries.
 # We use a single invocation of the compiler.  This will be adequate until
@@ -265,35 +324,27 @@ fi
 # accept any argument; we don't need to know if we've seen a source
 # file or an object file.  Instead, we just check for an option that
 # suppressing linking, such as -c or -M.
-if [ "$linking" = yes ] ; then
-    # Attempt to encode rpath info into the executable if the user has not
-    # disabled rpath usage and some flavor of rpath makes sense on this
-    # platform.
-    # TODO configure and config.rpath are computing more sophisticated rpath
-    # schemes than this simple one.  Consider updating this logic accordingly.
-    if test "X$enable_wrapper_rpath" = "Xyes" ; then
-        eval rpath_flags=\"${hardcode_libdir_flag_spec}\"
+if [ "$show_info" = compile ] ; then
+    echo ${final_cppflags} $PROFILE_INCPATHS ${final_cflags} -I$includedir
+elif [ "$show_info" = link ] ; then
+    if [ "$nativelinking" = yes ] ; then
+        echo ${final_ldflags}
     else
-	rpath_flags=""
+        if [ "$static_mpi" = no ] ; then
+            echo ${final_ldflags} -L$libdir $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_shared} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
+        else
+            echo ${final_ldflags} -L$libdir $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_static} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
+        fi
     fi
-
+elif [ "$linking" = yes ] ; then
     if [ "$nativelinking" = yes ] ; then
         $Show $CC ${final_cppflags} $PROFILE_INCPATHS ${final_cflags} ${final_ldflags} "${allargs[@]}" -I$includedir
         rc=$?
     else
         if [ "$static_mpi" = no ] ; then
-          $Show $CC ${final_cppflags} $PROFILE_INCPATHS ${final_cflags} ${final_ldflags} "${allargs[@]}" -I$includedir -L$libdir $PROFILE_PRELIB $PROFILE_FOO $rpath_flags -lmpi  $PROFILE_POSTLIB ${final_libs}
+          $Show $CC ${final_cppflags} $PROFILE_INCPATHS ${final_cflags} ${final_ldflags} "${allargs[@]}" -I$includedir -L$libdir $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_shared} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
         else
-          fabric_dep=""
-          if [ "" = yes ] ; then
-              fabric_dep=`pkg-config --static --libs $libdir/pkgconfig/libfabric.pc`
-              fabric_dep=`echo $fabric_dep | sed 's/-lfabric//'`
-          elif [ -f /lib/pkgconfig/libfabric.pc ] ; then
-              fabric_dep=`pkg-config --static --libs /lib/pkgconfig/libfabric.pc`
-          else
-              fabric_dep=`pkg-config --static --libs libfabric`
-          fi
-          $Show $CC ${final_cppflags} $PROFILE_INCPATHS ${final_cflags} ${final_ldflags} "${allargs[@]}" -I$includedir -L$libdir $PROFILE_PRELIB $PROFILE_FOO $rpath_flags $libdir/libmpi.a  $PROFILE_POSTLIB ${final_libs} ${fabric_dep}
+          $Show $CC ${final_cppflags} $PROFILE_INCPATHS ${final_cflags} ${final_ldflags} "${allargs[@]}" -I$includedir -L$libdir $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_static} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
         fi
         rc=$?
     fi

linx64/mpi/mpich/bin/mpicxx

@@ -0,0 +1,373 @@
+#! /usr/bin/bash
+##
+## Copyright (C) by Argonne National Laboratory
+##     See COPYRIGHT in top-level directory
+##
+
+# Simple script to compile and/or link MPI programs.
+# This script knows the default flags and libraries, and can handle
+# alternative C++ compilers and the associated flags and libraries.
+# The important terms are:
+#    includedir, libdir - Directories containing an *installed* mpich
+#    prefix, execprefix - Often used to define includedir and libdir
+#    CXX                - C compiler
+#    WRAPPER_CXXFLAGS      - Any special flags needed to compile 
+#    WRAPPER_LDFLAGS       - Any special flags needed to link
+#    WRAPPER_LIBS          - Any special libraries needed in order to link
+#    
+# We assume that (a) the C++ compiler can both compile and link programs
+#
+# Handling of command-line options:
+#   This is a little tricky because some options may contain blanks.
+#
+# Special issues with shared libraries - todo
+#
+# --------------------------------------------------------------------------
+# Set the default values of all variables.
+#
+# Directory locations: Fixed for any MPI implementation.
+# Set from the directory arguments to configure (e.g., --prefix=/usr/local)
+prefix=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0
+exec_prefix=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0
+sysconfdir=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/etc
+includedir=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/include
+libdir=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/lib
+
+# Default settings for compiler, flags, and libraries.
+# Determined by a combination of environment variables and tests within
+# configure (e.g., determining whether -lsocket is needed)
+CXX="g++"
+MPICH_VERSION="4.3.0"
+
+
+# How to pass a linker flag through the compiler.
+wl="-Wl,"
+
+# Static library suffix (normally "a").
+libext="a"
+
+# Shared library suffix (normally "so").
+shlibext="so"
+
+# Format of library name prefix.
+libname_spec="lib\$name"
+
+# Library names that the linker finds when passed -lNAME.
+library_names_spec="\$libname\$shrext"
+
+# Flag to hardcode $libdir into a binary during linking.
+# This must work even if $libdir does not exist.
+hardcode_libdir_flag_spec="\${wl}-rpath \${wl}\$libdir"
+
+# Flag to add dtags to allow using runpath instead of rpath
+enable_dtags_flag="-Wl,--enable-new-dtags"
+disable_dtags_flag="-Wl,--disable-new-dtags"
+
+# Whether we need a single -rpath flag with a separated argument.
+hardcode_libdir_separator=""
+
+# Set to yes if using DIR/libNAME.so during linking hardcodes DIR into the
+# resulting binary.
+hardcode_direct="no"
+
+# Set to yes if using the -LDIR flag during linking hardcodes DIR into the
+# resulting binary.
+hardcode_minus_L="no"
+
+
+# Attempt to construct dynamic loading info, based on the user
+# preference of rpath, runpath or none and on the detected libdir
+# flags.
+with_wrapper_dl_type=runpath
+if test "X${with_wrapper_dl_type}" = "Xrunpath" ; then
+    eval wrapper_dl_type_flags=\"${hardcode_libdir_flag_spec} ${enable_dtags_flag}\"
+elif test "X${with_wrapper_dl_type}" = "Xrpath" ; then
+    eval wrapper_dl_type_flags=\"${hardcode_libdir_flag_spec} ${disable_dtags_flag}\"
+else
+    wrapper_dl_type_flags=""
+fi
+
+# Internal variables
+# Show is set to echo to cause the compilation command to be echoed instead 
+# of executed.
+Show=
+#
+# End of initialization of variables
+#---------------------------------------------------------------------
+
+# Environment Variables.
+# The environment variables MPICH_CXX may be used to override the 
+# default choices.
+# In addition, if there is a file $sysconfdir/mpicxx-$CXXname.conf, 
+# where CXXname is the name of the compiler with all spaces replaced by hyphens
+# (e.g., "CC -64" becomes "CC--64", that file is sources, allowing other
+# changes to the compilation environment.  See the variables used by the 
+# script (defined above)
+# Added MPICH_CXX_OLD, MPICH_CXX can be used to prefix CXX
+# with external utility, e.g. setenv MPICH_CXX 'eval linkcache $MPICH_CXX_OLD'
+if [ -n "$MPICH_CXX" ] ; then
+    MPICH_CXX_OLD="$CXX"
+    CXX="$MPICH_CXX"
+    CXXname=`echo $CXX | sed 's/ /-/g'`
+    if [ -s $sysconfdir/mpicxx-$CXXname.conf ] ; then
+        . $sysconfdir/mpicxx-$CXXname.conf
+    fi
+fi
+# Allow a profiling option to be selected through an environment variable
+if [ -n "$MPICXX_PROFILE" ] ; then
+    profConf=$MPICXX_PROFILE
+fi
+#
+# ------------------------------------------------------------------------
+# Argument processing.
+# This is somewhat awkward because of the handling of arguments within
+# the shell.  We want to handle arguments that include spaces without 
+# losing the spacing (an alternative would be to use a more powerful
+# scripting language that would allow us to retain the array of values, 
+# which the basic (rather than enhanced) Bourne shell does not.  
+#
+# Look through the arguments for arguments that indicate compile only.
+# If these are *not* found, add the library options
+
+linking=yes
+allargs=("$@")
+argno=0
+interlib_deps=yes
+static_mpi=no
+showinfo=""
+delay_libs=
+mpi_abi=no
+for arg in "$@" ; do
+    # Set addarg to no if this arg should be ignored by the C compiler
+    addarg=yes
+    case "$arg" in 
+ 	# ----------------------------------------------------------------
+	# Compiler options that affect whether we are linking or no
+    -c|-S|-E|-M|-MM)
+    # The compiler links by default
+    linking=no
+    ;;
+	# ----------------------------------------------------------------
+	# Options that control how we use mpicxx (e.g., -show, 
+	# -cxx=* -config=*
+    -static)
+    interlib_deps=no
+    ;;
+    -static-mpi)
+    interlib_deps=no
+    static_mpi=yes
+    addarg=no
+    ;;
+    -echo)
+    addarg=no
+    set -x
+    ;;
+    -cxx=*)
+    CXX=`echo A$arg | sed -e 's/A-cxx=//g'`
+    addarg=no
+    ;;
+    # Backwards compatibility for MPICH1 - scripts
+    -CC=*)
+    CXX=`echo A$arg | sed -e 's/A-CC=//g'`
+    addarg=no
+    ;;
+    -show)
+    addarg=no
+    Show=echo
+    ;;
+    -show-compile-info)
+    addarg=no
+    Show=echo
+    show_info=compile
+    ;;
+    -show-link-info)
+    addarg=no
+    Show=echo
+    show_info=link
+    ;;
+    -config=*)
+    addarg=no
+    CXXname=`echo A$arg | sed -e 's/A-config=//g'`
+    if [ -s "$sysconfdir/mpicxx-$CXXname.conf" ] ; then
+        . "$sysconfdir/mpicxx-$CXXname.conf"
+    else
+	echo "Configuration file mpicxx-$CXXname.conf not found"
+    fi
+    ;;
+    -compile-info|-compile_info)
+    # -compile_info included for backward compatibility
+    Show=echo
+    addarg=no
+    ;;
+    -link-info|-link_info)
+    # -link_info included for backward compatibility
+    Show=echo
+    addarg=no
+    ;;
+    -v)
+    # Pass this argument to the compiler as well.
+    echo "mpicxx for MPICH version $MPICH_VERSION"
+    # if there is only 1 argument, it must be -v.
+    if [ "$#" -eq "1" ] ; then
+        linking=no
+    fi
+    ;;
+    -profile=*)
+    # Pass the name of a profiling configuration.  As
+    # a special case, lib<name>.so or lib<name>.la may be used
+    # if the library is in $libdir
+    profConf=`echo A$arg | sed -e 's/A-profile=//g'`
+    addarg=no
+    # Loading the profConf file is handled below
+    ;;
+    -nativelinking)
+    # Internal option to use native compiler for linking without MPI libraries
+    nativelinking=yes
+    addarg=no
+    ;;
+    -lcuda|-lcudart|-lze_loader)
+    delay_libs="$delay_libs $arg"
+    addarg=no
+    ;;
+    -mpi-abi|-mpi_abi)
+    mpi_abi=yes
+    addarg=no
+    ;;
+    -help)
+    NC=`echo "$CXX" | sed 's%\/% %g' | awk '{print $NF}' -`
+    if [ -f "$sysconfdir/mpixxx_opts.conf" ] ; then
+        . $sysconfdir/mpixxx_opts.conf
+        echo "    -cxx=xxx      - Reset the native compiler to xxx."
+    else
+        if [ -f "./mpixxx_opts.conf" ] ; then
+            . ./mpixxx_opts.conf
+            echo "    -cxx=xxx      - Reset the native compiler to xxx."
+        fi
+    fi
+    exit 0
+    ;;
+    *)
+    ;;
+
+    esac
+    if [ $addarg = no ] ; then
+        unset allargs[$argno]
+    fi
+    # Some versions of bash do not accept ((argno++))
+    argno=`expr $argno + 1`
+done
+
+if [ $# -eq 0 ] ; then
+    echo "Error: Command line argument is needed!"
+    "$0" -help
+    exit 1
+fi
+
+# Check recursive situations
+# User mistakes, e.g. setting MPICH_CXX=mpicxx, may end up recursively running
+# this script. A simple check to bail if that's the case.
+if [ -n "$MPICH_RECURSION_CHECK" ] ; then
+    echo "This script ($0) is being called recursively, check that MPICH_CXX does not refer to mpicxx."
+    exit 1
+fi
+MPICH_RECURSION_CHECK=1
+export MPICH_RECURSION_CHECK
+
+# -----------------------------------------------------------------------
+# Derived variables.  These are assembled from variables set from the
+# default, environment, configuration file (if any) and command-line
+# options (if any)
+
+PROFILE_FOO=
+# Handle the case of a profile switch
+if [ -n "$profConf" ] ; then
+    profConffile=
+    if [ -s "$libdir/lib$profConf.a" -o -s "$libdir/lib$profConf.so" ] ; then
+	PROFILE_FOO="-l$profConf"
+    elif [ -s "$sysconfdir/$profConf.conf" ] ; then
+	profConffile="$sysconfdir/$profConf.conf"
+    elif [ -s "$profConf.conf" ] ; then
+        profConffile="$profConf.conf"
+    else
+        echo "Profiling configuration file $profConf.conf not found in $sysconfdir"
+    fi
+    if [ -n "$profConffile" -a -s "$profConffile" ] ; then
+	. $profConffile
+    fi
+fi
+
+final_cxxflags=" "
+final_cppflags=" "
+final_ldflags=" "
+final_libs=""
+if test "yes" = "no" -o "${interlib_deps}" = "no" ; then
+    final_ldflags="${final_ldflags} "
+    final_libs="${final_libs}  -L/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/lib -lm -ludev -latomic -lpthread -ldl"
+fi
+
+if [ "${0%_abi}" != "$0" ] ; then
+    mpi_abi=yes
+fi
+if [ "$mpi_abi" = no ] ; then
+    mpi_libs_shared="-lmpi "
+    mpi_libs_static="$libdir/libmpi.a "
+else
+    final_cppflags="-DMPI_ABI ${final_cppflags}"
+    mpi_libs_shared="-lmpi_abi "
+    mpi_libs_static="$libdir/libmpi_abi.a "
+fi
+
+cxxlibs=
+if [ "$mpi_abi" = no ] ; then
+    if [ "mpi" != "mpi" ] ; then
+        cxxlibs="-lmpi"
+    fi
+else
+    true # TODO: Implement support for libmpicxx_abi
+    # if [ "@MPICXXABILIBNAME@" != "mpi_abi" ] ; then
+    #     cxxlibs="-l@MPICXXABILIBNAME@"
+    # fi
+fi
+
+# -----------------------------------------------------------------------
+#
+# A temporary statement to invoke the compiler
+# Place the -L before any args in case there are any mpi libraries in there.
+# Eventually, we'll want to move this after any non-MPI implementation 
+# libraries.
+# We use a single invocation of the compiler.  This will be adequate until
+# we run into a system that uses a separate linking command.  With any luck,
+# such archaic systems are no longer with us.  This also lets us
+# accept any argument; we don't need to know if we've seen a source
+# file or an object file.  Instead, we just check for an option that
+# suppressing linking, such as -c or -M.
+if [ "$show_info" = compile ] ; then
+    echo ${final_cppflags} $PROFILE_INCPATHS ${final_cxxflags} -I$includedir
+elif [ "$show_info" = link ] ; then
+    if [ "$nativelinking" = yes ] ; then
+        echo ${final_ldflags}
+    else
+        if [ "$static_mpi" = no ] ; then
+            echo ${final_ldflags} -L$libdir $cxxlibs $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_shared} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
+        else
+            echo ${final_ldflags} -L$libdir $cxxlibs $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_static} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
+        fi
+    fi
+elif [ "$linking" = yes ] ; then
+    if [ "$nativelinking" = yes ] ; then
+        $Show $CXX ${final_cppflags} $PROFILE_INCPATHS ${final_cxxflags} ${final_ldflags} "${allargs[@]}" -I$includedir
+        rc=$?
+    else
+      if [ "$static_mpi" = no ] ; then
+        $Show $CXX ${final_cppflags} $PROFILE_INCPATHS ${final_cxxflags} ${final_ldflags} "${allargs[@]}" -I$includedir -L$libdir $cxxlibs $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_shared} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
+      else
+        $Show $CXX ${final_cppflags} $PROFILE_INCPATHS ${final_cxxflags} ${final_ldflags} "${allargs[@]}" -I$includedir -L$libdir $cxxlibs $PROFILE_PRELIB $PROFILE_FOO ${wrapper_dl_type_flags} ${mpi_libs_static} $PROFILE_POSTLIB ${delay_libs} ${final_libs}
+      fi
+        rc=$?
+    fi
+else
+    $Show $CXX ${final_cppflags} $PROFILE_INCPATHS ${final_cxxflags} "${allargs[@]}" -I$includedir
+    rc=$?
+fi
+
+exit $rc

linx64/mpi/mpich/bin/parkill

@@ -1,6 +1,9 @@
 #! /usr/bin/perl
-# -*- Mode: perl; -*-
-#
+##
+## Copyright (C) by Argonne National Laboratory
+##     See COPYRIGHT in top-level directory
+##
+
 # Kill all processes running a specified command
 #
 # Many systems also have the "killall" command; this should be used instead
@@ -93,7 +96,7 @@ while (<FD>) {
     my $procfile = "/proc/$pid/cmdline";
     print "Looking for $procfile\n" if $debug;
     # We *CANNOT USE* -s with a file in /proc because -s filename returns
-    # false alwyas ! (BUG BUG BUG).  Instead, we try to open and read from it
+    # false always ! (BUG BUG BUG).  Instead, we try to open and read from it
     $rc = open PFD, "<$procfile";
     if ($rc) {
 	my $cmdline = <PFD>;

linx64/mpi/mpich/include/mpio.h

@@ -1,8 +1,6 @@
-/* -*- Mode: C; c-basic-offset:4 ; -*- */
-/* 
- *
- *   Copyright (C) 1997 University of Chicago. 
- *   See COPYRIGHT notice in top-level directory.
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
  */
 
 /* user include file for MPI-IO programs */
@@ -56,6 +54,8 @@ typedef long long MPI_Offset;
 #define HAVE_MPI_DATAREP_FUNCTIONS
 typedef int (MPI_Datarep_conversion_function)(void *, MPI_Datatype, int, 
              void *, MPI_Offset, void *);
+typedef int (MPI_Datarep_conversion_function_c)(void *, MPI_Datatype, MPI_Count,
+                                                void *, MPI_Offset, void *);
 typedef int (MPI_Datarep_extent_function)(MPI_Datatype datatype, MPI_Aint *,
 					  void *);
 #endif
@@ -267,6 +267,80 @@ int MPI_File_iread_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
 int MPI_File_iwrite_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype,
                          MPI_Request *request)
     MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+
+/* MPI 4.0 large count functions */
+int MPI_File_read_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_read_all_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_read_all_begin_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype,
+                     MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_read_at_all_c(MPI_File fh, MPI_Offset offset, void * buf, MPI_Count count,
+                         MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_read_at_all_begin_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count,
+                               MPI_Datatype datatype) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_read_ordered_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                          MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_read_ordered_begin_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_read_shared_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                         MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_write_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                   MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_write_all_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                       MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_write_all_begin_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+
+int MPI_File_write_at_c(MPI_File fh, MPI_Offset offset, const void * buf, MPI_Count count,
+                      MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                          MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_write_at_all_begin_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                                MPI_Datatype datatype) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                           MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_write_ordered_begin_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_write_shared_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                          MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_iread_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype, MPIO_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_iread_all_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                        MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_iread_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype,
+                      MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_iread_at_all_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count,
+                           MPI_Datatype datatype, MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_iread_shared_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                          MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_iwrite_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                    MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_iwrite_all_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                         MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int MPI_File_iwrite_at_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                       MPI_Datatype datatype, MPIO_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_iwrite_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                            MPI_Datatype datatype, MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int MPI_File_iwrite_shared_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                           MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+
+int MPI_File_get_type_extent_c(MPI_File fh, MPI_Datatype datatype, MPI_Count *extent) ROMIO_API_PUBLIC;
+
+int MPI_Register_datarep_c(const char *datarep, MPI_Datarep_conversion_function_c *read_conversion_fn,
+			 MPI_Datarep_conversion_function_c *write_conversion_fn,
+			 MPI_Datarep_extent_function *dtype_file_extent_fn, void *extra_state) ROMIO_API_PUBLIC;
+
 /* End Prototypes */
 
 #ifndef HAVE_MPI_DARRAY_SUBARRAY
@@ -497,6 +571,79 @@ int PMPI_File_iwrite_all(MPI_File fh, const void *buf, int count, MPI_Datatype d
                           MPI_Request *request)
     MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
 
+/* MPI 4.0 large count functions */
+int PMPI_File_read_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_read_all_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_read_all_begin_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_read_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype,
+                     MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_read_at_all_c(MPI_File fh, MPI_Offset offset, void * buf, MPI_Count count,
+                         MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_read_at_all_begin_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count,
+                               MPI_Datatype datatype) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_read_ordered_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                          MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_read_ordered_begin_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_read_shared_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                         MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_write_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                   MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_write_all_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                       MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_write_all_begin_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+
+int PMPI_File_write_at_c(MPI_File fh, MPI_Offset offset, const void * buf, MPI_Count count,
+                      MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_write_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                          MPI_Datatype datatype, MPI_Status *status)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_write_at_all_begin_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                                MPI_Datatype datatype) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_write_ordered_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                           MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_write_ordered_begin_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_write_shared_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                          MPI_Status *status) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_iread_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype, MPIO_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_iread_all_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                        MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_iread_at_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count, MPI_Datatype datatype,
+                      MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_iread_at_all_c(MPI_File fh, MPI_Offset offset, void *buf, MPI_Count count,
+                           MPI_Datatype datatype, MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_iread_shared_c(MPI_File fh, void *buf, MPI_Count count, MPI_Datatype datatype,
+                          MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_iwrite_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                    MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_iwrite_all_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                         MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+int PMPI_File_iwrite_at_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                       MPI_Datatype datatype, MPIO_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_iwrite_at_all_c(MPI_File fh, MPI_Offset offset, const void *buf, MPI_Count count,
+                            MPI_Datatype datatype, MPI_Request *request)
+    MPICH_ATTR_POINTER_WITH_TYPE_TAG(3,5) ROMIO_API_PUBLIC;
+int PMPI_File_iwrite_shared_c(MPI_File fh, const void *buf, MPI_Count count, MPI_Datatype datatype,
+                           MPIO_Request *request) MPICH_ATTR_POINTER_WITH_TYPE_TAG(2,4) ROMIO_API_PUBLIC;
+
+int PMPI_File_get_type_extent_c(MPI_File fh, MPI_Datatype datatype, MPI_Count *extent) ROMIO_API_PUBLIC;
+
+int PMPI_Register_datarep_c(const char *datarep, MPI_Datarep_conversion_function_c *read_conversion_fn,
+			 MPI_Datarep_conversion_function_c *write_conversion_fn,
+			 MPI_Datarep_extent_function *dtype_file_extent_fn, void *extra_state) ROMIO_API_PUBLIC;
+
 #ifndef HAVE_MPI_DARRAY_SUBARRAY
 /* Section 4.14.4 */
 int PMPI_Type_create_subarray(int, int *, int *, int *, int, 

linx64/mpi/mpich/include/mpiof.h

@@ -1,8 +1,8 @@
-! 
-!     Copyright (C) 1997 University of Chicago. 
-!     See COPYRIGHT notice in top-level directory.
 !
-! 
+! Copyright (C) by Argonne National Laboratory
+!     See COPYRIGHT in top-level directory
+!
+
 !    user include file for Fortran MPI-IO programs 
 !
       INTEGER MPI_MODE_RDONLY, MPI_MODE_RDWR, MPI_MODE_WRONLY

linx64/mpi/mpich/lib/libmpi.la

@@ -8,7 +8,7 @@
 dlname='libmpi.so.12'
 
 # Names of this library.
-library_names='libmpi.so.12.1.7 libmpi.so.12 libmpi.so'
+library_names='libmpi.so.12.5.0 libmpi.so.12 libmpi.so'
 
 # The name of the static archive.
 old_library='libmpi.a'
@@ -17,15 +17,15 @@ old_library='libmpi.a'
 inherited_linker_flags=''
 
 # Libraries that this one depends upon.
-dependency_libs=' -lm -lpthread -lrt'
+dependency_libs=' -L/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/lib -lm -ludev -latomic -lpthread -ldl'
 
 # Names of additional weak libraries provided by this library
 weak_library_names=''
 
 # Version information for libmpi.
-current=13
-age=1
-revision=7
+current=17
+age=5
+revision=0
 
 # Is this an already installed library?
 installed=yes
@@ -38,4 +38,4 @@ dlopen=''
 dlpreopen=''
 
 # Directory that this library needs to be installed in:
-libdir='/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1/lib64'
+libdir='/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/lib'

linx64/mpi/mpich/lib/libmpi.so

@@ -0,0 +1 @@
+libmpi.so.12.5.0

linx64/mpi/mpich/lib/libmpi.so.12

@@ -0,0 +1 @@
+libmpi.so.12.5.0

linx64/mpi/mpich/lib/libopa.so

@@ -0,0 +1 @@
+libmpi.so

linx64/mpi/mpich/lib/pkgconfig/mpich.pc

@@ -1,15 +1,15 @@
 # this gives access to the mpich header files
-prefix=/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1
+prefix=/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0
 exec_prefix=${prefix}
-libdir=${exec_prefix}/lib64
+libdir=${exec_prefix}/lib
 includedir=${prefix}/include
 
 Name: mpich
 Description: High Performance and portable MPI
-Version: 3.3.1
+Version: 4.3.0
 URL: http://www.mcs.anl.gov/research/projects/mpich
 Requires:
-Libs: -Wl,-rpath -Wl,${exec_prefix}/lib64  -L${libdir} -lmpi  -lm   -lpthread  -lrt 
+Libs: -Wl,-rpath -Wl,${exec_prefix}/lib -Wl,--enable-new-dtags  -L${libdir} -lmpi     -lpthread     -L/home/admin/Documents/code/projects/cpp-thirdparty-src/mpi/mpich/install/4.3.0/lib  -latomic -lpthread -ldl
 Cflags:   -I${includedir}
 
 # pkg-config does not understand Cxxflags, etc. So we allow users to

linx64/mpi/mpich/lib64/libmpi.so

@@ -1 +0,0 @@
-libmpi.so.12.1.7

linx64/mpi/mpich/lib64/libmpi.so.12

@@ -1 +0,0 @@
-libmpi.so.12.1.7

linx64/mpi/mpich/lib64/pkgconfig/openpa.pc

@@ -1,15 +0,0 @@
-prefix=/home/brgirgis/Documents/code/idiscovery/cpp-thirdparty-src/mpi/mpich/install/3.3.1
-exec_prefix=${prefix}
-libdir=${exec_prefix}/lib64
-includedir=${prefix}/include
-
-# Technically, the -lopa below isn't needed if you are only using the atomic
-# primitives (not the queue code) and you are not using mutex emulation.
-
-Name: OpenPA
-Description: Portable library for atomic operations
-Version: 1.0.3
-Libs: -L${libdir} -lopa
-Libs.private: -lpthread 
-Cflags: -I${includedir}
-

linx64/mpi/mpich/share/doc/mpich/www1/hydra_nameserver.html

@@ -0,0 +1,23 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>hydra_nameserver</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="hydra_nameserver">hydra_nameserver</H1>
+program to support MPI<tt>s name publishing features with hydra 
+<H2>Description</H2>
+Hydra is the default name service for MPI\</tt>s name publishing features
+(MPI_PUBLISH_NAME/MPI_LOOKUP_NAME). For programs launched by separate
+hydra instances, a separate nameserver mechanism need be launched in
+order for the name publishing to work across multiple process managers.
+<P>
+This can be achieved with hydra_nameserver. For example:
+<PRE>
+    shell@myhost1$ hydra_nameserver &amp;
+    shell@myhost1$ mpiexec -hosts myhost1,myhost2 -n 4 -nameserver myhost1 ./a.out
+</PRE>
+
+<P><B>Location:</B>/var/tmp/qbvJ6gPnF7/mpich-4.3.0/src/pm/hydra/mansrc/hydra_nameserver.txt<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www1/hydra_persist.html

@@ -0,0 +1,16 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>hydra_persist</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="hydra_persist">hydra_persist</H1>
+Internal executable used by Hydra 
+<H2>Description</H2>
+This executable is designed to not be run directly by users, but to
+support the Hydra process manager. As such, no documentation will be
+provided here.
+<P>
+<P><B>Location:</B>/var/tmp/qbvJ6gPnF7/mpich-4.3.0/src/pm/hydra/mansrc/hydra_persist.txt<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www1/hydra_pmi_proxy.html

@@ -0,0 +1,16 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>hydra_pmi_proxy</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="hydra_pmi_proxy">hydra_pmi_proxy</H1>
+Internal executable used by Hydra 
+<H2>Description</H2>
+This executable is designed to not be run directly by users, but to
+support the Hydra process manager. As such, no documentation will be
+provided here.
+<P>
+<P><B>Location:</B>/var/tmp/qbvJ6gPnF7/mpich-4.3.0/src/pm/hydra/mansrc/hydra_pmi_proxy.txt<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www1/index.htm

@@ -1,17 +1,21 @@
 <HTML>
 <HEAD>
 <TITLE>Manpages for MPICH</TITLE>
-<!-- This file generated by createhtmlindex on  -->
+<!-- This file generated by createhtmlindex on 2/3/2025 -->
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
 <H1>Manpages for MPICH</H1>
 <TABLE>
-<TR><TD><A HREF="mpicc.html">mpicc</A></TD>
-<TD><A HREF="mpiexec.html">mpiexec</A></TD>
+<TR><TD><A HREF="hydra_nameserver.html">hydra_nameserver</A></TD>
+<TD><A HREF="mpicc.html">mpicc</A></TD>
 <TD><A HREF="mpifort.html">mpifort</A></TD>
 </TR>
-<TR><TD><A HREF="mpicxx.html">mpicxx</A></TD>
-<TD><A HREF="mpif77.html">mpif77</A></TD>
+<TR><TD><A HREF="hydra_persist.html">hydra_persist</A></TD>
+<TD><A HREF="mpicxx.html">mpicxx</A></TD>
+<TD></TD>
+</TR>
+<TR><TD><A HREF="hydra_pmi_proxy.html">hydra_pmi_proxy</A></TD>
+<TD><A HREF="mpiexec.html">mpiexec</A></TD>
 <TD></TD>
 </TR>
 </TABLE>

linx64/mpi/mpich/share/doc/mpich/www1/mpicc.html

@@ -5,7 +5,7 @@
 <TITLE>mpicc</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="mpicc"><H1>mpicc</H1></A>
+<H1 id="mpicc">mpicc</H1>
 Compiles and links MPI programs written in C 
 <H2>Description</H2>
 This command can be used to compile and link MPI programs written in
@@ -17,67 +17,69 @@ as it provides the necessary libraries.
 <P>
 <H2>Command line arguments</H2>
 <DL>
-<DT><B>-show      </B><DD>Show the commands that would be used without
+<DT><B>-show      </B> <DD> Show the commands that would be used without
 running them
 
-<DT><B>-help      </B><DD>Give short help
+<DT><B>-help      </B> <DD> Give short help
 
-<DT><B>-cc=name   </B><DD>Use compiler <TT>name</TT> instead of the default choice.  Use
+<DT><B>-cc=name   </B> <DD> Use compiler <tt>name</tt> instead of the default choice.  Use
 this only if the compiler is compatible with the MPICH
 library (see below)
 
-<DT><B>-config=name </B><DD>Load a configuration file for a particular compiler.
-This allows a single <TT>mpicc</TT> command to be used with 
+<DT><B>-config=name </B> <DD> Load a configuration file for a particular compiler.
+This allows a single <tt>mpicc</tt> command to be used with 
 multiple compilers.
 
-<DT><B>-compile_info </B><DD>Show the steps for compiling a program.  This option
+<DT><B>-compile_info </B> <DD> Show the steps for compiling a program.  This option
 can be used to see what options and include paths are
 used by mpicc.
 
-<DT><B>-link_info </B><DD>Show the steps for linking a program.  This option
+<DT><B>-link_info </B> <DD> Show the steps for linking a program.  This option
 can be used to see what options and libraries are used by
 mpicc.
 
-<DT><B>-profile=name </B><DD>Use the MPI profiling given by name.  See below for
+<DT><B>-profile=name </B> <DD> Use the MPI profiling given by name.  See below for
 details
 
-<DT><B>-echo      </B><DD>Show exactly what this program is doing.
+<DT><B>-echo      </B> <DD> Show exactly what this program is doing.
 This option should normally not be used.
 
-<DT><B>-static</B><DD>mpi - Use a statically compile MPI library, but shared libraries
+<DT><B>-static</B> <DD> mpi - Use a statically compile MPI library, but shared libraries
 for all of the other dependencies.
 
-<DT><B>others     </B><DD>are passed to the compiler or linker.  For example, <TT>\-c
-</TT>causes files to be compiled, <TT>\-g</TT> selects compilation with
-debugging on most systems, and <TT>\-o name</TT> causes linking
-with the output executable given the name <TT>name</TT>.
+<DT><B>-mpi</B> <DD> abi  - Compile and link using the standard MPI ABI.
+
+<DT><B>others     </B> <DD> are passed to the compiler or linker.  For example, <tt>\-c
+</tt>causes files to be compiled, <tt>\-g</tt> selects compilation with
+debugging on most systems, and <tt>\-o name</tt> causes linking
+with the output executable given the name <tt>name</tt>.
 </DL>
 <P>
 <H2>Environment Variables</H2>
-The environment variable <TT>MPICH_CC</TT> may be used
+The environment variable <tt>MPICH_CC</tt> may be used
 to select different C compiler and linker.  Note that since
 MPICH is built with a particular C and Fortran compiler, changing the
 compilers used can cause problems.  Use this only if you could intermix
 code compiled with the different compilers.
 <P>
-The environment variable <TT>MPICC_PROFILE</TT> specifies a profile library
-and has the same effect as if <TT>\-profile=$MPICC_PROFILE</TT> were used as
-an argument to <TT>mpicc</TT>.  See the discussion of <TT>\-profile</TT> below for more
+The environment variable <tt>MPICC_PROFILE</tt> specifies a profile library
+and has the same effect as if <tt>\-profile=$MPICC_PROFILE</tt> were used as
+an argument to <tt>mpicc</tt>.  See the discussion of <tt>\-profile</tt> below for more
 details.
 <P>
 <H2>Compatible Compilers</H2>
 The MPI library may be used with any compiler that uses the same
-lengths for basic data objects (such as <TT>long double</TT>) and that
+lengths for basic data objects (such as <tt>long double</tt>) and that
 uses compatible run-time libraries.  On many systems, the various
-compilers are compatible and may be used interchangably.  There are
-exceptions; if you use the <TT>MPICH_CC</TT> environment variable or the
-<TT>\-cc=name</TT> command-line argument to override the choice of compiler
+compilers are compatible and may be used interchangeably.  There are
+exceptions; if you use the <tt>MPICH_CC</tt> environment variable or the
+<tt>\-cc=name</tt> command-line argument to override the choice of compiler
 and encounter problems, try reconfiguring MPICH with the new compiler
 and installing MPICH in a separate location.  See the installation manual
 for more details.
 <P>
 <H2>Examples</H2>
-To compile a single file <TT>foo.c</TT>, use
+To compile a single file <tt>foo.c</tt>, use
 <PRE>
    mpicc -c foo.c
 </PRE>
@@ -96,29 +98,29 @@ Combining compilation and linking in a single command
 is a convenient way to build simple programs.
 <P>
 <H2>Selecting a Profiling Library</H2>
-The <TT>\-profile=name</TT> argument allows you to specify an MPI profiling
-library to be used.  <TT>name</TT> can have two forms:
+The <tt>\-profile=name</tt> argument allows you to specify an MPI profiling
+library to be used.  <tt>name</tt> can have two forms:
 <P>
-<BR>A library in the same directory as the MPI library
-<BR>The name of a profile configuration file
-<BR>
+<br>A library in the same directory as the MPI library
+<br>The name of a profile configuration file
+<br>
 <P>
-If <TT>name</TT> is a library, then this library is included before the MPI
+If <tt>name</tt> is a library, then this library is included before the MPI
 library.  This allows the simple use of libraries that make use of the
 MPI profiling interface and that are installed in the same directory as
 the MPI library.
 <P>
-If <TT>name.conf</TT> is the name of a file in the sysconfdir directory, then this
+If <tt>name.conf</tt> is the name of a file in the sysconfdir directory, then this
 is read and may define the following variables:
 <DL>
-<DT><B>PROFILE_PRELIB </B><DD>Libraries (and paths) to include before the MPI library
+<DT><B>PROFILE_PRELIB </B> <DD> Libraries (and paths) to include before the MPI library
 
-<DT><B>PROFILE_POSTLIB </B><DD>Libraries to include after the MPI library
+<DT><B>PROFILE_POSTLIB </B> <DD> Libraries to include after the MPI library
 
-<DT><B>PROFILE_INCPATHS </B><DD>C preprocessor arguments for any include files
-For example, to add <TT>/usr/local/myprof/include</TT> to the include path and
-the library <TT>libmyprof.a</TT> in <TT>/usr/local/myprof/lib</TT> to the link step, 
-you could create the file <TT>myprof.conf</TT> with the lines
+<DT><B>PROFILE_INCPATHS </B> <DD> C preprocessor arguments for any include files
+For example, to add <tt>/usr/local/myprof/include</tt> to the include path and
+the library <tt>libmyprof.a</tt> in <tt>/usr/local/myprof/lib</tt> to the link step, 
+you could create the file <tt>myprof.conf</tt> with the lines
 </DL>
 <P>
 <PRE>
@@ -128,10 +130,10 @@ you could create the file <TT>myprof.conf</TT> with the lines
 
 and place it in the sysconfdir directory (this directory is set at
 configure time when MPICH is built).  Then using the command-line
-argument <TT>\-profile=myprof</TT> will cause these
+argument <tt>\-profile=myprof</tt> will cause these
 definitions to be added to the relevant compile commands.
 <P>
 <H2>See Also</H2>
  mpicxx, mpifort, mpiexec
-<BR>
+<br>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www1/mpicxx.html

@@ -5,7 +5,7 @@
 <TITLE>mpicxx</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="mpicxx"><H1>mpicxx</H1></A>
+<H1 id="mpicxx">mpicxx</H1>
 Compiles and links MPI programs written in C++ 
 <H2>Description</H2>
 This command can be used to compile and link MPI programs written in
@@ -17,67 +17,69 @@ as it provides the necessary libraries.
 <P>
 <H2>Command line arguments</H2>
 <DL>
-<DT><B>-show      </B><DD>Show the commands that would be used without
+<DT><B>-show      </B> <DD> Show the commands that would be used without
 running them
 
-<DT><B>-help      </B><DD>Give short help
+<DT><B>-help      </B> <DD> Give short help
 
-<DT><B>-cxx=name   </B><DD>Use compiler <TT>name</TT> instead of the default choice.  Use
+<DT><B>-cxx=name   </B> <DD> Use compiler <tt>name</tt> instead of the default choice.  Use
 this only if the compiler is compatible with the MPICH
 library (see below)
 
-<DT><B>-config=name </B><DD>Load a configuration file for a particular compiler.
-This allows a single <TT>mpicxx</TT> command to be used with 
+<DT><B>-config=name </B> <DD> Load a configuration file for a particular compiler.
+This allows a single <tt>mpicxx</tt> command to be used with 
 multiple compilers.
 
-<DT><B>-compile_info </B><DD>Show the steps for compiling a program.  This option
+<DT><B>-compile_info </B> <DD> Show the steps for compiling a program.  This option
 can be used to see what options and include paths are
 used by mpicxx.
 
-<DT><B>-link_info </B><DD>Show the steps for linking a program.  This option
+<DT><B>-link_info </B> <DD> Show the steps for linking a program.  This option
 can be used to see what options and libraries are used by
 mpicxx.
 
-<DT><B>-profile=name </B><DD>Use the MPI profiling given by name.  See below for
+<DT><B>-profile=name </B> <DD> Use the MPI profiling given by name.  See below for
 details
 
-<DT><B>-echo      </B><DD>Show exactly what this program is doing.
+<DT><B>-echo      </B> <DD> Show exactly what this program is doing.
 This option should normally not be used.
 
-<DT><B>-static</B><DD>mpi - Use a statically compile MPI library, but shared libraries
+<DT><B>-static</B> <DD> mpi - Use a statically compile MPI library, but shared libraries
 for all of the other dependencies.
 
-<DT><B>others     </B><DD>are passed to the compiler or linker.  For example, <TT>\-c
-</TT>causes files to be compiled, <TT>\-g</TT> selects compilation with
-debugging on most systems, and <TT>\-o name</TT> causes linking
-with the output executable given the name <TT>name</TT>.
+<DT><B>-mpi</B> <DD> abi  - Compile and link using the standard MPI ABI.
+
+<DT><B>others     </B> <DD> are passed to the compiler or linker.  For example, <tt>\-c
+</tt>causes files to be compiled, <tt>\-g</tt> selects compilation with
+debugging on most systems, and <tt>\-o name</tt> causes linking
+with the output executable given the name <tt>name</tt>.
 </DL>
 <P>
 <H2>Environment Variables</H2>
-The environment variables <TT>MPICH_CXX</TT> may be used
+The environment variables <tt>MPICH_CXX</tt> may be used
 to select different C++ compiler and linker.  Note that since
 MPICH is built with a particular C and Fortran compiler, changing the
 compilers used can cause problems.  Use this only if you could intermix
 code compiled with the different compilers.
 <P>
-The environment variable <TT>MPICC_PROFILE</TT> specifies a profile library
-and has the same effect as if <TT>\-profile=$MPICC_PROFILE</TT> were used as
-an argument to <TT>mpicc</TT>.  See the discussion of <TT>\-profile</TT> below for more
+The environment variable <tt>MPICC_PROFILE</tt> specifies a profile library
+and has the same effect as if <tt>\-profile=$MPICC_PROFILE</tt> were used as
+an argument to <tt>mpicc</tt>.  See the discussion of <tt>\-profile</tt> below for more
 details.
 <P>
 <H2>Compatible Compilers</H2>
 The MPI library may be used with any compiler that uses the same
-lengths for basic data objects (such as <TT>long double</TT>) and that
+lengths for basic data objects (such as <tt>long double</tt>) and that
 uses compatible run-time libraries.  On many systems, the various
-compilers are compatible and may be used interchangably.  There are
-exceptions; if you use the <TT>MPICH_CXX</TT> environment variable or the
-<TT>\-cxx=name</TT> command-line argument to override the choice of compiler
+compilers are compatible and may be used interchangeably.  There are
+exceptions; if you use the <tt>MPICH_CXX</tt> environment variable or the
+<tt>\-cxx=name</tt> command-line argument to override the choice of compiler
 and encounter problems, try reconfiguring MPICH with the new compiler,
 and installing MPICH in a separate location.  See the installation manual
 for more details.
 <P>
 <H2>Examples</H2>
-To compile a single file <TT>foo.c</TT>, use
+To compile a single file <tt>foo.c</tt>, use
 <PRE>
    mpicxx -c foo.cxx
 </PRE>
@@ -96,29 +98,29 @@ Combining compilation and linking in a single command
 is a convenient way to build simple programs.
 <P>
 <H2>Selecting a Profiling Library</H2>
-The <TT>\-profile=name</TT> argument allows you to specify an MPI profiling
-library to be used.  <TT>name</TT> can have two forms:
+The <tt>\-profile=name</tt> argument allows you to specify an MPI profiling
+library to be used.  <tt>name</tt> can have two forms:
 <P>
-<BR>A library in the same directory as the MPI library
-<BR>The name of a profile configuration file
-<BR>
+<br>A library in the same directory as the MPI library
+<br>The name of a profile configuration file
+<br>
 <P>
-If <TT>name</TT> is a library, then this library is included before the MPI
+If <tt>name</tt> is a library, then this library is included before the MPI
 library.  This allows the simple use of libraries that make use of the
 MPI profiling interface and that are installed in the same directory as
 the MPI library.
 <P>
-If <TT>name.conf</TT> is the name of a file in the sysconfdir directory, then this
+If <tt>name.conf</tt> is the name of a file in the sysconfdir directory, then this
 is read and may define the following variables:
 <DL>
-<DT><B>PROFILE_PRELIB </B><DD>Libraries (and paths) to include before the MPI library
+<DT><B>PROFILE_PRELIB </B> <DD> Libraries (and paths) to include before the MPI library
 
-<DT><B>PROFILE_POSTLIB </B><DD>Libraries to include after the MPI library
+<DT><B>PROFILE_POSTLIB </B> <DD> Libraries to include after the MPI library
 
-<DT><B>PROFILE_INCPATHS </B><DD>C preprocessor arguments for any include files
-For example, to add <TT>/usr/local/myprof/include</TT> to the include path and
-the library <TT>libmyprof.a</TT> in <TT>/usr/local/myprof/lib</TT> to the link step, 
-you could create the file <TT>myprof.conf</TT> with the lines
+<DT><B>PROFILE_INCPATHS </B> <DD> C preprocessor arguments for any include files
+For example, to add <tt>/usr/local/myprof/include</tt> to the include path and
+the library <tt>libmyprof.a</tt> in <tt>/usr/local/myprof/lib</tt> to the link step, 
+you could create the file <tt>myprof.conf</tt> with the lines
 </DL>
 <P>
 <PRE>
@@ -128,10 +130,10 @@ you could create the file <TT>myprof.conf</TT> with the lines
 
 and place it in the sysconfdir directory (this directory is set at
 configure time when MPICH is built).  Then using the command-line
-argument <TT>\-profile=myprof</TT> will cause these
+argument <tt>\-profile=myprof</tt> will cause these
 definitions to be added to the relevant compile commands.
 <P>
 <H2>See Also</H2>
  mpicc, mpifort, mpiexec
-<BR>
+<br>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www1/mpiexec.html

@@ -5,7 +5,7 @@
 <TITLE>mpiexec</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="mpiexec"><H1>mpiexec</H1></A>
+<H1 id="mpiexec">mpiexec</H1>
 Run an MPI program 
 <H2>Synopsis</H2>
 <PRE>
@@ -14,52 +14,52 @@ Run an MPI program
     mpiexec args executable pgmargs [ : args executable pgmargs ... ]
 </PRE>
 
-where <TT>args</TT> are command line arguments for <TT>mpiexec</TT> (see below),
-<TT>executable</TT> is the name of an executable MPI program, and <TT>pgmargs
-</TT>are command line arguments for the executable.  Multiple executables
+where <tt>args</tt> are command line arguments for <tt>mpiexec</tt> (see below),
+<tt>executable</tt> is the name of an executable MPI program, and <tt>pgmargs
+</tt>are command line arguments for the executable.  Multiple executables
 can be specified by using the colon notation (for MPMD - Multiple Program
 Multiple Data applications).   For example, the following command will run
-the MPI program <TT>a.out</TT> on 4 processes:
+the MPI program <tt>a.out</tt> on 4 processes:
 <PRE>
     mpiexec -n 4 a.out
 </PRE>
 
 <P>
-The MPI standard specifies the following arguments and their meanings:
+The MPI standard suggests the following arguments and their meanings:
 <P>
 <DL>
-<DT><B>-n &lt;np&gt; </B><DD>Specify the number of processes to use
+<DT><B>-n &lt;np&gt; </B> <DD> Specify the number of processes to use
 
-<DT><B>-host &lt;hostname&gt; </B><DD>Name of host on which to run processes
+<DT><B>-host &lt;hostname&gt; </B> <DD> Name of host on which to run processes
 
-<DT><B>-arch &lt;architecture name&gt; </B><DD>Pick hosts with this architecture type
+<DT><B>-wdir &lt;working directory&gt; </B> <DD> cd to this one <em>before</em> running executable
 
-<DT><B>-wdir &lt;working directory&gt; </B><DD>cd to this one <EM>before</EM> running executable
+<DT><B>-configfile &lt;name&gt; </B> <DD> file containing specifications of host/program,
+one per line, with # as a comment indicator, e.g., the usual
+mpiexec input. The lines excluding comments are concatenated with newlines
+replaced with spaces, and are then parsed as command line options.
+</DL>
+<P>
+The following options are reserved by the MPI standard, but they are not
+supported by MPICH (hydra):
+<P>
+<DL>
+<DT><B>-arch &lt;architecture name&gt; </B> <DD> Pick hosts with this architecture type
 
-<DT><B>-path &lt;pathlist&gt; </B><DD>use this to find the executable
+<DT><B>-path &lt;pathlist&gt; </B> <DD> use this to find the executable
 
-<DT><B>-soft &lt;triplets&gt; </B><DD>comma separated triplets that specify requested numbers
+<DT><B>-soft &lt;triplets&gt; </B> <DD> comma separated triplets that specify requested numbers
 of processes (see the MPI-2 specification for more details)
 
-<DT><B>-file &lt;name&gt; </B><DD>implementation-defined specification file
-
-<DT><B>-configfile &lt;name&gt; </B><DD>file containing specifications of host/program,
-one per line, with # as a comment indicator, e.g., the usual
-mpiexec input, but with ":" replaced with a newline.  That is,
-the configfile contains lines with -soft, -n etc.
+<DT><B>-file &lt;name&gt; </B> <DD> implementation-defined specification file
 </DL>
 <P>
 Additional arguments that are specific to the MPICH implementation
 are discussed below.
 <P>
-Note that not all of these parameters are meaningful for all
-systems.  For example, the <TT>gforker</TT> version of <TT>mpiexec</TT> creates all
-processes on the same system on which it is running; in that case, the
-<TT>\-arch</TT> and <TT>\-host</TT> options are ignored.
-<P>
-The colon character (<TT>:</TT>) may be used to separate different executables
+The colon character (<tt>:</tt>) may be used to separate different executables
 for MPMD (multiple program multiple data) programming.  For example,
-to run the program <TT>ocean</TT> on 4 processes and <TT>air</TT> on 8 processes, use:
+to run the program <tt>ocean</tt> on 4 processes and <tt>air</tt> on 8 processes, use:
 <P>
 <PRE>
     mpiexec -n 4 ocean : -n 8 air
@@ -68,85 +68,81 @@ to run the program <TT>ocean</TT> on 4 processes and <TT>air</TT> on 8 processes
 <P>
 <P>
 <H2>MPICH-Specific Arguments</H2>
+<P>
+MPICH implements a few process managers. Here we list a general set of
+arguments that many of the implementations support. More detailed options
+that are supported by a particular implementation, e.g. hydra, are
+available via:
+<PRE>
+    mpiexec -h
+</PRE>
+
 <P>
 Many of the implementations of process managers in MPICH support the
-following arguments to <TT>mpiexec</TT>:
+following arguments to <tt>mpiexec</tt>:
 <P>
 <DL>
-<DT><B>-np &lt;num&gt; </B><DD>A synonym for the standard <TT>\-n</TT> argument
+<DT><B>-np &lt;num&gt; </B> <DD> A synonym for the standard <tt>\-n</tt> argument
 
-<DT><B>-env &lt;name&gt; &lt;value&gt; </B><DD>Set the environment variable <TT>&lt;name&gt;</TT> to <TT>&lt;value&gt;</TT> for
-the processes being run by <TT>mpiexec
-</TT>
-<DT><B>-envnone </B><DD>Pass no environment variables (other than ones specified with
-other <TT>\-env</TT> or <TT>\-genv</TT> arguments) to the processes being run by <TT>mpiexec</TT>.
+<DT><B>-env &lt;name&gt; &lt;value&gt; </B> <DD> Set the environment variable <tt>&lt;name&gt;</tt> to <tt>&lt;value&gt;</tt> for
+the processes being run by <tt>mpiexec
+</tt>
+<DT><B>-envnone </B> <DD> Pass no environment variables (other than ones specified with
+other <tt>\-env</tt> or <tt>\-genv</tt> arguments) to the processes being run by <tt>mpiexec</tt>.
 By default, all environment
 variables are provided to each MPI process (rationale: principle of
 least surprise for the user)
 
-<DT><B>-envlist &lt;list&gt; </B><DD>Pass the listed environment variables (names separated
+<DT><B>-envlist &lt;list&gt; </B> <DD> Pass the listed environment variables (names separated
 by commas), with their current values, to the processes being run by
-<TT>mpiexec</TT>.
+<tt>mpiexec</tt>.
 
-<DT><B>-genv &lt;name&gt; &lt;value&gt; </B><DD>The <TT>\-genv</TT> options have the same meaning as their
-corresponding <TT>\-env</TT> version, except they apply to all executables, not just
+<DT><B>-genv &lt;name&gt; &lt;value&gt; </B> <DD> The <tt>\-genv</tt> options have the same meaning as their
+corresponding <tt>\-env</tt> version, except they apply to all executables, not just
 the current executable (in the case that the colon syntax is used to specify
-multiple execuables).
+multiple executables).
 
-<DT><B>-genvnone </B><DD>Like <TT>\-envnone</TT>, but for all executables
+<DT><B>-genvnone </B> <DD> Like <tt>\-envnone</tt>, but for all executables
 
-<DT><B>-genvlist &lt;list&gt; </B><DD>Like <TT>\-envlist</TT>, but for all executables
+<DT><B>-genvlist &lt;list&gt; </B> <DD> Like <tt>\-envlist</tt>, but for all executables
 
-<DT><B>-usize &lt;n&gt; </B><DD>Specify the value returned for the value of the attribute
-<TT>MPI_UNIVERSE_SIZE</TT>.
+<DT><B>-usize &lt;n&gt; </B> <DD> Specify the value returned for the value of the attribute
+<tt>MPI_UNIVERSE_SIZE</tt>.
 
-<DT><B>-l </B><DD>Label standard out and standard error (<TT>stdout</TT> and <TT>stderr</TT>) with
+<DT><B>-l </B> <DD> Label standard out and standard error (<tt>stdout</tt> and <tt>stderr</tt>) with
 the rank of the process
 
-<DT><B>-maxtime &lt;n&gt; </B><DD>Set a timelimit of <TT>&lt;n&gt;</TT> seconds.
+<DT><B>-maxtime &lt;n&gt; </B> <DD> Set a timelimit of <tt>&lt;n&gt;</tt> seconds.
 
-<DT><B>-exitinfo </B><DD>Provide more information on the reason each process exited if
+<DT><B>-exitinfo </B> <DD> Provide more information on the reason each process exited if
 there is an abnormal exit
 </DL>
 <P>
 <H2>Environment variables for mpiexec</H2>
 The following environment variables are understood by some versions of
-<TT>mpiexec</TT>.  The command line arguments have priority over these; that is,
+<tt>mpiexec</tt>.  The command line arguments have priority over these; that is,
 if both the environment variable and command line argument are used, the
 value specified by the command line argument is used.
 <P>
 <DL>
-<DT><B>MPIEXEC_TIMEOUT  </B><DD>Maximum running time in seconds.  <TT>mpiexec</TT> will
+<DT><B>MPIEXEC_TIMEOUT  </B> <DD> Maximum running time in seconds.  <tt>mpiexec</tt> will
 terminate MPI programs that take longer than the value specified by
-<TT>MPIEXEC_TIMEOUT</TT>.  
+<tt>MPIEXEC_TIMEOUT</tt>.  
 
-<DT><B>MPIEXEC_UNIVERSE_SIZE </B><DD>Set the universe size
+<DT><B>MPIEXEC_UNIVERSE_SIZE </B> <DD> Set the universe size
 
-<DT><B>MPIEXEC_PORT_RANGE </B><DD>Set the range of ports that <TT>mpiexec</TT> will use 
+<DT><B>MPIEXEC_PORT_RANGE </B> <DD> Set the range of ports that <tt>mpiexec</tt> will use 
 in communicating with the processes that it starts.  The format of 
-this is <TT>&lt;low&gt;:&lt;high&gt;</TT>.  For example, to specify any port between
-10000 and 10100, use <TT>10000:10100</TT>.  
+this is <tt>&lt;low&gt;:&lt;high&gt;</tt>.  For example, to specify any port between
+10000 and 10100, use <tt>10000:10100</tt>.  
 
-<DT><B>MPICH_PORT_RANGE </B><DD>Has the same meaning as <TT>MPIEXEC_PORT_RANGE</TT> and 
-is used if <TT>MPIEXEC_PORT_RANGE</TT> is not set.
+<DT><B>MPICH_PORT_RANGE </B> <DD> Has the same meaning as <tt>MPIEXEC_PORT_RANGE</tt> and 
+is used if <tt>MPIEXEC_PORT_RANGE</tt> is not set.
 
-<DT><B>MPIEXEC_PREFIX_DEFAULT </B><DD>If this environment variable is set, output
-to standard output is prefixed by the rank in <TT>MPI_COMM_WORLD</TT> of the 
-process and output to standard error is prefixed by the rank and the 
-text <TT>(err)</TT>; both are followed by an angle bracket (<TT>&gt;</TT>).  If
-this variable is not set, there is no prefix.
-
-<DT><B>MPIEXEC_PREFIX_STDOUT </B><DD>Set the prefix used for lines sent to standard
-output.  A <TT>%d</TT> is replaced with the rank in <TT>MPI_COMM_WORLD</TT>; a <TT>%w</TT> is 
-replaced with an indication of which <TT>MPI_COMM_WORLD</TT> in MPI jobs that
-involve multiple <TT>MPI_COMM_WORLD</TT>s (e.g., ones that use <TT>MPI_Comm_spawn</TT> or
-<TT>MPI_Comm_connect</TT>).
-
-<DT><B>MPIEXEC_PREFIX_STDERR </B><DD>Like <TT>MPIEXEC_PREFIX_STDOUT</TT>, but for standard error.
-</DL>
 <P>
 <H2>Return Status</H2>
-<TT>mpiexec</TT> returns the maximum of the exit status values of all of the
-processes created by <TT>mpiexec</TT>.
+<tt>mpiexec</tt> returns the maximum of the exit status values of all of the
+processes created by <tt>mpiexec</tt>.
 <P>
+<P><B>Location:</B>/var/tmp/qbvJ6gPnF7/mpich-4.3.0/src/pm/hydra/mansrc/mpiexec.txt<P>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www1/mpif77.html

@@ -1,132 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>mpif77</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="mpif77"><H1>mpif77</H1></A>
-Compiles and links MPI programs written in Fortran 77 
-<H2>Description</H2>
-This command can be used to compile and link MPI programs written in
-Fortran.  It provides the options and any special libraries that are
-needed to compile and link MPI programs.
-<P>
-It is important to use this command, particularly when linking programs,
-as it provides the necessary libraries.
-<P>
-<H2>Command line arguments</H2>
-<DL>
-<DT><B>-show      </B><DD>Show the commands that would be used without
-running them
-
-<DT><B>-help      </B><DD>Give short help
-
-<DT><B>-f77=name   </B><DD>Use compiler <TT>name</TT> instead of the default choice.  Use
-this only if the compiler is compatible with the MPICH
-library (see below)
-
-<DT><B>-config=name </B><DD>Load a configuration file for a particular compiler.
-This allows a single <TT>mpif77</TT> command to be used with 
-multiple compilers.
-
-<DT><B>-compile_info </B><DD>Show the steps for compiling a program.  This option
-can be used to see what options and include paths are
-used by mpif77.
-
-<DT><B>-link_info </B><DD>Show the steps for linking a program.  This option
-can be used to see what options and libraries are used by
-mpif77.
-
-<DT><B>-profile=name </B><DD>Use the MPI profiling given by name.  See below for
-details
-
-<DT><B>-echo      </B><DD>Show exactly what this program is doing.
-This option should normally not be used.
-
-<DT><B>-static</B><DD>mpi - Use a statically compile MPI library, but shared libraries
-for all of the other dependencies.
-
-<DT><B>others     </B><DD>are passed to the compiler or linker.  For example, <TT>\-c
-</TT>causes files to be compiled, <TT>\-g</TT> selects compilation with
-debugging on most systems, and <TT>\-o name</TT> causes linking
-with the output executable given the name <TT>name</TT>.
-</DL>
-<P>
-<H2>Environment Variables</H2>
-The environment variables <TT>MPICH_F77</TT> may be used
-to select different Fortran compiler and linker.  Note that since
-MPICH is built with a particular C and Fortran compiler, change the
-compilers used can cause problems.  Use this only if you could intermix
-code compiled with the different compilers.
-<P>
-<H2>Compatible Compilers</H2>
-The MPI library may be used with any compiler that uses the same
-lengths for basic data objects (such as <TT>long double</TT>) and that
-uses compatible run-time libraries.  On many systems, the various
-compilers are compatible and may be used interchangably.  There are
-exceptions; if you use the <TT>MPICH_F77</TT> environment variable or the
-<TT>\-f77=name</TT> command-line argument to override the choice of compiler
-and encounter problems, try reconfiguring MPICH with the new compiler
-and installing MPICH in a separate location.  See the installation manual
-for more details.
-<P>
-<H2>Examples</H2>
-To compile a single file <TT>foo.f</TT>, use
-<PRE>
-   mpif77 -c foo.f
-</PRE>
-
-<P>
-To link the output and make an executable, use
-<PRE>
-   mpif77 -o foo foo.o
-</PRE>
-
-Combining compilation and linking in a single command
-<PRE>
-   mpif77 -o foo foo.f
-</PRE>
-
-is a convenient way to build simple programs.
-<P>
-<H2>Selecting a Profiling Library</H2>
-The <TT>\-profile=name</TT> argument allows you to specify an MPI profiling
-library to be used.  <TT>name</TT> can have two forms:
-<P>
-<BR>A library in the same directory as the MPI library
-<BR>The name of a profile configuration file
-<BR>
-<P>
-If <TT>name</TT> is a library, then this library is included before the MPI
-library.  This allows the simple use of libraries that make use of the
-MPI profiling interface and that are installed in the same directory as
-the MPI library.
-<P>
-If <TT>name.conf</TT> is the name of a file in the sysconfdir directory, then this
-is read and may define the following variables:
-<DL>
-<DT><B>PROFILE_PRELIB </B><DD>Libraries (and paths) to include before the MPI library
-
-<DT><B>PROFILE_POSTLIB </B><DD>Libraries to include after the MPI library
-
-<DT><B>PROFILE_INCPATHS </B><DD>C preprocessor arguments for any include files
-For example, to add <TT>/usr/local/myprof/include</TT> to the include path and
-the library <TT>libmyprof.a</TT> in <TT>/usr/local/myprof/lib</TT> to the link step, 
-you could create the file <TT>myprof.conf</TT> with the lines
-</DL>
-<P>
-<PRE>
-    PROFILE_PRELIB="-L/usr/local/myprof/lib -lmyprof"
-    PROFILE_INCPATHS="-I/usr/local/myprof/include"
-</PRE>
-
-and place it in the sysconfdir directory (this directory is set at
-configure time when MPICH is built).  Then using the command-line
-argument <TT>\-profile=myprof</TT> will cause these
-definitions to be added to the relevant compile commands.
-<P>
-<H2>See Also</H2>
- mpicc, mpicxx, mpifort, mpiexec
-<BR>
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www1/mpifort.html

@@ -5,7 +5,7 @@
 <TITLE>mpifort</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="mpifort"><H1>mpifort</H1></A>
+<H1 id="mpifort">mpifort</H1>
 Compiles and links MPI programs written in Fortran 90 
 <H2>Description</H2>
 This command can be used to compile and link MPI programs written in
@@ -17,44 +17,44 @@ as it provides the necessary libraries.
 <P>
 <H2>Command line arguments</H2>
 <DL>
-<DT><B>-show      </B><DD>Show the commands that would be used without
+<DT><B>-show      </B> <DD> Show the commands that would be used without
 running them
 
-<DT><B>-help      </B><DD>Give short help
+<DT><B>-help      </B> <DD> Give short help
 
-<DT><B>-fc=name   </B><DD>Use compiler <TT>name</TT> instead of the default choice.  Use
+<DT><B>-fc=name   </B> <DD> Use compiler <tt>name</tt> instead of the default choice.  Use
 this only if the compiler is compatible with the MPICH
 library (see below)
 
-<DT><B>-config=name </B><DD>Load a configuration file for a particular compiler.
-This allows a single <TT>mpifort</TT> command to be used with
+<DT><B>-config=name </B> <DD> Load a configuration file for a particular compiler.
+This allows a single <tt>mpifort</tt> command to be used with
 multiple compilers.
 
-<DT><B>-compile_info </B><DD>Show the steps for compiling a program.  This option
+<DT><B>-compile_info </B> <DD> Show the steps for compiling a program.  This option
 can be used to see what options and include paths are
 used by mpifort.
 
-<DT><B>-link_info </B><DD>Show the steps for linking a program.  This option
+<DT><B>-link_info </B> <DD> Show the steps for linking a program.  This option
 can be used to see what options and libraries are used by
 mpifort.
 
-<DT><B>-profile=name </B><DD>Use the MPI profiling given by name.  See below for
+<DT><B>-profile=name </B> <DD> Use the MPI profiling given by name.  See below for
 details
 
-<DT><B>-echo      </B><DD>Show exactly what this program is doing.
+<DT><B>-echo      </B> <DD> Show exactly what this program is doing.
 This option should normally not be used.
 
-<DT><B>-static</B><DD>mpi - Use a statically compile MPI library, but shared libraries
+<DT><B>-static</B> <DD> mpi - Use a statically compile MPI library, but shared libraries
 for all of the other dependencies.
 
-<DT><B>others     </B><DD>are passed to the compiler or linker.  For example, <TT>\-c
-</TT>causes files to be compiled, <TT>\-g</TT> selects compilation with
-debugging on most systems, and <TT>\-o name</TT> causes linking
-with the output executable given the name <TT>name</TT>.
+<DT><B>others     </B> <DD> are passed to the compiler or linker.  For example, <tt>\-c
+</tt>causes files to be compiled, <tt>\-g</tt> selects compilation with
+debugging on most systems, and <tt>\-o name</tt> causes linking
+with the output executable given the name <tt>name</tt>.
 </DL>
 <P>
 <H2>Environment Variables</H2>
-The environment variables <TT>MPICH_FC</TT> may be used
+The environment variables <tt>MPICH_FC</tt> may be used
 to select different Fortran compiler and linker.  Note that since
 MPICH is built with a particular C and Fortran compiler, change the
 compilers used can cause problems.  Use this only if you could intermix
@@ -62,17 +62,17 @@ code compiled with the different compilers.
 <P>
 <H2>Compatible Compilers</H2>
 The MPI library may be used with any compiler that uses the same
-lengths for basic data objects (such as <TT>long double</TT>) and that
+lengths for basic data objects (such as <tt>long double</tt>) and that
 uses compatible run-time libraries.  On many systems, the various
-compilers are compatible and may be used interchangably.  There are
-exceptions; if you use the <TT>MPICH_FC</TT> environment variable or the
-<TT>\-fc=name</TT> command-line argument to override the choice of compiler
+compilers are compatible and may be used interchangeably.  There are
+exceptions; if you use the <tt>MPICH_FC</tt> environment variable or the
+<tt>\-fc=name</tt> command-line argument to override the choice of compiler
 and encounter problems, try reconfiguring MPICH with the new compiler
 and installing MPICH in a separate location.  See the installation manual
 for more details.
 <P>
 <H2>Examples</H2>
-To compile a single file <TT>foo.f</TT>, use
+To compile a single file <tt>foo.f</tt>, use
 <PRE>
    mpifort -c foo.f
 </PRE>
@@ -91,29 +91,29 @@ Combining compilation and linking in a single command
 is a convenient way to build simple programs.
 <P>
 <H2>Selecting a Profiling Library</H2>
-The <TT>\-profile=name</TT> argument allows you to specify an MPI profiling
-library to be used.  <TT>name</TT> can have two forms:
+The <tt>\-profile=name</tt> argument allows you to specify an MPI profiling
+library to be used.  <tt>name</tt> can have two forms:
 <P>
-<BR>A library in the same directory as the MPI library
-<BR>The name of a profile configuration file
-<BR>
+<br>A library in the same directory as the MPI library
+<br>The name of a profile configuration file
+<br>
 <P>
-If <TT>name</TT> is a library, then this library is included before the MPI
+If <tt>name</tt> is a library, then this library is included before the MPI
 library.  This allows the simple use of libraries that make use of the
 MPI profiling interface and that are installed in the same directory as
 the MPI library.
 <P>
-If <TT>name.conf</TT> is the name of a file in the sysconfdir directory, then this
+If <tt>name.conf</tt> is the name of a file in the sysconfdir directory, then this
 is read and may define the following variables:
 <DL>
-<DT><B>PROFILE_PRELIB </B><DD>Libraries (and paths) to include before the MPI library
+<DT><B>PROFILE_PRELIB </B> <DD> Libraries (and paths) to include before the MPI library
 
-<DT><B>PROFILE_POSTLIB </B><DD>Libraries to include after the MPI library
+<DT><B>PROFILE_POSTLIB </B> <DD> Libraries to include after the MPI library
 
-<DT><B>PROFILE_INCPATHS </B><DD>C preprocessor arguments for any include files
-For example, to add <TT>/usr/local/myprof/include</TT> to the include path and
-the library <TT>libmyprof.a</TT> in <TT>/usr/local/myprof/lib</TT> to the link step,
-you could create the file <TT>myprof.conf</TT> with the lines
+<DT><B>PROFILE_INCPATHS </B> <DD> C preprocessor arguments for any include files
+For example, to add <tt>/usr/local/myprof/include</tt> to the include path and
+the library <tt>libmyprof.a</tt> in <tt>/usr/local/myprof/lib</tt> to the link step,
+you could create the file <tt>myprof.conf</tt> with the lines
 </DL>
 <P>
 <PRE>
@@ -123,10 +123,10 @@ you could create the file <TT>myprof.conf</TT> with the lines
 
 and place it in the sysconfdir directory (this directory is set at
 configure time when MPICH is built).  Then using the command-line
-argument <TT>\-profile=myprof</TT> will cause these
+argument <tt>\-profile=myprof</tt> will cause these
 definitions to be added to the relevant compile commands.
 <P>
 <H2>See Also</H2>
  mpicc, mpicxx, mpifort, mpiexec
-<BR>
+<br>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIR_Type_commit.html

@@ -1,18 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>MPIR_Type_commit</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="MPIR_Type_commit"><H1>MPIR_Type_commit</H1></A>
-nput Parameters: . datatype_p - pointer to MPI datatype 
-<H2>Synopsis</H2>
-<PRE>
-int MPIR_Type_commit(MPI_Datatype * datatype_p)
-</PRE>
-<H2>Output Parameters</H2>
-<P>
-<H2>Return Value</H2>
-0 on success, -1 on failure.
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIR_Type_contiguous.html

@@ -1,27 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>MPIR_Type_contiguous</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="MPIR_Type_contiguous"><H1>MPIR_Type_contiguous</H1></A>
-create a contiguous datatype 
-<H2>Synopsis</H2>
-<PRE>
-int MPIR_Type_contiguous(int count, MPI_Datatype oldtype, MPI_Datatype * newtype)
-</PRE>
-<H2>Input Parameters</H2>
-<DL>
-<DT><B>count </B><DD>number of elements in the contiguous block
-
-<DT><B>oldtype </B><DD>type (using handle) of datatype on which vector is based
-</DL>
-<P>
-<H2>Output Parameters</H2>
-<DL><DT><B>newtype </B> <DD> handle of new contiguous datatype
-</DL>
-<P>
-<H2>Return Value</H2>
-MPI_SUCCESS on success, MPI error code on failure.
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIR_Type_dup.html

@@ -1,29 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>MPIR_Type_dup</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="MPIR_Type_dup"><H1>MPIR_Type_dup</H1></A>
-create a copy of a datatype 
-<H2>Synopsis</H2>
-<PRE>
-#undef FUNCNAME
-#define FUNCNAME MPIR_Type_dup
-#undef FCNAME
-#define FCNAME MPL_QUOTE(FUNCNAME)
-int MPIR_Type_dup(MPI_Datatype oldtype, MPI_Datatype * newtype)
-</PRE>
-<H2>Input Parameters</H2>
-<DL><DT><B>oldtype </B> <DD> handle of original datatype
-</DL>
-</DL>
-<P>
-<H2>Output Parameters</H2>
-<DL><DT><B>newtype </B> <DD> handle of newly created copy of datatype
-</DL>
-<P>
-<H2>Return Value</H2>
-0 on success, MPI error code on failure.
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIR_Type_get_contents.html

@@ -1,39 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>MPIR_Type_get_contents</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="MPIR_Type_get_contents"><H1>MPIR_Type_get_contents</H1></A>
-get content information from datatype 
-<H2>Synopsis</H2>
-<PRE>
-int MPIR_Type_get_contents(MPI_Datatype datatype,
-                           int max_integers,
-                           int max_addresses,
-                           int max_datatypes,
-                           int array_of_integers[],
-                           MPI_Aint array_of_addresses[], MPI_Datatype array_of_datatypes[])
-</PRE>
-<H2>Input Parameters</H2>
-<DL>
-<DT><B>datatype </B><DD>MPI datatype
-
-<DT><B>max_integers </B><DD>size of array_of_integers
-
-<DT><B>max_addresses </B><DD>size of array_of_addresses
-
-<DT><B>max_datatypes </B><DD>size of array_of_datatypes
-</DL>
-<P>
-<H2>Output Parameters</H2>
-<DL>
-<DT><B>array_of_integers </B><DD>integers used in creating type
-
-<DT><B>array_of_addresses </B><DD>MPI_Aints used in creating type
-
-<DT><B>array_of_datatypes </B><DD>MPI_Datatypes used in creating type
-</DL>
-<P>
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIR_Type_indexed.html

@@ -1,39 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>MPIR_Type_indexed</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="MPIR_Type_indexed"><H1>MPIR_Type_indexed</H1></A>
-create an indexed datatype 
-<H2>Synopsis</H2>
-<PRE>
-int MPIR_Type_indexed(int count,
-                      const int *blocklength_array,
-                      const void *displacement_array,
-                      int dispinbytes, MPI_Datatype oldtype, MPI_Datatype * newtype)
-</PRE>
-<H2>Input Parameters</H2>
-<DL>
-<DT><B>count </B><DD>number of blocks in type
-
-<DT><B>blocklength_array </B><DD>number of elements in each block
-
-<DT><B>displacement_array </B><DD>offsets of blocks from start of type (see next
-parameter for units)
-
-<DT><B>dispinbytes </B><DD>if nonzero, then displacements are in bytes (the
-displacement_array is an array of ints), otherwise they in terms of
-extent of oldtype (the displacement_array is an array of MPI_Aints)
-
-<DT><B>oldtype </B><DD>type (using handle) of datatype on which new type is based
-</DL>
-<P>
-<H2>Output Parameters</H2>
-<DL><DT><B>newtype </B> <DD> handle of new indexed datatype
-</DL>
-<P>
-<H2>Return Value</H2>
-0 on success, -1 on failure.
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIR_Type_struct.html

@@ -1,38 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>MPIR_Type_struct</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="MPIR_Type_struct"><H1>MPIR_Type_struct</H1></A>
-create a struct datatype 
-<H2>Synopsis</H2>
-<PRE>
-#undef FUNCNAME
-#define FUNCNAME MPIR_Type_struct
-#undef FCNAME
-#define FCNAME MPL_QUOTE(FUNCNAME)
-int MPIR_Type_struct(int count,
-                     const int *blocklength_array,
-                     const MPI_Aint * displacement_array,
-                     const MPI_Datatype * oldtype_array, MPI_Datatype * newtype)
-</PRE>
-<H2>Input Parameters</H2>
-<DL>
-<DT><B>count </B><DD>number of blocks in vector
-
-<DT><B>blocklength_array </B><DD>number of elements in each block
-
-<DT><B>displacement_array </B><DD>offsets of blocks from start of type in bytes
-
-<DT><B>oldtype_array </B><DD>types (using handle) of datatypes on which vector is based
-</DL>
-<P>
-<H2>Output Parameters</H2>
-<DL><DT><B>newtype </B> <DD> handle of new struct datatype
-</DL>
-<P>
-<H2>Return Value</H2>
-MPI_SUCCESS on success, MPI errno on failure.
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIR_Type_vector.html

@@ -1,38 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
-<HTML>
-<HEAD>
-<META NAME="GENERATOR" CONTENT="DOCTEXT">
-<TITLE>MPIR_Type_vector</TITLE>
-</HEAD>
-<BODY BGCOLOR="FFFFFF">
-<A NAME="MPIR_Type_vector"><H1>MPIR_Type_vector</H1></A>
-create a vector datatype 
-<H2>Synopsis</H2>
-<PRE>
-int MPIR_Type_vector(int count,
-                     int blocklength,
-                     MPI_Aint stride,
-                     int strideinbytes, MPI_Datatype oldtype, MPI_Datatype * newtype)
-</PRE>
-<H2>Input Parameters</H2>
-<DL>
-<DT><B>count </B><DD>number of blocks in vector
-
-<DT><B>blocklength </B><DD>number of elements in each block
-
-<DT><B>stride </B><DD>distance from beginning of one block to the next (see next
-parameter for units)
-
-<DT><B>strideinbytes </B><DD>if nonzero, then stride is in bytes, otherwise stride
-is in terms of extent of oldtype
-
-<DT><B>oldtype </B><DD>type (using handle) of datatype on which vector is based
-</DL>
-<P>
-<H2>Output Parameters</H2>
-<DL><DT><B>newtype </B> <DD> handle of new vector datatype
-</DL>
-<P>
-<H2>Return Value</H2>
-0 on success, MPI error code on failure.
-</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Allreduce_enqueue.html

@@ -0,0 +1,110 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Allreduce_enqueue</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Allreduce_enqueue">MPIX_Allreduce_enqueue</H1>
+Enqueue an allreduce operation to a GPU stream that is associated with the local stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Allreduce_enqueue(const void *sendbuf, void *recvbuf, int count,
+                           MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+</PRE>
+
+<PRE>
+int MPIX_Allreduce_enqueue_c(const void *sendbuf, void *recvbuf,
+                             MPI_Count count, MPI_Datatype datatype, MPI_Op op,
+                             MPI_Comm comm)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>sendbuf </B> <DD> starting address of send buffer (choice)
+
+<DT><B>count </B> <DD> number of elements in send buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> data type of elements of send buffer (handle)
+
+<DT><B>op </B> <DD> operation (handle)
+
+<DT><B>comm </B> <DD> communicator (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>recvbuf </B> <DD> starting address of receive buffer (choice)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OP </B> <DD> Invalid operation.  MPI operations (objects of type <tt>MPI_Op</tt>)
+must either be one of the predefined operations (e.g., <tt>MPI_SUM</tt>) or
+created with <tt>MPI_Op_create</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Allreduce_enqueue_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Allreduce_enqueue.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Async_get_state.html

@@ -0,0 +1,64 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Async_get_state</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Async_get_state">MPIX_Async_get_state</H1>
+[short description] 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+void * MPIX_Async_get_state(MPIX_Async_thing async_thing)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>async_thing </B> <DD> opaque pointer for async thing (None)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Async_spawn.html

@@ -0,0 +1,76 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Async_spawn</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Async_spawn">MPIX_Async_spawn</H1>
+[short description] 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Async_spawn(MPIX_Async_thing async_thing,
+                     MPIX_Async_poll_function *poll_fn, void *extra_state,
+                     MPIX_Stream stream)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>async_thing </B> <DD> opaque pointer for async thing (None)
+
+<DT><B>poll_fn </B> <DD> user defined poll function for progressing async things (function)
+
+<DT><B>extra_state </B> <DD> extra state for callback function (None)
+
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Async_start.html

@@ -0,0 +1,73 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Async_start</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Async_start">MPIX_Async_start</H1>
+[short description] 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Async_start(MPIX_Async_poll_function *poll_fn, void *extra_state,
+                     MPIX_Stream stream)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>poll_fn </B> <DD> user defined poll function for progressing async things (function)
+
+<DT><B>extra_state </B> <DD> extra state for callback function (None)
+
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_agree.html

@@ -5,57 +5,82 @@
 <TITLE>MPIX_Comm_agree</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="MPIX_Comm_agree"><H1>MPIX_Comm_agree</H1></A>
+<H1 id="MPIX_Comm_agree">MPIX_Comm_agree</H1>
 Performs agreement operation on comm 
 <H2>Synopsis</H2>
 <PRE>
+</PRE>
+<PRE>
 int MPIX_Comm_agree(MPI_Comm comm, int *flag)
 </PRE>
+
+<P>
 <H2>Input Parameters</H2>
-<DL><DT><B>comm </B> <DD> communicator (handle)
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
 </DL>
 <P>
 <H2>Output Parameters</H2>
-<DL><DT><B>newcomm </B> <DD> new communicator (handle)
+<DL>
+<DT><B>flag </B> <DD> new communicator (logical)
 </DL>
 <P>
+<H2>Notes</H2>
+The purpose of this collective communication is to agree on the integer value flag and on the group of failed processes in comm.
+<P>
+On completion, all non-failed MPI processes have agreed to set the output integer value of flag to the result of a bitwise AND operation over the contributed input values of flag. If comm is an intercommunicator, the value of flagis a bitwise AND operation over the values contributed by the remote group.
+<P>
+When an MPI process fails before contributing to the operation, the flag is computed ignoring its contribution, and MPIX_Comm_agree raises an error of class MPI_ERR_PROC_FAILED. However, if all MPI processes have acknowledged this failure priorto the call to MPIX_Comm_agree, using MPIX_Comm_failure_ack, the error related to this failure is not raised. When an error of class MPI_ERR_PROC_FAILED is raised, it is consistently raised at allMPI processes, in both the local and remote groups (if applicable).
+<P>
+After MPIX_Comm_agree raised an error of class MPI_ERR_PROC_FAILED, a subse-quent call to MPIX_Comm_failure_ack on comm acknowledges the failure of every MPI process that didn<tt>t contribute to the computation offlag.
+<P>
 <H2>Thread and Interrupt Safety</H2>
 <P>
 This routine is thread-safe.  This means that this routine may be
 safely used by multiple threads without the need for any user-provided
 thread locks.  However, the routine is not interrupt safe.  Typically,
-this is due to the use of memory allocation routines such as <TT>malloc
-</TT>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+this is due to the use of memory allocation routines such as </tt>malloc
+<tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
 <P>
 <H2>Notes for Fortran</H2>
-All MPI routines in Fortran (except for <TT>MPI_WTIME</TT> and <TT>MPI_WTICK</TT>) have
-an additional argument <TT>ierr</TT> at the end of the argument list.  <TT>ierr
-</TT>is an integer and has the same meaning as the return value of the routine
+All MPI routines in Fortran (except for </tt>MPI_WTIME<tt> and </tt>MPI_WTICK<tt>) have
+an additional argument </tt>ierr<tt> at the end of the argument list.  </tt>ierr
+<tt>is an integer and has the same meaning as the return value of the routine
 in C.  In Fortran, MPI routines are subroutines, and are invoked with the
-<TT>call</TT> statement.
+</tt>call<tt> statement.
 <P>
-All MPI objects (e.g., <TT>MPI_Datatype</TT>, <TT>MPI_Comm</TT>) are of type <TT>INTEGER
-</TT>in Fortran.
+All MPI objects (e.g., </tt>MPI_Datatype<tt>, </tt>MPI_Comm<tt>) are of type </tt>INTEGER
+<tt>in Fortran.
 <P>
 <H2>Errors</H2>
 <P>
-All MPI routines (except <TT>MPI_Wtime</TT> and <TT>MPI_Wtick</TT>) return an error value;
+All MPI routines (except </tt>MPI_Wtime<tt> and </tt>MPI_Wtick<tt>) return an error value;
 C routines as the value of the function and Fortran routines in the last
 argument.  Before the value is returned, the current MPI error handler is
 called.  By default, this error handler aborts the MPI job.  The error handler
-may be changed with <TT>MPI_Comm_set_errhandler</TT> (for communicators),
-<TT>MPI_File_set_errhandler</TT> (for files), and <TT>MPI_Win_set_errhandler</TT> (for
-RMA windows).  The MPI-1 routine <TT>MPI_Errhandler_set</TT> may be used but
+may be changed with </tt>MPI_Comm_set_errhandler<tt> (for communicators),
+</tt>MPI_File_set_errhandler<tt> (for files), and </tt>MPI_Win_set_errhandler<tt> (for
+RMA windows).  The MPI-1 routine </tt>MPI_Errhandler_set<tt> may be used but
 its use is deprecated.  The predefined error handler
-<TT>MPI_ERRORS_RETURN</TT> may be used to cause error values to be returned.
-Note that MPI does <EM>not</EM> guarentee that an MPI program can continue past
+</tt>MPI_ERRORS_RETURN<tt> may be used to cause error values to be returned.
+Note that MPI does </tt><em>not</em> guarantee that an MPI program can continue past
 an error; however, MPI implementations will attempt to continue whenever
 possible.
 <P>
-<DL><DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
 </DL>
-<DL><DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
-communicator in a call (not even allowed in <TT>MPI_Comm_rank</TT>).
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., MPI_ERR_RANK<tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in </tt>MPI_Comm_rank<tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use </tt>MPI_Error_string<tt> to get more information
+about this error code. 
 </DL>
 <P>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_create_errhandler_x.html

@@ -0,0 +1,80 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Comm_create_errhandler_x</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Comm_create_errhandler_x">MPIX_Comm_create_errhandler_x</H1>
+Creates an MPI-style communicator errorhandler with an extra_state 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Comm_create_errhandler_x(MPIX_Comm_errhandler_function_x *comm_errhandler_fn_x,
+                                  MPIX_Destructor_function *destructor_fn,
+                                  void *extra_state,
+                                  MPI_Errhandler *errhandler)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm_errhandler_fn_x </B> <DD> user defined error handling procedure with extra_state (function)
+
+<DT><B>destructor_fn </B> <DD> callback when op is freed (function)
+
+<DT><B>extra_state </B> <DD> extra state (None)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>errhandler </B> <DD> MPI error handler (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_failure_ack.html

@@ -5,57 +5,67 @@
 <TITLE>MPIX_Comm_failure_ack</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="MPIX_Comm_failure_ack"><H1>MPIX_Comm_failure_ack</H1></A>
+<H1 id="MPIX_Comm_failure_ack">MPIX_Comm_failure_ack</H1>
 Acknowledge the current group of failed processes 
 <H2>Synopsis</H2>
 <PRE>
+</PRE>
+<PRE>
 int MPIX_Comm_failure_ack(MPI_Comm comm)
 </PRE>
+
+<P>
 <H2>Input Parameters</H2>
-<DL><DT><B>comm </B> <DD> Communicator (handle)
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
 </DL>
 <P>
 <H2>Notes</H2>
-Because MPI specifies that null objects (e.g., <TT>MPI_COMM_NULL</TT>) are invalid
-as input to MPI routines unless otherwise specified, using <TT>MPI_COMM_NULL
-</TT>as input to this routine is an error.
+This local operation gives the users a way to acknowledge all locally notified failures on comm. After the call, unmatched MPI_ANY_SOURCE receive operations that would haveraised an error of class MPI_ERR_PROC_FAILED_PENDING due to MPI process failure proceed without further raising errors due to those acknowledged failures. Also after this call, MPIX_Comm_agree will not raise an error of class MPI_ERR_PROC_FAILED due to those acknowledged failures.
 <P>
 <H2>Thread and Interrupt Safety</H2>
 <P>
 This routine is thread-safe.  This means that this routine may be
 safely used by multiple threads without the need for any user-provided
 thread locks.  However, the routine is not interrupt safe.  Typically,
-this is due to the use of memory allocation routines such as <TT>malloc
-</TT>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
 <P>
 <H2>Notes for Fortran</H2>
-All MPI routines in Fortran (except for <TT>MPI_WTIME</TT> and <TT>MPI_WTICK</TT>) have
-an additional argument <TT>ierr</TT> at the end of the argument list.  <TT>ierr
-</TT>is an integer and has the same meaning as the return value of the routine
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
 in C.  In Fortran, MPI routines are subroutines, and are invoked with the
-<TT>call</TT> statement.
+<tt>call</tt> statement.
 <P>
-All MPI objects (e.g., <TT>MPI_Datatype</TT>, <TT>MPI_Comm</TT>) are of type <TT>INTEGER
-</TT>in Fortran.
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
 <P>
 <H2>Errors</H2>
 <P>
-All MPI routines (except <TT>MPI_Wtime</TT> and <TT>MPI_Wtick</TT>) return an error value;
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
 C routines as the value of the function and Fortran routines in the last
 argument.  Before the value is returned, the current MPI error handler is
 called.  By default, this error handler aborts the MPI job.  The error handler
-may be changed with <TT>MPI_Comm_set_errhandler</TT> (for communicators),
-<TT>MPI_File_set_errhandler</TT> (for files), and <TT>MPI_Win_set_errhandler</TT> (for
-RMA windows).  The MPI-1 routine <TT>MPI_Errhandler_set</TT> may be used but
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
 its use is deprecated.  The predefined error handler
-<TT>MPI_ERRORS_RETURN</TT> may be used to cause error values to be returned.
-Note that MPI does <EM>not</EM> guarentee that an MPI program can continue past
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
 an error; however, MPI implementations will attempt to continue whenever
 possible.
 <P>
-<DL><DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
 </DL>
-<DL><DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
-communicator in a call (not even allowed in <TT>MPI_Comm_rank</TT>).
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
 </DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_failure_get_acked.html

@@ -5,61 +5,76 @@
 <TITLE>MPIX_Comm_failure_get_acked</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="MPIX_Comm_failure_get_acked"><H1>MPIX_Comm_failure_get_acked</H1></A>
+<H1 id="MPIX_Comm_failure_get_acked">MPIX_Comm_failure_get_acked</H1>
 Get the group of acknowledged failures. 
 <H2>Synopsis</H2>
 <PRE>
-int MPIX_Comm_failure_get_acked(MPI_Comm comm, MPI_Group * failedgrp)
 </PRE>
+<PRE>
+int MPIX_Comm_failure_get_acked(MPI_Comm comm, MPI_Group *failedgrp)
+</PRE>
+
+<P>
 <H2>Input Parameters</H2>
-<DL><DT><B>comm </B> <DD> Communicator (handle)
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
 </DL>
 <P>
 <H2>Output Parameters</H2>
-<DL><DT><B>failed_group </B> <DD> Group (handle)
+<DL>
+<DT><B>failedgrp </B> <DD> group of failed processes (handle)
 </DL>
 <P>
 <H2>Notes</H2>
-Because MPI specifies that null objects (e.g., <TT>MPI_COMM_NULL</TT>) are invalid
-as input to MPI routines unless otherwise specified, using <TT>MPI_COMM_NULL
-</TT>as input to this routine is an error.
+This local operation returns the group failedgrp of processes, from the communicatorcomm, that have been locally acknowledged as failed by preceding calls to MPIX_Comm_failure_ack. The failedgrp can be empty, that is, equal to MPI_GROUP_EMPTY.
 <P>
 <H2>Thread and Interrupt Safety</H2>
 <P>
 This routine is thread-safe.  This means that this routine may be
 safely used by multiple threads without the need for any user-provided
 thread locks.  However, the routine is not interrupt safe.  Typically,
-this is due to the use of memory allocation routines such as <TT>malloc
-</TT>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
 <P>
 <H2>Notes for Fortran</H2>
-All MPI routines in Fortran (except for <TT>MPI_WTIME</TT> and <TT>MPI_WTICK</TT>) have
-an additional argument <TT>ierr</TT> at the end of the argument list.  <TT>ierr
-</TT>is an integer and has the same meaning as the return value of the routine
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
 in C.  In Fortran, MPI routines are subroutines, and are invoked with the
-<TT>call</TT> statement.
+<tt>call</tt> statement.
 <P>
-All MPI objects (e.g., <TT>MPI_Datatype</TT>, <TT>MPI_Comm</TT>) are of type <TT>INTEGER
-</TT>in Fortran.
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
 <P>
 <H2>Errors</H2>
 <P>
-All MPI routines (except <TT>MPI_Wtime</TT> and <TT>MPI_Wtick</TT>) return an error value;
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
 C routines as the value of the function and Fortran routines in the last
 argument.  Before the value is returned, the current MPI error handler is
 called.  By default, this error handler aborts the MPI job.  The error handler
-may be changed with <TT>MPI_Comm_set_errhandler</TT> (for communicators),
-<TT>MPI_File_set_errhandler</TT> (for files), and <TT>MPI_Win_set_errhandler</TT> (for
-RMA windows).  The MPI-1 routine <TT>MPI_Errhandler_set</TT> may be used but
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
 its use is deprecated.  The predefined error handler
-<TT>MPI_ERRORS_RETURN</TT> may be used to cause error values to be returned.
-Note that MPI does <EM>not</EM> guarentee that an MPI program can continue past
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
 an error; however, MPI implementations will attempt to continue whenever
 possible.
 <P>
-<DL><DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
 </DL>
-<DL><DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
-communicator in a call (not even allowed in <TT>MPI_Comm_rank</TT>).
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
 </DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_get_failed.html

@@ -0,0 +1,82 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Comm_get_failed</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Comm_get_failed">MPIX_Comm_get_failed</H1>
+This local operation returns the group of processes that are locally known to have failed. 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Comm_get_failed(MPI_Comm comm, MPI_Group *failedgrp)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>failedgrp </B> <DD> group of failed processes (handle)
+</DL>
+<P>
+<H2>Notes</H2>
+The returned failedgrp can be empty, that is, equal to MPI_GROUP_EMPTY.
+<P>
+For any two groups obtained from calls to this routine at the same MPI process with the same comm, the smaller group is a prefix of the larger group, that is, the same failed process will have the same rank in the returned failedgrp.
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_get_stream.html

@@ -0,0 +1,79 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Comm_get_stream</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Comm_get_stream">MPIX_Comm_get_stream</H1>
+Get the stream object that is attached to the communicator 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Comm_get_stream(MPI_Comm comm, int idx, MPIX_Stream *stream)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
+
+<DT><B>idx </B> <DD> idx (integer)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_revoke.html

@@ -5,46 +5,20 @@
 <TITLE>MPIX_Comm_revoke</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="MPIX_Comm_revoke"><H1>MPIX_Comm_revoke</H1></A>
+<H1 id="MPIX_Comm_revoke">MPIX_Comm_revoke</H1>
 Prevent a communicator from being used in the future 
 <H2>Synopsis</H2>
 <PRE>
+</PRE>
+<PRE>
 int MPIX_Comm_revoke(MPI_Comm comm)
 </PRE>
+
+<P>
 <H2>Input Parameters</H2>
-<DL><DT><B>comm </B> <DD> communicator to revoke
+<DL>
+<DT><B>comm </B> <DD> communicator to revoke (handle)
 </DL>
 <P>
 <H2>Notes</H2>
-Asynchronously notifies all MPI processes associated with the communicator <TT>comm</TT>.
-This will be manifest by returning the MPIX_ERR_REVOKED during a subsequent MPI
-call.
-<P>
-<H2>Notes for Fortran</H2>
-All MPI routines in Fortran (except for <TT>MPI_WTIME</TT> and <TT>MPI_WTICK</TT>) have
-an additional argument <TT>ierr</TT> at the end of the argument list.  <TT>ierr
-</TT>is an integer and has the same meaning as the return value of the routine
-in C.  In Fortran, MPI routines are subroutines, and are invoked with the
-<TT>call</TT> statement.
-<P>
-All MPI objects (e.g., <TT>MPI_Datatype</TT>, <TT>MPI_Comm</TT>) are of type <TT>INTEGER
-</TT>in Fortran.
-<P>
-<H2>Errors</H2>
-<P>
-All MPI routines (except <TT>MPI_Wtime</TT> and <TT>MPI_Wtick</TT>) return an error value;
-C routines as the value of the function and Fortran routines in the last
-argument.  Before the value is returned, the current MPI error handler is
-called.  By default, this error handler aborts the MPI job.  The error handler
-may be changed with <TT>MPI_Comm_set_errhandler</TT> (for communicators),
-<TT>MPI_File_set_errhandler</TT> (for files), and <TT>MPI_Win_set_errhandler</TT> (for
-RMA windows).  The MPI-1 routine <TT>MPI_Errhandler_set</TT> may be used but
-its use is deprecated.  The predefined error handler
-<TT>MPI_ERRORS_RETURN</TT> may be used to cause error values to be returned.
-Note that MPI does <EM>not</EM> guarentee that an MPI program can continue past
-an error; however, MPI implementations will attempt to continue whenever
-possible.
-<P>
-<DL><DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
-</DL>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_shrink.html

@@ -5,57 +5,25 @@
 <TITLE>MPIX_Comm_shrink</TITLE>
 </HEAD>
 <BODY BGCOLOR="FFFFFF">
-<A NAME="MPIX_Comm_shrink"><H1>MPIX_Comm_shrink</H1></A>
+<H1 id="MPIX_Comm_shrink">MPIX_Comm_shrink</H1>
 Creates a new communitor from an existing communicator while excluding failed processes 
 <H2>Synopsis</H2>
 <PRE>
-int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm * newcomm)
 </PRE>
+<PRE>
+int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm)
+</PRE>
+
+<P>
 <H2>Input Parameters</H2>
-<DL><DT><B>comm </B> <DD> communicator (handle)
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
 </DL>
 <P>
 <H2>Output Parameters</H2>
-<DL><DT><B>newcomm </B> <DD> new communicator (handle)
-</DL>
-<P>
-<H2>Thread and Interrupt Safety</H2>
-<P>
-This routine is thread-safe.  This means that this routine may be
-safely used by multiple threads without the need for any user-provided
-thread locks.  However, the routine is not interrupt safe.  Typically,
-this is due to the use of memory allocation routines such as <TT>malloc
-</TT>or other non-MPICH runtime routines that are themselves not interrupt-safe.
-<P>
-<H2>Notes for Fortran</H2>
-All MPI routines in Fortran (except for <TT>MPI_WTIME</TT> and <TT>MPI_WTICK</TT>) have
-an additional argument <TT>ierr</TT> at the end of the argument list.  <TT>ierr
-</TT>is an integer and has the same meaning as the return value of the routine
-in C.  In Fortran, MPI routines are subroutines, and are invoked with the
-<TT>call</TT> statement.
-<P>
-All MPI objects (e.g., <TT>MPI_Datatype</TT>, <TT>MPI_Comm</TT>) are of type <TT>INTEGER
-</TT>in Fortran.
-<P>
-<H2>Errors</H2>
-<P>
-All MPI routines (except <TT>MPI_Wtime</TT> and <TT>MPI_Wtick</TT>) return an error value;
-C routines as the value of the function and Fortran routines in the last
-argument.  Before the value is returned, the current MPI error handler is
-called.  By default, this error handler aborts the MPI job.  The error handler
-may be changed with <TT>MPI_Comm_set_errhandler</TT> (for communicators),
-<TT>MPI_File_set_errhandler</TT> (for files), and <TT>MPI_Win_set_errhandler</TT> (for
-RMA windows).  The MPI-1 routine <TT>MPI_Errhandler_set</TT> may be used but
-its use is deprecated.  The predefined error handler
-<TT>MPI_ERRORS_RETURN</TT> may be used to cause error values to be returned.
-Note that MPI does <EM>not</EM> guarentee that an MPI program can continue past
-an error; however, MPI implementations will attempt to continue whenever
-possible.
-<P>
-<DL><DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
-</DL>
-<DL><DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
-communicator in a call (not even allowed in <TT>MPI_Comm_rank</TT>).
+<DL>
+<DT><B>newcomm </B> <DD> new communicator (handle)
 </DL>
 <P>
+<H2>Notes</H2>
 </BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Comm_test_threadcomm.html

@@ -0,0 +1,77 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Comm_test_threadcomm</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Comm_test_threadcomm">MPIX_Comm_test_threadcomm</H1>
+Tests to see if a comm is a threadcomm 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Comm_test_threadcomm(MPI_Comm comm, int *flag)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>flag </B> <DD> true if comm is an inter-communicator (logical)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_File_create_errhandler_x.html

@@ -0,0 +1,80 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_File_create_errhandler_x</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_File_create_errhandler_x">MPIX_File_create_errhandler_x</H1>
+Creates an MPI-style file errorhandler with an extra_state 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_File_create_errhandler_x(MPIX_File_errhandler_function_x *comm_errhandler_fn_x,
+                                  MPIX_Destructor_function *destructor_fn,
+                                  void *extra_state,
+                                  MPI_Errhandler *errhandler)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm_errhandler_fn_x </B> <DD> user defined error handling procedure with extra_state (function)
+
+<DT><B>destructor_fn </B> <DD> callback when op is freed (function)
+
+<DT><B>extra_state </B> <DD> extra state (None)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>errhandler </B> <DD> MPI error handler (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_GPU_query_support.html

@@ -0,0 +1,73 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_GPU_query_support</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_GPU_query_support">MPIX_GPU_query_support</H1>
+Returns whether the specific type of GPU is supported 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_GPU_query_support(int gpu_type, int *is_supported)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>gpu_type </B> <DD> MPIX_GPU_SUPPORT_CUDA, MPIX_GPU_SUPPORT_ZE, or MPIX_GPU_SUPPORT_HIP (integer)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>is_supported </B> <DD> true if gpu of given type is supported, false otherwise. (logical)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Grequest_class_allocate.html

@@ -0,0 +1,76 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Grequest_class_allocate</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Grequest_class_allocate">MPIX_Grequest_class_allocate</H1>
+Create and return a user-defined extended request based on a generalized request class 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Grequest_class_allocate(MPIX_Grequest_class greq_class,
+                                 void *extra_state, MPI_Request *request)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>greq_class </B> <DD> generalized request class (None)
+
+<DT><B>extra_state </B> <DD> extra state (None)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>request </B> <DD> generalized request (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Grequest_class_create.html

@@ -0,0 +1,86 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Grequest_class_create</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Grequest_class_create">MPIX_Grequest_class_create</H1>
+[short description] 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Grequest_class_create(MPI_Grequest_query_function *query_fn,
+                               MPI_Grequest_free_function *free_fn,
+                               MPI_Grequest_cancel_function *cancel_fn,
+                               MPIX_Grequest_poll_function *poll_fn,
+                               MPIX_Grequest_wait_function *wait_fn,
+                               MPIX_Grequest_class *greq_class)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>query_fn </B> <DD> callback function invoked when request status is queried (function)
+
+<DT><B>free_fn </B> <DD> callback function invoked when request is freed (function)
+
+<DT><B>cancel_fn </B> <DD> callback function invoked when request is cancelled (function)
+
+<DT><B>poll_fn </B> <DD> callback function invoked when request completion is tested (function)
+
+<DT><B>wait_fn </B> <DD> callback function invoked when request is waited on (function)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>greq_class </B> <DD> generalized request class (None)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Grequest_start.html

@@ -0,0 +1,88 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Grequest_start</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Grequest_start">MPIX_Grequest_start</H1>
+Create and return a user-defined extended request 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Grequest_start(MPI_Grequest_query_function *query_fn,
+                        MPI_Grequest_free_function *free_fn,
+                        MPI_Grequest_cancel_function *cancel_fn,
+                        MPIX_Grequest_poll_function *poll_fn,
+                        MPIX_Grequest_wait_function *wait_fn, void *extra_state,
+                        MPI_Request *request)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>query_fn </B> <DD> callback function invoked when request status is queried (function)
+
+<DT><B>free_fn </B> <DD> callback function invoked when request is freed (function)
+
+<DT><B>cancel_fn </B> <DD> callback function invoked when request is cancelled (function)
+
+<DT><B>poll_fn </B> <DD> callback function invoked when request completion is tested (function)
+
+<DT><B>wait_fn </B> <DD> callback function invoked when request is waited on (function)
+
+<DT><B>extra_state </B> <DD> extra state (None)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>request </B> <DD> generalized request (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Info_set_hex.html

@@ -0,0 +1,78 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Info_set_hex</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Info_set_hex">MPIX_Info_set_hex</H1>
+[short description] 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Info_set_hex(MPI_Info info, const char *key, const void *value,
+                      int value_size)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>info </B> <DD> info object (handle)
+
+<DT><B>key </B> <DD> key (string)
+
+<DT><B>value </B> <DD> value (choice)
+
+<DT><B>value_size </B> <DD> size of value (integer)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_INFO </B> <DD> Invalid Info 
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Irecv_enqueue.html

@@ -0,0 +1,118 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Irecv_enqueue</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Irecv_enqueue">MPIX_Irecv_enqueue</H1>
+Enqueue a nonblocking receive operation to a GPU stream that is associated with the local stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Irecv_enqueue(void *buf, int count, MPI_Datatype datatype, int source,
+                       int tag, MPI_Comm comm, MPI_Request *request)
+</PRE>
+
+<PRE>
+int MPIX_Irecv_enqueue_c(void *buf, MPI_Count count, MPI_Datatype datatype,
+                         int source, int tag, MPI_Comm comm,
+                         MPI_Request *request)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>count </B> <DD> number of elements in receive buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each receive buffer element (handle)
+
+<DT><B>source </B> <DD> rank of source or MPI_ANY_SOURCE (integer)
+
+<DT><B>tag </B> <DD> message tag or MPI_ANY_TAG (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of receive buffer (choice)
+
+<DT><B>request </B> <DD> communication request (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Irecv_enqueue_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Irecv_enqueue.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Isend_enqueue.html

@@ -0,0 +1,118 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Isend_enqueue</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Isend_enqueue">MPIX_Isend_enqueue</H1>
+Enqueue a nonblocking send operation to a GPU stream that is associated with the local stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Isend_enqueue(const void *buf, int count, MPI_Datatype datatype,
+                       int dest, int tag, MPI_Comm comm, MPI_Request *request)
+</PRE>
+
+<PRE>
+int MPIX_Isend_enqueue_c(const void *buf, MPI_Count count,
+                         MPI_Datatype datatype, int dest, int tag,
+                         MPI_Comm comm, MPI_Request *request)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of send buffer (choice)
+
+<DT><B>count </B> <DD> number of elements in send buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each send buffer element (handle)
+
+<DT><B>dest </B> <DD> rank of destination (integer)
+
+<DT><B>tag </B> <DD> message tag (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>request </B> <DD> communication request (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Isend_enqueue_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Isend_enqueue.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Op_create_x.html

@@ -0,0 +1,94 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Op_create_x</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Op_create_x">MPIX_Op_create_x</H1>
+Creates a user-defined reduction op with an extra_state 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Op_create_x(MPIX_User_function_x *user_fn_x,
+                     MPIX_Destructor_function *destructor_fn, int commute,
+                     void *extra_state, MPI_Op *op)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>user_fn_x </B> <DD> user defined function with extra_state (function)
+
+<DT><B>destructor_fn </B> <DD> callback when op is freed (function)
+
+<DT><B>commute </B> <DD> true if commutative; false otherwise. (logical)
+
+<DT><B>extra_state </B> <DD> extra state (None)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>op </B> <DD> operation (handle)
+</DL>
+<P>
+<H2>Notes on MPIX_User_function_x</H2>
+The calling list for the user function type is
+<PRE>
+    typedef void (MPIX_User_function_x) (void * a, void * b,
+                                         MPI_Count len, MPI_Datatype datatype,
+                                         void *extra_state);
+</PRE>
+
+There are a few differences from MPI_User_function used in MPI_Op_create.
+1. It passes len and datatype as scalar input, rather than as addresses.
+2. The len is of MPI_Count, rather than int.
+3. It accepts a void *extra_state which is passed from users during MPIX_Op_create_x.
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Query_cuda_support.html

@@ -0,0 +1,36 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Query_cuda_support</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Query_cuda_support">MPIX_Query_cuda_support</H1>
+Returns whether CUDA is supported 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Query_cuda_support(void)
+</PRE>
+
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Query_hip_support.html

@@ -0,0 +1,36 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Query_hip_support</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Query_hip_support">MPIX_Query_hip_support</H1>
+Returns whether HIP (AMD GPU) is supported 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Query_hip_support(void)
+</PRE>
+
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Query_ze_support.html

@@ -0,0 +1,36 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Query_ze_support</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Query_ze_support">MPIX_Query_ze_support</H1>
+Returns whether ZE (Intel GPU) is supported 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Query_ze_support(void)
+</PRE>
+
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Recv_enqueue.html

@@ -0,0 +1,117 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Recv_enqueue</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Recv_enqueue">MPIX_Recv_enqueue</H1>
+Enqueue a receive operation to a GPU stream that is associated with the local stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Recv_enqueue(void *buf, int count, MPI_Datatype datatype, int source,
+                      int tag, MPI_Comm comm, MPI_Status *status)
+</PRE>
+
+<PRE>
+int MPIX_Recv_enqueue_c(void *buf, MPI_Count count, MPI_Datatype datatype,
+                        int source, int tag, MPI_Comm comm, MPI_Status *status)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>count </B> <DD> number of elements in receive buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each receive buffer element (handle)
+
+<DT><B>source </B> <DD> rank of source or MPI_ANY_SOURCE (integer)
+
+<DT><B>tag </B> <DD> message tag or MPI_ANY_TAG (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of receive buffer (choice)
+
+<DT><B>status </B> <DD> status object (Status)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Recv_enqueue_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Recv_enqueue.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Request_is_complete.html

@@ -0,0 +1,64 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Request_is_complete</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Request_is_complete">MPIX_Request_is_complete</H1>
+[short description] 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Request_is_complete(MPI_Request request)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>request </B> <DD> communication request (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Send_enqueue.html

@@ -0,0 +1,108 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Send_enqueue</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Send_enqueue">MPIX_Send_enqueue</H1>
+Enqueue a send operation to a GPU stream that is associated with the local stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Send_enqueue(const void *buf, int count, MPI_Datatype datatype,
+                      int dest, int tag, MPI_Comm comm)
+</PRE>
+
+<PRE>
+int MPIX_Send_enqueue_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                        int dest, int tag, MPI_Comm comm)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of send buffer (choice)
+
+<DT><B>count </B> <DD> number of elements in send buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each send buffer element (handle)
+
+<DT><B>dest </B> <DD> rank of destination (integer)
+
+<DT><B>tag </B> <DD> message tag (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Send_enqueue_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Send_enqueue.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Session_create_errhandler_x.html

@@ -0,0 +1,80 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Session_create_errhandler_x</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Session_create_errhandler_x">MPIX_Session_create_errhandler_x</H1>
+Creates an MPI-style session errorhandler with an extra_state 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Session_create_errhandler_x(MPIX_Session_errhandler_function_x *comm_errhandler_fn_x,
+                                     MPIX_Destructor_function *destructor_fn,
+                                     void *extra_state,
+                                     MPI_Errhandler *errhandler)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm_errhandler_fn_x </B> <DD> user defined error handling procedure with extra_state (function)
+
+<DT><B>destructor_fn </B> <DD> callback when op is freed (function)
+
+<DT><B>extra_state </B> <DD> extra state (None)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>errhandler </B> <DD> MPI error handler (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Start_progress_thread.html

@@ -0,0 +1,64 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Start_progress_thread</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Start_progress_thread">MPIX_Start_progress_thread</H1>
+Start a progress thread that will poll progress on the given stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Start_progress_thread(MPIX_Stream stream)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stop_progress_thread.html

@@ -0,0 +1,64 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stop_progress_thread</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stop_progress_thread">MPIX_Stop_progress_thread</H1>
+Stop the progress thread that polls progress on the given stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stop_progress_thread(MPIX_Stream stream)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_comm_create.html

@@ -0,0 +1,80 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_comm_create</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_comm_create">MPIX_Stream_comm_create</H1>
+Create a new communicator with local stream attached 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_comm_create(MPI_Comm comm, MPIX_Stream stream,
+                            MPI_Comm *newcomm)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
+
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>newcomm </B> <DD> new stream-associated communicator (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_comm_create_multiplex.html

@@ -0,0 +1,87 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_comm_create_multiplex</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_comm_create_multiplex">MPIX_Stream_comm_create_multiplex</H1>
+Create a new communicator with multiple local streams attached 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_comm_create_multiplex(MPI_Comm comm, int count,
+                                      MPIX_Stream array_of_streams[],
+                                      MPI_Comm *newcomm)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>comm </B> <DD> communicator (handle)
+
+<DT><B>count </B> <DD> list length (non-negative integer)
+
+<DT><B>array_of_streams </B> <DD> stream object array (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>newcomm </B> <DD> new stream-associated communicator (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_create.html

@@ -0,0 +1,76 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_create</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_create">MPIX_Stream_create</H1>
+Create a new stream object 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_create(MPI_Info info, MPIX_Stream *stream)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>info </B> <DD> info argument (handle)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>stream </B> <DD> stream object created (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_INFO </B> <DD> Invalid Info 
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_free.html

@@ -0,0 +1,64 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_free</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_free">MPIX_Stream_free</H1>
+Free a stream object 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_free(MPIX_Stream *stream)
+</PRE>
+
+<P>
+<H2>Input/Output Parameters</H2>
+<DL>
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_irecv.html

@@ -0,0 +1,124 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_irecv</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_irecv">MPIX_Stream_irecv</H1>
+Start a nonblocking receive from a specific source stream to a specific local stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_irecv(void *buf, int count, MPI_Datatype datatype, int source,
+                      int tag, MPI_Comm comm, int source_stream_index,
+                      int dest_stream_index, MPI_Request *request)
+</PRE>
+
+<PRE>
+int MPIX_Stream_irecv_c(void *buf, MPI_Count count, MPI_Datatype datatype,
+                        int source, int tag, MPI_Comm comm,
+                        int source_stream_index, int dest_stream_index,
+                        MPI_Request *request)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>count </B> <DD> number of elements in receive buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each receive buffer element (handle)
+
+<DT><B>source </B> <DD> rank of source or MPI_ANY_SOURCE (integer)
+
+<DT><B>tag </B> <DD> message tag or MPI_ANY_TAG (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+
+<DT><B>source_stream_index </B> <DD> source_stream_index (integer)
+
+<DT><B>dest_stream_index </B> <DD> dest_stream_index (integer)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of receive buffer (choice)
+
+<DT><B>request </B> <DD> communication request (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_irecv_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Stream_irecv.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_isend.html

@@ -0,0 +1,124 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_isend</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_isend">MPIX_Stream_isend</H1>
+Start a nonblocking send from a specific local stream to a specific remote stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_isend(const void *buf, int count, MPI_Datatype datatype,
+                      int dest, int tag, MPI_Comm comm, int source_stream_index,
+                      int dest_stream_index, MPI_Request *request)
+</PRE>
+
+<PRE>
+int MPIX_Stream_isend_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                        int dest, int tag, MPI_Comm comm,
+                        int source_stream_index, int dest_stream_index,
+                        MPI_Request *request)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of send buffer (choice)
+
+<DT><B>count </B> <DD> number of elements in send buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each send buffer element (handle)
+
+<DT><B>dest </B> <DD> rank of destination (integer)
+
+<DT><B>tag </B> <DD> message tag (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+
+<DT><B>source_stream_index </B> <DD> source_stream_index (integer)
+
+<DT><B>dest_stream_index </B> <DD> dest_stream_index (integer)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>request </B> <DD> communication request (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_isend_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Stream_isend.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_progress.html

@@ -0,0 +1,64 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_progress</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_progress">MPIX_Stream_progress</H1>
+Invoke progress on the given stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_progress(MPIX_Stream stream)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>stream </B> <DD> stream object (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_recv.html

@@ -0,0 +1,124 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_recv</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_recv">MPIX_Stream_recv</H1>
+Receive a message from a specific source stream to a specific local stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_recv(void *buf, int count, MPI_Datatype datatype, int source,
+                     int tag, MPI_Comm comm, int source_stream_index,
+                     int dest_stream_index, MPI_Status *status)
+</PRE>
+
+<PRE>
+int MPIX_Stream_recv_c(void *buf, MPI_Count count, MPI_Datatype datatype,
+                       int source, int tag, MPI_Comm comm,
+                       int source_stream_index, int dest_stream_index,
+                       MPI_Status *status)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>count </B> <DD> number of elements in receive buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each receive buffer element (handle)
+
+<DT><B>source </B> <DD> rank of source or MPI_ANY_SOURCE (integer)
+
+<DT><B>tag </B> <DD> message tag or MPI_ANY_TAG (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+
+<DT><B>source_stream_index </B> <DD> source_stream_index (integer)
+
+<DT><B>dest_stream_index </B> <DD> dest_stream_index (integer)
+</DL>
+<P>
+<H2>Output Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of receive buffer (choice)
+
+<DT><B>status </B> <DD> status object (Status)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_recv_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Stream_recv.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_send.html

@@ -0,0 +1,118 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Stream_send</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Stream_send">MPIX_Stream_send</H1>
+Send message from a specific local stream to a specific destination stream 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Stream_send(const void *buf, int count, MPI_Datatype datatype,
+                     int dest, int tag, MPI_Comm comm, int source_stream_index,
+                     int dest_stream_index)
+</PRE>
+
+<PRE>
+int MPIX_Stream_send_c(const void *buf, MPI_Count count, MPI_Datatype datatype,
+                       int dest, int tag, MPI_Comm comm,
+                       int source_stream_index, int dest_stream_index)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>buf </B> <DD> initial address of send buffer (choice)
+
+<DT><B>count </B> <DD> number of elements in send buffer (non-negative integer)
+
+<DT><B>datatype </B> <DD> datatype of each send buffer element (handle)
+
+<DT><B>dest </B> <DD> rank of destination (integer)
+
+<DT><B>tag </B> <DD> message tag (integer)
+
+<DT><B>comm </B> <DD> communicator (handle)
+
+<DT><B>source_stream_index </B> <DD> source_stream_index (integer)
+
+<DT><B>dest_stream_index </B> <DD> dest_stream_index (integer)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_ARG </B> <DD> Invalid argument.  Some argument is invalid and is not
+identified by a specific error class (e.g., <tt>MPI_ERR_RANK</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_BUFFER </B> <DD> Invalid buffer pointer.  Usually a null buffer where
+one is not valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_COUNT </B> <DD> Invalid count argument.  Count arguments must be 
+non-negative; a count of zero is often valid.
+</DL>
+<DL>
+<DT><B>MPI_ERR_RANK </B> <DD> Invalid source or destination rank.  Ranks must be between
+zero and the size of the communicator minus one; ranks in a receive
+(<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may also be <tt>MPI_ANY_SOURCE</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TAG </B> <DD> Invalid tag argument.  Tags must be non-negative; tags
+in a receive (<tt>MPI_Recv</tt>, <tt>MPI_Irecv</tt>, <tt>MPI_Sendrecv</tt>, etc.) may
+also be <tt>MPI_ANY_TAG</tt>.  The largest tag value is available through the 
+the attribute <tt>MPI_TAG_UB</tt>.
+</DL>
+<DL>
+<DT><B>MPI_ERR_TYPE </B> <DD> Invalid datatype argument.  Additionally, this error can
+occur if an uncommitted MPI_Datatype (see <tt>MPI_Type_commit</tt>) is used
+in a communication call.
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+</BODY></HTML>

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Stream_send_c.html

@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=MPIX_Stream_send.html">

linx64/mpi/mpich/share/doc/mpich/www3/MPIX_Threadcomm_finish.html

@@ -0,0 +1,71 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML3.2 EN">
+<HTML>
+<HEAD>
+<META NAME="GENERATOR" CONTENT="DOCTEXT">
+<TITLE>MPIX_Threadcomm_finish</TITLE>
+</HEAD>
+<BODY BGCOLOR="FFFFFF">
+<H1 id="MPIX_Threadcomm_finish">MPIX_Threadcomm_finish</H1>
+finish/deactiviate a thread communicator (inside thread parallel regions) 
+<H2>Synopsis</H2>
+<PRE>
+</PRE>
+<PRE>
+int MPIX_Threadcomm_finish(MPI_Comm threadcomm)
+</PRE>
+
+<P>
+<H2>Input Parameters</H2>
+<DL>
+<DT><B>threadcomm </B> <DD> thread communicator (handle)
+</DL>
+<P>
+<H2>Thread and Interrupt Safety</H2>
+<P>
+This routine is thread-safe.  This means that this routine may be
+safely used by multiple threads without the need for any user-provided
+thread locks.  However, the routine is not interrupt safe.  Typically,
+this is due to the use of memory allocation routines such as <tt>malloc
+</tt>or other non-MPICH runtime routines that are themselves not interrupt-safe.
+<P>
+<H2>Notes for Fortran</H2>
+All MPI routines in Fortran (except for <tt>MPI_WTIME</tt> and <tt>MPI_WTICK</tt>) have
+an additional argument <tt>ierr</tt> at the end of the argument list.  <tt>ierr
+</tt>is an integer and has the same meaning as the return value of the routine
+in C.  In Fortran, MPI routines are subroutines, and are invoked with the
+<tt>call</tt> statement.
+<P>
+All MPI objects (e.g., <tt>MPI_Datatype</tt>, <tt>MPI_Comm</tt>) are of type <tt>INTEGER
+</tt>in Fortran.
+<P>
+<H2>Errors</H2>
+<P>
+All MPI routines (except <tt>MPI_Wtime</tt> and <tt>MPI_Wtick</tt>) return an error value;
+C routines as the value of the function and Fortran routines in the last
+argument.  Before the value is returned, the current MPI error handler is
+called.  By default, this error handler aborts the MPI job.  The error handler
+may be changed with <tt>MPI_Comm_set_errhandler</tt> (for communicators),
+<tt>MPI_File_set_errhandler</tt> (for files), and <tt>MPI_Win_set_errhandler</tt> (for
+RMA windows).  The MPI-1 routine <tt>MPI_Errhandler_set</tt> may be used but
+its use is deprecated.  The predefined error handler
+<tt>MPI_ERRORS_RETURN</tt> may be used to cause error values to be returned.
+Note that MPI does <em>not</em> guarantee that an MPI program can continue past
+an error; however, MPI implementations will attempt to continue whenever
+possible.
+<P>
+<DL>
+<DT><B>MPI_SUCCESS </B> <DD> No error; MPI routine completed successfully.
+</DL>
+<DL>
+<DT><B>MPI_ERR_COMM </B> <DD> Invalid communicator.  A common error is to use a null
+communicator in a call (not even allowed in <tt>MPI_Comm_rank</tt>).
+</DL>
+<DL>
+<DT><B>MPI_ERR_OTHER </B> <DD> Other error; use <tt>MPI_Error_string</tt> to get more information
+about this error code. 
+</DL>
+<P>
+<H2>See Also</H2>
+ MPIX_Threadcomm_init, MPIX_Threadcomm_start, MPIX_Threadcomm_free
+<br>
+</BODY></HTML>