Upgrade MPIs on linux
This commit is contained in:
163
linx64/mpi/openmpi/share/doc/hwloc/dynamic_SVG_example.html
Normal file
163
linx64/mpi/openmpi/share/doc/hwloc/dynamic_SVG_example.html
Normal file
@@ -0,0 +1,163 @@
|
||||
<!-- Copyright © 2018-2019 Inria. All rights reserved. -->
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Dynamic SVG</title>
|
||||
</head>
|
||||
<body>
|
||||
<p>
|
||||
This page is an example of HTML/JS interacting with the lstopo SVG output.
|
||||
</p>
|
||||
<p>
|
||||
Load a SVG that was exported with lstopo's <b>native</b> SVG backend
|
||||
(e.g. with <tt>lstopo --of nativesvg foo.svg</tt>):
|
||||
</p>
|
||||
<input type="file" name="myFile" accept=".svg" onchange="openFile(event)">
|
||||
<br>
|
||||
<br>
|
||||
<object id="img" type="image/svg+xml">
|
||||
</object>
|
||||
<br>
|
||||
<p id='caption' style='display:none'>
|
||||
Click on a box to change its background color
|
||||
and replace its first text line (if any) with its SVG object id.
|
||||
</p>
|
||||
<p id='drag' style='display:none'>
|
||||
<input type='checkbox' id='draggable' onclick='enableDrag()'>Make boxes draggable</input>
|
||||
</p>
|
||||
</body>
|
||||
<footer>
|
||||
<script type="text/javascript">
|
||||
function openFile(event) {
|
||||
let image = document.getElementById('img')
|
||||
image.data = null
|
||||
var input = event.target;
|
||||
var TextReader = new FileReader()
|
||||
TextReader.onload = async function(){
|
||||
var parser = new DOMParser()
|
||||
let blob = new Blob([TextReader.result], {type: 'image/svg+xml'})
|
||||
let url = URL.createObjectURL(blob)
|
||||
image.data = await url
|
||||
setTimeout(function(){
|
||||
document.getElementById('caption').style = ''
|
||||
document.getElementById('drag').style = ''
|
||||
document.getElementById("draggable").checked = false
|
||||
|
||||
svgObject = document.getElementById('img').contentDocument
|
||||
let svgElement = svgObject.getElementById('Machine_0_rect')
|
||||
|
||||
if(svgObject.getElementById('Machine_0_rect')){
|
||||
start(svgObject)
|
||||
}else{
|
||||
const h1 = document.createElement('h1')
|
||||
h1.innerHTML = "Your svg file doesn't have the good format to load javascript"
|
||||
document.body.appendChild(h1)
|
||||
}
|
||||
},200)
|
||||
}
|
||||
TextReader.readAsText(input.files[0], "UTF-8")
|
||||
}
|
||||
|
||||
function enableDrag(){
|
||||
let svgObject = document.getElementById('img').contentDocument
|
||||
const elements = svgObject.getElementsByTagName('rect')
|
||||
for (let element of elements) {
|
||||
element.classList.toggle("draggable")
|
||||
}
|
||||
}
|
||||
|
||||
function start(svgObject){
|
||||
makeDraggable(svgObject)
|
||||
const texts = svgObject.getElementsByTagName('text')
|
||||
for (let text of texts) {
|
||||
text.setAttribute("style", text.getAttribute("style") + ";pointer-events:none;")
|
||||
}
|
||||
|
||||
const elements = svgObject.getElementsByTagName('rect')
|
||||
for (let element of elements) {
|
||||
if(!element.id.includes("Bridge")){
|
||||
element.addEventListener('click', function(e) {
|
||||
changeColor(svgObject, e.target)
|
||||
changeText(svgObject, e.target)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function changeColor(svgObject, element){
|
||||
oldColor = element.getAttribute("saveColor")
|
||||
|
||||
if(!oldColor){
|
||||
element.setAttribute("saveColor", element.getAttribute("fill"))
|
||||
element.setAttribute("fill", "red")
|
||||
}else{
|
||||
element.setAttribute("fill", oldColor)
|
||||
element.removeAttribute("saveColor")
|
||||
}
|
||||
}
|
||||
|
||||
function changeText(svgObject, element){
|
||||
if(element.id == "")
|
||||
return
|
||||
const text = svgObject.getElementById(element.id.replace('rect', 'text'))
|
||||
|
||||
let textContent = element.getAttribute("saveText")
|
||||
|
||||
if(!textContent){
|
||||
element.setAttribute("saveText", text.innerHTML)
|
||||
textContent = element.id
|
||||
}else{
|
||||
element.removeAttribute("saveText")
|
||||
}
|
||||
|
||||
let svg = svgObject.documentElement;
|
||||
let svgNS = svg.namespaceURI;
|
||||
let newText = text.cloneNode(false)
|
||||
|
||||
newText.appendChild(svgObject.createTextNode(textContent))
|
||||
svg.removeChild(text)
|
||||
svg.appendChild(newText)
|
||||
}
|
||||
|
||||
function makeDraggable(svgObject) {
|
||||
let selectedElement = null
|
||||
let text = null
|
||||
let marginTextX = null
|
||||
let marginTextY = null
|
||||
svgObject.addEventListener('mousedown', startDrag)
|
||||
svgObject.addEventListener('mousemove', drag)
|
||||
svgObject.addEventListener('mouseup', endDrag)
|
||||
svgObject.addEventListener('mouseleave', endDrag)
|
||||
|
||||
function startDrag(evt) {
|
||||
if (evt.target.classList.contains('draggable')) {
|
||||
selectedElement = evt.target;
|
||||
text = getText(svgObject, selectedElement.id)
|
||||
marginTextX = text.getAttribute("x") - selectedElement.getAttribute("x")
|
||||
marginTextY = text.getAttribute("y") - selectedElement.getAttribute("y")
|
||||
}
|
||||
}
|
||||
|
||||
function drag(evt) {
|
||||
if (selectedElement) {
|
||||
evt.preventDefault();
|
||||
let textDragX = evt.clientX + marginTextX
|
||||
let textDragY = evt.clientY + marginTextY
|
||||
let dragX = evt.clientX;
|
||||
let dragY = evt.clientY;
|
||||
|
||||
selectedElement.setAttribute("x", dragX);
|
||||
selectedElement.setAttribute("y", dragY);
|
||||
text.setAttribute("x", textDragX);
|
||||
text.setAttribute("y", textDragY);
|
||||
}
|
||||
}
|
||||
|
||||
function endDrag(evt) {
|
||||
selectedElement = null
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</footer>
|
||||
</html>
|
||||
BIN
linx64/mpi/openmpi/share/doc/hwloc/hwloc-a4.pdf
Normal file
BIN
linx64/mpi/openmpi/share/doc/hwloc/hwloc-a4.pdf
Normal file
Binary file not shown.
BIN
linx64/mpi/openmpi/share/doc/hwloc/hwloc-letter.pdf
Normal file
BIN
linx64/mpi/openmpi/share/doc/hwloc/hwloc-letter.pdf
Normal file
Binary file not shown.
4
linx64/mpi/openmpi/share/doc/openmpi/html/.buildinfo
Normal file
4
linx64/mpi/openmpi/share/doc/openmpi/html/.buildinfo
Normal file
@@ -0,0 +1,4 @@
|
||||
# Sphinx build info version 1
|
||||
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
|
||||
config: d6494f6822afd14ab07fe36f09f326c0
|
||||
tags: 645f666f9bcd5a90fca523b33c5a78b7
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
@@ -0,0 +1,61 @@
|
||||
Open MPI Runtime Debugging Options
|
||||
==================================
|
||||
|
||||
Open MPI has a series of MCA parameters for the MPI layer
|
||||
itself that are designed to help with debugging.
|
||||
These parameters :ref:`can be set <label-running-setting-mca-param-values>`
|
||||
in the usual ways. MPI-level MCA parameters can be
|
||||
displayed by invoking the following command:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Use "--level 9" to see all the MCA parameters
|
||||
# (the default is "--level 1"):
|
||||
shell$ ompi_info --param mpi all --level 9
|
||||
|
||||
Here is a summary of the debugging parameters for the MPI layer:
|
||||
|
||||
* ``mpi_param_check``: If set to true (any positive value), and when
|
||||
Open MPI is compiled with parameter checking enabled (the default),
|
||||
the parameters to each MPI function can be passed through a series
|
||||
of correctness checks. Problems such as passing illegal values
|
||||
(e.g., NULL or ``MPI_DATATYPE_NULL`` or other "bad" values) will be
|
||||
discovered at run time and an MPI exception will be invoked (the
|
||||
default of which is to print a short message and abort the entire
|
||||
MPI job). If set to false, these checks are disabled, slightly
|
||||
increasing performance.
|
||||
|
||||
* ``mpi_show_handle_leaks``: If set to true (any positive value),
|
||||
Open MPI will display lists of any MPI handles that were not freed before
|
||||
:ref:`MPI_Finalize(3) <mpi_finalize>` (e.g., communicators,
|
||||
datatypes, requests, etc.)
|
||||
|
||||
* ``mpi_no_free_handles``: If set to true (any positive value), do not
|
||||
actually free MPI objects when their corresponding MPI "free"
|
||||
function is invoked (e.g., do not free communicators when
|
||||
:ref:`MPI_Comm_free(3) <mpi_comm_free>`. This can be helpful in tracking down
|
||||
applications that accidentally continue to use MPI handles after
|
||||
they have been freed.
|
||||
|
||||
* ``mpi_show_mca_params``: If set to true (any positive value), show a
|
||||
list of all MCA parameters and their values when MPI is initialized.
|
||||
This can be quite helpful for reproducibility of MPI applications.
|
||||
|
||||
* ``mpi_show_mca_params_file``: If set to a non-empty value, and if
|
||||
the value of ``mpi_show_mca_params`` is true, then output the list
|
||||
of MCA parameters to the filename value. If this parameter is an
|
||||
empty value, the list is sent to ``stderr``.
|
||||
|
||||
* ``mpi_abort_delay``: If nonzero, print out an identifying message
|
||||
when :ref:`MPI_Abort(3) <mpi_abort>` is invoked showing the hostname and PID of the
|
||||
process that invoked :ref:`MPI_Abort(3) <mpi_abort>`, and then delay that many seconds
|
||||
before exiting. A negative value means to delay indefinitely. This
|
||||
allows a user to manually come in and attach a debugger when an
|
||||
error occurs. Remember that the default MPI error handler |mdash|
|
||||
``MPI_ERRORS_ABORT`` |mdash| invokes :ref:`MPI_Abort(3) <mpi_abort>`, so this
|
||||
parameter can be useful to discover problems identified by
|
||||
``mpi_param_check``.
|
||||
|
||||
* ``mpi_abort_print_stack``: If nonzero, print out a stack trace (on
|
||||
supported systems) when :ref:`MPI_Abort(3) <mpi_abort>` is invoked.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
Parallel Debugging Tools
|
||||
========================
|
||||
|
||||
There are two main categories of tools that can aid in
|
||||
parallel debugging:
|
||||
|
||||
* **Debuggers:** Both serial and parallel debuggers are useful. Serial
|
||||
debuggers are what most programmers are used to (e.g.,
|
||||
the GNU debugger, ``gdb``), while
|
||||
parallel debuggers can attach to all the individual processes in an
|
||||
MPI job simultaneously, treating the MPI application as a single
|
||||
entity. This can be an extremely powerful abstraction, allowing the
|
||||
user to control every aspect of the MPI job, manually replicate race
|
||||
conditions, etc.
|
||||
|
||||
* **Profilers:** Tools that analyze your usage of MPI and display
|
||||
statistics and meta information about your application's run. Some
|
||||
tools present the information "live" (as it occurs), while others
|
||||
collect the information and display it in a post mortem analysis.
|
||||
@@ -0,0 +1,30 @@
|
||||
.. Open MPI Application Debugging
|
||||
|
||||
Debugging Open MPI Parallel Applications
|
||||
========================================
|
||||
|
||||
Debugging a serial applications includes solving problems like
|
||||
logic errors, uninitialized variables, storage overlays and timing
|
||||
problems.
|
||||
|
||||
Debugging a parallel application can be further complicated
|
||||
by problems that can include additional race conditions and aysynchronous
|
||||
events, as well as understanding execution of multiple application
|
||||
processes running simultaneously.
|
||||
|
||||
This section of the documentation describes some techniques that can
|
||||
be useful for parallel debugging. This section also describes some
|
||||
tools that can be useful as well as some Open MPI runtime options
|
||||
that can aid debugging.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
debug-tools
|
||||
debug-options
|
||||
serial-debug
|
||||
parallel-debug
|
||||
lost-output
|
||||
memchecker
|
||||
valgrind
|
||||
mpir-tools
|
||||
@@ -0,0 +1,34 @@
|
||||
Application Output Lost with Abnormal Termination
|
||||
=================================================
|
||||
|
||||
There many be many reasons for application output to be lost when
|
||||
an application abnormally terminates. The Open MPI Team strongly
|
||||
encourages the use of tools (such as debuggers) whenever possible.
|
||||
|
||||
One of the reasons, however, may come from inside Open MPI itself. If
|
||||
your application fails due to memory corruption, Open MPI may
|
||||
subsequently fail to output an error message before terminating.
|
||||
Open MPI attempts to aggregate error
|
||||
messages from multiple processes in an attempt to show unique error
|
||||
messages only once (vs. once for each MPI process |mdash| which can be
|
||||
unwieldy, especially when running large MPI jobs).
|
||||
|
||||
However, this aggregation process requires allocating memory in the
|
||||
MPI process when it displays the error message. If the process's
|
||||
memory is already corrupted, Open MPI's attempt to allocate memory may
|
||||
fail and the process will simply terminate, possibly silently. When Open
|
||||
MPI does not attempt to aggregate error messages, most of its setup
|
||||
work is done when the MPI library is initiaized and no memory is allocated
|
||||
during the "print the error" routine. It therefore almost always successfully
|
||||
outputs error messages in real time |mdash| but at the expense that you'll
|
||||
potentially see the same error message for *each* MPI process that
|
||||
encountered the error.
|
||||
|
||||
Hence, the error message aggregation is *usually* a good thing, but
|
||||
sometimes it can mask a real error. You can disable Open MPI's error
|
||||
message aggregation with the ``opal_base_help_aggregate`` MCA
|
||||
parameter. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun --mca opal_base_help_aggregate 0 ...
|
||||
@@ -0,0 +1,193 @@
|
||||
Using Memchecker
|
||||
================
|
||||
|
||||
The Memchecker functionality in Open MPI provides MPI semantic
|
||||
checking for your application (as well as internals of Open MPI), with
|
||||
the help of memory checking tools such as the ``memcheck`` component of
|
||||
`the Valgrind suite <https://www.valgrind.org/>`_.
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Types of Errors Detected by Memchecker
|
||||
--------------------------------------
|
||||
|
||||
Open MPI's Memchecker is based on the ``memcheck`` tool included with
|
||||
Valgrind, so it takes all the advantages from it. Firstly, it checks
|
||||
all reads and writes of memory, and intercepts calls to
|
||||
``malloc(3)``/``free(3)`` and C++'s ``new``/``delete`` operators.
|
||||
Most importantly, Memchecker is able to detect
|
||||
the user buffer errors in both non-blocking and one-sided
|
||||
communications, e.g. reading or writing to buffers of active
|
||||
non-blocking receive operations and writing to buffers of active
|
||||
non-blocking send operations.
|
||||
|
||||
Here are some example problems that Memchecker can detect:
|
||||
|
||||
Accessing buffer under control of non-blocking communication:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
int buf;
|
||||
MPI_Irecv(&buf, 1, MPI_INT, 1, 0, MPI_COMM_WORLD, &req);
|
||||
// The following line will produce a memchecker warning
|
||||
buf = 4711;
|
||||
MPI_Wait (&req, &status);
|
||||
|
||||
Wrong input parameters, e.g., wrong-sized send buffers:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
char *send_buffer;
|
||||
send_buffer = malloc(5);
|
||||
memset(send_buffer, 0, 5);
|
||||
// The following line will produce a memchecker warning
|
||||
MPI_Send(send_buffer, 10, MPI_CHAR, 1, 0, MPI_COMM_WORLD);
|
||||
|
||||
Accessing a window in a one-sided communication:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
MPI_Get(A, 10, MPI_INT, 1, 0, 1, MPI_INT, win);
|
||||
A[0] = 4711;
|
||||
MPI_Win_fence(0, win);
|
||||
|
||||
Uninitialized input buffers:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
char *buffer;
|
||||
buffer = malloc(10);
|
||||
// The following line will produce a memchecker warning
|
||||
MPI_Send(buffer, 10, MPI_INT, 1, 0, MPI_COMM_WORLD);
|
||||
|
||||
Usage of the uninitialized ``MPI_Status`` field in ``MPI_ERROR``
|
||||
structure: (the MPI-1 standard defines the ``MPI ERROR`` field to be
|
||||
undefined for single-completion calls such as :ref:`MPI_Wait(3) <mpi_wait>` or
|
||||
:ref:`MPI_Test(3) <mpi_test>`, see MPI-1 p. 22):
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
MPI_Wait(&request, &status);
|
||||
// The following line will produce a memchecker warning
|
||||
if (status.MPI_ERROR != MPI_SUCCESS)
|
||||
return ERROR;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Building Open MPI with Memchecker Support
|
||||
-----------------------------------------
|
||||
|
||||
To use Memchecker, you need Valgrind 3.2.0 or later, and have an Open
|
||||
MPI that was configured with the ``--enable-memchecker`` and
|
||||
``--enable-debug`` flags.
|
||||
|
||||
.. note:: The Memchecker functionality is off by default, because it
|
||||
incurs a performance penalty.
|
||||
|
||||
When ``--enable-memchecker`` is specified, ``configure`` will check
|
||||
for a recent-enable valgrind distribution. If found, Open MPI will
|
||||
build Memchecker support.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure --prefix=/path/to/openmpi --enable-debug \
|
||||
--enable-memchecker --with-valgrind=/path/to/valgrind
|
||||
|
||||
You can check that Open MPI was built with Memchecker support by using
|
||||
the :ref:`ompi_info(1) <man1-ompi_info>` command.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# The exact version numbers shown may be different for your Open
|
||||
# MPI installation
|
||||
shell$ ompi_info | grep memchecker
|
||||
MCA memchecker: valgrind (MCA v1.0, API v1.0, Component v1.3)
|
||||
|
||||
If you do not see the "MCA memchecker: valgrind" line, you probably
|
||||
did not configure and install Open MPI correctly.
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Running an Open MPI Application with Memchecker
|
||||
-----------------------------------------------
|
||||
|
||||
After Open MPI was built and installed with Memchecker support,
|
||||
simply run your application with Valgrind, e.g.:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -n 2 valgrind ./my_app
|
||||
|
||||
If you enabled Memchecker, but you don't want to check the
|
||||
application at this time, then just run your application as
|
||||
usual. E.g.:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -n 2 ./my_app
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Application Performance Impacts Using Memchecker
|
||||
------------------------------------------------
|
||||
|
||||
The configure option ``--enable-memchecker`` (together with
|
||||
``--enable-debug``) *does* cause performance degradation, even if not
|
||||
running under Valgrind. The following explains the mechanism and may
|
||||
help in making the decision whether to provide a cluster-wide
|
||||
installation with ``--enable-memchecker``.
|
||||
|
||||
There are two cases:
|
||||
|
||||
#. If run without Valgrind, the Valgrind ClientRequests (assembler
|
||||
instructions added to the normal execution path for checking) do
|
||||
not affect overall MPI performance. Valgrind ClientRequests are
|
||||
explained in detail `in Valgrind's documentation
|
||||
<https://valgrind.org/docs/manual/manual-core-adv.html>`_.
|
||||
In the case of x86-64, ClientRequests boil down to the following
|
||||
four rotate-left (ROL) and one xchange (XCHG) assembler instructions
|
||||
from ``valgrind.h``:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#define __SPECIAL_INSTRUCTION_PREAMBLE \
|
||||
"rolq \$3, %%rdi; rolq \$13, %%rdi\\n\\t" \
|
||||
"rolq \$61, %%rdi; rolq \$51, %%rdi\\n\\t"
|
||||
|
||||
and
|
||||
|
||||
.. We do not make the code block below as "c" because the Sphinx C
|
||||
syntax highlighter fails to parse it as C and emits a warning.
|
||||
So we might as well just leave it as a plan verbatim block
|
||||
(i.e., not syntax highlighted).
|
||||
|
||||
.. code-block::
|
||||
|
||||
__asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \
|
||||
/* %RDX = client_request ( %RAX ) */ \
|
||||
"xchgq %%rbx,%%rbx" \
|
||||
: "=d" (_zzq_result) \
|
||||
: "a" (& _zzq_args``0``), "0" (_zzq_default) \
|
||||
: "cc", "memory" \
|
||||
);
|
||||
|
||||
for every single ClientRequest. In the case of not running
|
||||
Valgrind, these ClientRequest instructions do not change the
|
||||
arithmetic outcome (rotating a 64-bit register left by 128-Bits,
|
||||
exchanging a register with itself), except for the carry flag.
|
||||
|
||||
The first request is checking whether we're running under Valgrind.
|
||||
In case we're not running under Valgrind subsequent checks (a.k.a.
|
||||
ClientRequests) are not done.
|
||||
|
||||
#. If the application is run under Valgrind, performance is naturally reduced due
|
||||
to the Valgrind JIT and the checking tool employed.
|
||||
For costs and overheads of Valgrind's Memcheck tool on the SPEC 2000 Benchmark,
|
||||
please see the excellent paper
|
||||
`Valgrind: A Framework for Heavyweight Dynamic Binary Instrumentation
|
||||
<https://valgrind.org/docs/valgrind2007.pdf>`_.
|
||||
For an evaluation of various internal implementation alternatives of Shadow Memory, please see
|
||||
`Building Workload Characterization Tools with Valgrind
|
||||
<https://valgrind.org/docs/iiswc2006.pdf>`_.
|
||||
@@ -0,0 +1,127 @@
|
||||
.. _using-mpir-based-tools-label:
|
||||
|
||||
Using MPIR-based tools with Open MPI
|
||||
====================================
|
||||
|
||||
The `legacy MPIR specification
|
||||
<https://www.mpi-forum.org/docs/mpir-specification-03-01-2018.pdf>`_
|
||||
is the specification of an interface that debuggers and others tools
|
||||
can use to obtain the PID, hostname, and executable name for each MPI
|
||||
application process.
|
||||
|
||||
This specification has been replaced by the PMIx tools API that is part of the
|
||||
`OpenPMIx project <https://openpmix.github.io/>`_.
|
||||
|
||||
Debuggers and tools which use the legacy MPIR specification can
|
||||
continue functioning correctly with the `MPIR shim module
|
||||
<https://github.com/hpc/mpir-to-pmix-guide>`_ described in this
|
||||
section.
|
||||
|
||||
Obtaining Application Process Mapping Information
|
||||
-------------------------------------------------
|
||||
|
||||
The `MPIR shim module <https://github.com/hpc/mpir-to-pmix-guide>`_ is
|
||||
a stand-alone module which can be used by debuggers and tools that
|
||||
still utilize the legacy MPIR interface to attach to parallel MPI
|
||||
applications. The debugger or tool launches the shim module as an
|
||||
intermediate process between it and the application's :ref:`mpirun(1)
|
||||
<man1-mpirun>` command.
|
||||
|
||||
The MPIR shim module implements the ``MPIR_Breakpoint`` function, as a
|
||||
hook where debuggers and tools can set a breakpoint to read the MPIR
|
||||
processing mapping when it is available.
|
||||
|
||||
Instructions for use of the shim module are available at
|
||||
https://github.com/hpc/mpir-to-pmix-guide/blob/master/README.md.
|
||||
|
||||
You may need to build this shim module before you can use it. The
|
||||
source code for this module can be downloaded from
|
||||
`MPIR to PMIx Shim repository <https://github.com/hpc/mpir-to-pmix-guide>`_.
|
||||
|
||||
Once you have the MPIR shim module built and installed, follow the
|
||||
instructions included with the debugger or tool to tell it where to
|
||||
find the MPIR shim module.
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Building the MPIR Shim Module
|
||||
-----------------------------
|
||||
|
||||
Here is an example shell script that builds the MPIR shim module
|
||||
|mdash| you may need to customize it for your specific environment:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
cd $HOME
|
||||
mkdir -p git
|
||||
cd git
|
||||
|
||||
git clone https://github.com/hpc/mpir-to-pmix-guide.git
|
||||
cd mpir-to-pmix-guide
|
||||
|
||||
# PMIX_ROOT must be set to your PMIx install root directory
|
||||
# (which may be the same as your Open MPI installation directory)
|
||||
if test -z "$PMIX_ROOT" -o ! -d "$PMIX_ROOT"; then
|
||||
echo "Set the env variable PMIX_ROOT to the location of your PMIx installation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./autogen.sh
|
||||
./configure --prefix=$HOME/MPIR --with-pmix=$PMIX_ROOT
|
||||
make
|
||||
make install
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Verification of MPIR shim functionality using Open MPI
|
||||
------------------------------------------------------
|
||||
|
||||
Once installed, you may want to verify that the MPIR shim is
|
||||
functioning correctly. Below is a shell script that builds and runs a
|
||||
trivial MPI application, and tests the MPIR shim with it. Just like
|
||||
the prior shell script example, this script may need to be adapted for
|
||||
your environment.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
# This script assumes the same directories and locations as the
|
||||
# prior script.
|
||||
BUILD_DIR=$HOME/git/mpir-to-pmix-guide
|
||||
|
||||
# Add the previously-installed MPIR shim into PATH and
|
||||
# LD_LIBRARY_PATH (we assume that all relevant Open MPI / PMIx /
|
||||
# etc. values already exist in PATH and LD_LIBRARY_PATH)
|
||||
export PATH=$HOME/MPIR/bin:$PATH
|
||||
export LD_LIBRARY_PATH=$HOME/MPIR/lib:$LD_LIBRARY_PATH
|
||||
|
||||
cat > testprog.c <<EOF
|
||||
#include <mpi.h>
|
||||
#include <unistd.h>
|
||||
int main(int argc, char **argv) {
|
||||
MPI_Init(&argc, &argv);
|
||||
sleep(60);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
EOF
|
||||
mpicc -o testprog testprog.c
|
||||
|
||||
# Test the shim in proxy mode.
|
||||
# Manually verify that displayed process mapping is correct.
|
||||
$BUILD_DIR/test/mpirshim_test mpirun -n 2 ./testprog
|
||||
|
||||
# Launch mpirun for attach test and get its PID
|
||||
mpirun -n 2 ./testprog &
|
||||
PID=$!
|
||||
# Test shim attach mode.
|
||||
# Manually verify that displayed process mapping is correct.
|
||||
$BUILD_DIR/test/mpirshim_test -c $PID
|
||||
@@ -0,0 +1,41 @@
|
||||
Using Parallel Debuggers to Debug Open MPI Applications
|
||||
=======================================================
|
||||
|
||||
Debugging with TotalView
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Debugging MPI applications has changed with different releases of
|
||||
TotalView and Open MPI; it is best to consult TotalView's
|
||||
documentation for how you should debug Open MPI applications with
|
||||
TotalView.
|
||||
|
||||
Starting with Open MPI v5.0.0, debugging with Totalview requires the
|
||||
use of the MPIR shim module to attach to MPI processes. See the
|
||||
:ref:`using MPIR-based tools with Open MPI section
|
||||
<using-mpir-based-tools-label>` for instructions on building and
|
||||
installing the MPIR shim module.
|
||||
|
||||
Once TotalView is attached to Open MPI processes, it can also view
|
||||
Open MPI's message queues (for some transports), which can aid in
|
||||
debugging.
|
||||
|
||||
.. note:: The integration of Open MPI message queue support is
|
||||
problematic with 64-bit versions of TotalView prior to v8.3:
|
||||
|
||||
* The message queues view will be truncated.
|
||||
* Both the communicators and requests lists will be incomplete.
|
||||
* Both the communicators and requests lists may be filled with wrong
|
||||
values (such as an ``MPI_Send`` to the destination
|
||||
``MPI_ANY_SOURCE``).
|
||||
|
||||
There are two workarounds:
|
||||
|
||||
* Use a 32-bit version of TotalView
|
||||
* Upgrade to TotalView v8.3 or later
|
||||
|
||||
Debugging with DDT
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Debugging MPI application with DDT has changed with different releases
|
||||
of DDT and Open MPI; it is best to consult DDT's documentation for how
|
||||
you should debug Open MPI applications with DDT.
|
||||
@@ -0,0 +1,175 @@
|
||||
Using Serial Debuggers to Debug Open MPI Applications
|
||||
=====================================================
|
||||
|
||||
Since the GNU debugger (``gdb``) is fairly ubiquitiously
|
||||
available, it is common to use a serial debugger for debugging Open
|
||||
MPI applications. Parallel debuggers are generally *better*, but
|
||||
``gdb`` is free, and therefore quite common.
|
||||
|
||||
There are two common ways to use serial debuggers.
|
||||
|
||||
Attach to Individual Running MPI processes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
You can use multiple invocations of a serial debugger to individually
|
||||
attach to every application process, or to a subset of application process.
|
||||
|
||||
If you are using ``gdb``, then you would log onto the nodes where you want
|
||||
to debug application processes and invoke gdb specifying the
|
||||
gdb ``--pid`` option and the PID for each application process you want to
|
||||
attach to.
|
||||
|
||||
You can use a similar technique to attach to application processes in
|
||||
an application that was just invoked by :ref:`mpirun(1) <man1-mpirun>`.
|
||||
|
||||
An inelegant-but-functional technique commonly used with this method
|
||||
is to insert the following code in your application where you want to
|
||||
suspend the application process until the debugger is attached, such as
|
||||
just after MPI is initialized.
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
{
|
||||
volatile int i = 0;
|
||||
char hostname[256];
|
||||
gethostname(hostname, sizeof(hostname));
|
||||
printf("PID %d on %s ready for attach\n", getpid(), hostname);
|
||||
fflush(stdout);
|
||||
while (0 == i)
|
||||
sleep(5);
|
||||
}
|
||||
|
||||
This code will output a line to stdout outputting the name of the host
|
||||
where the process is running and the PID to attach to. It will then
|
||||
spin on the ``sleep(3)`` function forever waiting for you to attach
|
||||
with a debugger. Using ``sleep(3)`` as the inside of the loop means
|
||||
that the processor won't be pegged at 100% while waiting for you to
|
||||
attach.
|
||||
|
||||
Once you attach with a debugger, go up the function stack until you
|
||||
are in this block of code (you'll likely attach during the
|
||||
``sleep(3)``) then set the variable ``i`` to a nonzero value. With
|
||||
GDB, the syntax is:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
(gdb) set var i = 7
|
||||
|
||||
Then set a breakpoint after your block of code and continue execution
|
||||
until the breakpoint is hit. Now you have control of your live MPI
|
||||
application and use of the full functionality of the debugger.
|
||||
|
||||
You can even add conditionals to only allow this "pause" in the
|
||||
application for specific MPI processes (e.g., ``MPI_COMM_WORLD`` rank
|
||||
0, or whatever process is misbehaving).
|
||||
|
||||
Use ``mpirun`` to Launch Separate Instances of Serial Debuggers
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
This technique launches a separate window for each MPI process in
|
||||
``MPI_COMM_WORLD``, each one running a serial debugger (such as
|
||||
``gdb``) that will launch and run your MPI application. Having a
|
||||
separate window for each MPI process can be quite handy for low
|
||||
process-count MPI jobs, but requires a bit of setup and configuration
|
||||
that is outside of Open MPI to work properly. A naive approach would
|
||||
be to assume that the following would immediately work:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -n 4 xterm -e gdb my_mpi_application
|
||||
|
||||
If running on a personal computer, this will probably work. You can
|
||||
also use `tmpi <https://github.com/Azrael3000/tmpi>`_ to launch the
|
||||
debuggers in separate ``tmux`` panes instead of separate ``xterm``
|
||||
windows, which has the advantage of synchronizing keyboard input
|
||||
between all debugger instances.
|
||||
|
||||
Unfortunately, the ``tmpi`` or ``xterm`` approaches likely *won't*
|
||||
work on an computing cluster. Several factors must be considered:
|
||||
|
||||
#. What launcher is Open MPI using? In an ``ssh``-based environment,
|
||||
Open MPI will use ``ssh`` by default.
|
||||
But note that Open MPI closes the ``ssh``
|
||||
sessions when the MPI job starts for scalability reasons. This
|
||||
means that the built-in SSH X forwarding tunnels will be shut down
|
||||
before the ``xterms`` can be launched. Although it is possible to
|
||||
force Open MPI to keep its SSH connections active (to keep the X
|
||||
tunneling available), we recommend using non-SSH-tunneled X
|
||||
connections, if possible (see below).
|
||||
|
||||
#. In non-``ssh`` environments (such as when using resource managers),
|
||||
the environment of the process invoking ``mpirun`` may be copied to
|
||||
all nodes. In this case, the ``DISPLAY`` environment variable may
|
||||
not be suitable.
|
||||
|
||||
#. Some operating systems default to disabling the X11 server from
|
||||
listening for remote/network traffic. For example, see `this post
|
||||
on the Open MPI user's mailing list
|
||||
<https://www.open-mpi.org/community/lists/users/2008/02/4995.php>`_
|
||||
describing how to enable network access to the X11 server on Fedora
|
||||
Linux.
|
||||
|
||||
#. There may be intermediate firewalls or other network blocks that
|
||||
prevent X traffic from flowing between the hosts where the MPI
|
||||
processes (and ``xterm``) are running and the host connected to
|
||||
the output display.
|
||||
|
||||
The easiest way to get remote X applications (such as ``xterm``) to
|
||||
display on your local screen is to forego the security of SSH-tunneled
|
||||
X forwarding. In a closed environment such as an HPC cluster, this
|
||||
may be an acceptable practice (indeed, you may not even have the
|
||||
option of using SSH X forwarding if SSH logins to cluster nodes are
|
||||
disabled), but check with your security administrator to be sure.
|
||||
|
||||
If using non-encrypted X11 forwarding is permissible, we recommend the
|
||||
following:
|
||||
|
||||
#. For each non-local host where you will be running an MPI process,
|
||||
add it to your X server's permission list with the ``xhost``
|
||||
command. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my_hostfile
|
||||
inky
|
||||
blinky
|
||||
stinky
|
||||
clyde
|
||||
shell$ for host in `cat my_hostfile` ; do xhost +host ; done
|
||||
|
||||
#. Use the ``-x`` option to ``mpirun`` to export an appropriate
|
||||
DISPLAY variable so that the launched X applications know where to
|
||||
send their output. An appropriate value is *usually* (but not
|
||||
always) the hostname containing the display where you want the
|
||||
output and the ``:0`` (or ``:0.0``) suffix. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ hostname
|
||||
arcade.example.come
|
||||
shell$ mpirun -n 4 --hostfile my_hostfile \
|
||||
-x DISPLAY=arcade.example.com:0 xterm -e gdb my_mpi_application
|
||||
|
||||
.. warning:: X traffic is fairly "heavy" |mdash| if you are
|
||||
operating over a slow network connection, it may take
|
||||
some time before the ``xterm`` windows appear on your
|
||||
screen.
|
||||
|
||||
#. If your ``xterm`` supports it, the ``-hold`` option may be useful.
|
||||
``-hold`` tells ``xterm`` to stay open even when the application
|
||||
has completed. This means that if something goes wrong (e.g.,
|
||||
``gdb`` fails to execute, or unexpectedly dies, or ...), the
|
||||
``xterm`` window will stay open, allowing you to see what happened,
|
||||
instead of closing immediately and losing whatever error message
|
||||
may have been output.
|
||||
|
||||
#. When you have finished, you may wish to disable X11 network
|
||||
permissions from the hosts that you were using. Use ``xhost``
|
||||
again to disable these permissions:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ for host in `cat my_hostfile` ; do xhost -host ; done
|
||||
|
||||
.. note:: ``mpirun`` will not complete until all the ``xterm``
|
||||
instances are complete.
|
||||
@@ -0,0 +1,24 @@
|
||||
Using Valgrind to Find Open MPI Application Errors
|
||||
==================================================
|
||||
|
||||
The `Valgrind <https://valgrind.org/>`_ utilities may be useful in identifying application
|
||||
logic errors. However, there are many situations where Open MPI purposefully does
|
||||
not initialize and subsequently communicate memory, e.g., by calling ``writev(2)``.
|
||||
Furthermore, several cases are known where memory is not properly freed when
|
||||
MPI is finalized.
|
||||
|
||||
This certainly does not help distinguishing real errors from false positives.
|
||||
Valgrind provides functionality to suppress errors and warnings from certain
|
||||
function contexts.
|
||||
|
||||
In an attempt to ease debugging using Valgrind, Open MPI provides a
|
||||
Valgrind suppression file, that can be passed on the command
|
||||
line:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -n 2 valgrind --suppressions=$PREFIX/share/openmpi/openmpi-valgrind.supp
|
||||
|
||||
More information on suppression files and how to generate them can be
|
||||
found in `Valgrind's documentation
|
||||
<https://valgrind.org/docs/manual/manual-core.html#manual-core.suppress>`_.
|
||||
@@ -0,0 +1,59 @@
|
||||
.. _label-binary-compatibility:
|
||||
|
||||
ABI compatibility to previous versions of Open MPI
|
||||
==================================================
|
||||
|
||||
The Open MPI |ompi_series| series maintains Application Binary Interface (ABI)
|
||||
compatibility for the C MPI bindings to the last major Open MPI release. Specifically, an
|
||||
application compiled with Open MPI v4.x can be executed with Open MPI
|
||||
|ompi_series| without having to recompile the application.
|
||||
|
||||
.. important:: ABI is maintained for *most* of the Fortran MPI bindings, too |mdash| see below for additional information.
|
||||
|
||||
There are however a few scenarios where an application compiled with
|
||||
Open MPI v4.x might not execute correctly with Open MPI 5.0.
|
||||
|
||||
- Fortran compilers provide varying degrees of ABI guarantees between
|
||||
their releases. As such, Open MPI can only provide ABI guarantees
|
||||
with MPI applications that use the Fortran MPI bindings when all
|
||||
three entities |mdash| Open MPI v4.x, Open MPI |ompi_series|, and
|
||||
the MPI application |mdash| were built with exactly the same Fortran
|
||||
compiler and environment.
|
||||
|
||||
If these conditions are met, Open MPI's ABI guarantees between the
|
||||
Open MPI v4.x and |ompi_series| series are in effect, with the
|
||||
exception of the following cases:
|
||||
|
||||
* When using the Fortran ``mpi`` module bindings with GCC compiler
|
||||
versions prior to v4.8, non-compliant Fortran interfaces for
|
||||
``MPI_Comm_spawn_multiple``, ``MPI_Testall``, ``MPI_Testsome``,
|
||||
``MPI_Waitall``, and ``MPI_Waitsome`` were removed starting with
|
||||
Open MPI v5.0.0.
|
||||
|
||||
* When using the Fortran ``mpi`` module with modern Fortran
|
||||
compilers (e.g., GNU Fortran >= v4.9), Open MPI v5.0.0 removed the
|
||||
names from the MPI interfaces when there is only a single
|
||||
subroutine in the interface, and that subroutine name exactly
|
||||
matches the iterface name. This change is likely to make Open MPI
|
||||
|ompi_series|'s ``mpi`` module bindings *less* restrictive than
|
||||
Open MPI v4.x, but it *may* also have ABI implications, depending
|
||||
on your Fortran compiler.
|
||||
|
||||
`See this git commit message
|
||||
<https://github.com/open-mpi/ompi/commit/f34782fe0c493963ec44bcf7dde9d94b88a7ea91>`_
|
||||
for more details.
|
||||
|
||||
.. important:: This is likely a compiler-specific issue, and may
|
||||
need to be tested in your environment.
|
||||
|
||||
* When using the Fortran ``mpi_f08`` module bindings in an
|
||||
environment where a Fortran ``INTEGER`` is 8 bytes but a C ``int``
|
||||
is 4 bytes, the size of a ``Type(MPI_Status)`` was increased
|
||||
starting with Open MPI v5.0.0.
|
||||
|
||||
- Open MPI v5.0.0 removed support for the MPI C++ bindings. If an application
|
||||
was using the deprecated and now removed C++ bindings, it will not
|
||||
be able to compile or execute with Open MPI v5.0.0. For details on deprecated and
|
||||
removed functions see :ref:`Removed MPI constructs
|
||||
<label-removed-mpi-constructs>` and :ref:`Deprecation warnings
|
||||
<label-deprecated-functions>`
|
||||
@@ -0,0 +1,182 @@
|
||||
.. _label-building-fully-static-apps:
|
||||
|
||||
Building fully-static MPI applications
|
||||
======================================
|
||||
|
||||
Is fully-static really what you need?
|
||||
-------------------------------------
|
||||
|
||||
No one ends up in this documentation section by accident.
|
||||
|
||||
If you are reading this text, it is likely because you are looking to
|
||||
solve a problem, and fully-static MPI applications sound like the
|
||||
right answer. There are two common problems that people think that
|
||||
fully-static MPI applications will solve:
|
||||
|
||||
#. MPI applications fail to launch because dependent shared libraries
|
||||
are not found at run-time.
|
||||
|
||||
#. Filesystem performance when launching at scale is terrible.
|
||||
|
||||
If either of these are your problems, the Open MPI community *strongly
|
||||
encourages you to use other mechanisms to fix the problem*:
|
||||
fully-static MPI applications are *possible*, but are sub-optimal for
|
||||
other reasons.
|
||||
|
||||
The following sections discuss the above problems.
|
||||
|
||||
Finding dependent shared libraries at run-time
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
If you are running in to problems finding shared libraries at run time
|
||||
|mdash| particularly on nodes that are remote from where you have invoked
|
||||
``mpirun`` |mdash| your best bet is to set ``LD_LIBRARY_PATH`` (or
|
||||
equivalent mechanism) properly on *all* nodes (including remote
|
||||
nodes).
|
||||
|
||||
This is an involved topic, but even so, it is generally simpler to
|
||||
solve this problem than creating and maintaining static builds. See
|
||||
:ref:`the section on Running MPI applications
|
||||
<label-running-mpi-applications>` for more details.
|
||||
|
||||
Improving filesystem performance at scale
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Prior to v5.0.0, Open MPI compiled a large number of plugins as
|
||||
individual dynamic shared objects (DSOs) |mdash| i.e., individual
|
||||
files in the filesystem. Many of these DSOs would be opened by each
|
||||
MPI process at run time.
|
||||
|
||||
This could cause filesystem congestion, particularly when Open MPI is
|
||||
installed on a network filesystem and a large job is launched: many
|
||||
nodes will simultaneously communicate with the file server(s), and
|
||||
potentially need to transfer a large number of small(ish) files.
|
||||
|
||||
Starting with v5.0.0, by default, Open MPI's plugins are no longer
|
||||
built as DSOs. As such, Open MPI typically only opens a small number
|
||||
of shared libraries at launch time. Even if Open MPI is installed on
|
||||
a network filesystem, these libraries are likely to be cached on nodes
|
||||
over time, and therefore generate a fairly small amount network
|
||||
filesystem traffic when MPI jobs are launched.
|
||||
|
||||
In short: Open MPI |ompi_ver|'s impact on network filesystems is
|
||||
greatly diminished compared to prior versions. Compiling fully-static
|
||||
applications to eliminate the open-every-DSO-file-at-launch-time
|
||||
behavior is no longer necessary.
|
||||
|
||||
Other reasons fully-static applications are bad
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Here are a few other reasons that fully-static MPI applications are
|
||||
sub-optimal:
|
||||
|
||||
#. When applications link all of their dependencies statically, the
|
||||
operating system cannot share code between multiple copies of the
|
||||
process.
|
||||
|
||||
For example, if you launch N copies of your fully-statically-linked
|
||||
MPI application on a node, it will consume (N *
|
||||
size_of_the_application) bytes of RAM. Alternately, launching N
|
||||
copies of a dynamically-linked MPI application |mdash| where each
|
||||
of the copies have the same dependent libraries |mdash| will only
|
||||
load each shared dependent library into RAM once.
|
||||
|
||||
In other words: *using dynamic linking saves memory.*
|
||||
|
||||
#. *Fully*-static applications are not linked to the dynamic linking library,
|
||||
e.g. ``libdl`` on Linux, which provides ``dlopen(3)``, ``dlsym(3)``, etc.
|
||||
This will break Open MPI functionalities that depend on such interfaces.
|
||||
|
||||
.. warning:: Open MPI's memory management functionality, which provides
|
||||
important performance optimizations on OS-bypass networks
|
||||
such as InfiniBand, requires the ``dlsym(3)`` interface,
|
||||
and therefore does not work with fully-static applications.
|
||||
|
||||
Are you convinced yet? *Please try to avoid building fully-static MPI
|
||||
applications if at all possible.*
|
||||
|
||||
|
||||
Building fully-static MPI applications
|
||||
--------------------------------------
|
||||
|
||||
.. caution:: If, after reading all of the above, you are still of the
|
||||
mind that you want to build fully-static MPI
|
||||
applications, be aware that fully static linking is not
|
||||
for the meek, and it is not recommended. But it is
|
||||
possible, with some caveats.
|
||||
|
||||
#. You must have static libraries available for *everything* to which
|
||||
your program links. This includes Open MPI; you must have used the
|
||||
``--enable-static`` option to Open MPI's ``configure`` or otherwise
|
||||
have available the static versions of the Open MPI libraries.
|
||||
|
||||
.. note:: Some Linux distributions may not have static versions of
|
||||
popular Linux libraries by default (e.g., libnuma), or
|
||||
require additional RPMs to be installed to get the
|
||||
equivalent static libraries.
|
||||
|
||||
#. Open MPI must have been built without a memory manager. This means
|
||||
that Open MPI must have been configured with the
|
||||
``--without-memory-manager`` flag. This is irrelevant on some
|
||||
platforms for which Open MPI does not have a memory manager, but on
|
||||
some platforms it is necessary (Linux when using many OS-bypass
|
||||
networks). It is harmless to use this flag on platforms where Open
|
||||
MPI does not have a memory manager.
|
||||
|
||||
.. important:: Not including memory manager support can lead to
|
||||
lower performance when Open MPI is used with
|
||||
OS-bypass networks.
|
||||
|
||||
This is how to configure Open MPI to build fully-static libraries on
|
||||
Linux:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure --without-memory-manager --disable-dlopen \
|
||||
--enable-static --disable-shared ...
|
||||
|
||||
The ``--disable-shared`` flag is optional; it will prevent Open MPI
|
||||
from *also* building shared libraries.
|
||||
|
||||
Alternatively, you could build Open MPI with as many static libraries
|
||||
as possible, but still preserve ``dlopen`` functionality by omitting
|
||||
the ``--disable-dlopen`` flag:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure --without-memory-manager \
|
||||
--enable-static --disable-shared ...
|
||||
|
||||
This gives you a *mostly* static build of Open MPI, but has the
|
||||
advantage of preserving at least some dynamic libraries.
|
||||
|
||||
Including whole archives
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Some systems may have additional constraints about their support
|
||||
libraries that require additional steps to produce working
|
||||
fully-static MPI applications. For example, any library that has its
|
||||
own run-time plugin system (i.e., that opens dynamically shared
|
||||
objects ("DSOs") at run time) will have additional complications in
|
||||
producing fully-static builds.
|
||||
|
||||
In such cases, you generally want to run ``mpicc ... --showme`` to see
|
||||
the compiler / linker commands that Open MPI's wrapper commands will
|
||||
use, and then augment those commands with linker arguments for the
|
||||
static versions of the DSO plugins that you will need at run time.
|
||||
|
||||
For example, if you have ``libfoo.a`` that dynamically loads
|
||||
``plugin.so`` at run time, you'll need to have a ``plugin.a`` and
|
||||
|mdash| assuming the GNU linker |mdash| add arguments similar to the
|
||||
following:
|
||||
|
||||
* ``-static``: Tell the linker to generate a static executable.
|
||||
* ``-Wl,--whole-archive -lfoo /path/to/plugin.a -Wl,--no-whole-archive``:
|
||||
Tell the linker to include the entire ``foo`` library and the entire
|
||||
``plugin.a`` archive in the executable.
|
||||
|
||||
You can either add these arguments on the command line manually, or
|
||||
you can :ref:`modify the default behavior of the wrapper compilers
|
||||
<label-customizing-wrapper-compiler>` to hide this complexity from end
|
||||
users (but be aware that if you modify the wrapper compilers' default
|
||||
behavior, *all* users will be creating static applications!).
|
||||
@@ -0,0 +1,122 @@
|
||||
.. _label-customizing-wrapper-compiler:
|
||||
|
||||
Customizing wrapper compiler behavior
|
||||
=====================================
|
||||
|
||||
The Open MPI wrapper compilers are driven by text files that contain,
|
||||
among other things, the flags that are passed to the underlying
|
||||
compiler. These text files are generated automatically for Open MPI
|
||||
and are customized for the compiler set that was selected when Open
|
||||
MPI was configured; it is *not* recommended that users edit these
|
||||
files.
|
||||
|
||||
However, there are cases where it may be necessary or desirable to
|
||||
edit these files and add to or subtract from the flags that Open MPI
|
||||
selected. These files are installed in ``$pkgdatadir``, which
|
||||
defaults to ``$prefix/share/openmpi/WRAPPER_NAME-wrapper-data.txt``.
|
||||
Several environment variables are also available for run-time
|
||||
replacement of the wrapper's default values (from the text files):
|
||||
|
||||
.. note:: You may need to scroll right in the following table.
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Wrapper compiler
|
||||
- Compiler
|
||||
- Preprocessor flags
|
||||
- Compiler flags
|
||||
- Linker flags
|
||||
- Linker library flags
|
||||
- Data file
|
||||
|
||||
* - ``mpicc``
|
||||
- ``OMPI_CC``
|
||||
- ``OMPI_CPPFLAGS``
|
||||
- ``OMPI_CFLAGS``
|
||||
- ``OMPI_LDFLAGS``
|
||||
- ``OMPI_LIBS``
|
||||
- ``mpicc-wrapper-data.txt``
|
||||
|
||||
* - ``mpic++`` and ``mpiCC``
|
||||
- ``OMPI_CXX``
|
||||
- ``OMPI_CPPFLAGS``
|
||||
- ``OMPI_CXXFLAGS``
|
||||
- ``OMPI_LDFLAGS``
|
||||
- ``OMPI_LIBS``
|
||||
- ``mpic++-wrapper-data.txt`` and ``mpiCC-wrapper-data.txt``,
|
||||
respectively
|
||||
|
||||
* - ``mpifort``
|
||||
- ``OMPI_FC``
|
||||
- ``OMPI_CPPFLAGS``
|
||||
- ``OMPI_FCFLAGS``
|
||||
- ``OMPI_LDFLAGS``
|
||||
- ``OMPI_LIBS``
|
||||
- ``mpifort-wrapper-data.txt``
|
||||
|
||||
.. caution:: Note that changing the underlying compiler may not work
|
||||
at all.
|
||||
|
||||
For example, C++ and Fortran compilers are notoriously binary
|
||||
incompatible with each other (sometimes even within multiple
|
||||
releases of the same compiler). If you compile/install Open MPI
|
||||
with C++ compiler vX.Y.Z and then use the ``OMPI_CXX`` environment
|
||||
variable to change the ``mpicxx`` wrapper compiler to use the
|
||||
vA.B.C C++ compiler, your application code may not compile and/or
|
||||
link. The traditional method of using multiple different compilers
|
||||
with Open MPI is to install Open MPI multiple times; each
|
||||
installation should be built/installed with a different compiler.
|
||||
This is annoying, but it is beyond the scope of Open MPI to be able
|
||||
to fix.
|
||||
|
||||
Note that the values of these fields can be directly influenced by
|
||||
passing flags to Open MPI's ``configure`` script. :ref:`See this
|
||||
section in the Installation guide <install-wrapper-flags-label>` for
|
||||
more details.
|
||||
|
||||
The files cited in the above table use fairly simplistic "key=value"
|
||||
data formats. The following are several fields that are likely to be
|
||||
of interest to end-users:
|
||||
|
||||
* ``project_short``: Prefix for all environment variables. See
|
||||
below.
|
||||
|
||||
* ``compiler_env``: Specifies the base name of the environment
|
||||
variable that can be used to override the wrapper's underlying
|
||||
compiler at run-time. The full name of the environment variable is
|
||||
of the form ``<project_short>_<compiler_env>``; see table above.
|
||||
|
||||
* ``compiler_flags_env``: Specifies the base name of the environment
|
||||
variable that can be used to override the wrapper's compiler flags
|
||||
at run-time. The full name of the environment variable is of the
|
||||
form ``<project_short>_<compiler_flags_env>``; see table above.
|
||||
|
||||
* ``compiler``: The executable name of the underlying compiler.
|
||||
|
||||
* ``extra_includes``: Relative to ``$installdir``, a list of directories
|
||||
to also list in the preprocessor flags to find header files.
|
||||
|
||||
* ``preprocessor_flags``: A list of flags passed to the preprocessor.
|
||||
|
||||
* ``compiler_flags``: A list of flags passed to the compiler.
|
||||
|
||||
* ``linker_flags``: A list of flags passed to the linker.
|
||||
|
||||
* ``libs``: A list of libraries passed to the linker.
|
||||
|
||||
* ``required_file``: If non-empty, check for the presence of this file
|
||||
before continuing. If the file is not there, the wrapper will abort
|
||||
saying that the language is not supported.
|
||||
|
||||
* ``includedir``: Directory containing Open MPI's header files. The
|
||||
proper compiler "include" flag is prepended to this directory and
|
||||
added into the preprocessor flags.
|
||||
|
||||
* ``libdir``: Directory containing Open MPI's library files. The
|
||||
proper compiler "include" flag is prepended to this directory and
|
||||
added into the linker flags.
|
||||
|
||||
* ``module_option``: This field only appears in ``mpifort``. It is
|
||||
the flag that the Fortran compiler requires to declare where module
|
||||
files are located.
|
||||
@@ -0,0 +1,266 @@
|
||||
.. _label-deprecated-functions:
|
||||
|
||||
Deprecation warnings while compiling MPI applications
|
||||
=====================================================
|
||||
|
||||
If you see deprecation warnings when compiling MPI applications, it is
|
||||
because your application is symbols / functions that are deprecated in
|
||||
MPI. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpicc deprecated-example.c -c
|
||||
deprecated-example.c: In function 'foo':
|
||||
deprecated-example.c:6:5: warning: 'MPI_Attr_delete' is deprecated: MPI_Attr_delete was deprecated in MPI-2.0; use MPI_Comm_delete_attr instead [-Wdeprecated-declarations]
|
||||
MPI_Attr_delete(MPI_COMM_WORLD, 2);
|
||||
^~~~~~~~~~~~~~~
|
||||
In file included from deprecated-example.c:2:
|
||||
/usr/local/openmpi/include/mpi.h:2601:20: note: declared here
|
||||
OMPI_DECLSPEC int MPI_Attr_delete(MPI_Comm comm, int keyval)
|
||||
^~~~~~~~~~~~~~~
|
||||
|
||||
Note that the deprecation compiler warnings tells you how to upgrade
|
||||
your code to avoid the deprecation warnings. In this example, it
|
||||
advises you to use ``MPI_Comm_delete_attr()`` instead of
|
||||
``MPI_Attr_delete()``.
|
||||
|
||||
Also, note that even if Open MPI was configured with
|
||||
``--enable-mpi1-compatibility`` to re-enable removed MPI-1 symbols,
|
||||
you will still get compiler warnings when you use the removed symbols.
|
||||
|
||||
The following is a list of functions that have been deprecated in MPI,
|
||||
and the function that is replacing them. Some functions have been
|
||||
deprecated and removed from the MPI specification, these functions are
|
||||
listed :ref:`here <label-removed-mpi-constructs>`.
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Deprecated symbol
|
||||
|
||||
(click for more details, below)
|
||||
- Replaced with
|
||||
|
||||
(click to go to the corresponding man page)
|
||||
- MPI version deprecating the symbol
|
||||
|
||||
* - :ref:`MPI_KEYVAL_CREATE <label-mpi-keyval-create>`
|
||||
- :ref:`MPI_COMM_CREATE_KEYVAL <mpi_comm_create_keyval>`
|
||||
- MPI-2.0 (1996)
|
||||
|
||||
* - :ref:`MPI_KEYVAL_FREE <label-mpi-keyval-free>`
|
||||
- :ref:`MPI_COMM_FREE_KEYVAL <mpi_comm_free_keyval>`
|
||||
- MPI-2.0 (1996)
|
||||
|
||||
* - :ref:`MPI_COPY_FUNCTION <label-mpi-copy-delete-function>`
|
||||
- :ref:`MPI_COMM_COPY_ATTR_FUNCTION <mpi_comm_create_keyval>`
|
||||
- MPI-2.0 (1996)
|
||||
|
||||
* - :ref:`MPI_DELETE_FUNCTION <label-mpi-copy-delete-function>`
|
||||
- :ref:`MPI_COMM_DELETE_ATTR_FUNCTION <mpi_comm_create_keyval>`
|
||||
- MPI-2.0 (1996)
|
||||
|
||||
* - :ref:`MPI_ATTR_PUT <label-mpi-attr-put>`
|
||||
- :ref:`MPI_COMM_SET_ATTR <mpi_comm_set_attr>`
|
||||
- MPI-2.0 (1996)
|
||||
|
||||
* - :ref:`MPI_ATTR_GET <label-mpi-attr-get>`
|
||||
- :ref:`MPI_COMM_GET_ATTR <mpi_comm_get_attr>`
|
||||
- MPI-2.0 (1996)
|
||||
|
||||
* - :ref:`MPI_ATTR_DELETE <label-mpi-attr-delete>`
|
||||
- :ref:`MPI_COMM_DELETE_ATTR <mpi_comm_delete_attr>`
|
||||
- MPI-2.0 (1996)
|
||||
|
||||
* - :ref:`MPI_Comm_errhandler_fn <label-errhandler-fn>`
|
||||
- :ref:`MPI_Comm_errhandler_function <mpi_comm_create_errhandler>`
|
||||
- MPI-2.2 (2009)
|
||||
|
||||
* - :ref:`MPI_File_errhandler_fn <label-errhandler-fn>`
|
||||
- :ref:`MPI_File_errhandler_function <mpi_file_create_errhandler>`
|
||||
- MPI-2.2 (2009)
|
||||
|
||||
* - :ref:`MPI_Win_errhandler_fn <label-errhandler-fn>`
|
||||
- :ref:`MPI_Win_errhandler_function <mpi_win_create_errhandler>`
|
||||
- MPI-2.2 (2009)
|
||||
|
||||
* - :ref:`MPI_INFO_GET <label-mpi-info-get>`
|
||||
- :ref:`MPI_INFO_GET_STRING <mpi_info_get_string>`
|
||||
- MPI-4.0 (2021)
|
||||
|
||||
* - :ref:`MPI_INFO_GET_VALUELEN <label-mpi-info-get-valuelen>`
|
||||
- :ref:`MPI_INFO_GET_STRING <mpi_info_get_string>`
|
||||
- MPI-4.0 (2021)
|
||||
|
||||
* - :ref:`MPI_SIZEOF <label-mpi-sizeof>`
|
||||
- Fortran intrinsics``c_sizeof`` or ``storage_size``
|
||||
- MPI-4.0 (2021)
|
||||
|
||||
.. _label-mpi-keyval-create:
|
||||
|
||||
MPI_Keyval_create
|
||||
-----------------
|
||||
|
||||
``MPI_Keyval_create`` has been deprecated and replaced by
|
||||
``MPI_Comm_create_keyval``. The C binding of the new function is
|
||||
identical to the deprecated version. Hence, applications can simply
|
||||
replace the function that is being invoked.
|
||||
|
||||
The Fortran binding differs in that the ``extra_state`` argument is an
|
||||
address-sized integer in the new interfaces (vs. a regular integer in
|
||||
the old interfaces). Also, the copy and delete callback functions have
|
||||
Fortran bindings that are consistent with address-sized attributes.
|
||||
|
||||
.. code-block:: Fortran
|
||||
|
||||
USE mpi
|
||||
EXTERNAL my_copy_attr_function
|
||||
EXTERNAL my_copy_delete_function
|
||||
INTEGER ierror
|
||||
INTEGER comm_keyval
|
||||
INTEGER old_extra_state
|
||||
INTEGER(KIND=MPI_ADDRESS_KIND) new_extra_state
|
||||
|
||||
! Old way
|
||||
CALL MPI_KEYVAL_CREATE(my_copy_attr_function, my_copy_delete_function,
|
||||
comm_keyval, old_extra_state, ierror)
|
||||
|
||||
! New way
|
||||
CALL MPI_COMM_CREATE_KEYVAL(my_copy_attr_function, my_delete_attr_function,
|
||||
comm_keyval, new_extra_state, ierror)
|
||||
|
||||
|
||||
.. _label-mpi-keyval-free:
|
||||
|
||||
MPI_Keyval_free
|
||||
----------------
|
||||
|
||||
The binding of ``MPI_Keyval_free`` and ``MPI_Comm_free_keyval`` are identical
|
||||
for both C and Fortran. Users can directly replace the deprecated function with its
|
||||
new version.
|
||||
|
||||
.. _label-mpi-copy-delete-function:
|
||||
|
||||
MPI_Copy_function and MPI_Delete_function
|
||||
------------------------------------------
|
||||
|
||||
The ``MPI_Copy_function`` and ``MPI_Delete_function`` are only used in the
|
||||
deprecated function ``MPI_Keyval_create()``, as described in the
|
||||
:ref:`MPI_COMM_CREATE_KEYVAL <label-mpi-keyval-create>`.
|
||||
|
||||
For C codes, developers can simply use the new, exactly-equivalent
|
||||
type name (i.e., the return type, number, and type of parameters
|
||||
didn't change) ``MPI_Comm_copy_attr_function``, and
|
||||
``MPI_Comm_delete_attr_function`` respectively.
|
||||
|
||||
For Fortran applications, the only difference lies in required integer type for the
|
||||
``extra_state`` argument, which now has to be an address-sized integer.
|
||||
|
||||
.. _label-mpi-attr-put:
|
||||
|
||||
MPI_Attr_put
|
||||
------------
|
||||
|
||||
The C binding for the deprecated ``MPI_Attr_put`` is identical to its
|
||||
replacement, ``MPI_Comm_set_attr``. The Fortran binding differ in the
|
||||
usage of an addressed size integer for the attribute value in the new
|
||||
``MPI_Comm_set_attr`` vs. a regular integer in ``MPI_Attr_put``.
|
||||
|
||||
.. code-block:: Fortran
|
||||
|
||||
USE mpi
|
||||
INTEGER ierror
|
||||
INTEGER comm_keyval
|
||||
INTEGER old_attr_val
|
||||
INTEGER(KIND=MPI_ADDRESS_KIND) new_attr_val
|
||||
|
||||
! Old way
|
||||
CALL MPI_ATTR_PUT(MPI_COMM_WORLD, comm_keyval,
|
||||
old_attr_val, ierror)
|
||||
|
||||
! New way
|
||||
CALL MPI_COMM_SET_ATTR(MPI_COMM_WORLD, comm_keyval,
|
||||
new_attr_val, ierror)
|
||||
|
||||
.. _label-mpi-attr-get:
|
||||
|
||||
MPI_Attr_get
|
||||
------------
|
||||
|
||||
The C bindings of the old and the new interfaces are identical.
|
||||
Fortran binding differ in the usage of an addressed size integer for
|
||||
the attribute value in the new ``MPI_Comm_get_attr`` vs. a regular
|
||||
integer in ``MPI_Attr_get``.
|
||||
|
||||
.. _label-mpi-attr-delete:
|
||||
|
||||
MPI_Attr_delete
|
||||
---------------
|
||||
|
||||
C and Fortran bindings are identical for ``MPI_Attr_delete`` and
|
||||
``MPI_Comm_delete_attr``, hence developers should be able to just
|
||||
directly substitute one function call by the other.
|
||||
|
||||
|
||||
.. _label-mpi-info-get:
|
||||
|
||||
MPI_Info_get
|
||||
------------
|
||||
|
||||
Applications should replace the use of ``MPI_Info_get`` with ``MPI_Info_get_string``,
|
||||
but the usage differs slightly. See the example below.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
MPI_Info info;
|
||||
|
||||
// Create an info object using MPI_Info_create()
|
||||
...
|
||||
|
||||
// Retrieve the the value of a provided key later in the code
|
||||
char key[] = "my_key";
|
||||
char value[64];
|
||||
int valuelen=64;
|
||||
int flag;
|
||||
|
||||
// Old way
|
||||
MPI_Info_get(info, key, valuelen, &value, &flag);
|
||||
|
||||
// New way
|
||||
// Note that we pass the address of valuelen with
|
||||
// the new interfaces, since the variable will
|
||||
// contain the length of the value string after
|
||||
// the function call.
|
||||
MPI_Info_get_string(info, key, &valuelen, &value, &flag);
|
||||
}
|
||||
|
||||
.. _label-mpi-info-get-valuelen:
|
||||
|
||||
MPI_Info_get_valuelen
|
||||
---------------------
|
||||
|
||||
``MPI_Info_get_valuelen`` has been deprecated since the new function
|
||||
``MPI_Info_get_string`` also returns the length of the value string.
|
||||
Please refer to the example shown in :ref:`MPI_INFO_GET <label-mpi-info-get>`.
|
||||
|
||||
.. _label-mpi-sizeof:
|
||||
|
||||
MPI_Sizeof
|
||||
----------
|
||||
|
||||
The ``MPI_SIZEOF`` construct in Fortran has been deprected since there
|
||||
are standard Fortran language constructs such as ``c_sizeof`` and
|
||||
``storage_size`` that can be used instead.
|
||||
|
||||
.. _label-errhandler-fn:
|
||||
|
||||
MPI_Comm_errhandler_fn, MPI_File_errhandler_fn, MPI_Win_errhandler_fn
|
||||
---------------------------------------------------------------------
|
||||
|
||||
The following function typedefs have been deprecated and are superseded by new
|
||||
names. Other than the typedef names, the function signatures are exactly the same; the
|
||||
names were updated to match conventions of other function typedef names.
|
||||
|
||||
* ``MPI_Comm_errhandler_fn`` |rarrow| ``MPI_Comm_errhandler_function``
|
||||
* ``MPI_File_errhandler_fn`` |rarrow| ``MPI_File_errhandler_function``
|
||||
* ``MPI_Win_errhandler_fn`` |rarrow| ``MPI_Win_errhandler_function``
|
||||
@@ -0,0 +1,127 @@
|
||||
Extracting flags from the wrapper compilers
|
||||
===========================================
|
||||
|
||||
If you cannot use the wrapper compilers for some reason, there are
|
||||
multiple supported ways to extract the compiler/linker flags that you
|
||||
will need.
|
||||
|
||||
Using the ``--showme`` option
|
||||
-----------------------------
|
||||
|
||||
The wrapper compilers all support a ``--showme`` command line option
|
||||
that will show what commands would have been invoked.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Show the flags necessary to compile MPI C applications
|
||||
shell$ mpicc --showme:compile
|
||||
|
||||
# Show the flags necessary to link MPI C applications
|
||||
shell$ mpicc --showme:link
|
||||
|
||||
# Show all the flags necessary to build MPI C applications
|
||||
shell$ mpicc --showme
|
||||
|
||||
.. note:: If you pass ``--showme`` *and additional command line
|
||||
parameters* to the wrapper compiler, be sure to *also* pass in a
|
||||
filename. Otherwise, the ``--showme`` functionality will not
|
||||
display output as expected.
|
||||
|
||||
.. warning:: It is almost never a good idea to hard-code these results
|
||||
in a ``Makefile`` (or other build system). It is almost always
|
||||
best to run (for example) ``mpicc --showme:compile`` in a dynamic
|
||||
fashion to find out what you need. For example, GNU Make allows
|
||||
running commands and assigning their results to variables:
|
||||
|
||||
.. code-block:: make
|
||||
|
||||
MPI_COMPILE_FLAGS = $(shell mpicc --showme:compile)
|
||||
MPI_LINK_FLAGS = $(shell mpicc --showme:link)
|
||||
|
||||
my_app: my_app.c
|
||||
$(CC) $(MPI_COMPILE_FLAGS) my_app.c $(MPI_LINK_FLAGS) -o my_app
|
||||
|
||||
Using ``pkg-config``
|
||||
--------------------
|
||||
|
||||
Alternatively, Open MPI also installs ``pkg-config(1)`` configuration
|
||||
files under ``$libdir/pkgconfig``. If ``pkg-config`` is configured to
|
||||
find these files (e.g., if you add ``$libdir/pkgconfig`` |mdash| which
|
||||
is usually ``$prefix/lib/pkgconfig`` |mdash| to the
|
||||
``PKG_CONFIG_PATH`` environment variable), then compiling / linking
|
||||
Open MPI programs can be performed like this:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ export PKG_CONFIG_PATH=/opt/openmpi/lib/pkgconfig
|
||||
shell$ gcc hello_world_mpi.c -o hello_world_mpi -g \
|
||||
`pkg-config ompi-c --cflags --libs`
|
||||
shell$
|
||||
|
||||
Open MPI supplies multiple ``pkg-config`` configuration files; one for
|
||||
each different wrapper compiler (language):
|
||||
|
||||
* ``ompi``: Synonym for ``ompi-c``; Open MPI applications using the C
|
||||
MPI bindings
|
||||
* ``ompi-c``: Open MPI applications using the C MPI bindings
|
||||
* ``ompi-cxx``: Open MPI applications using the C MPI bindings
|
||||
* ``ompi-fort``: Open MPI applications using the Fortran MPI bindings
|
||||
|
||||
.. note:: Open MPI's ``pkg-config`` files *work properly*, but they
|
||||
probably aren't *technically correct*.
|
||||
|
||||
Specifically: Open MPI will list all of its dependent
|
||||
libraries that are necessary to link an MPI application,
|
||||
even if a given dependency has a ``.pc`` file and should
|
||||
therefore be listed as a ``Requires`` and/or
|
||||
``Requires.private`` in Open MPI's ``.pc`` files.
|
||||
|
||||
For example, Open MPI lists ``-lpmix`` in both ``Libs`` and
|
||||
``Libs.private``. But since PMIx provides its own
|
||||
``pmix.pc`` file, it would be more correct for Open MPI to
|
||||
*not* list ``-lpmix`` in ``Libs`` / ``Libs.private``, and
|
||||
instead include:
|
||||
|
||||
.. code-block::
|
||||
|
||||
Requires: pmix
|
||||
Requires.private: pmix
|
||||
|
||||
The end result is likely immaterial, but we document this
|
||||
just in case it ever becomes an issue someday.
|
||||
|
||||
|
||||
Using ``ompi_info``
|
||||
-------------------
|
||||
|
||||
This method is not directly suitable for getting all the compiler /
|
||||
linker flags needed to compile MPI applications because it does not
|
||||
include the relevant flags to find Open MPI's headers and libraries.
|
||||
But it does show a breakdown of all other flags.
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ompi_info --all | grep -i wrapper
|
||||
Wrapper extra CFLAGS:
|
||||
Wrapper extra CXXFLAGS:
|
||||
Wrapper extra FFLAGS:
|
||||
Wrapper extra FCFLAGS:
|
||||
Wrapper extra LDFLAGS:
|
||||
Wrapper extra LIBS: -lutil -lnsl -ldl -Wl,--export-dynamic -lm
|
||||
|
||||
This installation is *only* adding options in the ``xLIBS`` areas of the
|
||||
wrapper compilers; all other values are blank (remember: the ``-I``'s
|
||||
and ``-L``'s are implicit).
|
||||
|
||||
Note that the ``--parsable`` option can be used to obtain
|
||||
machine-parsable versions of this output. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ompi_info --all --parsable | grep wrapper:extra
|
||||
option:wrapper:extra_cflags:
|
||||
option:wrapper:extra_cxxflags:
|
||||
option:wrapper:extra_fflags:
|
||||
option:wrapper:extra_fcflags:
|
||||
option:wrapper:extra_ldflags:
|
||||
option:wrapper:extra_libs:-lutil -lnsl -ldl -Wl,--export-dynamic -lm
|
||||
@@ -0,0 +1,26 @@
|
||||
Building MPI applications
|
||||
=========================
|
||||
|
||||
The simplest way to compile and link MPI applications is to use the
|
||||
Open MPI "wrapper" compilers.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
quickstart
|
||||
customizing-wrappers
|
||||
extracting-wrapper-flags
|
||||
abi-compatibility
|
||||
removed-mpi-constructs
|
||||
deprecation-warnings
|
||||
building-static-apps
|
||||
|
||||
.. warning:: Open MPI does not attempt to interoperate with other MPI
|
||||
implementations, nor executables that were compiled for
|
||||
them. MPI applications need to be compiled and linked
|
||||
with Open MPI in order to run under Open MPI.
|
||||
|
||||
See the :doc:`section on Open MPI's version numbering
|
||||
scheme </version-numbering>` for information about binary
|
||||
compatibility between different versions of Open MPI for
|
||||
applications compiled with another version.
|
||||
@@ -0,0 +1,67 @@
|
||||
.. _label-quickstart-building-apps:
|
||||
|
||||
Quick start: Building MPI applications
|
||||
======================================
|
||||
|
||||
Although this section skips many details, it offers examples that will
|
||||
probably work in many environments.
|
||||
|
||||
.. caution:: Note that this section is a "Quick start" |mdash| it does
|
||||
not attempt to be comprehensive or describe how to build Open MPI
|
||||
in all supported environments. The examples below may therefore
|
||||
not work exactly as shown in your environment.
|
||||
|
||||
Please consult the other sections in this chapter for more details,
|
||||
if necessary.
|
||||
|
||||
Open MPI provides "wrapper" compilers that should be used for
|
||||
compiling MPI and OpenSHMEM applications:
|
||||
|
||||
+---------+-----------------------------------+
|
||||
| C | :ref:`mpicc(1) <man1-mpicc>`, |
|
||||
| | :ref:`oshcc(1) <man1-oshcc>` |
|
||||
+---------+-----------------------------------+
|
||||
| C++ | :ref:`mpic++(1) <man1-mpic++>`, |
|
||||
| | :ref:`oshc++(1) <man1-oshc++>` |
|
||||
| | (and :ref:`mpiCC(1) <man1-mpiCC>` |
|
||||
| | and :ref:`oshCC(1) <man1-oshCC>` |
|
||||
| | if you filesystem is |
|
||||
| | case-sensitive) |
|
||||
+---------+-----------------------------------+
|
||||
| Fortran | :ref:`mpifort(1) <man1-mpifort>`, |
|
||||
| | :ref:`oshfort(1) <man1-oshfort>` |
|
||||
+---------+-----------------------------------+
|
||||
|
||||
.. caution:: The legacy names ``mpif77`` and ``mpif90`` still exist,
|
||||
and are simply symbolic links to the ``mpifort`` wrapper
|
||||
compiler. Users are strongly encouraged to update all
|
||||
build scripts to use ``mpifort`` instead of ``mpif77``
|
||||
and ``mpif90``.
|
||||
|
||||
The intent is that users can simply invoke the Open MPI wrapper
|
||||
compiler instead of their usual language compiler. For example,
|
||||
instead of invoking your usual C compiler to build your MPI C
|
||||
appliance, use ``mpicc``:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpicc hello_world_mpi.c -o hello_world_mpi -g
|
||||
shell$
|
||||
|
||||
For OpenSHMEM applications:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ oshcc hello_shmem.c -o hello_shmem -g
|
||||
shell$
|
||||
|
||||
All the wrapper compilers do is add a variety of compiler and linker
|
||||
flags to the command line and then invoke a back-end compiler. To be
|
||||
specific: the wrapper compilers do not parse source code at all; they
|
||||
are solely command-line manipulators, and have nothing to do with the
|
||||
actual compilation or linking of programs. The end result is an MPI
|
||||
executable that is properly linked to all the relevant libraries.
|
||||
|
||||
.. caution:: It is *absolutely not sufficient* to simply add ``-lmpi``
|
||||
to your link line and assume that you will obtain a valid
|
||||
Open MPI executable.
|
||||
@@ -0,0 +1,531 @@
|
||||
.. _label-removed-mpi-constructs:
|
||||
|
||||
Removed MPI constructs
|
||||
======================
|
||||
|
||||
Starting with v4.0.0, Open MPI |mdash| by default |mdash| removes the
|
||||
prototypes from ``mpi.h`` for MPI symbols that were deprecated in 1996
|
||||
in the MPI-2.0 standard, and finally removed from the MPI-3.0 standard
|
||||
(2012).
|
||||
|
||||
Specifically, the following symbols (specified in the MPI
|
||||
language-neutral names) are no longer prototyped in ``mpi.h`` by
|
||||
default:
|
||||
|
||||
.. note:: You may need to scroll right in the following table.
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Removed symbol
|
||||
|
||||
(click for more details, below)
|
||||
- Replaced with
|
||||
|
||||
(click to go to the corresponding man page)
|
||||
- Deprecated
|
||||
- Removed
|
||||
|
||||
* - :ref:`MPI_ADDRESS <label-mpi-address>`
|
||||
- :ref:`MPI_GET_ADDRESS <mpi_get_address>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_ERRHANDLER_CREATE <label-mpi-errhandler-create>`
|
||||
- :ref:`MPI_COMM_CREATE_ERRHANDLER <mpi_comm_create_errhandler>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_ERRHANDLER_GET <label-mpi-errhandler-get>`
|
||||
- :ref:`MPI_COMM_GET_ERRHANDLER <mpi_comm_get_errhandler>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_ERRHANDLER_SET <label-mpi-errhandler-set>`
|
||||
- :ref:`MPI_COMM_SET_ERRHANDLER <mpi_comm_set_errhandler>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_TYPE_EXTENT <label-mpi-type-extent>`
|
||||
- :ref:`MPI_TYPE_GET_EXTENT <mpi_type_get_extent>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_TYPE_HINDEXED <label-mpi-type-hindexed>`
|
||||
- :ref:`MPI_TYPE_CREATE_HINDEXED <mpi_type_create_hindexed>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_TYPE_HVECTOR <label-mpi-type-hvector>`
|
||||
- :ref:`MPI_TYPE_CREATE_HVECTOR <mpi_type_create_hvector>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_TYPE_LB <label-mpi-type-lb>`
|
||||
- :ref:`MPI_TYPE_GET_EXTENT <mpi_type_get_extent>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_TYPE_STRUCT <label-mpi-type-struct>`
|
||||
- :ref:`MPI_TYPE_CREATE_STRUCT <mpi_type_create_struct>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_TYPE_UB <label-mpi-type-ub>`
|
||||
- :ref:`MPI_TYPE_GET_EXTENT <mpi_type_get_extent>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_LB <label-mpi-lb-ub>`
|
||||
- :ref:`MPI_TYPE_CREATE_RESIZED <mpi_type_create_resized>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_UB <label-mpi-lb-ub>`
|
||||
- :ref:`MPI_TYPE_CREATE_RESIZED <mpi_type_create_resized>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_COMBINED_HINDEXED_INTEGER <label-mpi-combiner-fortran-integers>`
|
||||
- :ref:`MPI_COMBINER_HINDEXED <mpi_type_get_envelope>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_COMBINED_HVECTOR_INTEGER <label-mpi-combiner-fortran-integers>`
|
||||
- :ref:`MPI_COMBINER_HVECTOR <mpi_type_get_envelope>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_COMBINED_STRUCT_INTEGER <label-mpi-combiner-fortran-integers>`
|
||||
- :ref:`MPI_COMBINER_STRUCT <mpi_type_get_envelope>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
* - :ref:`MPI_HANDLER_FUNCTION <label-mpi-handler-function>`
|
||||
- :ref:`MPI_COMM_ERRHANDLER_FUNCTION <mpi_comm_create_errhandler>`
|
||||
- MPI-2.0 (1996)
|
||||
- MPI-3.0 (2012)
|
||||
|
||||
Although these symbols are no longer prototyped in ``mpi.h``, *they are
|
||||
still present in the MPI library in Open MPI* |ompi_series|. This enables
|
||||
legacy MPI applications to *link and run* successfully with Open MPI
|
||||
|ompi_series|, even though they will fail to *compile*.
|
||||
|
||||
Furthermore, the MPI C++ interfaces were deprecated in version
|
||||
2.2, and then were removed in MPI-3.0. Starting
|
||||
from v5.0.0, Open MPI does not support the C++ interfaces
|
||||
any more. Users who would like to continue using the C++ interfaces of
|
||||
MPI will need to use an older release of Open MPI.
|
||||
|
||||
.. warning:: The Open MPI team **strongly** encourages all
|
||||
MPI application developers to stop using these constructs that were
|
||||
first deprecated over 20 years ago, and finally removed from the MPI
|
||||
specification in MPI-3.0 (in 2012).
|
||||
|
||||
The FAQ items in this category show how to update your application to
|
||||
stop using these removed symbols.
|
||||
|
||||
All that being said, if you are unable to immediately update your
|
||||
application to stop using these removed MPI-1 symbols, you can
|
||||
re-enable them in ``mpi.h`` by configuring Open MPI with the
|
||||
``--enable-mpi1-compatibility`` flag.
|
||||
|
||||
.. note:: Future releases of Open MPI may remove these symbols
|
||||
altogether.
|
||||
|
||||
Why is Open MPI breaking the compilation of MPI applications?
|
||||
------------------------------------------------------------------
|
||||
|
||||
The Open MPI developer community decided to take a first step of
|
||||
removing prototypes of deprecated functions from ``mpi.h`` starting
|
||||
with the Open MPI v4.0.x series for the following reasons:
|
||||
|
||||
#. The first set of symbols have been deprecated since *1996.* It's
|
||||
time to start raising awareness for developers who are
|
||||
inadvertently still using these removed symbols.
|
||||
#. The MPI Forum removed a substantial set of symbols from the MPI-3.0
|
||||
specification in 2012. This is a sign that the Forum itself
|
||||
recognizes that these removed symbols are no longer needed.
|
||||
#. More functions have been deprecated in MPI 2.2 and MPI 4.0, and
|
||||
additional functions are expected to be deprecated and removed in
|
||||
future MPI versions. It is in the interest of both, developers and
|
||||
end-users, to minimize confusion as much as possible, and stick
|
||||
closely to the MPI specification.
|
||||
#. Note that Open MPI *did not fully remove* these removed symbols: we
|
||||
just made it slightly more painful to get to them. This is an
|
||||
attempt to raise awareness so that MPI application developers can
|
||||
update their applications (it's easy!).
|
||||
|
||||
In short: the only way to finally be able to remove these removed
|
||||
symbols from Open MPI someday is to have a "grace period" where the
|
||||
MPI application developers are a) made aware that they are using
|
||||
removed symbols, and b) educated how to update their applications.
|
||||
|
||||
We, the Open MPI developers, recognize that your MPI application
|
||||
failing to compile with Open MPI may be a nasty surprise. We
|
||||
apologize for that.
|
||||
|
||||
Our intent is simply to use this minor shock to raise awareness and
|
||||
use it as an educational opportunity to show you how to update your
|
||||
application (or direct your friendly neighborhood MPI application
|
||||
developer to this FAQ) to stop using these removed MPI symbols.
|
||||
|
||||
Thank you!
|
||||
|
||||
.. _label-mpi-address:
|
||||
|
||||
Stop using MPI_ADDRESS
|
||||
----------------------
|
||||
|
||||
In C, the only thing that changed was the function name:
|
||||
``MPI_Address()`` |rarrow| ``MPI_Get_address()``. Nothing else needs
|
||||
to change:
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
char buffer[30];
|
||||
MPI_Aint address;
|
||||
|
||||
// Old way
|
||||
MPI_Address(buffer, &address);
|
||||
|
||||
// New way
|
||||
MPI_Get_address(buffer, &address);
|
||||
|
||||
In Fortran, the type of the parameter changed from ``INTEGER``
|
||||
|rarrow| ``INTEGER(KIND=MPI_ADDRESS_KIND)`` so that it can hold
|
||||
larger values (e.g., 64 bit pointers):
|
||||
|
||||
.. code-block:: Fortran
|
||||
|
||||
USE mpi
|
||||
REAL buffer
|
||||
INTEGER ierror
|
||||
INTEGER old_address
|
||||
INTEGER(KIND = MPI_ADDRESS_KIND) new_address
|
||||
|
||||
! Old way
|
||||
CALL MPI_ADDRESS(buffer, old_address, ierror)
|
||||
|
||||
! New way
|
||||
CALL MPI_GET_ADDRESS(buffer, new_address, ierror)
|
||||
|
||||
.. _label-mpi-errhandler-create:
|
||||
|
||||
Stop using MPI_ERRHANDLER_CREATE
|
||||
--------------------------------
|
||||
|
||||
In C, *effectively* the only thing that changed was the name of the
|
||||
function: ``MPI_Errhandler_create()`` |rarrow|
|
||||
``MPI_Comm_create_errhandler()``.
|
||||
|
||||
*Technically*, the type of the first parameter also changed
|
||||
(``MPI_Handler_function`` |rarrow| ``MPI_Comm_errhandler_function``),
|
||||
but most applications do not use this type directly and may not even
|
||||
notice the change.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
void my_errhandler_function(MPI_Comm *comm, int *code, ...)
|
||||
{
|
||||
// Do something useful to handle the error
|
||||
}
|
||||
|
||||
void some_function(void)
|
||||
{
|
||||
MPI_Errhandler my_handler;
|
||||
|
||||
// Old way
|
||||
MPI_Errhandler_create(my_errhandler_function, &my_handler);
|
||||
|
||||
// New way
|
||||
MPI_Comm_create_errhandler(my_errhandler_function, &my_handler);
|
||||
}
|
||||
|
||||
In Fortran, only the subroutine name changed:
|
||||
``MPI_ERRHANDLER_CREATE`` |rarrow| ``MPI_COMM_CREATE_ERRHANDLER``.
|
||||
|
||||
.. code-block:: Fortran
|
||||
|
||||
USE mpi
|
||||
EXTERNAL my_errhandler_function
|
||||
INTEGER ierror
|
||||
INTEGER my_handler
|
||||
|
||||
! Old way
|
||||
CALL MPI_ERRHANDLER_CREATE(my_errhandler_function, my_handler, ierror)
|
||||
|
||||
! New way
|
||||
CALL MPI_COMM_CREATE_ERRHANDLER(my_errhandler_function, my_handler, ierror)
|
||||
|
||||
.. _label-mpi-errhandler-get:
|
||||
|
||||
Stop using MPI_ERRHANDLER_GET
|
||||
-----------------------------
|
||||
|
||||
In both C and Fortran, the only thing that changed with regards to
|
||||
``MPI_ERRHANDLER_GET`` is the name: ``MPI_ERRHANDLER_GET`` |rarrow|
|
||||
``MPI_COMM_GET_ERRHANDLER``.
|
||||
|
||||
All parameter types stayed the same.
|
||||
|
||||
.. _label-mpi-errhandler-set:
|
||||
|
||||
Stop using MPI_ERRHANDLER_SET
|
||||
-----------------------------
|
||||
|
||||
In both C and Fortran, the only thing that changed with regards to
|
||||
``MPI_ERRHANDLER_SET`` is the name: ``MPI_ERRHANDLER_SET`` |rarrow|
|
||||
``MPI_COMM_SET_ERRHANDLER``.
|
||||
|
||||
All parameter types stayed the same.
|
||||
|
||||
.. _label-mpi-type-hindexed:
|
||||
|
||||
Stop using MPI_TYPE_HINDEXED
|
||||
----------------------------
|
||||
|
||||
In both C and Fortran, *effectively* the only change is the name of
|
||||
the function: ``MPI_TYPE_HINDEXED`` |rarrow|
|
||||
``MPI_TYPE_CREATE_HINDEXED``.
|
||||
|
||||
In C, the new function also has a ``const`` attribute on the two array
|
||||
parameters, but most applications won't notice the difference.
|
||||
|
||||
All other parameter types stayed the same.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
int count = 2;
|
||||
int block_lengths[] = { 1, 2 };
|
||||
MPI_Aint displacements[] = { 0, sizeof(int) };
|
||||
MPI_Datatype newtype;
|
||||
|
||||
// Old way
|
||||
MPI_Type_hindexed(count, block_lengths, displacements, MPI_INT, &newtype);
|
||||
|
||||
// New way
|
||||
MPI_Type_create_hindexed(count, block_lengths, displacements, MPI_INT, &newtype);
|
||||
|
||||
.. _label-mpi-type-hvector:
|
||||
|
||||
Stop using MPI_TYPE_HVECTOR
|
||||
---------------------------
|
||||
|
||||
In both C and Fortran, the only change is the name of the function:
|
||||
``MPI_TYPE_HVECTOR`` |rarrow| ``MPI_TYPE_CREATE_HVECTOR``.
|
||||
|
||||
All parameter types stayed the same.
|
||||
|
||||
.. _label-mpi-type-struct:
|
||||
|
||||
Stop using MPI_TYPE_STRUCT
|
||||
--------------------------
|
||||
|
||||
In both C and Fortran, *effectively* the only change is the name of
|
||||
the function: ``MPI_TYPE_STRUCT`` |rarrow| ``MPI_TYPE_CREATE_STRUCT``.
|
||||
|
||||
In C, the new function also has a ``const`` attribute on the three
|
||||
array parameters, but most applications won't notice the difference.
|
||||
|
||||
All other parameter types stayed the same.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
int count = 2;
|
||||
int block_lengths[] = { 1, 2 };
|
||||
MPI_Aint displacements[] = { 0, sizeof(int) };
|
||||
MPI_Datatype datatypes[] = { MPI_INT, MPI_DOUBLE };
|
||||
MPI_Datatype newtype;
|
||||
|
||||
// Old way
|
||||
MPI_Type_struct(count, block_lengths, displacements, datatypes, &newtype);
|
||||
|
||||
// New way
|
||||
MPI_Type_create_struct(count, block_lengths, displacements, datatypes, &newtype);
|
||||
|
||||
.. _label-mpi-type-extent:
|
||||
|
||||
Stop using MPI_TYPE_EXTENT
|
||||
--------------------------
|
||||
|
||||
In both C and Fortran, the ``MPI_TYPE_EXTENT`` function is superseded
|
||||
by the slightly-different ``MPI_TYPE_GET_EXTENT`` function: the new
|
||||
function also returns the lower bound.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
MPI_Aint lb;
|
||||
MPI_Aint extent;
|
||||
|
||||
// Old way
|
||||
MPI_Type_extent(MPI_INT, &extent);
|
||||
|
||||
// New way
|
||||
MPI_Type_get_extent(MPI_INT, &lb, &extent);
|
||||
|
||||
.. _label-mpi-type-lb:
|
||||
|
||||
Stop using MPI_TYPE_LB
|
||||
----------------------
|
||||
|
||||
In both C and Fortran, the ``MPI_TYPE_LB`` function is superseded by
|
||||
the slightly-different ``MPI_TYPE_GET_EXTENT`` function: the new
|
||||
function also returns the extent.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
MPI_Aint lb;
|
||||
MPI_Aint extent;
|
||||
|
||||
// Old way
|
||||
MPI_Type_lb(MPI_INT, &lb);
|
||||
|
||||
// New way
|
||||
MPI_Type_get_extent(MPI_INT, &lb, &extent);
|
||||
|
||||
.. _label-mpi-type-ub:
|
||||
|
||||
Stop using MPI_TYPE_UB
|
||||
----------------------
|
||||
|
||||
In both C and Fortran, the ``MPI_TYPE_UB`` function is superseded by
|
||||
the slightly-different ``MPI_TYPE_GET_EXTENT`` function: the new
|
||||
function returns the lower bound and the extent, which can be used to
|
||||
compute the upper bound.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
MPI_Aint lb, ub;
|
||||
MPI_Aint extent;
|
||||
|
||||
// Old way
|
||||
MPI_Type_ub(MPI_INT, &ub);
|
||||
|
||||
// New way
|
||||
MPI_Type_get_extent(MPI_INT, &lb, &extent);
|
||||
ub = lb + extent
|
||||
|
||||
Note the ``ub`` calculation after calling ``MPI_Type_get_extent()``.
|
||||
|
||||
.. _label-mpi-lb-ub:
|
||||
|
||||
Stop using MPI_LB / MPI_UB
|
||||
--------------------------
|
||||
|
||||
The ``MPI_LB`` and ``MPI_UB`` positional markers were fully replaced
|
||||
with ``MPI_TYPE_CREATE_RESIZED`` in MPI-2.0.
|
||||
|
||||
Prior to MPI-2.0, ``MPI_UB`` and ``MPI_LB`` were intended to be used
|
||||
as input to ``MPI_TYPE_STRUCT`` (which, itself, has been deprecated
|
||||
and renamed to ``MPI_TYPE_CREATE_STRUCT``). The same end effect can
|
||||
now be achieved with ``MPI_TYPE_CREATE_RESIZED``. For example, using
|
||||
the old method:
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
int count = 3;
|
||||
int block_lengths[] = { 1, 1, 1 };
|
||||
MPI_Aint displacements[] = { -2, 0, 10 };
|
||||
MPI_Datatype datatypes[] = { MPI_LB, MPI_INT, MPI_UB };
|
||||
MPI_Datatype newtype;
|
||||
|
||||
MPI_Type_struct(count, block_lengths, displacements, datatypes, &newtype);
|
||||
MPI_Type_commit(&newtype);
|
||||
|
||||
MPI_Aint ub, lb, extent;
|
||||
MPI_Type_lb(newtype, &lb);
|
||||
MPI_Type_ub(newtype, &ub);
|
||||
MPI_Type_extent(newtype, &extent);
|
||||
printf("OLD: LB=%d, UB=%d, extent=%d\n",
|
||||
lb, ub, extent);
|
||||
|
||||
If we run the above, we get an output of:
|
||||
|
||||
.. code-block::
|
||||
|
||||
OLD: LB=-2, UB=10, extent=12
|
||||
|
||||
The ``MPI_TYPE_RESIZED`` function allows us to take any arbitrary
|
||||
datatype and set the lower bound and extent directly (which indirectly
|
||||
sets the upper bound), without needing to setup the arrays and
|
||||
computing the displacements necessary to invoke
|
||||
``MPI_TYPE_CREATE_STRUCT``.
|
||||
|
||||
Aside from the ``printf`` statement, the following example is exactly
|
||||
equivalent to the prior example (:ref:`see the MPI_TYPE_UB section
|
||||
<label-mpi-type-ub>` for a mapping of ``MPI_TYPE_UB`` to
|
||||
``MPI_TYPE_GET_EXTENT``):
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
MPI_Datatype newtype;
|
||||
|
||||
MPI_Type_create_resized(MPI_INT, -2, 12, &newtype);
|
||||
MPI_Type_commit(&newtype);
|
||||
|
||||
MPI_Aint ub, lb, extent;
|
||||
MPI_Type_get_extent(newtype, &lb, &extent);
|
||||
ub = lb + extent;
|
||||
printf("NEW: LB=%d, UB=%d, extent=%d\n",
|
||||
lb, ub, extent);
|
||||
|
||||
If we run the above, we get an output of:
|
||||
|
||||
.. code-block::
|
||||
|
||||
NEW: LB=-2, UB=10, extent=12
|
||||
|
||||
.. _label-mpi-combiner-fortran-integers:
|
||||
|
||||
Stop using MPI_COMBINER_HINDEXED_INTEGER, MPI_COMBINER_HVECTOR_INTEGER, and MPI_COMBINER_STRUCT_INTEGER
|
||||
-------------------------------------------------------------------------------------------------------
|
||||
|
||||
The ``MPI_COMBINER_HINDEXED_INTEGER``,
|
||||
``MPI_COMBINER_HVECTOR_INTEGER``, and ``MPI_COMBINER_STRUCT_INTEGER``
|
||||
constants could previously be returned from ``MPI_TYPE_GET_ENVELOPE``.
|
||||
|
||||
Starting with MPI-3.0, these values will never be returned. Instead,
|
||||
they will just return the same names, but without the ``_INTEGER``
|
||||
suffix. Specifically:
|
||||
|
||||
* ``MPI_COMBINER_HINDEXED_INTEGER`` |rarrow| ``MPI_COMBINER_HINDEXED``
|
||||
* ``MPI_COMBINER_HVECTOR_INTEGER`` |rarrow| ``MPI_COMBINER_HVECTOR``
|
||||
* ``MPI_COMBINER_STRUCT_INTEGER`` |rarrow| ``MPI_COMBINER_STRUCT``
|
||||
|
||||
If your Fortran code is using any of the ``_INTEGER``-suffixed names,
|
||||
you can just delete the ``_INTEGER`` suffix.
|
||||
|
||||
.. _label-mpi-handler-function:
|
||||
|
||||
Stop using MPI_Handler_function
|
||||
-------------------------------
|
||||
|
||||
The ``MPI_Handler_function`` C type is only used in the
|
||||
deprecated/removed function ``MPI_Errhandler_create()``, as described
|
||||
:ref:`in the MPI_ERRHANDLER_CREATE section
|
||||
<label-mpi-errhandler-create>`.
|
||||
|
||||
Most MPI applications likely won't use this type at all. But if they
|
||||
do, they can simply use the new, exactly-equivalent type name (i.e.,
|
||||
the return type, number, and type of parameters didn't change):
|
||||
``MPI_Comm_errhandler_function``.
|
||||
|
||||
.. code-block:: c++
|
||||
|
||||
void my_errhandler_function(MPI_Comm *comm, int *code, ...)
|
||||
{
|
||||
// Do something useful to handle the error
|
||||
}
|
||||
|
||||
void some_function(void)
|
||||
{
|
||||
// Old way
|
||||
MPI_Handler_function *old_ptr = my_errhandler_function;
|
||||
|
||||
// New way
|
||||
MPI_Comm_errhandler_function *new_ptr = my_errhandler_function;
|
||||
}
|
||||
|
||||
The ``MPI_Handler_function`` type isn't used at all in the Fortran
|
||||
bindings.
|
||||
@@ -0,0 +1,208 @@
|
||||
Contributing to Open MPI
|
||||
========================
|
||||
|
||||
There are many ways to contribute. Here are a few:
|
||||
|
||||
#. Subscribe to `the mailing lists
|
||||
<https://www.open-mpi.org/community/lists/ompi.php>`_ and become
|
||||
active in the discussions.
|
||||
|
||||
#. Obtain `a Git clone <https://github.com/open-mpi/ompi/>`_ of Open
|
||||
MPI's code base and start looking through the code.
|
||||
|
||||
.. note:: Be sure to see the :doc:`Developers guide
|
||||
</developers/index>` for technical details about the
|
||||
code base and how to build it).
|
||||
|
||||
#. Report bug fixes to the main code base.
|
||||
|
||||
* First, ensure the bug was not already reported by searching
|
||||
`existing GitHub Issues
|
||||
<https://github.com/open-mpi/ompi/issues>`_.
|
||||
* If you're unable to find an open issue addressing the problem,
|
||||
`open a new one <https://github.com/open-mpi/ompi/issues/new>`_.
|
||||
|
||||
#. Submit bug fixes to the main code base.
|
||||
|
||||
* Awesome! Open a new GitHub pull request with the patch.
|
||||
|
||||
* Please submit bug fixes / new features on the ``main`` branch
|
||||
first, and then port them to the relevant release branch(es)
|
||||
after they have been accepted on ``main``. :ref:`See this
|
||||
section for more information <git-github-branch-scheme-label>`.
|
||||
|
||||
* Ensure the PR description clearly describes the problem and
|
||||
solution. If there is an existing GitHub issue open describing
|
||||
this bug, please include it in the description so we can track
|
||||
them together.
|
||||
* Be sure to see the :ref:`Open source contributions
|
||||
<contributing-open-source-label>` section, below.
|
||||
|
||||
#. Submit feature enhancements and/or new components to the main code
|
||||
base.
|
||||
|
||||
* Awesome! We love new ideas!
|
||||
* You might want to suggest your change on the `devel mail list
|
||||
<https://www.open-mpi.org/community/lists/ompi.php>`_ before you
|
||||
start writing code. The `developer level technical information
|
||||
on the internals of Open MPI
|
||||
<https://www.open-mpi.org/faq/?category=developers>`_ may also be
|
||||
useful for large scale features.
|
||||
* If you're contributing a large new piece of functionality, that
|
||||
will be best viewed if you |mdash| or someone, anyone |mdash| is
|
||||
also stepping up to help maintain that functionality over time.
|
||||
We love new ideas and new features, but we do need to be
|
||||
realistic in what we can reliably test and deliver to our users.
|
||||
* Be sure to see the :ref:`Open source contributions
|
||||
<contributing-open-source-label>` section, below.
|
||||
|
||||
#. Submit fixes and/or entirely new content to this documentation.
|
||||
|
||||
* Docs are great! We always need help with documentation.
|
||||
* These docs are authored using ReStructured Text markup with the
|
||||
Sphinx rendering tool under the ``docs/`` directory in the
|
||||
repository. This means you can submit a pull request with your
|
||||
docs updates, just like you would for any Open MPI code contribution.
|
||||
* :ref:`See this section <developers-installing-sphinx-label>` and
|
||||
:ref:`also this section <developers-rst-for-markdown-expats>` for
|
||||
information about how to install Sphinx locally and some
|
||||
high-level logistical instructions on how to write these docs.
|
||||
|
||||
#. Provide testing resources:
|
||||
|
||||
#. For Github Pull Request Continuous Integration (CI)
|
||||
#. For nightly snapshot builds and testing
|
||||
|
||||
.. _contributing-open-source-label:
|
||||
|
||||
Open source contributions
|
||||
-------------------------
|
||||
|
||||
All code contributions are submitted as pull requests on the `Open
|
||||
MPI GitHub repository <https://github.com/open-mpi/ompi/>`_.
|
||||
|
||||
.. important:: All commits must include a ``Signed-off-by:`` line,
|
||||
indicating the submitter's agreement to the :ref:`Open
|
||||
MPI Contributor's Declaration
|
||||
<contributing-contributors-declaration-label>`.
|
||||
|
||||
.. _contributing-contributors-declaration-label:
|
||||
|
||||
Contributor's Declaration
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
In order to ensure that we can keep distributing Open MPI under our
|
||||
:doc:`open source license </license/index>`, we need to ensure that
|
||||
all contributions are compatible with that license. Put differently:
|
||||
we need to have an established intellectual property pedigree of the
|
||||
code in Open MPI. This means being able to ensure that all code
|
||||
included in Open MPI is free, open source, and able to be distributed
|
||||
under :doc:`the BSD license </license/index>`.
|
||||
|
||||
Open MPI has therefore adopted requirements based on the signed-off-by
|
||||
process as described in Section 11 of the Linux kernel document on
|
||||
`Submitting Patches
|
||||
<https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>`_.
|
||||
Each proposed contribution to the Open MPI code base must include the
|
||||
text ``Signed-off-by:`` followed by the contributor's name and email
|
||||
address.
|
||||
|
||||
.. admonition:: Pro tip
|
||||
:class: tip
|
||||
|
||||
You can use the ``-s`` flag to the ``git commit`` command (i.e.,
|
||||
``git commit -s ...``) to automatically add the appropriate
|
||||
``Signed-off-by:`` line to your commit message.
|
||||
|
||||
The ``Signed-off-by:`` line is a developer's certification that he or
|
||||
she has the right to submit the patch for inclusion into the project,
|
||||
and indicates agreement to the Developer's Certificate of Origin:
|
||||
|
||||
By making a contribution to this project, I certify that:
|
||||
|
||||
#. The contribution was created in whole or in part by me and I
|
||||
have the right to submit it under the :doc:`Open MPI open
|
||||
source license </license/index>`; or
|
||||
|
||||
#. The contribution is based upon previous work that, to the best
|
||||
of my knowledge, is covered under an appropriate open source
|
||||
license and I have the right under that license to submit that
|
||||
work with modifications, whether created in whole or in part by
|
||||
me, under the :doc:`Open MPI open source license
|
||||
</license/index/>` (unless I am permitted to submit under a
|
||||
different license); or
|
||||
|
||||
#. The contribution was provided directly to me by some other
|
||||
person who certified (1) or (2) and I have not modified it.
|
||||
|
||||
#. I understand and agree that this project and the contribution
|
||||
are public and that a record of the contribution (including all
|
||||
personal information I submit with it, including my sign-off)
|
||||
is maintained indefinitely and may be redistributed consistent
|
||||
with this project and the open source license(s) involved.
|
||||
|
||||
Proposed contributions failing to include the ``Signed-off-by:``
|
||||
certification will not be accepted into any Open MPI code
|
||||
repository. The community reserves the right to revert any commit
|
||||
inadvertently made without the required certification.
|
||||
|
||||
.. note:: This policy prevents a situation where intellectual property
|
||||
gets into the Open MPI code base and then someone later
|
||||
claims that we owe them money for it. Open MPI is a free,
|
||||
open source code base. We intend it to remain that way.
|
||||
|
||||
If you have not already done so, please ensure that *every* commit in
|
||||
your pull request contains the ``Signed-off-by:`` line.
|
||||
|
||||
Git commit messages
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Please write a good Git commit message, with a short first line
|
||||
describing *what* was done, and then a description of *why* it was
|
||||
done.
|
||||
|
||||
.. admonition:: Pro tip
|
||||
:class: tip
|
||||
|
||||
`See this blog entry <https://cbea.ms/git-commit/>`_ for an
|
||||
excellent description of how to write a good Git commit message.
|
||||
|
||||
Code style
|
||||
^^^^^^^^^^
|
||||
|
||||
There are a small number of style rules for Open MPI:
|
||||
|
||||
#. For all code:
|
||||
|
||||
* 4 space tabs. No more, no less.
|
||||
* No tab characters **at all**. 2 indentations are 8 spaces
|
||||
|mdash| not a tab.
|
||||
* m4 code is a bit weird in terms of indentation: we don't have a
|
||||
good, consistent indentation style in our existing code. But
|
||||
still: no tab characters at all.
|
||||
|
||||
#. For C code:
|
||||
|
||||
* We prefer if all blocks are enclosed in ``{}`` (even 1-line
|
||||
blocks).
|
||||
* We prefer that if you are testing equality with a constant, put
|
||||
the constant on the **left** of the ``==``. E.g., ``if (NULL ==
|
||||
ptr)``.
|
||||
* If there are no parameters to a C function, declare it with
|
||||
``(void)`` (vs. ``()``).
|
||||
|
||||
Closed source contributions
|
||||
---------------------------
|
||||
|
||||
While we are creating free / open-source software, and we would prefer
|
||||
if everyone's contributions to Open MPI were also free / open-source,
|
||||
we certainly recognize that other organizations have different goals
|
||||
from us. Such is the reality of software development in today's
|
||||
global economy.
|
||||
|
||||
As such, it is perfectly acceptable to make non-free / non-open-source
|
||||
contributions to Open MPI. We obviously cannot accept such
|
||||
contributions into the main code base, but you are free to distribute
|
||||
plugins, enhancements, etc. as you see fit. Indeed, the :doc:`the BSD
|
||||
license </license/index>` is extremely liberal in its redistribution
|
||||
provisions.
|
||||
@@ -0,0 +1,32 @@
|
||||
Running ``autogen.pl``
|
||||
======================
|
||||
|
||||
You can now run OMPI's top-level ``autogen.pl`` script. This script
|
||||
will invoke the GNU Autoconf, Automake, and Libtool commands in the
|
||||
proper order and do a bunch of component discovery and housekeeping to
|
||||
setup to run OMPI's top-level ``configure`` script.
|
||||
|
||||
Running ``autogen.pl`` may take a few minutes, depending on your
|
||||
system. It's not very exciting to watch.
|
||||
|
||||
If you have a multi-processor system, enabling the multi-threaded
|
||||
behavior in Automake 1.11 (or newer) can result in ``autogen.pl``
|
||||
running faster. Do this by setting the ``AUTOMAKE_JOBS`` environment
|
||||
variable to the number of processors (threads) that you want it to use
|
||||
before invoking ``autogen.pl``. For example (you can put this in your
|
||||
shell startup files)::
|
||||
|
||||
# For bash/sh/zsh:
|
||||
export AUTOMAKE_JOBS=4
|
||||
|
||||
# For csh/tcsh:
|
||||
setenv AUTOMAKE_JOBS 4
|
||||
|
||||
.. important:: You generally need to run ``autogen.pl`` whenever the
|
||||
top-level file ``configure.ac`` changes, or any files in the
|
||||
``config/`` or ``<project>/config/`` directories change (these
|
||||
directories are where a lot of "include" files for Open MPI's
|
||||
``configure`` script live).
|
||||
|
||||
.. note:: You do *NOT* need to re-run ``autogen.pl`` if you modify a
|
||||
``Makefile.am``.
|
||||
@@ -0,0 +1,9 @@
|
||||
Building Open MPI
|
||||
=================
|
||||
|
||||
Once you have run ``autogen.pl`` successfully, you can configure and
|
||||
build Open MPI just like end users do with official distribution Open
|
||||
MPI tarballs.
|
||||
|
||||
See the :doc:`general "Install Open MPI" documentation for more
|
||||
details. </installing-open-mpi/index>`
|
||||
@@ -0,0 +1,29 @@
|
||||
Compiler Pickyness by Default
|
||||
=============================
|
||||
|
||||
If you are building Open MPI from a Git clone (i.e., there is a
|
||||
``.git`` directory in your build tree), the default build includes
|
||||
extra compiler pickyness, which will result in more compiler warnings
|
||||
than in non-developer builds. Getting these extra compiler warnings
|
||||
is helpful to Open MPI developers in making the code base as clean as
|
||||
possible.
|
||||
|
||||
Developers can disable this picky-by-default behavior by using the
|
||||
``--disable-picky`` configure option. Also note that extra-picky
|
||||
compiles do *not* happen automatically when you do a VPATH build
|
||||
(e.g., if ``.git`` is in your source tree, but not in your build
|
||||
tree).
|
||||
|
||||
Prior versions of Open MPI would automatically activate a lot of
|
||||
(performance-reducing) debugging code by default if ``.git`` was found
|
||||
in your build tree. This is no longer true. You can manually enable
|
||||
these (performance-reducing) debugging features in the Open MPI code
|
||||
base with these configure options:
|
||||
|
||||
* ``--enable-debug``
|
||||
* ``--enable-mem-debug``
|
||||
* ``--enable-mem-profile``
|
||||
|
||||
.. note:: These options are really only relevant to those who are
|
||||
developing Open MPI itself. They are not generally helpful for
|
||||
debugging general MPI applications.
|
||||
@@ -0,0 +1,142 @@
|
||||
.. _label-frameworks:
|
||||
|
||||
Internal frameworks
|
||||
===================
|
||||
|
||||
The Modular Component Architecture (MCA) is the backbone of Open MPI
|
||||
-- most services and functionality are implemented through MCA
|
||||
components.
|
||||
|
||||
MPI layer frameworks
|
||||
--------------------
|
||||
|
||||
Here is a list of all the component frameworks in the MPI layer of
|
||||
Open MPI:
|
||||
|
||||
* ``bml``: BTL management layer
|
||||
* ``coll``: MPI collective algorithms
|
||||
* ``fbtl``: point to point file byte transfer layer: abstraction for
|
||||
individual read: collective read and write operations for MPI I/O
|
||||
* ``fcoll``: collective file system functions for MPI I/O
|
||||
* ``fs``: file system functions for MPI I/O
|
||||
* ``hook``: Generic hooks into Open MPI
|
||||
* ``io``: MPI I/O
|
||||
* ``mtl``: Matching transport layer, used for MPI point-to-point
|
||||
messages on some types of networks
|
||||
* ``op``: Back end computations for intrinsic MPI_Op operators
|
||||
* ``osc``: MPI one-sided communications
|
||||
* ``pml``: MPI point-to-point management layer
|
||||
* ``sharedfp``: shared file pointer operations for MPI I/O
|
||||
* ``topo``: MPI topology routines
|
||||
* ``vprotocol``: Protocols for the "v" PML
|
||||
|
||||
OpenSHMEM component frameworks
|
||||
------------------------------
|
||||
|
||||
* ``atomic``: OpenSHMEM atomic operations
|
||||
* ``memheap``: OpenSHMEM memory allocators that support the
|
||||
PGAS memory model
|
||||
* ``scoll``: OpenSHMEM collective operations
|
||||
* ``spml``: OpenSHMEM "pml-like" layer: supports one-sided,
|
||||
point-to-point operations
|
||||
* ``sshmem``: OpenSHMEM shared memory backing facility
|
||||
|
||||
Miscellaneous frameworks
|
||||
------------------------
|
||||
|
||||
* ``allocator``: Memory allocator
|
||||
* ``backtrace``: Debugging call stack backtrace support
|
||||
* ``btl``: Point-to-point Byte Transfer Layer
|
||||
* ``dl``: Dynamic loading library interface
|
||||
* ``hwloc``: Hardware locality (hwloc) versioning support
|
||||
* ``if``: OS IP interface support
|
||||
* ``installdirs``: Installation directory relocation services
|
||||
* ``memchecker``: Run-time memory checking
|
||||
* ``memcpy``: Memory copy support
|
||||
* ``memory``: Memory management hooks
|
||||
* ``mpool``: Memory pooling
|
||||
* ``patcher``: Symbol patcher hooks
|
||||
* ``pmix``: Process management interface (exascale)
|
||||
* ``rcache``: Memory registration cache
|
||||
* ``reachable``: Network reachability determination
|
||||
* ``shmem``: Shared memory support (NOT related to OpenSHMEM)
|
||||
* ``smsc``: Shared memory single-copy support
|
||||
* ``threads``: OS and userspace thread support
|
||||
* ``timer``: High-resolution timers
|
||||
|
||||
Framework notes
|
||||
---------------
|
||||
|
||||
Each framework typically has one or more components that are used at
|
||||
run-time. For example, the ``btl`` framework is used by the MPI layer
|
||||
to send bytes across different types underlying networks. The ``tcp``
|
||||
``btl``, for example, sends messages across TCP-based networks; the
|
||||
``ucx`` ``pml`` sends messages across InfiniBand-based networks.
|
||||
|
||||
MCA parameter notes
|
||||
-------------------
|
||||
|
||||
Each component typically has some tunable parameters that can be
|
||||
changed at run-time. Use the :ref:`ompi_info(1) <man1-ompi_info>`
|
||||
command to check a component to see what its tunable parameters are.
|
||||
For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ompi_info --param btl tcp
|
||||
|
||||
shows some of the parameters (and default values) for the ``tcp`` ``btl``
|
||||
component (use ``--all`` or ``--level 9`` to show *all* the parameters).
|
||||
|
||||
Note that ``ompi_info`` (without ``--all`` or a specified level) only
|
||||
shows a small number a component's MCA parameters by default. Each
|
||||
MCA parameter has a "level" value from 1 to 9, corresponding to the
|
||||
MPI-3 MPI_T tool interface levels. :ref:`See the LEVELS section in
|
||||
the ompi_info(1) man page <man1-ompi_info-levels>` for an explanation
|
||||
of the levels and how they correspond to Open MPI's code.
|
||||
|
||||
Here's rules of thumb to keep in mind when using Open MPI's levels:
|
||||
|
||||
* Levels 1-3:
|
||||
|
||||
* These levels should contain only a few MCA parameters.
|
||||
* Generally, only put MCA parameters in these levels that matter to
|
||||
users who just need to *run* Open MPI applications (and don't
|
||||
know/care anything about MPI). Examples (these are not
|
||||
comprehensive):
|
||||
|
||||
* Selection of which network interfaces to use.
|
||||
* Selection of which MCA components to use.
|
||||
* Selective disabling of warning messages (e.g., show warning
|
||||
message XYZ unless a specific MCA parameter is set, which
|
||||
disables showing that warning message).
|
||||
* Enabling additional stderr logging verbosity. This allows a
|
||||
user to run with this logging enabled, and then use that output
|
||||
to get technical assistance.
|
||||
|
||||
* Levels 4-6:
|
||||
|
||||
* These levels should contain any other MCA parameters that are
|
||||
useful to expose to end users.
|
||||
* There is an expectation that "power users" will utilize these MCA
|
||||
parameters |mdash| e.g., those who are trying to tune the system
|
||||
and extract more performance.
|
||||
* Here's some examples of MCA parameters suitable for these levels
|
||||
(these are not comprehensive):
|
||||
|
||||
* When you could have hard-coded a constant size of a resource
|
||||
(e.g., a resource pool size or buffer length), make it an MCA
|
||||
parameter instead.
|
||||
* When there are multiple different algorithms available for a
|
||||
particular operation, code them all up and provide an MCA
|
||||
parameter to let the user select between them.
|
||||
|
||||
* Levels 7-9:
|
||||
|
||||
* Put any other MCA parameters here.
|
||||
* It's ok for these MCA parameters to be esoteric and only relevant
|
||||
to deep magic / the internals of Open MPI.
|
||||
* There is little expectation of users using these MCA parameters.
|
||||
|
||||
See :ref:`this section <label-running-setting-mca-param-values>` for
|
||||
details on how to set MCA parameters at run time.
|
||||
@@ -0,0 +1,196 @@
|
||||
GitHub, Git, and related topics
|
||||
===============================
|
||||
|
||||
GitHub
|
||||
------
|
||||
|
||||
Open MPI's Git repositories are `hosted at GitHub
|
||||
<https://github.com/open-mpi/ompi>`_.
|
||||
|
||||
#. First, you will need a Git client. We recommend getting the latest
|
||||
version available. If you do not have the command ``git`` in your
|
||||
path, you will likely need to download and install Git.
|
||||
#. `ompi <https://github.com/open-mpi/ompi/>`_ is the main Open MPI
|
||||
repository where most active development is done. Git clone this
|
||||
repository. Note that the use of the ``--recursive`` CLI option is
|
||||
necessary because Open MPI uses Git submodules::
|
||||
|
||||
shell$ git clone --recursive https://github.com/open-mpi/ompi.git
|
||||
|
||||
Note that Git is natively capable of using many forms of web
|
||||
proxies. If your network setup requires the user of a web proxy,
|
||||
`consult the Git documentation for more details
|
||||
<https://git-scm.com/>`_.
|
||||
|
||||
Git commits: open source / contributor's declaration
|
||||
----------------------------------------------------
|
||||
|
||||
In order to remain open source, all new commits to the Open MPI
|
||||
repository must include a ``Signed-off-by:`` line, indicating the
|
||||
submitter's agreement to the :ref:`Open MPI Contributor's Declaration
|
||||
<contributing-contributors-declaration-label>`.
|
||||
|
||||
.. tip:: You can use the ``-s`` option to ``git commit`` to
|
||||
automatically add the ``Signed-off-by:`` line to your commit
|
||||
message.
|
||||
|
||||
.. _git-github-branch-scheme-label:
|
||||
|
||||
Git branch scheme
|
||||
-----------------
|
||||
|
||||
Generally, Open MPI has two types of branches in its Git repository:
|
||||
|
||||
#. ``main``:
|
||||
|
||||
* All active development occurs on the ``main`` branch (new features,
|
||||
bug fixes, etc.).
|
||||
|
||||
#. Release branches of the form ``vMAJOR.MINOR.x`` (e.g., ``v4.0.x``,
|
||||
``v4.1.x``, ``v5.0.x``).
|
||||
|
||||
* The ``.x`` suffix indicates that this branch is used to create
|
||||
all releases in the Open MPI vMAJOR.MINOR series.
|
||||
* Periodically, the Open MPI community will make a new release
|
||||
branch, typically from ``main``.
|
||||
* A Git tag of the form ``vMAJOR.MINOR.RELEASE`` is used to
|
||||
indicate the specific commit on a release branch from where
|
||||
official Open MPI release tarball was created (e.g., ``v4.1.0``,
|
||||
``v4.1.1``, ``v4.1.2``, etc.).
|
||||
|
||||
Once a bug is fixed or a new feature is implemented on ``main``, it is
|
||||
cherry-picked over to the relevant release branch(es).
|
||||
|
||||
.. attention:: It may seem odd to some, but the Open MPI community
|
||||
development model does *not* PR bug fixes or new
|
||||
features directly to release branches. Instead,
|
||||
initial bug-fix / feature PRs are generally first made
|
||||
to ``main``.
|
||||
|
||||
This helps us ensure that future releases (with
|
||||
``main`` as a Git ancestor) will contain the bug fix /
|
||||
feature.
|
||||
|
||||
For example:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ git checkout main
|
||||
shell$ git pull --rebase
|
||||
shell$ git checkout pr/bug-fix
|
||||
|
||||
# ... make changes / fix a bug / etc. ...
|
||||
|
||||
shell$ git add ...
|
||||
shell$ git commit -s ...
|
||||
shell$ git push myfork
|
||||
|
||||
At this point, you go create a PR from your fork's ``pr/bug-fix``
|
||||
branch to the Open MPI community GitHub repo ``main`` branch. Work
|
||||
with the community to get the PR completed and merged. Then you can
|
||||
open a new PR to cherry pick the Git commits from that bug fix to each
|
||||
of the relevant release branches.
|
||||
|
||||
Depending on how far the release branch has diverged from ``main``,
|
||||
there may be some porting effort involved in the cherry-pick.
|
||||
|
||||
For example, if your bug fix on ``main`` is comprised of a single Git
|
||||
commit hash ``123abc``:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
# Fetch all upstream git activity, including the merge of the "main" PR.
|
||||
shell$ get fetch --all
|
||||
|
||||
# Check out the target release branch, and advance to the most recent commit.
|
||||
shell$ git checkout v5.0.x
|
||||
shell$ git pull --rebase
|
||||
|
||||
# Make a branch for your bug fix
|
||||
shell$ git checkout -b pr/v5.0.x/bug-fix
|
||||
# Cherry pick the commit from the "main" branch
|
||||
shell$ git cherry-pick -x 123abc
|
||||
# Push to your fork
|
||||
shell$ git push myfork
|
||||
|
||||
The Open MPI development community *requires* adding the following
|
||||
line to the commit message of cherry-picked commits on release
|
||||
branches:
|
||||
|
||||
.. code:: text
|
||||
|
||||
(cherry picked from commit [git_hash_of_original_commit])
|
||||
|
||||
.. note:: Note the use of the ``-x`` option to ``git cherry-pick``.
|
||||
This option automatically adds the ``(cherry picked from
|
||||
...)`` line to your commit message.
|
||||
|
||||
.. admonition:: Rationale
|
||||
:class: tip
|
||||
|
||||
Git does not actually store any meta data about Git cherry-picks in
|
||||
the commit. Having a standardized text line containing the source
|
||||
Git commit hash in the commit messages helps the Open MPI
|
||||
development community track where commits came from on release
|
||||
branches, and therefore allows us to check whether all relevant
|
||||
commits have been ported to a given release branch.
|
||||
|
||||
Once your commits are ready and pushed up to your fork, make a PR to
|
||||
the target release branch.
|
||||
|
||||
.. warning:: A GitHub PR CI job checks all commits on release branches
|
||||
for the ``(cherry picked from...)`` line. It will also
|
||||
ensure that the Git hash cited in that line actually
|
||||
exists on the ``main`` branch.
|
||||
|
||||
This check ensures that commits are not made to release
|
||||
branches before their corresponding ``main`` PR was
|
||||
merged.
|
||||
|
||||
All this being said, sometimes there is a need for a non-cherry-picked
|
||||
commit on a release branch. E.g., sometimes a release branch has
|
||||
diverged so much that the bug no longer exists on ``main``. It would
|
||||
therefore not make sense |mdash| or even be impossible |mdash| to
|
||||
commit the bug fix in question to ``main``.
|
||||
|
||||
In such cases, make a regular PR to the target branch (with commits
|
||||
that do *not* include ``(cherry picked from ...)`` lines). In the PR
|
||||
description, add a line with the following token:
|
||||
|
||||
.. code:: text
|
||||
|
||||
bot:notacherrypick
|
||||
|
||||
This tells the GitHub CI job that this PR contains commits that are
|
||||
not cherry-picked from ``main``.
|
||||
|
||||
.. warning:: ``bot:notacherrypick`` should only be used when
|
||||
absolutely necessary. It is not a license to avoid
|
||||
the process of PR'ing to ``main`` first.
|
||||
|
||||
CI (testing)
|
||||
------------
|
||||
|
||||
The Open MPI community generally runs two flavors of testing:
|
||||
|
||||
#. A bunch of tests on each PR (Continuous Integration / CI). These
|
||||
tests are a mixture of GitHub Actions and other CI systems (e.g.,
|
||||
Jenkins). Examples include (but are not limited to):
|
||||
|
||||
* Check each Git commit for bozo email addresses
|
||||
* Check that each Git commit contains a ``Signed-off-by`` line
|
||||
* Check that commits on release branches contain a cherry-pick
|
||||
notice
|
||||
* Build and publish the docs
|
||||
* Build Open MPI in a variety of environments and run sanity tests
|
||||
with that installation
|
||||
|
||||
#. Daily testing via the MPI Testing Tool (MTT).
|
||||
|
||||
* These are generally tests that take much longer to run than on a
|
||||
per-PR basis. `A "nightly snapshot" tarball
|
||||
<https://www.open-mpi.org/nightly/>`_ is created for ``main`` and
|
||||
each relevant release branch.
|
||||
* MTT tests are run with this snapshot tarball so that all
|
||||
organizations are testing with the same snapshots.
|
||||
* `Results are available here <https://mtt.open-mpi.org/>`_.
|
||||
@@ -0,0 +1,312 @@
|
||||
.. _developers-installing-autotools-label:
|
||||
|
||||
Manually installing the GNU Autootools
|
||||
======================================
|
||||
|
||||
There is enough detail in building the GNU Autotools that it warrants
|
||||
its own section.
|
||||
|
||||
.. note:: As noted above, you only need to read/care about this
|
||||
section if you are building Open MPI from a Git clone. End
|
||||
users installing an Open MPI distribution tarball do *not*
|
||||
need to have the GNU Autotools installed.
|
||||
|
||||
Use a package manager
|
||||
---------------------
|
||||
|
||||
It is recommended that you use your Linux distribution's package
|
||||
manager, or Homebrew or MacPorts on macOS to install recent versions
|
||||
of GNU Autoconf, Automake, and Libtool.
|
||||
|
||||
If you cannot, keep reading in this section to see how to build and
|
||||
install these applications manually (i.e., download the source from
|
||||
the internet and build/install it yourself).
|
||||
|
||||
Autotools versions
|
||||
------------------
|
||||
|
||||
The following tools are required for developers to compile Open MPI
|
||||
from its repository sources (users who download Open MPI tarballs do
|
||||
not need these tools - they are only required for developers working
|
||||
on the internals of Open MPI itself):
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Software package
|
||||
- Notes
|
||||
- URL
|
||||
|
||||
* - GNU m4
|
||||
- See version chart below
|
||||
- https://ftp.gnu.org/gnu/m4/
|
||||
* - GNU Autoconf
|
||||
- See version chart below
|
||||
- https://ftp.gnu.org/gnu/autoconf/
|
||||
* - GNU Automake
|
||||
- See version chart below
|
||||
- https://ftp.gnu.org/gnu/automake/
|
||||
* - GNU Libtool
|
||||
- See version chart below
|
||||
- https://ftp.gnu.org/gnu/libtool/
|
||||
|
||||
The table below lists the versions that are used to make nightly
|
||||
snapshot and official release Open MPI tarballs. Other versions of the
|
||||
tools *may* work for some (but almost certainly not all) platforms;
|
||||
the ones listed below are the versions that we know work across an
|
||||
extremely wide variety of platforms and environments.
|
||||
|
||||
To strengthen the above point: the core Open MPI developers typically
|
||||
use very, very recent versions of the GNU tools. There are known bugs
|
||||
in older versions of the GNU tools that Open MPI no longer compensates
|
||||
for (it seemed senseless to indefinitely support patches for ancient
|
||||
versions of Autoconf, for example).
|
||||
|
||||
.. warning:: You **will** have problems if you do not use recent
|
||||
versions of the GNU Autotools.
|
||||
|
||||
That being said, ``autogen.pl`` and ``configure.ac`` scripts tend to
|
||||
be a bit lenient and enforce slightly older minimum versions than the
|
||||
ones listed below. This is because such older versions still make
|
||||
usable Open MPI builds on many platforms - especially Linux on x86_64
|
||||
with GNU compilers - and are convenient for developers whose Linux
|
||||
distribution may not have as recent as the versions listed below (but are
|
||||
recent enough to produce a working version for their platform).
|
||||
|
||||
To be clear: the versions listed below are required to support a wide
|
||||
variety of platforms and environments, and are used to make nightly
|
||||
and official release tarballs. When building Open MPI, YMMV when using
|
||||
versions older than those listed below |mdash| especially if you are
|
||||
not building on Linux x86_64 with the GNU compilers.
|
||||
|
||||
Using older versions is unsupported. If you run into problems, upgrade
|
||||
to at least the versions listed below.
|
||||
|
||||
.. note:: You may need to scroll right in the following table.
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Open MPI
|
||||
- M4
|
||||
- Autoconf
|
||||
- Automake
|
||||
- Libtool
|
||||
- Flex
|
||||
- Sphinx
|
||||
|
||||
* - v1.0.x
|
||||
- NA
|
||||
- 2.58 - 2.59
|
||||
- 1.7 - 1.9.6
|
||||
- 1.5.16 - 1.5.22
|
||||
- 2.5.4
|
||||
- NA
|
||||
* - v1.1.x
|
||||
- NA
|
||||
- 2.59
|
||||
- 1.9.6
|
||||
- 1.5.16 - 1.5.22
|
||||
- 2.5.4
|
||||
- NA
|
||||
* - v1.2.x
|
||||
- NA
|
||||
- 2.59
|
||||
- 1.9.6
|
||||
- 1.5.22 - 2.1a
|
||||
- 2.5.4
|
||||
- NA
|
||||
* - v1.3.x
|
||||
- 1.4.11
|
||||
- 2.63
|
||||
- 1.10.1
|
||||
- 2.2.6b
|
||||
- 2.5.4
|
||||
- NA
|
||||
* - v1.4.x
|
||||
- 1.4.11
|
||||
- 2.63
|
||||
- 1.10.3
|
||||
- 2.2.6b
|
||||
- 2.5.4
|
||||
- NA
|
||||
* - v1.5.x for x=0-4
|
||||
- 1.4.13
|
||||
- 2.65
|
||||
- 1.11.1
|
||||
- 2.2.6b
|
||||
- 2.5.4
|
||||
- NA
|
||||
* - v1.5.x for x>=5
|
||||
- 1.4.16
|
||||
- 2.68
|
||||
- 1.11.3
|
||||
- 2.4.2
|
||||
- 2.5.35
|
||||
- NA
|
||||
* - v1.6.x
|
||||
- 1.4.16
|
||||
- 2.68
|
||||
- 1.11.3
|
||||
- 2.4.2
|
||||
- 2.5.35
|
||||
- NA
|
||||
* - v1.7.x
|
||||
- 1.4.16
|
||||
- 2.69
|
||||
- 1.12.2
|
||||
- 2.4.2
|
||||
- 2.5.35
|
||||
- NA
|
||||
* - v1.8.x
|
||||
- 1.4.16
|
||||
- 2.69
|
||||
- 1.12.2
|
||||
- 2.4.2
|
||||
- 2.5.35
|
||||
- NA
|
||||
* - v1.10.x
|
||||
- 1.4.16
|
||||
- 2.69
|
||||
- 1.12.2
|
||||
- 2.4.2
|
||||
- 2.5.35
|
||||
- NA
|
||||
* - v2.0.x through v4.y
|
||||
- 1.4.17
|
||||
- 2.69
|
||||
- 1.15
|
||||
- 2.4.6
|
||||
- 2.5.35
|
||||
- NA
|
||||
* - v5.0.x
|
||||
- 1.4.17
|
||||
- 2.69
|
||||
- 1.15
|
||||
- 2.4.6
|
||||
- 2.5.35
|
||||
- 4.2.0
|
||||
* - Git main
|
||||
- 1.4.17
|
||||
- 2.69
|
||||
- 1.15
|
||||
- 2.4.6
|
||||
- 2.5.35
|
||||
- 4.2.0
|
||||
|
||||
Checking your versions
|
||||
----------------------
|
||||
|
||||
You can check what versions of the Autotools you have installed with
|
||||
the following:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ m4 --version
|
||||
shell$ autoconf --version
|
||||
shell$ automake --version
|
||||
shell$ libtoolize --version
|
||||
|
||||
Installing the GNU Autotools from source
|
||||
----------------------------------------
|
||||
|
||||
.. note:: Most operating system packaging systems (to include Homebrew
|
||||
and MacPorts on MacOS) install recent-enough versions of the
|
||||
GNU Autotools. You should generally only install the GNU
|
||||
Autotools manually if you can't use your operating system
|
||||
packaging system to install them for you.
|
||||
|
||||
The GNU Autotools sources can be can be downloaded from:
|
||||
|
||||
* https://ftp.gnu.org/gnu/autoconf/
|
||||
* https://ftp.gnu.org/gnu/automake/
|
||||
* https://ftp.gnu.org/gnu/libtool/
|
||||
* And if you need it: https://ftp.gnu.org/gnu/m4/
|
||||
|
||||
It is certainly easiest to download/build/install all four of these
|
||||
tools together. But note that Open MPI has no specific m4
|
||||
requirements; it is only listed here because Autoconf requires minimum
|
||||
versions of GNU m4. Hence, you may or may not *need* to actually
|
||||
install a new version of GNU m4. That being said, if you are confused
|
||||
or don't know, just install the latest GNU m4 with the rest of the GNU
|
||||
Autotools and everything will work out fine.
|
||||
|
||||
|
||||
Build and install ordering
|
||||
--------------------------
|
||||
|
||||
You must build and install the GNU Autotools in the following order:
|
||||
|
||||
#. m4
|
||||
#. Autoconf
|
||||
#. Automake
|
||||
#. Libtool
|
||||
|
||||
.. important:: You *must* install the last three tools (Autoconf,
|
||||
Automake, Libtool) into the same prefix directory.
|
||||
These three tools are somewhat inter-related, and if
|
||||
they're going to be used together, they *must* share a
|
||||
common installation prefix.
|
||||
|
||||
You can install m4 anywhere as long as it can be found in the path;
|
||||
it may be convenient to install it in the same prefix as the other
|
||||
three. Or you can use any recent-enough m4 that is in your path.
|
||||
|
||||
.. warning:: It is *strongly* encouraged that you do **not** install
|
||||
your new versions over the OS-installed versions. This could cause
|
||||
other things on your system to break. Instead, install into
|
||||
``$HOME/local``, or ``/usr/local``, or wherever else you tend to
|
||||
install "local" kinds of software.
|
||||
|
||||
In doing so, be sure to prefix your ``$PATH`` with the directory
|
||||
where they are installed. For example, if you install into
|
||||
``$HOME/local``, you may want to edit your shell startup file
|
||||
(``.bashrc``, ``.cshrc``, ``.tcshrc``, etc.) to have something
|
||||
like
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# For bash/sh:
|
||||
export PATH=$HOME/local/bin:$PATH
|
||||
# For csh/tcsh:
|
||||
set path = ($HOME/local/bin $path)
|
||||
|
||||
Ensure to set your ``$PATH`` *before* you configure/build/install
|
||||
the four packages.
|
||||
|
||||
All four packages require two simple commands to build and
|
||||
install:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cd M4_DIRECTORY
|
||||
shell$ ./configure --prefix=PREFIX
|
||||
shell$ make all install
|
||||
|
||||
.. important:: If you are using a shell that does not automatically
|
||||
re-index the ``$PATH`` (e.g., the ``csh`` or ``tcsh``
|
||||
shells), be sure to run the ``rehash`` command before
|
||||
you install the next package so that the executables
|
||||
that were just installed can be found by the next
|
||||
package.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Make $PATH be re-indexed if necessary, e.g., via "rehash"
|
||||
shell$ cd AUTOCONF_DIRECTORY
|
||||
shell$ ./configure --prefix=PREFIX
|
||||
shell$ make all install
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Make $PATH be re-indexed if necessary, e.g., via "rehash"
|
||||
shell$ cd AUTOMAKE_DIRECTORY
|
||||
shell$ ./configure --prefix=PREFIX
|
||||
shell$ make all install
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Make $PATH be re-indexed if necessary, e.g., via "rehash"
|
||||
shell$ cd LIBTOOL_DIRECTORY
|
||||
shell$ ./configure --prefix=PREFIX
|
||||
shell$ make all install
|
||||
@@ -0,0 +1,24 @@
|
||||
Developer's guide
|
||||
=================
|
||||
|
||||
This section is here for those who are building/exploring the
|
||||
internals of Open MPI in its source code form, most likely through a
|
||||
developer's tree (i.e., a Git clone).
|
||||
|
||||
If you are writing MPI applications that simply *use* Open MPI, you
|
||||
probably don't need to read this section.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
prerequisites
|
||||
git-github
|
||||
compiler-pickyness
|
||||
autogen
|
||||
building-open-mpi
|
||||
terminology
|
||||
source-code
|
||||
frameworks
|
||||
gnu-autotools
|
||||
sphinx
|
||||
rst-for-markdown-expats.rst
|
||||
@@ -0,0 +1,136 @@
|
||||
Prerequisites
|
||||
=============
|
||||
|
||||
Compilers
|
||||
---------
|
||||
|
||||
Although it should probably be assumed, you'll need a C compiler that
|
||||
supports C99.
|
||||
|
||||
You'll also need a Fortran compiler if you want to build the Fortran
|
||||
MPI bindings (the more recent the Fortran compiler, the better), and a
|
||||
Java compiler if you want to build the (unofficial) Java MPI bindings.
|
||||
|
||||
GNU Autotools
|
||||
-------------
|
||||
|
||||
When building Open MPI from its repository sources, the GNU Autotools
|
||||
must be installed (i.e., `GNU Autoconf
|
||||
<https://www.gnu.org/software/autoconf/>`_, `GNU Automake
|
||||
<https://www.gnu.org/software/automake/>`_, and `GNU Libtool
|
||||
<https://www.gnu.org/software/libtool/>`_).
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 10 10
|
||||
|
||||
* - Tool
|
||||
- Minimum version
|
||||
* - Autoconf
|
||||
- |autoconf_min_version|
|
||||
* - Automake
|
||||
- |automake_min_version|
|
||||
* - Libtool
|
||||
- |libtool_min_version|
|
||||
|
||||
.. note:: The GNU Autotools are *not* required when building Open MPI
|
||||
from distribution tarballs. Open MPI distribution tarballs
|
||||
are bootstrapped such that end-users do not need to have the
|
||||
GNU Autotools installed.
|
||||
|
||||
You can generally install GNU Autoconf, Automake, and Libtool via your
|
||||
Linux distribution native package system, or via Homebrew or MacPorts
|
||||
on MacOS. This usually "just works."
|
||||
|
||||
If you run into problems with the GNU Autotools, or need to download /
|
||||
build them manually, see the :ref:`how to build and install GNU
|
||||
Autotools section <developers-installing-autotools-label>` for much
|
||||
more detail.
|
||||
|
||||
Perl
|
||||
----
|
||||
|
||||
Open MPI still uses Perl for a few of its build scripts (most notably,
|
||||
``autogen.pl``).
|
||||
|
||||
Generally speaking, any recent-ish release of Perl 5 should be
|
||||
sufficient to correctly execute Open MPI's Perl scripts.
|
||||
|
||||
Flex
|
||||
----
|
||||
|
||||
Minimum supported version: |flex_min_version|.
|
||||
|
||||
`Flex <https://github.com/westes/flex>`_ is used during the
|
||||
compilation of a developer's checkout (it is not used to build
|
||||
official distribution tarballs). Other flavors of lex are *not*
|
||||
supported: given the choice of making parsing code portable between
|
||||
all flavors of lex and doing more interesting work on Open MPI, we
|
||||
greatly prefer the latter.
|
||||
|
||||
Note that no testing has been performed to see what the minimum
|
||||
version of Flex is required by Open MPI. We suggest that you use
|
||||
v2.5.35 at the earliest.
|
||||
|
||||
For now, Open MPI will allow developer builds with Flex |flex_min_version|. This
|
||||
is primarily motivated by the fact that RedHat/CentOS 5 ships with
|
||||
Flex 2.5.4. It is likely that someday Open MPI developer builds will
|
||||
require Flex version >=2.5.35.
|
||||
|
||||
Note that the ``flex``-generated code generates some compiler warnings
|
||||
on some platforms, but the warnings do not seem to be consistent or
|
||||
uniform on all platforms, compilers, and flex versions. As such, we
|
||||
have done little to try to remove those warnings.
|
||||
|
||||
If you do not have Flex installed and cannot easily install it via
|
||||
your operating system's packaging system (to include Homebrew or
|
||||
MacPorts on MacOS), see `the Flex Github repository
|
||||
<https://github.com/westes/flex>`_.
|
||||
|
||||
|
||||
Sphinx (and therefore Python)
|
||||
-----------------------------
|
||||
|
||||
`Sphinx <https://www.sphinx-doc.org/>`_ is a Python-based tool used to
|
||||
generate both the HTML version of the documentation (that you are
|
||||
reading right now) and the nroff man pages.
|
||||
|
||||
Official Open MPI distribution tarballs contain pre-built HTML
|
||||
documentation and man pages. This means that |mdash| similar to the GNU
|
||||
Autotools |mdash| end users do not need to have Sphinx installed, but will
|
||||
still have both the HTML documentation and man pages installed as part
|
||||
of the normal configure / build / install process.
|
||||
|
||||
However, the HTML documentation and man pages are *not* stored in Open
|
||||
MPI's Git repository; only the ReStructred Text source code of the
|
||||
documentation is in the Git repository. Hence, if you are building
|
||||
Open MPI from a Git clone, you will need Sphinx (and some Python
|
||||
modules) in order to build the HTML documentation and man pages.
|
||||
|
||||
.. important:: Most systems do not have Sphinx and/or the required
|
||||
Python modules installed by default. :ref:`See the
|
||||
Installing Sphinx section
|
||||
<developers-installing-sphinx-label>` for details on
|
||||
how to install Sphinx and the required Python modules.
|
||||
|
||||
If ``configure`` is able to find Sphinx and the required Python
|
||||
modules, it will automatically generate the HTML documentation and man
|
||||
pages during the normal build procedure (i.e., during ``make all``).
|
||||
If ``configure`` is *not* able to find Sphinx and/or the required
|
||||
Python modules, it will simply skip building the documentation.
|
||||
|
||||
.. note:: If you have built/installed Open MPI from a Git clone and
|
||||
unexpectedly did not have the man pages installed, it is
|
||||
likely that you do not have Sphinx and/or the required
|
||||
Python modules available.
|
||||
|
||||
:ref:`See the Installing Sphinx section
|
||||
<developers-installing-sphinx-label>` for details on how
|
||||
to install Sphinx and the required Python modules.
|
||||
|
||||
.. important:: ``make dist`` will fail if ``configure`` did not find
|
||||
Sphinx and/or the required Python modules.
|
||||
Specifically: if ``make dist`` is not able to generate
|
||||
the most up-to-date HTML documentation and man pages,
|
||||
you cannot build a distribution tarball. **This is an
|
||||
intentional design decision.**
|
||||
@@ -0,0 +1,404 @@
|
||||
.. _developers-rst-for-markdown-expats:
|
||||
|
||||
ReStructured Text for those who know Markdown
|
||||
=============================================
|
||||
|
||||
You can think of RST as "Markdown, but much better". Meaning:
|
||||
|
||||
#. RST is basically just as simple as Markdown
|
||||
#. But RST is both more precise, and has more available formatting
|
||||
constructs (without getting crazy complicated)
|
||||
|
||||
The full Sphinx / RST documentation is available here:
|
||||
https://www.sphinx-doc.org/en/master/index.html
|
||||
|
||||
If you're familiar with Markdown, the following sections contain some
|
||||
tips to get you started in RST.
|
||||
|
||||
Whitespace and indenting
|
||||
------------------------
|
||||
|
||||
* MD: Whitespace and indenting generally doesn't matter in most
|
||||
cases. It does matter with bullets and sub bullets, but the rules
|
||||
get pretty weird, and vary between different Markdown renderers.
|
||||
|
||||
* RST: **Indenting matters**. A lot. Just like Python. In
|
||||
general, you indent all RST text to keep it within the same level.
|
||||
For example, all this text would be a single paragraph
|
||||
|
||||
**Blank lines also matter**. A lot. You use blank lines to
|
||||
delimit sections within an indenting level. For example, the
|
||||
blank line before this paragraph denotes a paragraph break.
|
||||
|
||||
.. note:: RST was created by the Python community. Hence,
|
||||
whitespace is quite important.
|
||||
|
||||
* Indenting matters
|
||||
* Blank lines between content matter
|
||||
|
||||
Using a blank line and outdenting indicates the end of the previous
|
||||
item. For example, this paragraph is not part of the MD/RST
|
||||
bulleted list.
|
||||
|
||||
Fixed width font
|
||||
----------------
|
||||
|
||||
* MD: Use single quotes:
|
||||
|
||||
.. code-block:: md
|
||||
|
||||
`hello world`
|
||||
|
||||
* RST: Use a pair of single quotes:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
``hello world``
|
||||
|
||||
Italics
|
||||
-------
|
||||
|
||||
* MD: ``*hello world*`` or ``_hello world_``
|
||||
* RST: ``*hello world*``
|
||||
|
||||
Boldface
|
||||
--------
|
||||
|
||||
* MD: ``**hello world**``
|
||||
* RST: Same as MD
|
||||
|
||||
Chapter and section delimiters
|
||||
------------------------------
|
||||
|
||||
* MD: Either use one or more pound signs (#, ##, ###) to the left of
|
||||
the line of text, or underline the line of text with pound signs
|
||||
|
||||
* RST: Have a single line of text, underlined by non-ASCII
|
||||
characters.
|
||||
|
||||
* The length of the underlying *must* be at least as long as the
|
||||
line of text
|
||||
* Which non-ASCII character is used for the underlying does not
|
||||
matter, but the order in which they are used denotes chapters
|
||||
/ sections / subsections / etc.
|
||||
|
||||
In these OMPI docs, the sequence of underline characters we use
|
||||
are:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
Chapter 1: hello world
|
||||
======================
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
Section 1: hello world
|
||||
----------------------
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
Subsection 1: hello world
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
Subsubsection 1: hello world
|
||||
++++++++++++++++++++++++++++
|
||||
|
||||
Meaning: underlines made of ``=`` denotes chapters, underlines
|
||||
made of ``-`` denotes sections, underlines made of ``^`` denotes
|
||||
subsections, and underlines made of ``+`` denote subsubsections.
|
||||
|
||||
Multi-line code/fixed-width font
|
||||
--------------------------------
|
||||
|
||||
* MD: Use three single quotes to delimit blocks of text. Optionally
|
||||
include a token keyword to denote the syntax highlighting to use
|
||||
inside that block.
|
||||
|
||||
.. code-block:: md
|
||||
|
||||
```c
|
||||
int main() { printf("Hello world\n"); return 0 }
|
||||
```
|
||||
|
||||
* RST: Use ``.. code-block:: KEYWORD`` to start a section of code.
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
int main() { printf("Hello world\n"); return 0 }
|
||||
|
||||
* KEYWORD indicates which syntax highlighting to use (e.g., ``c``,
|
||||
``c++`` ``make``, ``sh``, ``ini``, ``Fortran``, ``diff``,
|
||||
``python``, ``java``, ``rst``, ... etc.).
|
||||
* KEYWORD can be omitted if no specific highlighting is to be
|
||||
used.
|
||||
* There *MUST* be a blank line after the ``code-block`` line.
|
||||
* The lines in the block must be indented to the same column as the
|
||||
first ``c`` in ``code-block``. For example:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ tar xf openmpi-<version>.tar.bz2
|
||||
shell$ cd openmpi-<version>
|
||||
shell$ ./configure --prefix=<path> |& tee config.out
|
||||
|
||||
Note that the code block will be rendered at the same level as
|
||||
where the first ``.`` of ``.. code-block::`` starts. In this
|
||||
case, the example code block will be rendered in the bulleted
|
||||
item.
|
||||
|
||||
Whereas this parargraph and code block will be outside of the
|
||||
above bulleted list:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ tar xf openmpi-<version>.tar.bz2
|
||||
shell$ cd openmpi-<version>
|
||||
shell$ ./configure --prefix=<path> |& tee config.out
|
||||
|
||||
# Fun note: the code-block can contain blank lines.
|
||||
|
||||
The code-block is terminated by a blank line and then outdent back
|
||||
to the same level as the first ``.`` in ``.. code-block::``.
|
||||
|
||||
Un-numbered bullets
|
||||
-------------------
|
||||
|
||||
* MD: Start lines with ``*`` or ``-``
|
||||
* RST: Start lines with ``*``. You can wrap lines at the same
|
||||
indenting level to make paragraphs in the same bullet.
|
||||
|
||||
Having a blank line and then more text at the same indenting level
|
||||
makes another paragraph in the same bullet. You even put other
|
||||
directives in this same indenting level.
|
||||
|
||||
* For example, you can start a sub bullet.
|
||||
|
||||
This text is the next paragraph in the same sub bullet.
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
This is a verbatim code block within this same sub bullet.
|
||||
More about code-blocks below.
|
||||
|
||||
This is the next paragraph (after the code block) in the same
|
||||
sub bullet.
|
||||
|
||||
* If you start a new bullet, that terminates the previous bullet.
|
||||
|
||||
* You **MUST** put blank lines between bullets!
|
||||
|
||||
Numbered bullets:
|
||||
-----------------
|
||||
|
||||
* MD: Start lines with ``#``
|
||||
* RST: Start lines with ``#.``
|
||||
|
||||
.. important:: Yes, the trailing ``.`` is important
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
#. Item number 1
|
||||
#. The second item
|
||||
#. A third item
|
||||
|
||||
All the same rules for indentation apply as described above.
|
||||
|
||||
Comments
|
||||
--------
|
||||
|
||||
* MD: Enclose content in ``<!--`` and ``-->`` (i.e., HTML comments,
|
||||
but they are included in the output)
|
||||
* RST: Start a line with two periods and a space.
|
||||
|
||||
For example, the following block is a comment, and will not be
|
||||
included in the output:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
.. Hello world. This is a comment. This whole block is a
|
||||
comment. You can leave it here in the final document, and it
|
||||
will not be included in the rendered output.
|
||||
|
||||
Your comment can even include blank lines. You terminate a
|
||||
comment |mdash| just like most other things in RST |mdash| by a blank
|
||||
line and then outdenting back out to the same column as the
|
||||
first ".".
|
||||
|
||||
This line is no longer part of the comment.
|
||||
|
||||
Including files
|
||||
---------------
|
||||
|
||||
* MD: You cannot include files in Markdown.
|
||||
* RST: Use the ``.. include:: FILENAME`` directive. For example:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
.. include:: features-extensions.rst
|
||||
.. include:: features-java.rst
|
||||
|
||||
Those directives include those 2 files right here in this RST file.
|
||||
|
||||
.. important:: Chapter/section/subsection delimiters will be
|
||||
continued in those files as part of rendering this
|
||||
file.
|
||||
|
||||
Hyperlinks to URLs
|
||||
------------------
|
||||
|
||||
* MD:
|
||||
|
||||
.. code-block:: md
|
||||
|
||||
[this is the link text](https://example.com/)
|
||||
|
||||
* RST:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
`this is the link text <https://example.com/>`_
|
||||
|
||||
.. important:: Yes, the trailing underscore in RST is important.
|
||||
It's a little weird, but you'll cope.
|
||||
|
||||
Hyperlinks to anchors
|
||||
---------------------
|
||||
|
||||
* MD: I forget offhand how to make anchors and links to them in MD.
|
||||
* RST: Use the ``:ref:`` directive.
|
||||
|
||||
Make an anchor like this:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
.. _ANCHOR_NAME:
|
||||
|
||||
It *must* start with and underscore and end with a colon.
|
||||
|
||||
I've typically used anchor names that either begin with ``label-``
|
||||
or end in ``-label`` to make it blatantly obvious that it's a
|
||||
label. For example:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
.. _building-and-installing-section-label:
|
||||
|
||||
Then you can use the ``:ref:`` directive:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
be sure to see :ref:`the VPATH build section
|
||||
<building-and-installing-section-label>`.
|
||||
|
||||
Hyperlinks to other (RST) pages
|
||||
-------------------------------
|
||||
|
||||
* MD:
|
||||
|
||||
.. code-block:: md
|
||||
|
||||
[link text](page_name)
|
||||
|
||||
* RST: Use the ``:doc:`` directive.
|
||||
|
||||
General format:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
:doc:`link text <PAGE_PATH>`
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
You should read :doc:`the Developer's Guide </developers>`.
|
||||
|
||||
The page path is relative to the ``docs`` dir in the OMPI git tree.
|
||||
|
||||
Macros
|
||||
------
|
||||
|
||||
* MD: There are no macros in Markdown.
|
||||
* RST: We have defined a few OMPI-specific macros in RST. You can
|
||||
insert these macros anywhere in RST content text.
|
||||
|
||||
* ``|ompi_ver|`` is the full Open MPI version number, including
|
||||
alpha/beta/rc/greek denotation. For example ``5.0.0rc1``.
|
||||
|
||||
* ``|ompi_series|`` is the major/minor Open MPI version, e.g.,
|
||||
``5.0.x``.
|
||||
|
||||
.. important:: Think twice about hard-coding the Open MPI version
|
||||
number or series when referring to the current
|
||||
version or series. It can be appropriate to
|
||||
hard-code an "x.y.0" version to denote a
|
||||
generational epoch, but in most other cases, you
|
||||
probably want to use one of the macros.
|
||||
|
||||
* ``|mdash|`` is a unicode long dash, an "em" dash. Use it instead
|
||||
of ``--``.
|
||||
|
||||
* ``|rarrow|`` is a unicode right arrow. Use it instead of ``->``
|
||||
or ``-->``.
|
||||
|
||||
Brightly-colored boxes
|
||||
----------------------
|
||||
|
||||
* MD: There are no brightly-colored boxes in MD.
|
||||
|
||||
* RST: You can use various directives to make brightly-colored
|
||||
"note" boxes (Called admonitions) in RST. For example:
|
||||
|
||||
.. important:: a green box with a "!" icon
|
||||
|
||||
Standard indenting rules apply for the content in the box. You
|
||||
can have multiple lines and multiple paragraphs, for example.
|
||||
|
||||
Yippee.
|
||||
|
||||
* You can even have bullets.
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
You can even have code blocks inside the bullet inside the
|
||||
caution box.
|
||||
|
||||
* All the standard indenting rules apply.
|
||||
|
||||
.. hint:: a green box with a "!" icon
|
||||
|
||||
.. note:: a blue box with a "!" icon
|
||||
|
||||
.. caution:: an orange box with a "!" icon
|
||||
|
||||
.. attention:: an orange box with a "!" icon
|
||||
|
||||
.. warning:: an orange box with a "!" icon
|
||||
|
||||
.. error:: a red box with a "!" icon
|
||||
|
||||
.. danger:: a red box with a "!" icon
|
||||
|
||||
.. admonition:: Custom title
|
||||
:class: tip
|
||||
|
||||
You can name this box whatever you want:
|
||||
|
||||
.. code-block:: rst
|
||||
|
||||
.. admonition:: Custom title
|
||||
:class: tip
|
||||
|
||||
Content of your box here.
|
||||
|
||||
Custom text for this custom admonition. Note that the ``:class: <type>``
|
||||
will change the coloring to the color for the basic admonition of that
|
||||
type. E.g., ``:class: tip`` makes the box be green.
|
||||
@@ -0,0 +1,295 @@
|
||||
Source code
|
||||
===========
|
||||
|
||||
Code style
|
||||
----------
|
||||
|
||||
We intentionally do not have too many code conventions in the Open MPI
|
||||
code base.
|
||||
|
||||
All languages
|
||||
^^^^^^^^^^^^^
|
||||
|
||||
* 4 space tabs. No more, no less.
|
||||
* **NEVER** use actual tab characters; always use spaces. Both emacs
|
||||
and vim have secret mojo that can automatically use spaces when you
|
||||
hit the ``<TAB>`` key. This makes the code look the same in every
|
||||
browser, regardless of individual tab display settings.
|
||||
|
||||
C / C++
|
||||
^^^^^^^
|
||||
|
||||
* When comparing constants for equality or inequality, always put the
|
||||
constant on the left. This is defensive programming: if you have a
|
||||
typo in the test and miss a ``!`` or ``=``, you'll get a compiler error.
|
||||
For example:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
/* Do this */
|
||||
if (NULL == foo) { ... }
|
||||
|
||||
/* Because if you have a typo (i.e., = instead of ==), this will
|
||||
be a compile error rather than a subtle bug */
|
||||
if (NULL = foo) { ... }
|
||||
|
||||
* More defensive programming: *always* include blocks in curly braces
|
||||
``{ }``, even if they are only one line long. For example:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
/* Do this */
|
||||
if (whatever) {
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/* Not this */
|
||||
if (whatever)
|
||||
return OMPI_SUCCESS;
|
||||
|
||||
* Starting with Open MPI 1.7, Open MPI requires a C99-compliant
|
||||
compiler.
|
||||
|
||||
* C++-style comments are now allowed (and preferred).
|
||||
* C99-style mixing declarations are allow allowable (and preferred).
|
||||
|
||||
* **ALWAYS** include ``<level>_config.h`` as your first #include file,
|
||||
where ``<level>`` is one of ``ompi``, ``oshmem``, or ``opal`` |mdash| the
|
||||
level that you're writing in. There are very, very few cases where
|
||||
this is not true (E.g., some bizarre Windows scenarios). But in
|
||||
99.9999% of cases, this file should be included **first** so that it
|
||||
can affect system-level #include files if necessary.
|
||||
* Filenames and symbols must follow the **prefix rule** (see [e-mail
|
||||
thread](http://www.open-mpi.org/community/lists/devel/2009/07/6389.php)):
|
||||
|
||||
* Filenames must be prefixed with ``<framework>_<component>``.
|
||||
* Public symbols must be prefixed in components with
|
||||
``<project>_<framework>_<component>``, where ``<project>`` is one
|
||||
of ``mca``, ``ompi``, ``oshmem``, or ``opal``. Note that `mca`
|
||||
used to be the most common, but it has fallen out of favor
|
||||
compared to the other ``<project>`` prefixes. When in doubt about
|
||||
whether a symbol is public, be safe and add the prefix.
|
||||
* Non-public symbols must be declared ``static`` or otherwise made to
|
||||
not appear in the global scope.
|
||||
|
||||
* **ALWAYS** #define macros, even for logical values.
|
||||
|
||||
* The GNU Way is to ``#define`` a macro when it is "true" and to
|
||||
``#undef`` it when it is "false".
|
||||
* In Open MPI, we **always** ``#define`` a logical macro to be
|
||||
either 0 or 1 |mdash| we never ``#undef`` it.
|
||||
* The reason for this is defensive programming: if you are only
|
||||
checking if a preprocessor macro is defined (via ``#ifdef FOO`` or
|
||||
``#if defined(FOO)``), you will get no warning when compiling if
|
||||
you accidentally misspell the macro name. However, if you use the
|
||||
logic test ``#if FOO`` with an undefined macro (e.g., because you
|
||||
misspelled it), you'll get a compiler warning or error.
|
||||
|
||||
.. admonition:: Rationale
|
||||
:class: tip
|
||||
|
||||
Misspelled macro names can be tremendously difficult to find
|
||||
when they are buried in thousands of lines of code; we will
|
||||
take all the help from the preprocessor/compiler that we can
|
||||
get!
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
/* GNU Way - you will get no warning from the compiler if you
|
||||
misspell "FOO"; the test will simply be false */
|
||||
#ifdef FOO
|
||||
...
|
||||
#else
|
||||
...
|
||||
#endif
|
||||
|
||||
/* Open MPI Way - you will get a warning from the compiler if you
|
||||
misspell "FOO"; the result of the test is a different value
|
||||
than whether you spelled the macro name right or not */
|
||||
#if FOO
|
||||
...
|
||||
#else
|
||||
...
|
||||
#endif
|
||||
|
||||
Fortran
|
||||
^^^^^^^
|
||||
|
||||
We do not have specific coding style guidelines for Fortran. Please
|
||||
read some of the existing Fortran code in the source code tree and try
|
||||
to use a similar style.
|
||||
|
||||
Shell scripting
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
Please read some of the existing shell code in the source code tree
|
||||
and try to use a similar style.
|
||||
|
||||
* Always enclose evaluated shell variables in quotes to ensure that
|
||||
multi-token values are handled properly.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# This is bad
|
||||
if test $foo = bar; then
|
||||
|
||||
# This is good
|
||||
if test "$foo" = "bar"; then
|
||||
|
||||
* The one exception to this is that when doing an assignment to a
|
||||
shell variable from another shell variable, it is not necessary to
|
||||
use quotes on the right hand side:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# This is harmless, but unnecessary
|
||||
foo="$bar"
|
||||
|
||||
# This is actually sufficient, even for multi-token values of $bar
|
||||
foo=$bar
|
||||
|
||||
* Do not use the ``==`` operator for ``test`` |mdash| this is a GNU
|
||||
extension and can cause portability problems on BSD systems.
|
||||
Instead, use the single ``=`` operator.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# This is bad
|
||||
if test "$foo" == "bar"; then
|
||||
|
||||
# This is good
|
||||
if test "$foo" = "bar"; then
|
||||
|
||||
m4
|
||||
^^^
|
||||
|
||||
We do not have specific coding style guidelines for m4 (the language
|
||||
used to create the ``configure`` script). Please read some of the
|
||||
existing m4 code in the source code tree and try to use a similar
|
||||
style.
|
||||
|
||||
Tree layout
|
||||
-----------
|
||||
|
||||
There are a few notable top-level directories in the source
|
||||
tree:
|
||||
|
||||
* The main sub-projects:
|
||||
|
||||
* ``oshmem``: Top-level OpenSHMEM code base
|
||||
* ``ompi``: The Open MPI code base
|
||||
* ``opal``: The OPAL code base
|
||||
|
||||
* ``config``: M4 scripts supporting the top-level ``configure`` script
|
||||
``mpi.h``
|
||||
* ``etc``: Some miscellaneous text files
|
||||
* ``docs``: Source code for Open MPI documentation
|
||||
* ``examples``: Trivial MPI / OpenSHMEM example programs
|
||||
* ``3rd-party``: Included copies of required core libraries (either
|
||||
via Git submodules in Git clones or via binary tarballs).
|
||||
|
||||
.. note:: While it may be considered unusual, we include binary
|
||||
tarballs (instead of Git submodules) for 3rd party projects that
|
||||
are:
|
||||
|
||||
#. Needed by Open MPI for correct operation, and
|
||||
#. Not universally included in OS distributions, and
|
||||
#. Rarely updated.
|
||||
|
||||
Each of the three main source directories (``oshmem``, ``ompi``, and
|
||||
``opal``) generate at least a top-level library named ``liboshmem``,
|
||||
``libmpi``, and ``libopen-pal``, respectively. They can be built as
|
||||
either static or shared libraries. Executables are also produced in
|
||||
subdirectories of some of the trees.
|
||||
|
||||
The ``libopen-pal`` top-level library is built internally in two parts:
|
||||
|
||||
* ``libopen-pal_core`` Internal "core" portion of OPAL containing the essential source and MCA needed for tools like mpicc/mpirun to link against. The "core" library is not installed.
|
||||
|
||||
* Includes the following MCA frameworks: ``backtrace``, ``dl``, ``installdirs``, ``threads``, ``timer``
|
||||
* Includes all of the source under ``opal/class`` and most of ``opal/util``
|
||||
* Includes the files suffixed with ``_core`` in ``opal/runtime``
|
||||
|
||||
* ``libopen-pal`` Includes "core" plus all of the other OPAL project sources. This is installed.
|
||||
|
||||
Each of the sub-project source directories have similar (but not
|
||||
identical) directory structures under them:
|
||||
|
||||
* ``class``: C++-like "classes" (using the OPAL class system)
|
||||
specific to this project
|
||||
* ``include``: Top-level include files specific to this project
|
||||
* ``mca``: MCA frameworks and components specific to this project
|
||||
* ``runtime``: Startup and shutdown of this project at runtime
|
||||
* ``tools``: Executables specific to this project
|
||||
* ``util``: Random utility code
|
||||
|
||||
There are other top-level directories in each of the sub-projects,
|
||||
each having to do with specific logic and code for that project. For
|
||||
example, the MPI API implementations can be found under
|
||||
``ompi/mpi/LANGUAGE``, where ``LANGUAGE`` is ``c``, ``fortran``, or
|
||||
``java``.
|
||||
|
||||
The layout of the ``mca`` trees are strictly defined. They are of the
|
||||
form:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
PROJECT/mca/FRAMEWORK/COMPONENT
|
||||
|
||||
To be explicit: it is forbidden to have a directory under the ``mca``
|
||||
trees that does not meet this template (with the exception of ``base``
|
||||
directories, explained below). Hence, only framework and component
|
||||
code can be in the ``mca`` trees.
|
||||
|
||||
That is, framework and component names must be valid directory names
|
||||
(and C variables; more on that later). For example, the TCP BTL
|
||||
component is located in ``opal/mca/btl/tcp/``.
|
||||
|
||||
The name ``base`` is reserved; there cannot be a framework or component
|
||||
named ``base``. Directories named ``base`` are reserved for the
|
||||
implementation of the MCA and frameworks. Here are a few examples (as
|
||||
of the |ompi_series| source tree):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Main implementation of the MCA
|
||||
opal/mca/base
|
||||
|
||||
# Implementation of the btl framework
|
||||
opal/mca/btl/base
|
||||
|
||||
# Implementation of the sysv framework
|
||||
oshmem/mcs/sshmem/sysv
|
||||
|
||||
# Implementation of the pml framework
|
||||
ompi/mca/pml/base
|
||||
|
||||
Under these mandated directories, frameworks and/or components may have
|
||||
arbitrary directory structures, however.
|
||||
|
||||
Symbol Visibility
|
||||
-----------------
|
||||
|
||||
The ``*_DECLSPEC`` macros provide a method to annotate symbols to indicate
|
||||
their intended visibility when compiling dynamically shared object files
|
||||
(e.g., ``libmpi.so``). The macros are defined on a per project basis:
|
||||
|
||||
* Open MPI: ``OMPI_DECLSPEC``
|
||||
* Open PAL: ``OPAL_DECLSPEC``
|
||||
* OpenSHMEM: ``OSHMEM_DECLSPEC``
|
||||
|
||||
The macros expand to the appropriate compiler and platform flags for marking
|
||||
whether a symbol should be explicitly made public in the target project's
|
||||
library namespace.
|
||||
The ``*_DECLSPEC`` attributes are used to declare that a symbol is to be
|
||||
visible outside of that library/DSO's scope. For example, ``OMPI_DECLSPEC``
|
||||
is used to control what symbols are visible in the ``libmpi.so`` scope.
|
||||
|
||||
.. note:: This is entirely related to dynamic library compilation and does not
|
||||
apply to static compilation.
|
||||
|
||||
.. note:: The macros were originally introduced when Open MPI supported
|
||||
Windows (circa Open MPI v1.0.0) and are motivated by the Windows
|
||||
`__declspec <https://docs.microsoft.com/en-us/cpp/cpp/declspec?view=msvc-170>`_.
|
||||
While support for Windows has been dropped from Open MPI, the symbol
|
||||
visibility macros remain.
|
||||
@@ -0,0 +1,242 @@
|
||||
.. _developers-installing-sphinx-label:
|
||||
|
||||
Installing and running Sphinx (building the Open MPI docs)
|
||||
==========================================================
|
||||
|
||||
As with all content in the Developer's Guide, this section is only
|
||||
relevant for developers who work in the Open MPI code base itself.
|
||||
End users who install a binary Open MPI package or build from an
|
||||
official Open MPI distribution tarball do not need to have Sphinx
|
||||
installed.
|
||||
|
||||
Installing Python
|
||||
-----------------
|
||||
|
||||
The `Sphinx tool <https://www.sphinx-doc.org/>`_ is written in Python,
|
||||
and therefore needs to have Python available. As of late 2022, Sphinx
|
||||
requires Python >= v3.7.
|
||||
|
||||
This documentation does not contain detailed instructions for
|
||||
installing a Python version sufficient for using Sphinx. Consult your
|
||||
local OS documentation for how to obtain Python >= v3.7, or search the
|
||||
internet for further information.
|
||||
|
||||
Installing Sphinx
|
||||
-----------------
|
||||
|
||||
`The Sphinx documentation
|
||||
<https://www.sphinx-doc.org/en/master/usage/installation.html>`_
|
||||
recommends installing Sphinx (and its required Python dependencies)
|
||||
via ``pip``, which typically requires connectivity to the general
|
||||
internet.
|
||||
|
||||
.. note:: If you are running on MacOS, you may be tempted to use
|
||||
Homebrew or MacPorts to install Sphinx. The Sphinx documentation
|
||||
recommends **against** this. Instead, you should use ``pip`` to
|
||||
install Sphinx.
|
||||
|
||||
There are three general ways to install Sphinx; you only need one of
|
||||
them.
|
||||
|
||||
Install Sphinx in a Python virtual environment
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The preferred method of installing Sphinx for Open MPI documentation
|
||||
development is to install Sphinx in a Python virtual environment.
|
||||
This places Sphinx in a sandbox that will not conflict with other
|
||||
``pip``-installed Python modules. This example installs Sphinx and
|
||||
other Python modules in the ``ompi-docs-venv`` tree under your Open
|
||||
MPI Git clone directory:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Create the Python virtual environment
|
||||
shell$ cd TOP_OF_OPEN_MPI_GIT_CLONE
|
||||
shell$ python3 -m venv ompi-docs-venv
|
||||
# Or: python3 -m virtualenv ompi-docs-venv
|
||||
# Or: virtualenv --python=python3 ompi-docs-venv
|
||||
|
||||
# Activate the virtual environment
|
||||
shell$ . ./ompi-docs-venv/bin/activate
|
||||
|
||||
# Notice that the shell prompt changes
|
||||
# Now install the required Python modules
|
||||
(ompi-docs-venv) shell$ pip3 install -r docs/requirements.txt
|
||||
# Or: python3 -m pip install install -r docs/requirements.txt
|
||||
|
||||
Note that sourcing the ``activate`` script will change your prompt to
|
||||
put the name of your virtual environment directory at the front, just
|
||||
as a visual reminder that you are operating in a Python virtual
|
||||
environment. You can run ``deactivate`` to leave the virtual
|
||||
environment.
|
||||
|
||||
.. important:: You will need to source the ``activate`` script to put
|
||||
Sphinx in your ``PATH`` *before* running ``configure``.
|
||||
|
||||
Install Sphinx globally
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
If Python virtual environments are not desirable on your system, you
|
||||
can install Sphinx globally on your system (you may need to run with
|
||||
root privileges):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cd TOP_OF_OPEN_MPI_GIT_CLONE
|
||||
shell$ pip3 install -r docs/requirements.txt
|
||||
# Or: python3 -m pip install install -r docs/requirements.txt
|
||||
|
||||
This will install Sphinx and some Python modules required for building
|
||||
the Open MPI documentation in a system-wide location.
|
||||
|
||||
This will likely install the ``sphinx-build`` executable in a location
|
||||
that is already in your ``PATH``. If the location is not already in
|
||||
your ``PATH``, then you need to add it to your ``PATH``.
|
||||
|
||||
.. important:: You will need to ensure that Sphinx is in your ``PATH``
|
||||
*before* running ``configure``.
|
||||
|
||||
|
||||
Install Sphinx locally
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
If you cannot or do not want to install Sphinx globally on your
|
||||
system, the following will install Sphinx somewhere under your
|
||||
``$HOME``. It is the same ``pip`` command as shown above, but with
|
||||
the addition of the ``--user`` flag (you should not need ``root``
|
||||
permissions to run this command):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cd TOP_OF_OPEN_MPI_GIT_CLONE
|
||||
shell$ pip3 install --user -r docs/requirements.txt
|
||||
# Or: python3 -m pip install install -r docs/requirements.txt
|
||||
|
||||
This will install Sphinx and some Python modules required for building
|
||||
the Open MPI documentation in a user-specific location, likely
|
||||
somewhere under ``$HOME``
|
||||
|
||||
You will likely need to find the location where ``sphinx-build`` was
|
||||
installed and add it to your ``PATH`` (e.g., on macOS, it might appear
|
||||
under ``$HOME/Library/Python/PYTHON_VERSION/bin/sphinx-build``).
|
||||
|
||||
.. note:: On MacOS, look for ``sphinx-build`` under
|
||||
``$HOME/Library/Python/VERSION/bin`` (where ``VERSION`` is
|
||||
the version number of Python). Or it may have installed to
|
||||
``/usr/local/bin/sphinx-build``. YMMV.
|
||||
|
||||
.. important:: You will need to ensure that Sphinx is in your ``PATH``
|
||||
*before* running ``configure``.
|
||||
|
||||
Running Sphinx
|
||||
--------------
|
||||
|
||||
Open MPI's build environment is setup to invoke Sphinx automatically;
|
||||
you should not need to invoke Sphinx manually.
|
||||
|
||||
.. important:: You will need to ensure that Sphinx is in your ``PATH``
|
||||
*before* running ``configure``.
|
||||
|
||||
As long as ``configure`` found Sphinx, ``make`` will invoke Sphinx to
|
||||
build the documentation. You can also run ``make`` directly in the
|
||||
``docs/`` directory to build *just* the docs and skip building the
|
||||
rest of the Open MPI software. This can be a huge time-saver when
|
||||
iteratively writing, rendering, and viewing/proofing documentation.
|
||||
|
||||
.. note:: The fully-built HTML and man page docs are included in
|
||||
official Open MPI distribution tarballs. Meaning: if you
|
||||
download an Open MPI tarball from
|
||||
https://www.open-mpi.org/software/ompi (version v5.0.0 or
|
||||
later), the pre-built HTML and man page files are included
|
||||
in the tarball.
|
||||
|
||||
Sphinx is a requirement for *developers* who want to build
|
||||
the Open MPI docs. End users do *not* need to have Sphinx
|
||||
available to build Open MPI or have its docs installed from
|
||||
an official distribution tarball.
|
||||
|
||||
Sphinx execution time
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The first time you invoke Sphinx on a clean tree, it takes a little
|
||||
time to render all the docs.
|
||||
|
||||
However, Sphinx is stateful: subsequent runs can be significantly
|
||||
faster because Sphinx will only re-render HTML files that have
|
||||
changes. This is an enormous time saver for Open MPI (e.g., if you
|
||||
are iterating over writing the docs and running ``make`` to see how
|
||||
they rendered in HTML).
|
||||
|
||||
.. caution:: Sphinx is only *somewhat* smart in its partial
|
||||
re-rendering. If you change a title in an RST file, for
|
||||
example, Sphinx will (by default) only re-render *that*
|
||||
page. The Tables of Contents / left hand navigation on
|
||||
other pages may not be updated.
|
||||
|
||||
You can always force a full re-render via:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ cd docs
|
||||
shell$ rm -rf _build
|
||||
shell$ make
|
||||
|
||||
Checking Sphinx HTML links
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
``make linkcheck`` will invoke Sphinx's functionality to check all the
|
||||
external links in the documentation:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ cd docs
|
||||
shell$ make linkcheck
|
||||
|
||||
.. important:: You will need to be on a computer that has good access
|
||||
to the internet when running this command.
|
||||
|
||||
Viewing docs locally
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Once you have built the docs in your local Git clone, you can view
|
||||
them locally in the build tree:
|
||||
|
||||
#. Open ``docs/_build/html/index.html`` in a browser to view the HTML
|
||||
docs. For example, on MacOS, the following command opens the build
|
||||
tree docs in the default web browser:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ open docs/_build/html/index.html
|
||||
|
||||
#. Use the ``man`` command to view the Nroff files in
|
||||
``docs/_build/man`` (you may need to use an absolute or relative
|
||||
filename to prevent ``man`` from using its search paths). For
|
||||
example:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ cd docs/_build/man
|
||||
shell$ man ./MPI_Send.3
|
||||
|
||||
Alternatively, you can view these files in their installed locations
|
||||
after running ``make install``:
|
||||
|
||||
#. The HTML docs are installed (by default) to
|
||||
``$prefix/share/doc/openmpi/html``. You can use a web browser to
|
||||
open the ``index.html`` in that directory to view the docs locally.
|
||||
For example, on MacOS, the following command opens the installed
|
||||
docs in the default web browser:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ open $prefix/share/doc/openmpi/html/index.html
|
||||
|
||||
#. The man pages are installed (by default) to ``$preix/share/man``.
|
||||
If your man page search path includes this location, you can invoke
|
||||
commands similar to the following to see the same content that you
|
||||
see in these HTML pages:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ man MPI_Send
|
||||
@@ -0,0 +1,81 @@
|
||||
Open MPI terminology
|
||||
====================
|
||||
|
||||
Open MPI is a large project containing many different sub-systems and
|
||||
a relatively large code base. Let's first cover some fundamental
|
||||
terminology in order to make the rest of the discussion easier.
|
||||
|
||||
Modular Component Architecture (MCA)
|
||||
------------------------------------
|
||||
|
||||
:ref:`See this section <label-mca>` for a discussion of the Modular
|
||||
Component Architecture (MCA). Seriously. Go read it now. From
|
||||
reading that section, you should understand the following terms before
|
||||
continuing reading these docs:
|
||||
|
||||
* Project
|
||||
* Framework
|
||||
* Component
|
||||
* Module
|
||||
* Parameters (variables)
|
||||
|
||||
Notes on projects
|
||||
-----------------
|
||||
|
||||
Projects are strict abstraction barriers in the code. That is, they
|
||||
are compiled into separate libraries: ``liboshmem``, ``libmpi``,
|
||||
``libopen-pal`` with a strict dependency order: OSHMEM depends on
|
||||
OMPI, OMPI depends on OPAL. For example, MPI executables are linked
|
||||
with:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpicc myapp.c -o myapp
|
||||
# This actually turns into:
|
||||
shell$ cc myapp.c -o myapp -lmpi ...
|
||||
|
||||
More system-level libraries may listed after ``-lmpi``, but you get
|
||||
the idea. ``libmpi`` will implicitly pull ``libopen-pal`` into the
|
||||
overall link step.
|
||||
|
||||
Strictly speaking, these are not "layers" in the classic software
|
||||
engineering sense (even though it is convenient to refer to them as
|
||||
such). They are listed above in dependency order, but that does not
|
||||
mean that, for example, the OMPI code must go through the OPAL code in
|
||||
order to reach the operating system or a network interface.
|
||||
|
||||
As such, this code organization more reflects abstractions and
|
||||
software engineering, not a strict hierarchy of functions that must be
|
||||
traversed in order to reach a lower layer. For example, OMPI can
|
||||
directly call the operating system as necessary (and not go through
|
||||
OPAL). Indeed, many top-level MPI API functions are quite performance
|
||||
sensitive; it would not make sense to force them to traverse an
|
||||
arbitrarily deep call stack just to move some bytes across a network.
|
||||
|
||||
Frameworks, components, and modules can be dynamic or static. That is,
|
||||
they can be available as plugins or they may be compiled statically
|
||||
into libraries (e.g., ``libmpi``).
|
||||
|
||||
In Open MPI |ompi_ver|, ``configure`` defaults to:
|
||||
|
||||
* Building projects as dynamic libraries
|
||||
* Linking all components into their parent project libraries
|
||||
(vs. compiling them as independent DSOs)
|
||||
|
||||
Although these defaults can be modified by :doc:`command line
|
||||
arguments to configure
|
||||
</installing-open-mpi/configure-cli-options/index>`.
|
||||
|
||||
Required 3rd party libraries
|
||||
----------------------------
|
||||
|
||||
Note that Open MPI also uses some third-party libraries for core
|
||||
functionality:
|
||||
|
||||
* PMIx
|
||||
* PRRTE
|
||||
* Libevent
|
||||
* Hardware Locality ("hwloc")
|
||||
|
||||
These are discussed in detail in the :ref:`required support libraries
|
||||
section <label-install-required-support-libraries>`.
|
||||
@@ -0,0 +1,91 @@
|
||||
.. _ompi-features-extentions-label:
|
||||
|
||||
Open MPI extensions
|
||||
===================
|
||||
|
||||
Open MPI contains a framework for extending the MPI API that is
|
||||
available to applications. Each extension is usually a standalone set
|
||||
of functionality that is distinct from other extensions (similar to
|
||||
how Open MPI's plugins are usually unrelated to each other). These
|
||||
extensions provide new functions and/or constants that are available
|
||||
to MPI applications.
|
||||
|
||||
.. warning:: These extensions are neither standard nor portable to
|
||||
other MPI implementations!
|
||||
|
||||
They are a mechanism for the Open MPI developer community to
|
||||
provide new functionality to users, typically before it becomes
|
||||
standardized by the MPI Forum.
|
||||
|
||||
Available extensions
|
||||
--------------------
|
||||
|
||||
The following extensions are included in this version of Open MPI:
|
||||
|
||||
#. ``shortfloat``: Provides MPI datatypes ``MPIX_C_FLOAT16``,
|
||||
``MPIX_SHORT_FLOAT``, ``MPIX_SHORT_FLOAT``, and
|
||||
``MPIX_CXX_SHORT_FLOAT_COMPLEX`` if corresponding language types are
|
||||
available. See ``ompi/mpiext/shortfloat/README.txt`` for details.
|
||||
#. ``affinity``: Provides the ``OMPI_Affinity_str()`` API, which returns
|
||||
a string indicating the resources which a process is bound. For
|
||||
more details, see its man page.
|
||||
#. ``cuda``: When the library is compiled with CUDA-aware support, it
|
||||
provides two things. First, a macro
|
||||
``MPIX_CUDA_AWARE_SUPPORT``. Secondly, the function
|
||||
``MPIX_Query_cuda_support()`` that can be used to query for support.
|
||||
#. ``example``: A non-functional extension; its only purpose is to
|
||||
provide an example for how to create other extensions.
|
||||
#. ``ftmpi``: An implementation of the User Level Fault Mitigation
|
||||
(ULFM) proposal. :ref:`See its documentation section <ulfm-label>`
|
||||
for more details.
|
||||
|
||||
Compiling the extensions
|
||||
------------------------
|
||||
|
||||
Open MPI extensions are all enabled by default; they can be disabled
|
||||
via the ``--disable-mpi-ext`` command line switch.
|
||||
|
||||
Since extensions are meant to be used by advanced users only, this
|
||||
file does not document which extensions are available or what they do.
|
||||
Look in the ``ompi/mpiext`` directory in a distribution Open MPI
|
||||
tarball to see the extensions; each subdirectory of that directory
|
||||
contains an extension. Each has a ``README`` file that describes what
|
||||
it does.
|
||||
|
||||
Using the extensions
|
||||
--------------------
|
||||
|
||||
To reinforce the fact that these extensions are non-standard, you must
|
||||
include a separate header file after ``<mpi.h>`` to obtain the function
|
||||
prototypes, constant declarations, etc. For example:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#include <mpi.h>
|
||||
#if defined(OPEN_MPI) && OPEN_MPI
|
||||
#include <mpi-ext.h>
|
||||
#endif
|
||||
|
||||
int main() {
|
||||
MPI_Init(NULL, NULL);
|
||||
|
||||
#if defined(OPEN_MPI) && OPEN_MPI
|
||||
char ompi_bound[OMPI_AFFINITY_STRING_MAX];
|
||||
char current_binding[OMPI_AFFINITY_STRING_MAX];
|
||||
char exists[OMPI_AFFINITY_STRING_MAX];
|
||||
|
||||
OMPI_Affinity_str(OMPI_AFFINITY_LAYOUT_FMT, ompi_bound,
|
||||
current_bindings, exists);
|
||||
#endif
|
||||
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
|
||||
Notice that the Open MPI-specific code is surrounded by the ``#if``
|
||||
statement to ensure that it is only ever compiled by Open MPI.
|
||||
|
||||
The Open MPI wrapper compilers (``mpicc`` and friends) should
|
||||
automatically insert all relevant compiler and linker flags necessary
|
||||
to use the extensions. No special flags or steps should be necessary
|
||||
compared to "normal" MPI applications.
|
||||
@@ -0,0 +1,17 @@
|
||||
Open MPI-specific features
|
||||
==========================
|
||||
|
||||
Open MPI has multiple features that are above and beyond what is
|
||||
specified in the `MPI Standard <https://www.mpi-forum.org/docs/>`_.
|
||||
|
||||
This section does not list *all* Open MPI features that are not
|
||||
specified in the official MPI Standard, but does list two major
|
||||
categories of Open MPI-specific features.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
profiling
|
||||
extensions
|
||||
ulfm
|
||||
java
|
||||
@@ -0,0 +1,350 @@
|
||||
.. _open-mpi-java-label:
|
||||
|
||||
Open MPI Java bindings
|
||||
======================
|
||||
|
||||
Open MPI |ompi_ver| provides support for Java-based MPI applications.
|
||||
|
||||
.. warning:: The Open MPI Java bindings are provided on a
|
||||
"provisional" basis |mdash| i.e., they are not part of the current or
|
||||
proposed MPI standards. Thus, inclusion of Java support is not
|
||||
required by the standard. Continued inclusion of the Java bindings
|
||||
is contingent upon active user interest and continued developer
|
||||
support.
|
||||
|
||||
The rest of this document provides step-by-step instructions on
|
||||
building OMPI with Java bindings, and compiling and running Java-based
|
||||
MPI applications. Also, part of the functionality is explained with
|
||||
examples. Further details about the design, implementation and usage
|
||||
of Java bindings in Open MPI can be found in its canonical reference
|
||||
paper [#ompijava]_. The bindings follow a JNI approach, that is, we do
|
||||
not provide a pure Java implementation of MPI primitives, but a thin
|
||||
layer on top of the C implementation. This is the same approach as in
|
||||
mpiJava [#mpijava]_; in fact, mpiJava was taken as a starting point
|
||||
for Open MPI Java bindings, but they were later totally rewritten.
|
||||
|
||||
Building the Java bindings
|
||||
--------------------------
|
||||
|
||||
Java support requires that Open MPI be built at least with shared
|
||||
libraries (i.e., ``--enable-shared``). Note that this is the default
|
||||
for Open MPI, so you don't have to explicitly add the option. The Java
|
||||
bindings will build only if ``--enable-mpi-java`` is specified, and a
|
||||
JDK is found in a typical system default location.
|
||||
|
||||
If the JDK is not in a place where we automatically find it, you can
|
||||
specify the location. For example, this is required on the Mac
|
||||
platform as the JDK headers are located in a non-typical location. Two
|
||||
options are available for this purpose:
|
||||
|
||||
#. ``--with-jdk-bindir=<foo>``: the location of ``javac`` and ``javah``
|
||||
#. ``--with-jdk-headers=<bar>``: the directory containing ``jni.h``
|
||||
|
||||
Some example configurations are provided in Open MPI configuration
|
||||
platform files under ``contrib/platform/hadoop``. These examples can
|
||||
provide a starting point for your own custom configuration.
|
||||
|
||||
In summary, therefore, you can configure the system using the
|
||||
following Java-related options::
|
||||
|
||||
$ ./configure --with-platform=contrib/platform/hadoop/<your-platform> ...
|
||||
|
||||
or::
|
||||
|
||||
$ ./configure --enable-mpi-java --with-jdk-bindir=<foo> --with-jdk-headers=<bar> ...
|
||||
|
||||
or simply::
|
||||
|
||||
$ ./configure --enable-mpi-java ...
|
||||
|
||||
if JDK is in a "standard" place that ``configure`` can automatically
|
||||
find.
|
||||
|
||||
Building Java MPI applications
|
||||
------------------------------
|
||||
|
||||
The ``mpijavac`` wrapper compiler is available for compiling
|
||||
Java-based MPI applications. It ensures that all required Open MPI
|
||||
libraries and classpaths are defined. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ mpijavac Hello.java
|
||||
|
||||
You can use the ``--showme`` option to see the full command line of
|
||||
the Java compiler that is invoked:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ mpijavac Hello.java --showme
|
||||
/usr/bin/javac -cp /opt/openmpi/lib/mpi.jar Hello.java
|
||||
|
||||
Note that if you are specifying a ``-cp`` argument on the command line
|
||||
to pass your application-specific classpaths, Open MPI will *extend*
|
||||
that argument to include the ``mpi.jar``:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ mpijavac -cp /path/to/my/app.jar Hello.java --showme
|
||||
/usr/bin/javac -cp /path/to/my/app.jar:/opt/openmpi/lib/mpi.jar Hello.java
|
||||
|
||||
Similarly, if you have a ``CLASSPATH`` environment variable defined,
|
||||
``mpijavac`` will convert that into a ``-cp`` argument and extend it
|
||||
to include the ``mpi.jar``:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ export CLASSPATH=/path/to/my/app.jar
|
||||
$ mpijavac Hello.java --showme
|
||||
/usr/bin/javac -cp /path/to/my/app.jar:/opt/openmpi/lib/mpi.jar Hello.java
|
||||
|
||||
|
||||
Running Java MPI applications
|
||||
-----------------------------
|
||||
|
||||
Once your application has been compiled, you can run it with the
|
||||
standard ``mpirun`` command line::
|
||||
|
||||
$ mpirun <options> java <your-java-options> <my-app>
|
||||
|
||||
``mpirun`` will detect the ``java`` token and ensure that the required
|
||||
MPI libraries and class paths are defined to support execution. You
|
||||
therefore do **not** need to specify the Java library path to the MPI
|
||||
installation, nor the MPI classpath. Any classpath definitions
|
||||
required for your application should be specified either on the
|
||||
command line or via the ``CLASSPATH`` environment variable. Note that
|
||||
the local directory will be added to the classpath if nothing is
|
||||
specified.
|
||||
|
||||
.. note:: The ``java`` executable, all required libraries, and your
|
||||
application classes must be available on all nodes.
|
||||
|
||||
Basic usage of the Java bindings
|
||||
--------------------------------
|
||||
|
||||
There is an MPI package that contains all classes of the MPI Java
|
||||
bindings: ``Comm``, ``Datatype``, ``Request``, etc. These classes have a
|
||||
direct correspondence with handle types defined by the MPI standard. MPI
|
||||
primitives are just methods included in these classes. The convention
|
||||
used for naming Java methods and classes is the usual camel-case
|
||||
convention, e.g., the equivalent of ``MPI_File_set_info(fh,info)`` is
|
||||
``fh.setInfo(info)``, where ``fh`` is an object of the class ``File``.
|
||||
|
||||
Apart from classes, the MPI package contains predefined public
|
||||
attributes under a convenience class ``MPI``. Examples are the
|
||||
predefined communicator ``MPI.COMM_WORLD`` and predefined datatypes such
|
||||
as ``MPI.DOUBLE``. Also, MPI initialization and finalization are methods
|
||||
of the ``MPI`` class and must be invoked by all MPI Java
|
||||
applications. The following example illustrates these concepts:
|
||||
|
||||
.. code-block:: java
|
||||
|
||||
import mpi.*;
|
||||
|
||||
class ComputePi {
|
||||
|
||||
public static void main(String args[]) throws MPIException {
|
||||
|
||||
MPI.Init(args);
|
||||
|
||||
int rank = MPI.COMM_WORLD.getRank(),
|
||||
size = MPI.COMM_WORLD.getSize(),
|
||||
nint = 100; // Intervals.
|
||||
double h = 1.0/(double)nint, sum = 0.0;
|
||||
|
||||
for (int i=rank+1; i<=nint; i+=size) {
|
||||
double x = h * ((double)i - 0.5);
|
||||
sum += (4.0 / (1.0 + x * x));
|
||||
}
|
||||
|
||||
double sBuf[] = { h * sum },
|
||||
rBuf[] = new double[1];
|
||||
|
||||
MPI.COMM_WORLD.reduce(sBuf, rBuf, 1, MPI.DOUBLE, MPI.SUM, 0);
|
||||
|
||||
if (rank == 0) {
|
||||
System.out.println("PI: " + rBuf[0]);
|
||||
}
|
||||
MPI.Finalize();
|
||||
}
|
||||
}
|
||||
|
||||
Exception handling
|
||||
------------------
|
||||
|
||||
The Java bindings in Open MPI support exception handling. By default,
|
||||
errors are fatal, but this behavior can be changed. The Java API will
|
||||
throw exceptions if the ``MPI.ERRORS_RETURN`` error handler is set:
|
||||
|
||||
.. code-block:: java
|
||||
|
||||
MPI.COMM_WORLD.setErrhandler(MPI.ERRORS_RETURN);
|
||||
|
||||
If you add this statement to your program, it will show the line
|
||||
where it breaks, instead of just crashing in case of an error.
|
||||
Error-handling code can be separated from main application code by
|
||||
means of try-catch blocks, for instance:
|
||||
|
||||
.. code-block:: java
|
||||
|
||||
try
|
||||
{
|
||||
File file = new File(MPI.COMM_SELF, "filename", MPI.MODE_RDONLY);
|
||||
}
|
||||
catch(MPIException ex)
|
||||
{
|
||||
System.err.println("Error Message: "+ ex.getMessage());
|
||||
System.err.println(" Error Class: "+ ex.getErrorClass());
|
||||
ex.printStackTrace();
|
||||
System.exit(-1);
|
||||
}
|
||||
|
||||
How to specify buffers
|
||||
----------------------
|
||||
|
||||
In MPI primitives that require a buffer (either send or receive), the
|
||||
Java API admits a Java array. Since Java arrays can be relocated by
|
||||
the Java runtime environment, the MPI Java bindings need to make a
|
||||
copy of the contents of the array to a temporary buffer, then pass the
|
||||
pointer to this buffer to the underlying C implementation. From the
|
||||
practical point of view, this implies an overhead associated to all
|
||||
buffers that are represented by Java arrays. The overhead is small for
|
||||
small buffers but increases for large arrays.
|
||||
|
||||
There is a pool of temporary buffers with a default capacity of 64K.
|
||||
If a temporary buffer of 64K or less is needed, then the buffer will
|
||||
be obtained from the pool. But if the buffer is larger, then it will
|
||||
be necessary to allocate the buffer and free it later.
|
||||
|
||||
The default capacity of pool buffers can be modified with an Open MPI
|
||||
MCA parameter::
|
||||
|
||||
$ mpirun --mca ompi_mpi_java_eager SIZE ...
|
||||
|
||||
The value of ``SIZE`` can be:
|
||||
|
||||
* ``N``: An integer number of bytes
|
||||
* ``Nk``: An integer number (suffixed with ``k``) of kilobytes
|
||||
* ``Nm``: An integer number (suffixed with ``m``) of megabytes
|
||||
|
||||
An alternative is to use "direct buffers" provided by standard classes
|
||||
available in the Java SDK such as ``ByteBuffer``. For convenience,
|
||||
Open MPI provides a few static methods ``new[Type]Buffer`` in the
|
||||
``MPI`` class to create direct buffers for a number of basic
|
||||
datatypes. Elements of the direct buffer can be accessed with methods
|
||||
``put()`` and ``get()``, and the number of elements in the buffer can
|
||||
be obtained with the method ``capacity()``. This example illustrates
|
||||
its use:
|
||||
|
||||
.. code-block:: java
|
||||
|
||||
int myself = MPI.COMM_WORLD.getRank();
|
||||
int tasks = MPI.COMM_WORLD.getSize();
|
||||
|
||||
IntBuffer in = MPI.newIntBuffer(MAXLEN * tasks),
|
||||
out = MPI.newIntBuffer(MAXLEN);
|
||||
|
||||
for (int i = 0; i < MAXLEN; i++)
|
||||
out.put(i, myself); // fill the buffer with the rank
|
||||
|
||||
Request request = MPI.COMM_WORLD.iAllGather(
|
||||
out, MAXLEN, MPI.INT, in, MAXLEN, MPI.INT);
|
||||
request.waitFor();
|
||||
request.free();
|
||||
|
||||
for (int i = 0; i < tasks; i++) {
|
||||
for (int k = 0; k < MAXLEN; k++) {
|
||||
if (in.get(k + i * MAXLEN) != i)
|
||||
throw new AssertionError("Unexpected value");
|
||||
}
|
||||
}
|
||||
|
||||
Direct buffers are available for: ``BYTE``, ``CHAR``, ``SHORT``,
|
||||
``INT``, ``LONG``, ``FLOAT``, and ``DOUBLE``.
|
||||
|
||||
.. note:: There is no direct buffer for booleans.
|
||||
|
||||
Direct buffers are not a replacement for arrays, because they have
|
||||
higher allocation and deallocation costs than arrays. In some cases
|
||||
arrays will be a better choice. You can easily convert a buffer into
|
||||
an array and vice versa.
|
||||
|
||||
.. important:: All non-blocking methods *must* use direct buffers.
|
||||
Only blocking methods can choose between arrays and
|
||||
direct buffers.
|
||||
|
||||
The above example also illustrates that it is necessary to call the
|
||||
``free()`` method on objects whose class implements the ``Freeable``
|
||||
interface. Otherwise, a memory leak will occur.
|
||||
|
||||
Specifying offsets in buffers
|
||||
-----------------------------
|
||||
|
||||
In a C program, it is common to specify an offset in a array with
|
||||
``&array[i]`` or ``array+i`` to send data starting from a given
|
||||
position in the array. The equivalent form in the Java bindings is to
|
||||
``slice()`` the buffer to start at an offset. Making a ``slice()`` on
|
||||
a buffer is only necessary, when the offset is not zero. Slices work
|
||||
for both arrays and direct buffers.
|
||||
|
||||
.. code-block:: java
|
||||
|
||||
import static mpi.MPI.slice;
|
||||
// ...
|
||||
int numbers[] = new int[SIZE];
|
||||
// ...
|
||||
MPI.COMM_WORLD.send(slice(numbers, offset), count, MPI.INT, 1, 0);
|
||||
|
||||
|
||||
Supported APIs
|
||||
--------------
|
||||
|
||||
Complete MPI-3.1 coverage is provided in the Open MPI Java bindings,
|
||||
with a few exceptions:
|
||||
|
||||
* The bindings for the ``MPI_Neighbor_alltoallw`` and
|
||||
``MPI_Ineighbor_alltoallw`` functions are not implemented.
|
||||
|
||||
* Also excluded are functions that incorporate the concepts of
|
||||
explicit virtual memory addressing, such as
|
||||
``MPI_Win_shared_query``.
|
||||
|
||||
|
||||
Known issues
|
||||
------------
|
||||
|
||||
There exist issues with the Omnipath (PSM2) interconnect involving
|
||||
Java. The problems definitely exist in PSM2 v10.2; we have not tested
|
||||
previous versions.
|
||||
|
||||
As of November 2016, there is not yet a PSM2 release that completely
|
||||
fixes the issue.
|
||||
|
||||
The following ``mpirun`` command options will disable PSM2::
|
||||
|
||||
shell$ mpirun ... --mca mtl ^psm2 java ...your-java-options... your-app-class
|
||||
|
||||
|
||||
Questions? Problems?
|
||||
---------------------
|
||||
|
||||
The Java API documentation is generated at build time in
|
||||
``$prefix/share/doc/openmpi/javadoc``.
|
||||
|
||||
Additionally, `this Cisco blog post
|
||||
<https://blogs.cisco.com/performance/java-bindings-for-open-mpi>`_ has
|
||||
quite a bit of information about the Open MPI Java bindings.
|
||||
|
||||
If you have any problems, or find any bugs, please feel free to report
|
||||
them to `Open MPI user's mailing list
|
||||
<https://www.open-mpi.org/community/lists/ompi.php>`_.
|
||||
|
||||
.. rubric:: Footnotes
|
||||
|
||||
.. [#ompijava] O. Vega-Gisbert, J. E. Roman, and J. M. Squyres. "Design
|
||||
and implementation of Java bindings in Open MPI". Parallel Comput.
|
||||
59: 1-20 (2016).
|
||||
|
||||
.. [#mpijava] M. Baker et al. "mpiJava: An object-oriented Java
|
||||
interface to MPI". In Parallel and Distributed Processing, LNCS
|
||||
vol. 1586, pp. 748-762, Springer (1999).
|
||||
@@ -0,0 +1,81 @@
|
||||
.. _open-mpi-profileing-label:
|
||||
|
||||
Open MPI profiling interface
|
||||
============================
|
||||
|
||||
Open MPI |ompi_ver| supportings the "PMPI" profiling interface as
|
||||
perscribed by the MPI standard for the C and Fortran bindings (*not*
|
||||
the :ref:`Open MPI Java binding extensions <open-mpi-java-label>`).
|
||||
|
||||
Per MPI-4.0 section 15.2.1, MPI implementations must document which
|
||||
bindings layer on top of each other, so that profile developers know
|
||||
whether to implement the profiling interface for each binding, or
|
||||
whether they can economize by implementing it only for the lowest
|
||||
level routines.
|
||||
|
||||
In general, Open MPI's Fortran bindings are implemented on top of the
|
||||
C bindings. Hence, a profile developer who implements ``MPI_Init()``
|
||||
in C will also intecept all Fortran calls to ``MPI_INIT`` regardless
|
||||
of whether the user is utilizing the ``mpif.h``, ``use mpi``, or ``use
|
||||
mpi_f08`` Fortran interfaces.
|
||||
|
||||
However, there are a handful of routines where Open MPI's Fortran
|
||||
bindings are *not* a simple wrapper around the back-end C MPI binding.
|
||||
Profile developers must therefore intercept the APIs listed below in
|
||||
their source language interface bindings in order to receive full
|
||||
profiling coverage. Note, however, that it is only necessary for
|
||||
profile developers to intercept the ``mpif.h`` binding of each of the
|
||||
routines listed below; the ``use mpi`` and ``use mpi_f08`` bindings
|
||||
will ultimately invoke the ``mpif.h`` binding.
|
||||
|
||||
* APIs that contain function pointer parameters:
|
||||
|
||||
.. admonition:: Rationale
|
||||
:class: Hint
|
||||
|
||||
In order for Open MPI to invoke callbacks through the function
|
||||
pointer with the proper language-specific calling conventions, it
|
||||
must know the source language from which the function pointer was
|
||||
passed.
|
||||
|
||||
* :ref:`MPI_COMM_CREATE_KEYVAL`
|
||||
* :ref:`MPI_COMM_CREATE_ERRHANDLER`
|
||||
* :ref:`MPI_ERRHANDLER_CREATE`
|
||||
|
||||
.. warning:: This function was deprecated by MPI-3.0.
|
||||
|
||||
* :ref:`MPI_FILE_CREATE_ERRHANDLER`
|
||||
* :ref:`MPI_GREQUEST_START`
|
||||
* :ref:`MPI_KEYVAL_CREATE`
|
||||
|
||||
.. warning:: This function was deprecated by MPI-3.0.
|
||||
|
||||
* :ref:`MPI_OP_CREATE`
|
||||
* :ref:`MPI_REGISTER_DATAREP`
|
||||
* :ref:`MPI_SESSION_CREATE_ERRHANDLER`
|
||||
* :ref:`MPI_TYPE_CREATE_KEYVAL`
|
||||
* :ref:`MPI_WIN_CREATE_KEYVAL`
|
||||
* :ref:`MPI_WIN_CREATE_ERRHANDLER`
|
||||
|
||||
* APIs dealing with MPI attributes:
|
||||
|
||||
.. admonition:: Rationale
|
||||
:class: Hint
|
||||
|
||||
The MPI standard's treatment of attributes differs depending on
|
||||
which language / interface was used to create the attribute
|
||||
keyval, get the attribute value, or set the attribute value.
|
||||
|
||||
* :ref:`MPI_COMM_GET_ATTR`
|
||||
* :ref:`MPI_COMM_SET_ATTR`
|
||||
* :ref:`MPI_TYPE_GET_ATTR`
|
||||
* :ref:`MPI_TYPE_SET_ATTR`
|
||||
* :ref:`MPI_WIN_GET_ATTR`
|
||||
* :ref:`MPI_WIN_SET_ATTR`
|
||||
|
||||
Note that there is no harm in intercepting *all* routines in *all*
|
||||
interfaces. Indeed, that is the most portable way to implement a
|
||||
profiling interface. Since Open MPI's Fortran bindings are |mdash|
|
||||
for the most part |mdash| implemented on top of its C bindings,
|
||||
profile developers can ignore all Fortran interfaces except for the
|
||||
ones enumated above.
|
||||
@@ -0,0 +1,591 @@
|
||||
.. _ulfm-label:
|
||||
|
||||
User-Level Fault Mitigation (ULFM)
|
||||
==================================
|
||||
|
||||
This chapter documents the features and options specific to the **User
|
||||
Level Failure Mitigation (ULFM)** Open MPI implementation.
|
||||
|
||||
Quick Start
|
||||
-----------
|
||||
|
||||
This is an extremely terse summary of how to use ULFM:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ./configure --with-ft=ulfm [...options...]
|
||||
shell$ make [-j N] all install
|
||||
shell$ mpicc my-ft-program.c -o my-ft-program
|
||||
shell$ mpirun -n 4 --with-ft ulfm my-ft-program
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
This implementation conforms to the User Level Failure Mitigation
|
||||
(ULFM) MPI Standard draft proposal. The ULFM proposal is developed by
|
||||
the MPI Forum's Fault Tolerance Working Group to support the continued
|
||||
operation of MPI programs after any type of failures, hard or soft,
|
||||
have impacted the execution. The key principle is that no MPI call
|
||||
(point-to-point, collective, RMA, IO, ...) can block indefinitely
|
||||
after a failure, but must either succeed or raise an MPI
|
||||
error. Accordingly, the errors are not all fatal, the MPI
|
||||
implementations will do a best-effort approach to maintain the
|
||||
execution environment up and running.
|
||||
|
||||
This implementation produces the three supplementary error codes and
|
||||
five supplementary interfaces defined in the communicator section of
|
||||
the `ULFM chapter
|
||||
<https://fault-tolerance.org/wp-content/uploads/2012/10/20170221-ft.pdf>`_
|
||||
standard draft document.
|
||||
|
||||
* ``MPIX_ERR_PROC_FAILED`` when a process failure prevents the
|
||||
completion of an MPI operation (error code).
|
||||
* ``MPIX_ERR_PROC_FAILED_PENDING`` when a potential sender matching a
|
||||
non-blocking wildcard source receive has failed (error code).
|
||||
* ``MPIX_ERR_REVOKED`` when the communicator is revoked (error
|
||||
code).
|
||||
* ``MPIX_Comm_revoke(MPI_Comm comm)`` Interrupts any communication
|
||||
pending on the communicator at all ranks (API). See :ref:`MPIX_Comm_revoke`.
|
||||
* ``MPIX_Comm_is_revoked(MPI_Comm comm, int *flag)`` Test if a Communicator
|
||||
is currently revoked (API). See :ref:`MPIX_Comm_is_revoked`.
|
||||
* ``MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm* newcomm)`` creates a new
|
||||
communicator where dead processes in comm were removed, and the
|
||||
remaining processes are renamed to cover all the gaps in the naming
|
||||
from the original communicator (API). See :ref:`MPIX_Comm_shrink`,
|
||||
:ref:`MPIX_Comm_ishrink`.
|
||||
* ``MPIX_Comm_agree(MPI_Comm comm, int *flag)`` performs a consensus
|
||||
(i.e. fault tolerant allreduce operation) on flag (with the
|
||||
operation bitwise AND) (API). Absorbs all new failures, and
|
||||
propagate the knowledge about failures among the participants. see
|
||||
:ref:`MPIX_Comm_agree`, :ref:`MPIX_Comm_iagree`.
|
||||
* ``MPIX_Comm_get_failed(MPI_Comm comm, MPI_Group* failedgrp)`` obtains the
|
||||
group of currently failed processes (API). See :ref:`MPIX_Comm_get_failed`.
|
||||
* ``MPIX_Comm_ack_failed(MPI_Comm comm, int num_to_ack, int* num_acked)``
|
||||
acknowledges that the application intends to ignore the effect of currently
|
||||
known failures on wildcard receive completions and agreement return values
|
||||
(API). See :ref:`MPIX_Comm_ack_failed`.
|
||||
|
||||
Supported Systems
|
||||
-----------------
|
||||
|
||||
There are several MPI communication engines available in Open MPI,
|
||||
notably:
|
||||
|
||||
* PML: ``ob1``, ``cm``, ``ucx``
|
||||
* MTL: ``ofi``, ``portals4``, ``psm2``
|
||||
|
||||
However, in Open MPI |ompi_ver|, only ``ob1`` is fully adapted to support
|
||||
fault tolerance. The UCX PML has been successfully tested in some setups,
|
||||
but at this point we cannot confirm that all UCT devices are fully capable
|
||||
to provide the necessary features.
|
||||
|
||||
``ob1`` uses BTL ("Byte Transfer Layer") components for each supported
|
||||
network. ``ob1`` supports a variety of networks that can be used in
|
||||
combination with each other. Collective operations (blocking and
|
||||
non-blocking) use an optimized implementation on top of ``ob1``.
|
||||
|
||||
- Loopback (send-to-self)
|
||||
- TCP
|
||||
- UCT (InfiniBand)
|
||||
- uGNI (Cray Gemini, Aries)
|
||||
- Shared Memory (FT supported with CMA and XPMEM; KNEM is untested)
|
||||
- Tuned and non-blocking collective communications
|
||||
|
||||
A full list of supported, untested and disabled components is provided
|
||||
later in this document.
|
||||
|
||||
ULFM web site
|
||||
-------------
|
||||
|
||||
More information (tutorials, examples, build instructions for leading
|
||||
top500 systems) is also available in the Fault Tolerance Research
|
||||
Hub website: https://fault-tolerance.org
|
||||
|
||||
Bibliographic References
|
||||
------------------------
|
||||
|
||||
If you are looking for, or want to cite a general reference for ULFM,
|
||||
please use:
|
||||
|
||||
*Wesley Bland, Aurelien Bouteiller, Thomas Herault, George Bosilca, Jack
|
||||
J. Dongarra: Post-failure recovery of MPI communication
|
||||
capability: Design and rationale. IJHPCA 27(3): 244-254 (2013).*
|
||||
|
||||
Available from DOI :doi:`10.1177/1094342013488238`.
|
||||
|
||||
Building ULFM support in Open MPI
|
||||
---------------------------------
|
||||
|
||||
In Open MPI |ompi_ver|, ULFM support is **built-in by default** |mdash|
|
||||
that is, when you build Open MPI, unless you specify ``--without-ft``, ULFM
|
||||
support is automatically available (but is inactive unless enabled at
|
||||
runtime).
|
||||
|
||||
Optionally, you can specify ``--with-ft ulfm`` to ensure that ULFM support
|
||||
is definitely built.
|
||||
|
||||
.. note:: ULFM Fault Tolerance does not apply to OpenSHMEM. It is recommended
|
||||
that if you are going to use ULFM, you should disable building OpenSHMEM
|
||||
with ``--disable-oshmem``.
|
||||
|
||||
Running ULFM Open MPI
|
||||
---------------------
|
||||
|
||||
Building your application
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
As ULFM is still an extension to the MPI standard, you will need to
|
||||
``#include <mpi-ext.h>`` in C, or ``use mpi_ext`` in Fortran to access
|
||||
the supplementary error codes and functions.
|
||||
|
||||
Compile your application as usual, using the provided ``mpicc`` or
|
||||
``mpifort`` wrappers.
|
||||
|
||||
Running your application
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
You can launch your application with fault tolerance by simply using
|
||||
the normal Open MPI ``mpirun`` launcher, with the
|
||||
``--with-ft ulfm`` CLI option (or its synonym ``--with-ft mpi``):
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun --with-ft ulfm ...
|
||||
|
||||
.. important:: By default, fault tolerance is not active at run time.
|
||||
It must be enabled via ``--with-ft ulfm``.
|
||||
|
||||
Running under a batch scheduler
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ULFM can operate under a job/batch scheduler, and is tested routinely
|
||||
with ALPS, PBS, and Slurm. One difficulty comes from the fact that
|
||||
many job schedulers handle failures by triggering an immediate "cleanup"
|
||||
of the application as soon as any process fails. In addition, failure
|
||||
detection subsystems integrated into PRTE are not active in direct launch
|
||||
scenarios and may not have a launcher specific alternative. This may cause
|
||||
the application to not detect failures and lock. In order to avoid these
|
||||
problems, it is preferred that you use ``mpirun`` within an allocation
|
||||
(e.g., ``salloc``, ``sbatch``, ``qsub``) rather than a direct launch.
|
||||
|
||||
* SLURM is tested and supported with fault tolerance.
|
||||
|
||||
.. important:: Use ``mpirun`` in an ``salloc`` or ``sbatch`` allocation.
|
||||
Direct launch with ``srun`` is not supported.
|
||||
|
||||
* PBS/Torque is tested and supported with fault tolerance.
|
||||
|
||||
.. important:: Use ``mpirun`` in a ``qsub`` allocation. Direct launch
|
||||
with ``aprun`` is not supported.
|
||||
|
||||
* LSF is untested with fault tolerance.
|
||||
|
||||
Run-time tuning knobs
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ULFM comes with a variety of knobs for controlling how it runs. The
|
||||
default parameters are sane and should result in good performance in
|
||||
most cases. You can change the default settings with ``--mca
|
||||
mpi_ft_foo <value>`` for Open MPI options, and with ``--prtemca
|
||||
errmgr_detector_bar <value>`` for PRTE options.
|
||||
|
||||
.. important:: The main control for enabling/disabling fault tolerance
|
||||
at runtime is the ``--with-ft ulfm`` (or its synomym ``--with-ft mpi``)
|
||||
``mpirun`` CLI option. This option sets up multiple subsystems in
|
||||
Open MPI to enable fault tolerance. The options described below are
|
||||
best used to override the default behavior after the ``--with-ft ulfm``
|
||||
opion is used.
|
||||
|
||||
PRTE level options
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* ``prrte_enable_ft <true|false> (default: false)`` controls
|
||||
automatic cleanup of apps with failed processes within
|
||||
``mpirun``. This option is automatically set to ``true`` when using
|
||||
``--with-ft ulfm``.
|
||||
* ``errmgr_detector_priority <int> (default 1005``) selects the
|
||||
PRRTE-based failure detector. Only available when
|
||||
``prte_enable_recovery`` is ``true``. You can set this to ``0`` when
|
||||
using the (experimental) Open MPI detector instead.
|
||||
* ``errmgr_detector_heartbeat_period <float> (default: 5e0)`` controls
|
||||
the heartbeat period. Recommended value is 1/2 of the timeout.
|
||||
* ``errmgr_detector_heartbeat_timeout <float> (default: 1e1 seconds)``
|
||||
heartbeat timeout (i.e. failure detection speed). Recommended value
|
||||
is 2 times the heartbeat period. The default setup is tuned for
|
||||
failure-free performance at the expense of fault detection
|
||||
reactivity. In environments where faults are expected to be common,
|
||||
less conservative values can be used (e.g., 100ms); Values lower
|
||||
than the TCP poll rate (typically 10ms) can cause false positive.
|
||||
|
||||
Open MPI level options
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default values are applied to some Open MPI parameters when using
|
||||
``mpirun --with-ft ulfm``. These defaults are obtained from the ``ft-mpi``
|
||||
aggregate MCA param file
|
||||
``$installdir/share/openmpi/amca-param-sets/ft-mpi``. You can tune the
|
||||
runtime behavior of ULFM by either setting or unsetting variables in
|
||||
this file, or by overriding the variable on the command line (e.g.,
|
||||
``--mca btl ofi,self``).
|
||||
|
||||
.. important:: Note that if fault tolerance is disabled at runtime,
|
||||
(that is, when not using ``--with-ft ulfm``), the ``ft-mpi`` AMCA
|
||||
param file is not loaded, thus components that are unsafe for fault
|
||||
tolerance will load normally (this may change observed performance
|
||||
when comparing with and without fault tolerance).
|
||||
|
||||
* ``mpi_ft_enable <true|false> (default: false)``
|
||||
permits turning on/off fault tolerance at runtime. This option is
|
||||
automatically set to ``true`` from the aggregate MCA param file
|
||||
``ft-mpi`` loaded when using ``--with-ft ulfm``. When false, failure
|
||||
detection is disabled; Interfaces defined by the fault tolerance extensions
|
||||
are substituted with dummy non-fault tolerant implementations (e.g.,
|
||||
``MPIX_Comm_agree`` is implemented with ``MPI_Allreduce``); All other
|
||||
controls below become irrelevant.
|
||||
* ``mpi_ft_verbose <int> (default: 0)`` increases the output of the
|
||||
fault tolerance activities. A value of 1 will report detected
|
||||
failures.
|
||||
* ``mpi_ft_detector <true|false> (default: false)``, **DEPRECATED**
|
||||
controls the activation of the Open MPI level failure detector. When
|
||||
this detector is turned off, all failure detection is delegated to
|
||||
PRTE (see above). The Open MPI level fault detector is
|
||||
experimental. There is a tradeoff between failure detection accuracy
|
||||
and performance with this detector. Users that experience accuracy
|
||||
issues may enable a more precise mode. See the tuning knobs below
|
||||
to adjust to taste; The Open MPI failure detector operates on
|
||||
``MPI_COMM_WORLD`` exclusively. Processes connected from
|
||||
``MPI_COMM_CONNECT``/``ACCEPT`` and ``MPI_COMM_SPAWN`` may
|
||||
occasionally not be detected when they fail.
|
||||
|
||||
.. caution:: This component is deprecated. Failure detection is now
|
||||
performed at the PRTE level. See the section above on controlling
|
||||
PRTE behavior for information about how to tune the failure detector.
|
||||
|
||||
* ``mpi_ft_detector_thread <true|false> (default: false)`` controls
|
||||
the use of a thread to emit and receive failure detector's
|
||||
heartbeats. *Setting this value to "true" will also set
|
||||
MPI_THREAD_MULTIPLE support, which has a noticeable effect on
|
||||
latency (typically 1us increase).* You may want to **enable this
|
||||
option if you experience false positive** processes incorrectly
|
||||
reported as failed with the Open MPI failure detector.
|
||||
|
||||
.. important:: This option is only relevant when ``mpi_ft_detector`` is ``true``.
|
||||
|
||||
* ``mpi_ft_detector_period <float> (default: 3e0 seconds)`` heartbeat
|
||||
period. Recommended value is 1/3 of the timeout. _Values lower than
|
||||
100us may impart a noticeable effect on latency (typically a 3us
|
||||
increase)._
|
||||
|
||||
.. important:: This option is only relevant when ``mpi_ft_detector`` is ``true``.
|
||||
|
||||
* ``mpi_ft_detector_timeout <float> (default: 1e1 seconds)`` heartbeat
|
||||
timeout (i.e. failure detection speed). Recommended value is 3 times
|
||||
the heartbeat period.
|
||||
|
||||
.. important:: This option is only relevant when ``mpi_ft_detector`` is ``true``.
|
||||
|
||||
Known Limitations in ULFM
|
||||
-------------------------
|
||||
|
||||
* InfiniBand support is provided through the UCT BTL; fault tolerant
|
||||
operation over the UCX PML is not yet supported for production runs.
|
||||
* TOPO, FILE, RMA are not fault tolerant. They are expected to work
|
||||
properly before the occurrence of the first failure.
|
||||
|
||||
Modified, Untested and Disabled Components
|
||||
------------------------------------------
|
||||
|
||||
Frameworks and components are listed below and categorized into one of
|
||||
three classifications:
|
||||
|
||||
1. **Modified:** This framework/component has been specifically modified
|
||||
such that it will continue to work after a failure.
|
||||
2. **Untested:** This framework/component has not been modified and/or
|
||||
tested with fault tolerance scenarios, and _may_ malfunction
|
||||
after a failure.
|
||||
3. **Disabled:** This framework/component will cause unspecified behavior when
|
||||
fault tolerance is enabled. As a consequence, it will be disabled when the
|
||||
``--with-ft ulfm`` option is used (see above for defails about implicit
|
||||
parameters loaded from the ``ft-mpi`` aggregate param file).
|
||||
|
||||
Any framework or component not listed below are categorized as **Unmodified**,
|
||||
meaning that it is unmodified for fault tolerance, but will continue to work
|
||||
correctly after a failure.
|
||||
|
||||
* ``pml``: MPI point-to-point management layer
|
||||
|
||||
* ``monitoring``, ``v``: **untested** (they have not been modified to handle
|
||||
faults)
|
||||
* ``cm``, ``crcpw``, ``ucx``: **disabled**
|
||||
|
||||
* ``btl``: Point-to-point Byte Transfer Layer
|
||||
|
||||
* ``ofi``, ``portals4``, ``smcuda``, ``usnic``, ``sm(+knem)``: **untested**
|
||||
(they may work properly, please report)
|
||||
|
||||
* ``mtl``: Matching transport layer Used for MPI point-to-point messages on
|
||||
some types of networks
|
||||
|
||||
* All ``mtl`` components are **disabled**
|
||||
|
||||
* ``coll``: MPI collective algorithms
|
||||
|
||||
* ``cuda``, ``inter``, ``sync``, ``sm``: **untested** (they have not
|
||||
been modified to handle faults, but we expect correct post-fault
|
||||
behavior)
|
||||
* ``hcoll``, ``portals4`` **disabled** (they have not been modified
|
||||
to handle faults, and we expect unspecified post-fault behavior)
|
||||
|
||||
* ``osc``: MPI one-sided communications
|
||||
|
||||
* All ``osc`` components are **untested** (they have not been
|
||||
modified to handle faults, and we expect unspecified post-fault
|
||||
behavior)
|
||||
|
||||
* ``io``: MPI I/O and dependent components
|
||||
|
||||
* ``fs``: File system functions for MPI I/O
|
||||
* ``fbtl``: File byte transfer layer: abstraction for individual
|
||||
read/write operations for OMPIO
|
||||
* ``fcoll``: Collective read and write operations for MPI I/O
|
||||
* ``sharedfp``: Shared file pointer operations for MPI I/O
|
||||
* All components in these frameworks are unmodified, **untested**
|
||||
(we expect clean post-failure abort)
|
||||
|
||||
* ``vprotocol``: Checkpoint/Restart components
|
||||
|
||||
* All ``vprotocol`` components are **untested**
|
||||
|
||||
* ``threads``, ``wait-sync``: Multithreaded wait-synchronization
|
||||
object
|
||||
|
||||
* ``argotbots``, ``qthreads``: **disabled** (these components have
|
||||
not been modified to handle faults; we expect post-failure
|
||||
deadlock)
|
||||
|
||||
Changelog
|
||||
---------
|
||||
|
||||
ULFM Integrated in Open MPI
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
As of |ompi_ver|, ULFM is now integrated directly in to the community
|
||||
release of Open MPI. The following sections describe previous ULFM
|
||||
standlone releases.
|
||||
|
||||
ULFM Standalone Release 4.0.2u1
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
This is a stability and upstream parity upgrade. It is based on the
|
||||
most current Open MPI Release (v4.0.2, October 2019).
|
||||
|
||||
* This release is based on Open MPI release v4.0.2 (ompi #cb5f4e737a).
|
||||
* This release is based on ULFM master (ulfm #0e249ca1).
|
||||
* New features
|
||||
|
||||
* Support for the UCT BTL enters beta stage.
|
||||
|
||||
* Bugfixes
|
||||
|
||||
* High sensitivity to noise in the failure detector.
|
||||
* Deadlocks when revoking while BTL progress threads are updating messages.
|
||||
* A case where the failure detector would keep observing a dead
|
||||
process forever.
|
||||
* Disable the use of external pmix/libevent by default (the
|
||||
internals are modified to handle error cases).
|
||||
* Clean error paths leaving some rdma registration dangling.
|
||||
* Do not remove the orte job/proc session dir prematurely upon
|
||||
error.
|
||||
|
||||
ULFM Standalone Release 4.0.1u1
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
This is a stability and upstream parity upgrade. It improves
|
||||
stability, performance and is based on the most current Open MPI
|
||||
Release (v4.0.1, May 2019).
|
||||
|
||||
* This release is based on Open MPI release v4.0.1 (ompi #b780667).
|
||||
* This release is based on ULFM master (ulfm #cf8dc43f).
|
||||
* New features
|
||||
|
||||
* Addition of the ``MPI_Comm_is_revoked`` function
|
||||
* Renamed ``ftbasic`` collective component to ``ftagree``
|
||||
* Restored the ``pcollreq`` extension
|
||||
|
||||
* Bugfixes
|
||||
|
||||
* Failures of node-local siblings were not always detected
|
||||
* Failure propagation and detection was slowed down by trying to
|
||||
notify known dead processes
|
||||
* There were deadlocks in multithreaded programs
|
||||
* There were issues with PMPI when compiling Fortran Interfaces
|
||||
* There were deadlocks on OS-X
|
||||
|
||||
ULFM Standalone Release 2.1
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
This release is a bugfix and upstream parity upgrade. It improves
|
||||
stability, performance and is based on the most current Open MPI
|
||||
main (November 2018).
|
||||
|
||||
* ULFM is now based upon Open MPI main branch (#37954b5f).
|
||||
* ULFM tuning MCA parameters are exposed by ``ompi_info``.
|
||||
* Fortran 90 bindings have been updated
|
||||
* Bugfixes:
|
||||
|
||||
* Correct the behavior of process placement during an MPI_COMM_SPAWN
|
||||
when some slots were occcupied by failed processes.
|
||||
* MPI_COMM_SPAWN accepts process placement directives in the Info object.
|
||||
* Fixed deadlocks in some NBC collective operations.
|
||||
* Crashes and deadlocks in MPI_FINALIZE have been resolved.
|
||||
* Any-source requests that returned with an error status of
|
||||
MPIX_PROC_FAILED_PENDING can now correctly complete during later
|
||||
MPI_WAIT/TEST.
|
||||
|
||||
ULFM Standalone Release 2.0
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Focus has been toward integration with current Open MPI main
|
||||
(November 2017), performance, and stability.
|
||||
|
||||
* ULFM is now based upon Open MPI main branch (#689f1be9). It will
|
||||
be regularly updated until it will eventually be merged.
|
||||
* Fault Tolerance is enabled by default and is controlled with MCA variables.
|
||||
* Added support for multithreaded modes (MPI_THREAD_MULTIPLE, etc.)
|
||||
* Added support for non-blocking collective operations (NBC).
|
||||
* Added support for CMA shared memory transport (Vader).
|
||||
* Added support for advanced failure detection at the MPI level.
|
||||
Implements the algorithm described in "Failure detection and
|
||||
propagation in HPC systems," DOI :doi:`10.1109/SC.2016.26`.
|
||||
* Removed the need for special handling of CID allocation.
|
||||
* Non-usable components are automatically removed from the build
|
||||
during configure
|
||||
* RMA, FILES, and TOPO components are enabled by default, and usage in
|
||||
a fault tolerant execution warns that they may cause undefined
|
||||
behavior after a failure.
|
||||
* Bugfixes:
|
||||
|
||||
* Code cleanup and performance cleanup in non-FT builds; --without-ft at
|
||||
configure time gives an almost stock Open MPI.
|
||||
* Code cleanup and performance cleanup in FT builds with FT runtime disabled;
|
||||
--mca ft_enable_mpi false thoroughly disables FT runtime activities.
|
||||
* Some error cases would return ERR_PENDING instead of ERR_PROC_FAILED in
|
||||
collective operations.
|
||||
* Some test could set ERR_PENDING or ERR_PROC_FAILED instead of
|
||||
ERR_PROC_FAILED_PENDING for ANY_SOURCE receptions.
|
||||
|
||||
ULFM Standalone Release 1.1
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Focus has been toward improving stability, feature coverage for
|
||||
intercomms, and following the updated specification for
|
||||
MPI_ERR_PROC_FAILED_PENDING.
|
||||
|
||||
* Forked from Open MPI 1.5.5 devel branch
|
||||
* Addition of the MPI_ERR_PROC_FAILED_PENDING error code, as per newer
|
||||
specification revision. Properly returned from point-to-point,
|
||||
non-blocking ANY_SOURCE operations.
|
||||
* Alias MPI_ERR_PROC_FAILED, MPI_ERR_PROC_FAILED_PENDING and
|
||||
MPI_ERR_REVOKED to the corresponding standard blessed -extension-
|
||||
names MPIX_ERR_xxx.
|
||||
* Support for Intercommunicators:
|
||||
|
||||
* Support for the blocking version of the agreement, MPI_COMM_AGREE
|
||||
on Intercommunicators.
|
||||
* MPI_COMM_REVOKE tested on intercommunicators.
|
||||
|
||||
* Disabled completely (.ompi_ignore) many untested components.
|
||||
* Changed the default ORTE failure notification propagation
|
||||
aggregation delay from 1s to 25ms.
|
||||
* Added an Open MPI internal failure propagator; failure propagation
|
||||
between SM domains is now immediate.
|
||||
* Bugfixes:
|
||||
|
||||
* SendRecv would not always report MPI_ERR_PROC_FAILED correctly.
|
||||
* SendRecv could incorrectly update the status with errors
|
||||
pertaining to the Send portion of the Sendrecv.
|
||||
* Revoked send operations are now always completed or remote
|
||||
cancelled and may not deadlock anymore.
|
||||
* Cancelled send operations to a dead peer will not trigger an
|
||||
assert when the BTL reports that same failure.
|
||||
* Repeat calls to operations returning MPI_ERR_PROC_FAILED will
|
||||
eventually return MPI_ERR_REVOKED when another process revokes the
|
||||
communicator.
|
||||
|
||||
ULFM Standalone Release 1.0
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Focus has been toward improving performance, both before and after the
|
||||
occurrence of failures. The list of new features includes:
|
||||
|
||||
* Support for the non-blocking version of the agreement, MPI_COMM_IAGREE.
|
||||
* Compliance with the latest ULFM specification draft. In particular,
|
||||
the MPI_COMM_(I)AGREE semantic has changed.
|
||||
* New algorithm to perform agreements, with a truly logarithmic
|
||||
complexity in number of ranks, which translates into huge
|
||||
performance boosts in MPI_COMM_(I)AGREE and MPI_COMM_SHRINK.
|
||||
* New algorithm to perform communicator revocation. MPI_COMM_REVOKE
|
||||
performs a reliable broadcast with a fixed maximum output degree,
|
||||
which scales logarithmically with the number of ranks.
|
||||
* Improved support for our traditional network layer:
|
||||
|
||||
* TCP: fully tested
|
||||
* SM: fully tested (with the exception of XPMEM, which remains unsupported)
|
||||
|
||||
* Added support for High Performance networks
|
||||
|
||||
* Open IB: reasonably tested
|
||||
* uGNI: reasonably tested
|
||||
|
||||
* The tuned collective module is now enabled by default (reasonably
|
||||
tested), expect a huge performance boost compared to the former
|
||||
basic default setting
|
||||
|
||||
* Back-ported PBS/ALPS fixes from Open MPI
|
||||
* Back-ported OpenIB bug/performance fixes from Open MPI
|
||||
* Improve Context ID allocation algorithm to reduce overheads of
|
||||
Shrink
|
||||
* Miscellaneous bug fixes
|
||||
|
||||
Binary Compatibility
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ULFM Open MPI is binary compatible with any version of Open MPI
|
||||
compatible with the underlying Open MPI main branch or release (see
|
||||
the binary compatibility and version number section in the upstream
|
||||
Open MPI README). That is, applications compiled with a compatible
|
||||
Open MPI can run with the ULFM Open MPI ``mpirun`` and MPI
|
||||
libraries. Conversely, *as long as the application does not employ one
|
||||
of the MPIX functions,* which are exclusively defined in ULFM Open
|
||||
MPI, an application compiled with ULFM Open MPI can be launched with a
|
||||
compatible Open MPI ``mpirun`` and run with the non-fault tolerant MPI
|
||||
library.
|
||||
|
||||
Contacting the Authors
|
||||
----------------------
|
||||
|
||||
Found a bug? Got a question? Want to make a suggestion? Want to
|
||||
contribute to ULFM Open MPI? Working on a cool use-case?
|
||||
Please let us know!
|
||||
|
||||
The best way to report bugs, send comments, or ask questions is to
|
||||
sign up on the user's mailing list: ulfm+subscribe@googlegroups.com
|
||||
|
||||
Because of spam, only subscribers are allowed to post to these lists
|
||||
(ensure that you subscribe with and post from exactly the same e-mail
|
||||
address |mdash| joe@example.com is considered different than
|
||||
joe@mycomputer.example.com!). Visit these pages to subscribe to the
|
||||
lists: https://groups.google.com/forum/#!forum/ulfm
|
||||
|
||||
When submitting questions and problems, be sure to include as much
|
||||
extra information as possible. See the :doc:`Getting help
|
||||
</getting-help>` section for more details.
|
||||
|
||||
Thanks for your time.
|
||||
|
||||
|
||||
ULFM Copyright
|
||||
--------------
|
||||
|
||||
Copyright (c) 2012-|year| The University of Tennessee and The
|
||||
University of Tennessee Research Foundation. All rights reserved.
|
||||
@@ -0,0 +1,263 @@
|
||||
Getting help
|
||||
============
|
||||
|
||||
If you have a problem or question, it is highly recommended that you
|
||||
execute the following steps **in order**. Many people have similar
|
||||
problems with configuration and initial setup of Open MPI |mdash| it
|
||||
is possible that your question has already been answered.
|
||||
|
||||
#. :doc:`Validate your Open MPI installation </validate>`. This
|
||||
ensures that you have a nominally-correct Open MPI installation.
|
||||
|
||||
#. `Check prior GitHub issues
|
||||
<https://github.com/open-mpi/ompi/issues>`_ and see if others have
|
||||
asked the same question and had it answered.
|
||||
|
||||
#. `Check the mailing list archives
|
||||
<https://www.open-mpi.org/community/lists/ompi.php>`_ "search"
|
||||
features (or use Google) to check old posts and see if others have
|
||||
asked the same question and had it answered.
|
||||
|
||||
#. If you do not find a solution to your problem in the above
|
||||
resources, proceed to the :ref:`Where to send?
|
||||
<getting-help-where-to-send-label>` section.
|
||||
|
||||
.. _getting-help-where-to-send-label:
|
||||
|
||||
Where to send?
|
||||
--------------
|
||||
|
||||
Different types of questions and problems should be sent to different
|
||||
places. If you have:
|
||||
|
||||
#. **A general end-user question or problem:** you should probably
|
||||
subscribe to the `Open MPI user's mailing list
|
||||
<https://www.open-mpi.org/community/lists/ompi.php>`_ and post it
|
||||
there.
|
||||
|
||||
.. note:: Because of spam, only subscribers to the mailing list are
|
||||
allowed to post to the mailing list. Specifically: **you must
|
||||
subscribe to the mailing list before posting.**
|
||||
|
||||
* If you have a compile-time question or problem, see the :ref:`For
|
||||
problems building or installing Open MPI
|
||||
<getting-help-compile-time-label>` section below for the content
|
||||
of what to include in your email.
|
||||
|
||||
* If you have problems launching your MPI or OpenSHMEM application
|
||||
successfully, see the :ref:`For problems launching MPI or
|
||||
OpenSHMEM applications <getting-help-launching-label>` section
|
||||
below for the content of what to include in your email.
|
||||
|
||||
* If you have other questions or problems about running your MPI or
|
||||
OpenSHMEM application, see the :ref:`For problems running MPI or
|
||||
OpenSHMEM applications <getting-help-running-label>` section
|
||||
below for the content of what to include in your email.
|
||||
|
||||
.. important:: The more information you include in your report, the
|
||||
better. E-mails/bug reports simply stating, "It doesn't work!"
|
||||
are not helpful; we need to know as much information about your
|
||||
environment as possible in order to provide meaningful
|
||||
assistance.
|
||||
|
||||
**The best way to get help** is to provide a "recipe" for
|
||||
reproducing the problem. This will allow the Open MPI developers
|
||||
to see the error for themselves, and therefore be able to fix it.
|
||||
|
||||
.. important:: Please **use a descriptive "subject" line in your
|
||||
email!** Some Open MPI question-answering people decide whether
|
||||
to read a mail based on its subject line (e.g., to see if it's a
|
||||
question that they can answer). So please *please* **please**
|
||||
use a good subject line that succinctly describes your problem.
|
||||
|
||||
#. **A bug report:** you should probably post it to `Open MPI's Github
|
||||
issue tracker <https://github.com/open-mpi/ompi/issues>`_. Follow
|
||||
the template to submit all the requested information.
|
||||
|
||||
#. **A patch, bug fix, or other code submission:** please post a Github
|
||||
Pull Request to the `Open MPI Github repository
|
||||
<https://github.com/open-mpi/ompi/pulls>`_.
|
||||
|
||||
#. **A developer-level / internal question about Open MPI itself:** you
|
||||
should probably subscribe to the `Open MPI developer's mailing list
|
||||
<https://www.open-mpi.org/community/lists/ompi.php>`_ and post it
|
||||
there.
|
||||
|
||||
If you're unsure where to send your question, subscribe and send an
|
||||
email to the user's mailing list (i.e., option #1, above).
|
||||
|
||||
.. _getting-help-compile-time-label:
|
||||
|
||||
For problems building or installing Open MPI
|
||||
--------------------------------------------
|
||||
|
||||
If you cannot successfully configure, build, or install Open MPI,
|
||||
please provide *all* of the following information:
|
||||
|
||||
#. The version of Open MPI that you're using.
|
||||
|
||||
#. The stdout and stderr from running ``configure``.
|
||||
|
||||
#. All ``config.log`` files from the Open MPI build tree.
|
||||
|
||||
#. Output from when you ran ``make V=1 all`` to build Open MPI.
|
||||
|
||||
#. Output from when you ran ``make install`` to install Open MPI.
|
||||
|
||||
The script below may be helpful to gather much of the above
|
||||
information (adjust as necessary for your specific environment):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
# Make a directory for the output files
|
||||
dir="`pwd`/ompi-output"
|
||||
mkdir $dir
|
||||
|
||||
# Fill in the options you want to pass to configure here
|
||||
options=""
|
||||
./configure $options 2>&1 | tee $dir/config.out
|
||||
tar -cf - `find . -name config.log` | tar -x -C $dir
|
||||
|
||||
# Build and install Open MPI
|
||||
make V=1 all 2>&1 | tee $dir/make.out
|
||||
make install 2>&1 | tee $dir/make-install.out
|
||||
|
||||
# Bundle up all of these files into a tarball
|
||||
filename="ompi-output.tar.bz2"
|
||||
tar -jcf $filename `basename $dir`
|
||||
echo "Tarball $filename created"
|
||||
|
||||
Then attach the resulting ``ompi-output.tar.bz2`` file to your report.
|
||||
|
||||
.. caution:: The mailing lists have **a 150 KB size limit on
|
||||
messages** (this is a limitation of the mailing list web archives).
|
||||
If attaching the tarball makes your message larger than 150 KB, you
|
||||
may need to post the tarball elsewhere and include a link to that
|
||||
tarball in your mail to the list.
|
||||
|
||||
.. _getting-help-launching-label:
|
||||
|
||||
For problems launching MPI or OpenSHMEM applications
|
||||
----------------------------------------------------
|
||||
|
||||
If you cannot successfully launch simple applications across multiple
|
||||
nodes (e.g., the non-MPI ``hostname`` command, or the MPI "hello world"
|
||||
or "ring" sample applications in the ``examples/`` directory), please
|
||||
provide *all* of the information from the :ref:`For problems building
|
||||
or installing Open MPI <getting-help-compile-time-label>` section, and
|
||||
*all* of the following additional information:
|
||||
|
||||
#. The output of the ``ompi_info --all`` command from the node where
|
||||
you are invoking :ref:`mpirun(1) <man1-mpirun>`.
|
||||
|
||||
#. If you have questions or problems about process mapping or binding,
|
||||
send the output from running the ``lstopo -v`` and ``lstopo --of
|
||||
xml`` commands from a recent version of `Hwloc
|
||||
<https://www.open-mpi.org/projects/hwloc/>`_.
|
||||
|
||||
#. If running on more than one node, also include the output of the
|
||||
``ompi_info --version`` command **from each node on which you are
|
||||
trying to run**.
|
||||
|
||||
#. The output of running ``mpirun --map-by ppr:1:node --prtemca
|
||||
plm_base_verbose 100 --prtemca rmaps_base_verbose 100 --display
|
||||
alloc hostname``. Add in a ``--hostfile`` argument if needed for
|
||||
your environment.
|
||||
|
||||
The script below may be helpful to gather much of the above
|
||||
information (adjust as necessary for your specific environment).
|
||||
|
||||
.. note:: It is safe to run this script after running the script from
|
||||
the :ref:`building and installing
|
||||
<getting-help-compile-time-label>` section.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
# Make a directory for the output files
|
||||
dir="`pwd`/ompi-output"
|
||||
mkdir -p $dir
|
||||
|
||||
# Get installation and system information
|
||||
ompi_info --all 2>&1 | tee $dir/ompi-info-all.out
|
||||
lstopo -v | tee $dir/lstopo-v.txt
|
||||
lstopo --of xml | tee $dir/lstopo.xml
|
||||
|
||||
# Have a text file "my_hostfile.txt" containing the hostnames on
|
||||
# which you are trying to launch
|
||||
for host in `cat my_hostfile.txt`; do
|
||||
ssh $host ompi_info --version 2>&1 | tee $dir/ompi_info-version-$host.out
|
||||
ssh $host lstopo -v | tee $dir/lstopo-v-$host.txt
|
||||
ssh $host lstopo --of xml | tee $dir/lstopo-$host.xml
|
||||
done
|
||||
|
||||
# Have a my_hostfile.txt file if needed for your environment, or
|
||||
# remove the --hostfile argument altogether if not needed.
|
||||
set +e
|
||||
mpirun \
|
||||
--hostfile my_hostfile.txt \
|
||||
--map-by ppr:1:node \
|
||||
--prtemca plm_base_verbose 100 \
|
||||
--prtemca rmaps_base_verbose 100 \
|
||||
--display alloc \
|
||||
hostname 2>&1 | tee $dir/mpirun-hostname.out
|
||||
|
||||
# Bundle up all of these files into a tarball
|
||||
filename="ompi-output.tar.bz2"
|
||||
tar -jcf $filename `basename $dir`
|
||||
echo "Tarball $filename created"
|
||||
|
||||
.. _getting-help-running-label:
|
||||
|
||||
For problems running MPI or OpenSHMEM applications
|
||||
--------------------------------------------------
|
||||
|
||||
If you can successfully launch parallel MPI or OpenSHMEM applications,
|
||||
but the jobs fail during the run, please provide *all* of the
|
||||
information from the :ref:`For problems building or installing Open
|
||||
MPI <getting-help-compile-time-label>` section, *all* of the
|
||||
information from the :ref:`For problems launching MPI or OpenSHMEM
|
||||
applications <getting-help-launching-label>` section, and then *all*
|
||||
of the following additional information:
|
||||
|
||||
#. A *detailed* description of what is failing. *The more details
|
||||
that you provide, the better.* Please include at least the
|
||||
following information:
|
||||
|
||||
* The exact command used to run your application.
|
||||
|
||||
* Any relevant MCA parameters that were set (or unset) when
|
||||
you ran (from either the command line, environment,
|
||||
parameter file, etc.).
|
||||
|
||||
* The value of the ``PATH`` and ``LD_LIBRARY_PATH``
|
||||
environment variables (did you set them correctly to point
|
||||
to all relevant executables, the Open MPI libraries, and
|
||||
any required support libraries, such as libraries required
|
||||
for high-speed networks such as InfiniBand).
|
||||
|
||||
#. The source code of a short sample program (preferably in C or
|
||||
Fortran) that exhibits the problem.
|
||||
|
||||
#. If you are experiencing networking problems, include detailed
|
||||
information about your network.
|
||||
|
||||
.. error:: TODO Update link to IB FAQ entry.
|
||||
|
||||
#. For RoCE- or InfiniBand-based networks, include the information
|
||||
:ref:`in this FAQ entry <faq-ib-troubleshoot-label>`.
|
||||
|
||||
#. For Ethernet-based networks (including RoCE-based networks),
|
||||
include the output of the ``ip addr`` command (or the legacy
|
||||
``ifconfig`` command) on all relevant nodes.
|
||||
|
||||
.. note:: Some Linux distributions do not put ``ip`` or
|
||||
``ifconfig`` in the default ``PATH`` of normal users.
|
||||
Try looking for it in ``/sbin`` or ``/usr/sbin``.
|
||||
@@ -0,0 +1,75 @@
|
||||
History of Open MPI
|
||||
===================
|
||||
|
||||
Open MPI represents the merger of three prior MPI implementations:
|
||||
|
||||
#. LAM/MPI: originally from the Ohio State University supercomputing
|
||||
center and later migrated to the University of Notre Dame.
|
||||
#. LA-MPI: from the US Department of Energy Los Alamos National
|
||||
Laboratory.
|
||||
#. FT-MPI: from the University of Tennassee at Knoxville. One of the
|
||||
UTK developers moved back to the University of Stuttgart in late
|
||||
2004, which effectively added their team into the project.
|
||||
|
||||
The lead developers of these projects kept bumping into each other at
|
||||
various HPC conferences in 2003. At each conference, our
|
||||
lunch/dinner-table conversations got more and more technically
|
||||
involved when it finally dawned on us that we are doing a *lot* of the
|
||||
same things in each of our respective implementations. Although each
|
||||
MPI implementation focused on different areas of excellence, we all
|
||||
shared the same common core values:
|
||||
|
||||
* A full MPI implementation
|
||||
* Production quality code |mdash| it has to "just work"
|
||||
* A desire to explore lots of things that an MPI implementation can do
|
||||
that we've never had the time/resources to investigate because we
|
||||
are bound to bug fixing, etc.
|
||||
|
||||
Hence, we decided to collaborate and pool our resources. At SC2003,
|
||||
we decided to start an entire new code base |mdash| leaving all the cruft
|
||||
and legacy code of our prior implementations behind. Take the best,
|
||||
leave the rest. The source tree's first commit was on November 22,
|
||||
2003; development work started in earnest on January 5, 2004. Since
|
||||
then, we have met together as a group once a month (for at least a
|
||||
week) to meet our goal of a world-class MPI implementation, bar none.
|
||||
|
||||
|
||||
Goals of the Open MPI Project
|
||||
-----------------------------
|
||||
|
||||
We have several top-level goals:
|
||||
|
||||
#. Create a free, open source, peer-reviewed, production-quality
|
||||
complete MPI implementation.
|
||||
#. Provide extremely high, competitive performance (latency,
|
||||
bandwidth, ...pick your favorite metric).
|
||||
#. Directly involve the HPC community with external development
|
||||
and feedback (vendors, 3rd party researchers, users, etc.).
|
||||
#. Provide a stable platform for 3rd party research and commercial
|
||||
development.
|
||||
#. Support a wide variety of HPC platforms and environments.
|
||||
|
||||
In short, we want to work *with* and *for* the HPC community to make a
|
||||
world-class MPI implementation that can be used on a huge number and
|
||||
kind of systems.
|
||||
|
||||
|
||||
Community
|
||||
---------
|
||||
|
||||
Bringing together smart researchers and developers to work on a common
|
||||
product is not only a good idea, it's the open source model. The Open
|
||||
MPI project started by multiple MPI implementation teams, and that
|
||||
proved to work *extremely* well; extending this concept to the HPC
|
||||
open source community is the next logical step.
|
||||
|
||||
The component architecture upon which Open MPI is founded is designed
|
||||
to foster 3rd party collaboration by enabling independent developers
|
||||
to use Open MPI as a production quality research platform. Although
|
||||
Open MPI is a relatively large code base, it is not necessary to learn
|
||||
the entirety of it; it may be sufficient to learn the interfaces for
|
||||
the component type which you are implementing and some of the
|
||||
surrounding infrastructure. Specifically, the component architecture
|
||||
was designed to allow small, discrete implementations of major
|
||||
portions of MPI functionality (e.g., point-to-point messaging,
|
||||
collective communications, run-time environment support, etc.).
|
||||
@@ -0,0 +1,92 @@
|
||||
.. image:: openmpi_logo.png
|
||||
:align: right
|
||||
|
||||
Open MPI |ompi_ver|
|
||||
===================
|
||||
|
||||
`The Open MPI Project <https://www.open-mpi.org/>`_ is an open source
|
||||
implementation of the `Message Passing Interface (MPI) specification
|
||||
<https://www.mpi-forum.org/docs/>`_ that is developed and maintained
|
||||
by a consortium of academic, research, and industry partners. Open
|
||||
MPI is therefore able to combine the expertise, technologies, and
|
||||
resources from all across the High Performance Computing community in
|
||||
order to build the best MPI library available. Open MPI offers
|
||||
advantages for system and software vendors, application developers and
|
||||
computer science researchers.
|
||||
|
||||
Documentation locations
|
||||
=======================
|
||||
|
||||
Documentation for Open MPI can be found in the following locations:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Open MPI version
|
||||
- Documentation location
|
||||
|
||||
* - v5.0.0 and later
|
||||
- Web: https://docs.open-mpi.org/
|
||||
|
||||
Included in tarball: ``docs/html/index.html``
|
||||
|
||||
Built in source tree (if Sphinx available): ``docs/_build/html/index.html``
|
||||
|
||||
Installed: ``$docdir/html/index.html``
|
||||
|
||||
(which defaults to: ``$prefix/share/doc/openmpi/html/index.html``)
|
||||
|
||||
* - v4.1.x and earlier
|
||||
- See the `legacy Open MPI FAQ <https://www.open-mpi.org/faq/>`_
|
||||
and the README file in the source tarball.
|
||||
|
||||
For example:
|
||||
|
||||
* `v4.1.x README file <https://github.com/open-mpi/ompi/blob/v4.1.x/README>`_
|
||||
* `v4.0.x README file <https://github.com/open-mpi/ompi/blob/v4.0.x/README>`_
|
||||
|
||||
Release announcements
|
||||
=====================
|
||||
|
||||
The best way to hear about new Open MPI releases is via the
|
||||
`low-volume announcement mailing list
|
||||
<https://lists.open-mpi.org/mailman/listinfo/announce>`_.
|
||||
|
||||
Additionally, if you are a downstream packager of Open MPI (e.g., you
|
||||
package Open MPI for an operating system distribution), you may wish
|
||||
to sign up for the `low-volume packagers list
|
||||
<https://lists.open-mpi.org/mailman/listinfo/ompi-packagers>`_.
|
||||
|
||||
Table of contents
|
||||
=================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:numbered:
|
||||
|
||||
quickstart
|
||||
getting-help
|
||||
release-notes/index
|
||||
installing-open-mpi/index
|
||||
features/index
|
||||
validate
|
||||
version-numbering
|
||||
mca
|
||||
building-apps/index
|
||||
launching-apps/index
|
||||
tuning-apps/index
|
||||
app-debug/index
|
||||
developers/index
|
||||
contributing
|
||||
license/index
|
||||
history
|
||||
man-openmpi/index
|
||||
man-openshmem/index
|
||||
|
||||
Contributors
|
||||
============
|
||||
|
||||
A gigantic "thank you!" to all of our contributors:
|
||||
|
||||
.. image:: https://contrib.rocks/image?repo=open-mpi/ompi&max=999
|
||||
:target: https://github.com/open-mpi/ompi/graphs/contributors
|
||||
@@ -0,0 +1,152 @@
|
||||
.. _install-configure-compilers-and-flags-label:
|
||||
|
||||
Specifying compilers and flags
|
||||
==============================
|
||||
|
||||
Changing the compilers that Open MPI uses to build itself uses the
|
||||
standard GNU Autoconf mechanism of setting special environment variables
|
||||
either before invoking ``configure`` or on the ``configure`` command
|
||||
line itself.
|
||||
|
||||
The following environment variables are recognized by ``configure``:
|
||||
|
||||
* ``CC``: C compiler to use
|
||||
* ``CFLAGS``: Compile flags to pass to the C compiler
|
||||
* ``CPPFLAGS``: Preprocessor flags to pass to the C compiler
|
||||
* ``CXX``: C++ compiler to use
|
||||
* ``CXXCFLAGS``: Compile flags to pass to the C++ compiler
|
||||
* ``CXXCPPFLAGS``: Preprocessor flags to pass to the C++ compiler
|
||||
* ``FC``: Fortran compiler to use
|
||||
* ``FCFLAGS``: Compile flags to pass to the Fortran compiler
|
||||
* ``LDFLAGS``: Linker flags to pass to all compilers
|
||||
* ``LIBS``: Libraries to pass to all compilers (it is rarely
|
||||
necessary for users to need to specify additional ``LIBS``)
|
||||
* ``PKG_CONFIG``: Path to the ``pkg-config`` utility
|
||||
|
||||
.. note:: Open MPI |ompi_ver| does not contain any C++ code. The only
|
||||
tests that ``configure`` runs with the C++ compiler is for
|
||||
the purposes of determining an appropriate value for ``CXX``
|
||||
to use in the ``mpic++`` :ref:`wrapper compiler
|
||||
<label-quickstart-building-apps>`. The ``CXXCFLAGS`` and
|
||||
``CXXCPPFLAGS`` values are *only* used in these
|
||||
``configure`` checks to ensure that the C++ compiler works.
|
||||
|
||||
For example, to build with a specific instance of ``gcc``, ``g++``,
|
||||
and ``gfortran``:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure \
|
||||
CC=/opt/gcc-a.b.c/bin/gcc \
|
||||
CXX=/opt/gcc-a.b.c/bin/g++ \
|
||||
FC=/opt/gcc-a.b.c/bin/gfortran ...
|
||||
|
||||
Here's another example, this time showing building with the Intel
|
||||
compiler suite:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure \
|
||||
CC=icc \
|
||||
CXX=icpc \
|
||||
FC=ifort ...
|
||||
|
||||
.. note:: The Open MPI community generally suggests using the above
|
||||
command line form for setting different compilers (vs. setting
|
||||
environment variables and then invoking ``./configure``). The
|
||||
above form will save all variables and values in the ``config.log``
|
||||
file, which makes post-mortem analysis easier if problems occur.
|
||||
|
||||
Note that the flags you specify must be compatible across all the
|
||||
compilers. In particular, flags specified to one language compiler
|
||||
must generate code that can be compiled and linked against code that
|
||||
is generated by the other language compilers. For example, on a 64
|
||||
bit system where the compiler default is to build 32 bit executables:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Assuming the GNU compiler suite
|
||||
shell$ ./configure CFLAGS=-m64 ...
|
||||
|
||||
will produce 64 bit C objects, but 32 bit objects for Fortran. These
|
||||
codes will be incompatible with each other, and Open MPI will not build
|
||||
successfully. Instead, you must specify building 64 bit objects for
|
||||
*all* languages:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Assuming the GNU compiler suite
|
||||
shell$ ./configure CFLAGS=-m64 FCFLAGS=-m64 ...
|
||||
|
||||
The above command line will pass ``-m64`` to all the compilers, and
|
||||
therefore will produce 64 bit objects for all languages.
|
||||
|
||||
.. warning:: Note that setting ``CFLAGS`` (etc.) does *not* affect the
|
||||
flags used by the :ref:`wrapper compilers
|
||||
<label-quickstart-building-apps>`. In the above, example, you may
|
||||
also need to add ``-m64`` to various ``--with-wrapper-FOO``
|
||||
options:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ./configure CFLAGS=-m64 FCFLAGS=-m64 \
|
||||
--with-wrapper-cflags=-m64 \
|
||||
--with-wrapper-cxxflags=-m64 \
|
||||
--with-wrapper-fcflags=-m64 ...
|
||||
|
||||
Failure to do this will result in MPI applications
|
||||
failing to compile / link properly.
|
||||
|
||||
See the :ref:`Customizing wrapper compiler behavior
|
||||
<label-customizing-wrapper-compiler>` section for more
|
||||
details.
|
||||
|
||||
Note that if you intend to compile Open MPI with a ``make`` other than
|
||||
the default one in your ``PATH``, then you must either set the ``$MAKE``
|
||||
environment variable before invoking Open MPI's ``configure`` script, or
|
||||
pass ``MAKE=your_make_prog`` to configure. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure MAKE=/path/to/my/make ...
|
||||
|
||||
This could be the case, for instance, if you have a shell alias for
|
||||
``make``, or you always type ``gmake`` out of habit. Failure to tell
|
||||
``configure`` which non-default ``make`` you will use to compile Open MPI
|
||||
can result in undefined behavior (meaning: don't do that).
|
||||
|
||||
Note that you may also want to ensure that the value of
|
||||
``LD_LIBRARY_PATH`` is set appropriately (or not at all) for your build
|
||||
(or whatever environment variable is relevant for your operating
|
||||
system). For example, some users have been tripped up by setting to
|
||||
use a non-default Fortran compiler via the ``FC`` environment variable,
|
||||
but then failing to set ``LD_LIBRARY_PATH`` to include the directory
|
||||
containing that non-default Fortran compiler's support libraries.
|
||||
This causes Open MPI's ``configure`` script to fail when it tries to
|
||||
compile / link / run simple Fortran programs.
|
||||
|
||||
It is required that the compilers specified be compile and link
|
||||
compatible, meaning that object files created by one compiler must be
|
||||
able to be linked with object files from the other compilers and
|
||||
produce correctly functioning executables.
|
||||
|
||||
Statically linking to the libraries of Intel compiler suite
|
||||
-----------------------------------------------------------
|
||||
|
||||
The Intel compiler suite, by default, dynamically links its runtime libraries
|
||||
against the Open MPI binaries and libraries. This can cause problems if the
|
||||
Intel compiler libraries are installed in non-standard locations. For example,
|
||||
you might get errors like:
|
||||
|
||||
.. code-block::
|
||||
|
||||
error while loading shared libraries: libimf.so: cannot open shared object file:
|
||||
No such file or directory
|
||||
|
||||
To avoid such problems, you can pass flags to Open MPI's configure
|
||||
script that instruct the Intel compiler suite to statically link its
|
||||
runtime libraries with Open MPI:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ./configure CC=icc CXX=icpc FC=ifort LDFLAGS=-Wc,-static-intel ...
|
||||
@@ -0,0 +1,58 @@
|
||||
.. _building-ompi-cli-options-conventions-label:
|
||||
|
||||
``configure`` CLI option conventions
|
||||
------------------------------------
|
||||
|
||||
``configure`` will, by default, search for header files and/or
|
||||
libraries for various optional features (e.g., various HPC network
|
||||
API/support libraries). If the relevant files are found, Open MPI
|
||||
will built support for that feature. If they are not found, Open MPI
|
||||
will skip building support for that feature.
|
||||
|
||||
However, if you specify ``--with-FOO`` (where ``FOO`` is the
|
||||
corresponding CLI option name for the feature) on the ``configure``
|
||||
command line and Open MPI is unable to find relevant support for
|
||||
``FOO``, ``configure`` will assume that it was unable to provide a
|
||||
feature that was specifically requested, and will therefore abort so
|
||||
that a human can resolve out the issue.
|
||||
|
||||
.. note:: Using ``--with-FOO`` to force Open MPI's ``configure``
|
||||
script to abort it if can't find support for a given feature
|
||||
may be preferable to unexpectedly discovering at run-time
|
||||
that Open MPI is missing support for a critical feature.
|
||||
|
||||
Additionally, if a search directory is specified for ``FOO`` in the
|
||||
form ``--with-FOO=DIR``, Open MPI will:
|
||||
|
||||
#. Search for ``FOO``'s header files in ``DIR/include``.
|
||||
#. Search for ``FOO``'s library files:
|
||||
|
||||
#. If ``--with-FOO-libdir=LIBDIR`` was specified, search in
|
||||
``LIBDIR``.
|
||||
#. Otherwise, search in ``DIR/lib``, and if they are not found
|
||||
there, search again in ``DIR/lib64``.
|
||||
|
||||
#. If both the relevant header files and libraries are found:
|
||||
|
||||
#. Open MPI will build support for ``FOO``.
|
||||
#. If the root path where the FOO libraries are found is neither
|
||||
``/usr`` nor ``/usr/local``, Open MPI will compile itself with
|
||||
RPATH flags pointing to the directory where ``FOO``'s libraries
|
||||
are located.
|
||||
|
||||
.. important:: Open MPI does not RPATH ``/usr/lib[64]`` and
|
||||
``/usr/local/lib[64]`` because many systems
|
||||
already search these directories for run-time
|
||||
libraries by default; adding RPATH for them could
|
||||
have unintended consequences for the search path
|
||||
ordering.
|
||||
|
||||
.. caution:: The ``--with-FOO-libdir=LIBDIR`` options are not usually
|
||||
needed; they are typically only needed when ``FOO``'s libraries are
|
||||
installed in an "unexpected" location.
|
||||
|
||||
Also note the difference between ``--with-FOO=DIR`` and
|
||||
``--with-FOO-libdir=LIBDIR``: the former is a directory to which
|
||||
suffixes such as ``/include`` and ``/lib`` are added, whereas the
|
||||
latter is assumed to be a full library directory name (e.g.,
|
||||
``/opt/some_library/lib``).
|
||||
@@ -0,0 +1,22 @@
|
||||
``configure`` command line options
|
||||
==================================
|
||||
|
||||
There are many available ``configure`` command line options; see
|
||||
``./configure --help`` for a full list.
|
||||
|
||||
Conventions for the command line options and many commonly-used
|
||||
options (but not all of them!) are described in the sections listed
|
||||
below.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
conventions
|
||||
installation
|
||||
networking
|
||||
runtime
|
||||
required-support-libraries
|
||||
mpi
|
||||
oshmem
|
||||
misc
|
||||
rpath-and-runpath
|
||||
@@ -0,0 +1,334 @@
|
||||
.. _label-building-installation-cli-options:
|
||||
|
||||
Installation options
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The following are general installation command line options that can
|
||||
be used with ``configure``:
|
||||
|
||||
* ``--prefix=DIR``:
|
||||
Install Open MPI into the base directory named ``DIR``. Hence, Open
|
||||
MPI will place its executables in ``DIR/bin``, its header files in
|
||||
``DIR/include``, its libraries in ``DIR/lib``, etc.
|
||||
|
||||
.. note:: Also see the section on :ref:`installation location
|
||||
<building-open-mpi-installation-location-label>` for more
|
||||
information on the installation prefix.
|
||||
|
||||
* ``--disable-shared``: By default, Open MPI and OpenSHMEM build
|
||||
shared libraries, and all components are included as part of those
|
||||
shared libraries. This switch disables this default; it is really
|
||||
only useful when used with ``--enable-static``. Specifically, this
|
||||
option does *not* imply ``--enable-static``; enabling static
|
||||
libraries and disabling shared libraries are two independent
|
||||
options.
|
||||
|
||||
.. tip::
|
||||
|
||||
:ref:`See this section <label-install-packagers-dso-or-not>` for
|
||||
advice to packagers about this CLI option.
|
||||
|
||||
* ``--enable-static``:
|
||||
Build MPI and OpenSHMEM as static libraries, and statically link in
|
||||
all components. Note that this option does *not* imply
|
||||
``--disable-shared``; enabling static libraries and disabling shared
|
||||
libraries are two independent options.
|
||||
|
||||
Be sure to read the description of ``--without-memory-manager``,
|
||||
below; it may have some effect on ``--enable-static``.
|
||||
|
||||
.. tip::
|
||||
|
||||
:ref:`See this section <label-install-packagers-dso-or-not>` for
|
||||
advice to packagers about this CLI option.
|
||||
|
||||
* ``--disable-wrapper-runpath`` / ``--disable-wrapper-rpath``: By
|
||||
default, the wrapper compilers (e.g., ``mpicc``) will explicitly add
|
||||
"runpath" and "rpath" linker flags when linking user executables on
|
||||
systems that support them. That is, the created executables will
|
||||
include a filesystem path reference to the location of Open MPI's
|
||||
libraries in the application executable itself. This means that the
|
||||
user does not have to set ``LD_LIBRARY_PATH`` to find Open MPI's
|
||||
libraries, which can be helpful if they are installed in a location
|
||||
that the run-time linker does not search by default.
|
||||
|
||||
Using the ``--disable-wrapper-r*path`` options will prevent the
|
||||
wrappers from explicitly adding one or both of these linker flags.
|
||||
|
||||
.. note:: By default, the wrapper compilers prefer "runpath"
|
||||
behavior over "rpath" behavior.
|
||||
|
||||
* Using ``--disable-wrapper-runpath`` alters this
|
||||
preference: explicit "runpath" linker flags will not be
|
||||
added by the wrappers. However, "rpath" flags may still
|
||||
be added, if the platform supports them.
|
||||
* Using both ``--disable-wrapper-runpath`` *and*
|
||||
``--disable-wrapper-rpath`` will prevent the wrappers
|
||||
from explicitly adding "runpath" *and* "rpath" linker
|
||||
flags.
|
||||
|
||||
.. caution:: Even if the wrapper compilers do not explicitly add
|
||||
"runpath" or "rpath" linker flags, the local compiler,
|
||||
linker, and/or operating system may implicitly enable
|
||||
either "runpath" or "rpath" behavior when linking.
|
||||
|
||||
.. important:: The ``--disable-wrapper-runpath`` and
|
||||
``--disable-wrapper-rpath`` CLI options *only* affect
|
||||
the flags that the wrapper compilers use when
|
||||
building MPI/SHMEM applications. These options do
|
||||
not affect how Open MPI or OpenSHMEM are built (to
|
||||
include the wrapper compilers themselves).
|
||||
|
||||
See the :ref:`Linker "rpath" and "runpath"
|
||||
functionality
|
||||
<building-ompi-cli-options-rpath-and-runpath-label>`
|
||||
section for details on how "rpath" and "runpath"
|
||||
affect the building and linking of Open MPI itself.
|
||||
|
||||
When either of "runpath" or "rpath" behaviors are enabled, the
|
||||
applications will have the filesystem path location of the Open MPI
|
||||
libraries hard-coded into the Open MPI/OpenSHMEM application. The
|
||||
most notable differences between "runpath" and "rpath" behavior are:
|
||||
|
||||
* runpath
|
||||
|
||||
#. The run-time linker first searches the paths in the
|
||||
``LD_LIBRARY_PATH`` environment variable for the relevant Open
|
||||
MPI/OpenSHMEM libraries.
|
||||
#. If not found there, the run-time linker falls back to checking
|
||||
the hard-coded location for the relevant Open MPI/OpenSHMEM
|
||||
libraries.
|
||||
|
||||
* rpath
|
||||
|
||||
#. The run-time linker first checks the hard-coded location for
|
||||
the relevant Open MPI/OpenSHMEM libraries.
|
||||
#. If not found there, the run-time linker falls back to
|
||||
searching the paths in the ``LD_LIBRARY_PATH`` environment
|
||||
variable for the relevant Open MPI/OpenSHMEM libraries.
|
||||
|
||||
.. warning:: There are other, subtle differences between "runpath"
|
||||
and "rpath" which are out of scope for this
|
||||
documentation. You may wish to consult other sources
|
||||
for more information.
|
||||
|
||||
For example, a decent set of explanations can be found
|
||||
in the slides for a Linux course entitled "Building and
|
||||
Using Shared Libraries on Linux // `Shared Libraries:
|
||||
The Dynamic Linker
|
||||
<https://man7.org/training/download/shlib_dynlinker_slides.pdf>`_".
|
||||
|
||||
For example, consider that you install Open MPI vA.B.0 and
|
||||
compile/link your MPI/OpenSHMEM application against it. Later, you
|
||||
install Open MPI vA.B.1 to a different installation prefix (e.g.,
|
||||
``/opt/openmpi/A.B.1`` vs. ``/opt/openmpi/A.B.0``), and you leave
|
||||
the old installation intact.
|
||||
|
||||
In the runpath case, you can set the ``LD_LIBRARY_PATH`` environment
|
||||
variable to point to the A.B.1 installation, and then your MPI
|
||||
application will use those libraries, since the runtime will search
|
||||
the paths in ``LD_LIBRARY_PATH`` first.
|
||||
|
||||
In the rpath case, since the run-time linker searches the
|
||||
``/opt/openmpi/A.B.0`` location that is hard-coded in your MPI
|
||||
application first, your application will use the libraries from your
|
||||
A.B.0 installation (regardless of the value of the
|
||||
``LD_LIBRARY_PATH`` environment variable).
|
||||
|
||||
Note that in both cases, however, if you remove the original A.B.0
|
||||
installation and set ``LD_LIBRARY_PATH`` to point to the A.B.1
|
||||
installation, your application will use the A.B.1 libraries.
|
||||
|
||||
As noted above, both runpath/rpath behaviors can be disabled via
|
||||
``--disable-wrapper-rpath``.
|
||||
|
||||
.. note:: You can also :ref:`customize the compiler/linker flags
|
||||
that are used by the wrapper compilers
|
||||
<label-customizing-wrapper-compiler>` to build Open
|
||||
MPI/OpenSHMEM applications.
|
||||
|
||||
.. _building-ompi-cli-options-diable-dlopen-label:
|
||||
|
||||
* ``--enable-dlopen``: Enable Open MPI to load components as
|
||||
standalone Dynamic Shared Objects (DSOs) at run-time. This option
|
||||
is enabled by default.
|
||||
|
||||
The opposite of this option, ``--disable-dlopen``, causes the following:
|
||||
|
||||
#. Open MPI will not attempt to open any DSOs at run-time.
|
||||
#. configure behaves as if the ``--enable-mca-static`` argument was set.
|
||||
#. configure will ignore the ``--enable-mca-dso`` argument.
|
||||
|
||||
See the description of ``--enable-mca-static`` / ``--enable-mca-dso`` for
|
||||
more information.
|
||||
|
||||
.. note:: This option does *not* change how Open MPI's libraries
|
||||
(``libmpi``, for example) will be built. You can change
|
||||
whether Open MPI builds static or dynamic libraries via
|
||||
the ``--enable|disable-static`` and
|
||||
``--enable|disable-shared`` arguments.
|
||||
|
||||
.. _building-ompi-cli-options-mca-dso-label:
|
||||
|
||||
* ``--enable-mca-dso[=LIST]`` and ``--enable-mca-static[=LIST]``
|
||||
These two options, along with ``--enable-mca-no-build``, govern the
|
||||
behavior of how Open MPI's frameworks and components are built.
|
||||
|
||||
.. tip::
|
||||
|
||||
:ref:`See this section <label-install-packagers-dso-or-not>` for
|
||||
advice to packagers about these CLI options.
|
||||
|
||||
The ``--enable-mca-dso`` option specifies which frameworks and/or
|
||||
components are built as Dynamic Shared Objects (DSOs).
|
||||
Specifically, DSOs are built as "plugins" outside of the core Open
|
||||
MPI libraries, and are loaded by Open MPI at run time.
|
||||
|
||||
The ``--enable-mca-static`` option specifies which frameworks and/or
|
||||
components are built as part of the core Open MPI libraries (i.e.,
|
||||
they are not built as DSOs, and therefore do not need to be
|
||||
separately discovered and opened at run time).
|
||||
|
||||
Both options can be used one of two ways:
|
||||
|
||||
#. ``--enable-mca-OPTION`` (with no value)
|
||||
#. ``--enable-mca-OPTION=LIST``
|
||||
|
||||
``--enable-mca-OPTION=no`` or ``--disable-mca-OPTION`` are both legal
|
||||
options, but have no impact on the selection logic described below.
|
||||
Only affirmative options change the selection process.
|
||||
|
||||
If ``LIST`` is not specified (e.g., ``--enable-mca-dso`` with no
|
||||
``LIST``), or if ``LIST`` is the special value ``yes``, then *all*
|
||||
components will be selected. If ``LIST`` is specified, it is a
|
||||
comma-delimited list of Open MPI frameworks and/or
|
||||
framework+component tuples. Examples:
|
||||
|
||||
* ``btl`` specifies the entire BTL framework
|
||||
* ``btl-tcp`` specifies just the TCP component in the BTL framework
|
||||
* ``mtl,btl-tcp`` specifies the entire MTL framework and the TCP
|
||||
component in the BTL framework
|
||||
|
||||
Open MPI's ``configure`` script uses the values of these two options
|
||||
when evaluating each component to determine how it should be built
|
||||
by evaluating these conditions in order:
|
||||
|
||||
#. If an individual component's build behavior has been specified
|
||||
via these two options, ``configure`` uses that behavior.
|
||||
#. Otherwise, if the component is in a framework whose build
|
||||
behavior has been specified via these two options, ``configure``
|
||||
uses that behavior.
|
||||
#. Otherwise, ``configure`` uses the global default build behavior.
|
||||
|
||||
At each level of the selection process, if the component is
|
||||
specified to be built as both a static and DSO component, the static
|
||||
option will win.
|
||||
|
||||
.. note:: As of Open MPI |ompi_ver|, ``configure``'s global default
|
||||
is to build all components as static (i.e., part of the
|
||||
Open MPI core libraries, not as DSOs). Prior to Open MPI
|
||||
v5.0.0, the global default behavior was to build
|
||||
most components as DSOs.
|
||||
|
||||
.. important:: If the ``--disable-dlopen`` option is specified, then
|
||||
Open MPI will not be able to search for DSOs at run
|
||||
time, and the value of the ``--enable-mca-dso``
|
||||
option will be silently ignored.
|
||||
|
||||
Some examples:
|
||||
|
||||
#. Default to building all components as static (i.e., as part of
|
||||
the Open MPI core libraries |mdash| no DSOs)::
|
||||
|
||||
shell$ ./configure
|
||||
|
||||
#. Build all components as DSOs::
|
||||
|
||||
shell$ ./configure --enable-mca-dso
|
||||
|
||||
#. Build all components as static, except the TCP BTL, which will be
|
||||
built as a DSO::
|
||||
|
||||
shell$ ./configure --enable-mca-dso=btl-tcp
|
||||
|
||||
#. Build all components as static, except all BTL components, which
|
||||
will be built as DSOs::
|
||||
|
||||
shell$ ./configure --enable-mca-dso=btl
|
||||
|
||||
#. Build all components as static, except all MTL components and the
|
||||
TCP BTL component, which will be built as DSOs::
|
||||
|
||||
shell$ ./configure --enable-mca-dso=mtl,btl-tcp
|
||||
|
||||
#. Build all BTLs as static, except the TCP BTL, as the
|
||||
``<framework-component>`` option is more specific than the
|
||||
``<framework>`` option::
|
||||
|
||||
shell$ ./configure --enable-mca-dso=btl --enable-mca-static=btl-tcp
|
||||
|
||||
#. Build the TCP BTL as static, because the static option at the
|
||||
same level always wins::
|
||||
|
||||
shell$ ./configure --enable-mca-dso=btl-tcp --enable-mca-static=btl-tcp
|
||||
|
||||
* ``--enable-mca-no-build=LIST``: Comma-separated list of
|
||||
``<framework>-<component>`` pairs that will not be built. For
|
||||
example, ``--enable-mca-no-build=threads-qthreads,pml-monitoring`` will
|
||||
disable building both the ``qthreads`` threading component and the
|
||||
``monitoring`` PML.
|
||||
|
||||
.. note:: This option is typically only useful for components that
|
||||
would otherwise be built. For example, if you are on a
|
||||
machine without Libfabric support, it is not necessary to
|
||||
specify::
|
||||
|
||||
shell$ ./configure --enable-mca-no-build=cm-ofi
|
||||
|
||||
because the ``configure`` script will naturally see that
|
||||
you do not have support for Libfabric and will
|
||||
automatically skip the ``ofi`` CM component.
|
||||
|
||||
* ``--with-show-load-errors=VALUE``:
|
||||
Set the default value of the ``mca_base_component_show_load_errors``
|
||||
MCA variable. The MCA ``mca_base_component_show_load_errors`` variable
|
||||
can still be overridden at run time via the usual MCA-variable-setting
|
||||
mechanisms; this configure option simply sets the default value.
|
||||
|
||||
The ``no``/``none`` value of this option is intended for Open MPI
|
||||
packagers who tend to enable support for many different types of
|
||||
networks and systems in their packages. For example, consider a
|
||||
packager who includes support for both the FOO and BAR networks in
|
||||
their Open MPI package, both of which require support libraries
|
||||
(``libFOO.so`` and ``libBAR.so``). If an end user only has BAR
|
||||
hardware, they likely only have ``libBAR.so`` available on their
|
||||
systems |mdash| not ``libFOO.so``. Disabling load errors by default will
|
||||
prevent the user from seeing potentially confusing warnings about
|
||||
the FOO components failing to load because ``libFOO.so`` is not
|
||||
available on their systems.
|
||||
|
||||
Conversely, the ``yes``/``all`` value of this option is intended for
|
||||
system administrators who tend to build an Open MPI that is
|
||||
targeted at their specific environment, and contains few (if any)
|
||||
components that are not needed. In such cases, they might want
|
||||
their users to be warned that the FOO network components failed to
|
||||
load (e.g., if ``libFOO.so`` was mistakenly unavailable), because Open
|
||||
MPI may otherwise silently failover to a slower network path for MPI
|
||||
traffic.
|
||||
|
||||
.. note:: See the section on :ref:`common MCA parameters
|
||||
<label-mca-common-parameters>` for details related to the
|
||||
``mca_base_component_show_load_errors`` MCA variable.
|
||||
|
||||
* ``--with-platform=FILE``:
|
||||
Load configure options for the build from ``FILE``. Options on the
|
||||
command line that are not in ``FILE`` are also used. Options on the
|
||||
command line and in ``FILE`` are replaced by what is in ``FILE``.
|
||||
|
||||
* ``--with-libmpi-name=STRING``:
|
||||
Replace ``libmpi.*`` and ``libmpi_FOO.*`` (where ``FOO`` is one of the
|
||||
fortran supporting libraries installed in lib) with ``libSTRING.*``
|
||||
and ``libSTRING_FOO.*``. This is provided as a convenience mechanism
|
||||
for third-party packagers of Open MPI that might want to rename
|
||||
these libraries for their own purposes. This option is *not*
|
||||
intended for typical users of Open MPI.
|
||||
@@ -0,0 +1,97 @@
|
||||
.. This file is included by building-open-mpi.rst
|
||||
|
||||
Miscellaneous functionality
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The following are command line options that don't fit any any of the
|
||||
above categories that can be used with ``configure``:
|
||||
|
||||
* ``--without-memory-manager``:
|
||||
Disable building Open MPI's memory manager. Open MPI's memory
|
||||
manager is usually built on Linux based platforms, and is generally
|
||||
only used for optimizations with some OpenFabrics-based networks (it
|
||||
is not *necessary* for OpenFabrics networks, but some performance
|
||||
loss may be observed without it).
|
||||
|
||||
.. warning:: Open MPI's memory management functionality, which provides
|
||||
important performance optimizations on OS-bypass networks
|
||||
such as InfiniBand, requires the ``dlsym(3)`` interface,
|
||||
and therefore does not work with fully-static applications.
|
||||
|
||||
* ``--with-ft=TYPE``:
|
||||
Specify the type of fault tolerance to enable. The only allowed
|
||||
values are ``ulfm`` and ``none``. See :ref:`the ULFM section
|
||||
<ulfm-label>` for more details.
|
||||
|
||||
* ``--enable-peruse``:
|
||||
Enable the PERUSE MPI data analysis interface.
|
||||
|
||||
* ``--enable-heterogeneous``:
|
||||
Enable support for running on heterogeneous clusters (e.g., machines
|
||||
with different endian representations). Heterogeneous support is
|
||||
disabled by default because it imposes a minor performance penalty.
|
||||
|
||||
.. danger:: The heterogeneous functionality is currently broken |mdash|
|
||||
do not use.
|
||||
|
||||
.. _install-wrapper-flags-label:
|
||||
|
||||
* ``--with-wrapper-cflags=CFLAGS``
|
||||
* ``--with-wrapper-cxxflags=CXXFLAGS``
|
||||
* ``--with-wrapper-fcflags=FCFLAGS``
|
||||
* ``--with-wrapper-ldflags=LDFLAGS``
|
||||
* ``--with-wrapper-libs=LIBS``:
|
||||
Add the specified flags to the default flags that are used in Open
|
||||
MPI's "wrapper" compilers (e.g., ``mpicc`` |mdash| see below for more
|
||||
information about Open MPI's wrapper compilers). By default, Open
|
||||
MPI's wrapper compilers use the same compilers used to build Open
|
||||
MPI and specify a minimum set of additional flags that are necessary
|
||||
to compile/link MPI applications. These configure options give
|
||||
system administrators the ability to embed additional flags in
|
||||
OMPI's wrapper compilers (which is a local policy decision). The
|
||||
meanings of the different flags are:
|
||||
|
||||
* ``CFLAGS``: Flags passed by the ``mpicc`` wrapper to the C
|
||||
compiler
|
||||
* ``CXXFLAGS``: Flags passed by the ``mpic++`` and ``mpiCC``
|
||||
wrappers to the C++ compiler
|
||||
* ``FCFLAGS``: Flags passed by the ``mpifort`` wrapper to the
|
||||
Fortran compiler
|
||||
* ``LDFLAGS``: Flags passed by all the wrappers to the linker
|
||||
* ``LIBS``: Flags passed by all the wrappers to the linker
|
||||
|
||||
See the section on :ref:`customizing wrapper compiler behavior
|
||||
<label-customizing-wrapper-compiler>` to see how to alter the
|
||||
wrapper compiler behavior at run time.
|
||||
|
||||
* ``--with-mpi-moduledir``: Specify the directory where the Fortran
|
||||
MPI module files are installed.
|
||||
|
||||
For historical reasons, Open MPI's Fortran MPI modulefiles are
|
||||
installed into ``$libdir`` by default. This option lets you change
|
||||
where they are installed; some users prefer Fortran module files
|
||||
installed into ``$installdir``, for example.
|
||||
|
||||
.. note:: If you intend to make your Open MPI installation
|
||||
relocatable :ref:`via the OPAL_PREFIX mechanism
|
||||
<install-location-opal-prefix>`, you will want to ensure
|
||||
to specify the module directory relative to the
|
||||
``prefix``. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ ./configure --prefix=/opt/openmpi --with-mpi-moduledir='${includedir}/modules`...
|
||||
|
||||
Note the additional shell quoting that is likely necessary
|
||||
to prevent shell variable expansion, and the additional
|
||||
``${}`` around ``includedir`` that is necessary for Open MPI
|
||||
to recognize that it is a special name that needs to be
|
||||
expanded.
|
||||
|
||||
Finally, note that the Fortran module installation
|
||||
directory is *not* one of the :ref:`recognized directories
|
||||
that can be specified via environment variable at run time
|
||||
<install-location-overriding-individual-directories>`.
|
||||
Instead, to make the Fortran module directory relocatable,
|
||||
it needs to be relative to one of the other recognized
|
||||
directories.
|
||||
@@ -0,0 +1,100 @@
|
||||
.. This file is included by building-open-mpi.rst
|
||||
|
||||
MPI functionality
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
The following are command line options to set the default for various
|
||||
MPI API behaviors that can be used with ``configure``:
|
||||
|
||||
* ``--with-mpi-param-check[=VALUE]``:
|
||||
Whether or not to check MPI function parameters for errors at
|
||||
runtime. The following ``VALUE``\s are permitted:
|
||||
|
||||
* ``always``: MPI function parameters are always checked for errors
|
||||
* ``never``: MPI function parameters are never checked for errors
|
||||
* ``runtime``: Whether MPI function parameters are checked depends on
|
||||
the value of the MCA parameter ``mpi_param_check`` (default: yes).
|
||||
* ``yes``: Synonym for "always" (same as ``--with-mpi-param-check``).
|
||||
* ``no``: Synonym for "never" (same as ``--without-mpi-param-check``).
|
||||
|
||||
If ``--with-mpi-param`` is not specified, ``runtime`` is the default.
|
||||
|
||||
* ``--disable-mpi-thread-multiple``:
|
||||
Disable the MPI thread level ``MPI_THREAD_MULTIPLE`` (it is enabled by
|
||||
default).
|
||||
|
||||
* ``--disable-ft``:
|
||||
Disable the User-Level Fault Mitigation (ULFM) support in Open MPI
|
||||
(it is enabled by default).
|
||||
|
||||
:ref:`See the ULFM section <ulfm-label>` for more information.
|
||||
|
||||
* ``--enable-mpi-java``:
|
||||
Enable building of an **EXPERIMENTAL** Java MPI interface (disabled
|
||||
by default). You may also need to specify ``--with-jdk-dir``,
|
||||
``--with-jdk-bindir``, and/or ``--with-jdk-headers``.
|
||||
|
||||
.. warning:: Note that this Java interface is **INCOMPLETE**
|
||||
(meaning: it does not support all MPI functionality) and **LIKELY
|
||||
TO CHANGE**. The Open MPI developers would very much like to
|
||||
hear your feedback about this interface.
|
||||
|
||||
:ref:`See the Java section <open-mpi-java-label>` for many more
|
||||
details.
|
||||
|
||||
* ``--enable-mpi-fortran[=VALUE]``:
|
||||
By default, Open MPI will attempt to build all 3 Fortran bindings:
|
||||
``mpif.h``, the ``mpi`` module, and the ``mpi_f08`` module. The following
|
||||
``VALUE``\s are permitted:
|
||||
|
||||
* ``all``: Synonym for ``yes``.
|
||||
* ``yes``: Attempt to build all 3 Fortran bindings; skip
|
||||
any binding that cannot be built (same as
|
||||
``--enable-mpi-fortran``).
|
||||
* ``mpifh``: Only build ``mpif.h`` support.
|
||||
* ``usempi``: Only build ``mpif.h`` and ``mpi`` module support.
|
||||
* ``usempif08``: Build ``mpif.h``, ``mpi`` module, and ``mpi_f08``
|
||||
module support.
|
||||
* ``none``: Synonym for ``no``.
|
||||
* ``no``: Do not build any MPI Fortran support (same as
|
||||
``--disable-mpi-fortran``). This is mutually exclusive
|
||||
with building the OpenSHMEM Fortran interface.
|
||||
|
||||
* ``--with-mpi-moduledir=DIR``:
|
||||
Specify a specific ``DIR`` directory where to install the MPI
|
||||
Fortran bindings modulefiles. By default, Open MPI will install
|
||||
Fortran modulefiles into ``$libdir``.
|
||||
|
||||
* ``--enable-mpi-ext[=LIST]``:
|
||||
Enable Open MPI's non-portable API extensions. ``LIST`` is a
|
||||
comma-delmited list of extensions. If no ``LIST`` is specified, all
|
||||
of the extensions are enabled.
|
||||
|
||||
See the "Open MPI API Extensions" section for more details.
|
||||
|
||||
* ``--disable-mpi-io``:
|
||||
Disable built-in support for MPI-2 I/O, likely because an
|
||||
externally-provided MPI I/O package will be used. Default is to use
|
||||
the internal framework system that uses the ompio component and a
|
||||
specially modified version of ROMIO that fits inside the romio
|
||||
component
|
||||
|
||||
* ``--disable-io-romio``:
|
||||
Disable the :ref:`ROMIO MPI-IO <label-romio-reference>` component
|
||||
from being compiled. This is a deprecated mechanism; it is effectively
|
||||
the same as adding ``io-romio341`` to the list of items passed to
|
||||
``--enable-mca-no-build``.
|
||||
|
||||
* ``--with-io-romio-flags=FLAGS``:
|
||||
Pass ``FLAGS`` to the ROMIO distribution configuration script. This
|
||||
option is usually only necessary to pass
|
||||
parallel-filesystem-specific preprocessor/compiler/linker flags back
|
||||
to the ROMIO system.
|
||||
|
||||
* ``--disable-io-ompio``:
|
||||
Disable the ompio MPI-IO component
|
||||
|
||||
* ``--enable-sparse-groups``:
|
||||
Enable the usage of sparse groups. This would save memory
|
||||
significantly especially if you are creating large
|
||||
communicators. (Disabled by default)
|
||||
@@ -0,0 +1,109 @@
|
||||
.. This file is included by building-open-mpi.rst
|
||||
|
||||
.. _install-network-support-label:
|
||||
|
||||
Networking support / options
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The following are command line options for various network types that
|
||||
can be used with ``configure``:
|
||||
|
||||
* ``--with-fca=DIR``:
|
||||
Specify the directory where the Mellanox FCA library and
|
||||
header files are located.
|
||||
|
||||
FCA is the support library for Mellanox switches and HCAs.
|
||||
|
||||
* ``--with-hcoll=DIR``:
|
||||
Specify the directory where the Mellanox hcoll library and header
|
||||
files are located. This option is generally only necessary if the
|
||||
hcoll headers and libraries are not in default compiler/linker
|
||||
search paths.
|
||||
|
||||
hcoll is the support library for MPI collective operation offload on
|
||||
Mellanox ConnectX-3 HCAs (and later).
|
||||
|
||||
* ``--with-knem=DIR``:
|
||||
Specify the directory where the knem libraries and header files are
|
||||
located. This option is generally only necessary if the knem headers
|
||||
and libraries are not in default compiler/linker search paths.
|
||||
|
||||
knem is a Linux kernel module that allows direct process-to-process
|
||||
memory copies (optionally using hardware offload), potentially
|
||||
increasing bandwidth for large messages sent between messages on the
|
||||
same server. See `the Knem web site
|
||||
<https://knem.gitlabpages.inria.fr/>`_ for details.
|
||||
|
||||
* ``--with-libfabric=DIR``:
|
||||
Specify the directory where the OpenFabrics Interfaces ``libfabric``
|
||||
library and header files are located. This option is generally only
|
||||
necessary if the libfabric headers and libraries are not in default
|
||||
compiler/linker search paths.
|
||||
|
||||
Libfabric is the support library for OpenFabrics Interfaces-based
|
||||
network adapters, such as Cisco usNIC, Intel True Scale PSM, Cray
|
||||
uGNI, etc.
|
||||
|
||||
* ``--with-libfabric-libdir=DIR``:
|
||||
Look in directory for the libfabric libraries. By default, Open MPI
|
||||
will look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases.
|
||||
This option is only needed for special configurations.
|
||||
|
||||
* ``--with-portals4=DIR``:
|
||||
Specify the directory where the Portals4 libraries and header files
|
||||
are located. This option is generally only necessary if the Portals4
|
||||
headers and libraries are not in default compiler/linker search
|
||||
paths.
|
||||
|
||||
Portals is a low-level network API for high-performance networking
|
||||
on high-performance computing systems developed by Sandia National
|
||||
Laboratories, Intel Corporation, and the University of New Mexico.
|
||||
The Portals 4 Reference Implementation is a complete implementation
|
||||
of Portals 4, with transport over InfiniBand verbs and UDP.
|
||||
|
||||
* ``--with-portals4-libdir=DIR``:
|
||||
Location of libraries to link with for Portals4 support.
|
||||
|
||||
* ``--with-portals4-max-md-size=SIZE`` and
|
||||
``--with-portals4-max-va-size=SIZE``:
|
||||
Set configuration values for Portals 4
|
||||
|
||||
* ``--with-psm=<directory>``:
|
||||
Specify the directory where the QLogic InfiniPath / Intel True Scale
|
||||
PSM library and header files are located. This option is generally
|
||||
only necessary if the PSM headers and libraries are not in default
|
||||
compiler/linker search paths.
|
||||
|
||||
PSM is the support library for QLogic InfiniPath and Intel True Scale
|
||||
network adapters.
|
||||
|
||||
* ``--with-psm-libdir=DIR``:
|
||||
Look in directory for the PSM libraries. By default, Open MPI will
|
||||
look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases. This
|
||||
option is only needed for special configurations.
|
||||
|
||||
* ``--with-psm2=DIR``:
|
||||
Specify the directory where the Intel Omni-Path PSM2 library and
|
||||
header files are located. This option is generally only necessary
|
||||
if the PSM2 headers and libraries are not in default compiler/linker
|
||||
search paths.
|
||||
|
||||
PSM is the support library for Intel Omni-Path network adapters.
|
||||
|
||||
* ``--with-psm2-libdir=DIR``:
|
||||
Look in directory for the PSM2 libraries. By default, Open MPI will
|
||||
look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases. This
|
||||
option is only needed for special configurations.
|
||||
|
||||
* ``--with-ucx=DIR``:
|
||||
Specify the directory where the UCX libraries and header files are
|
||||
located. This option is generally only necessary if the UCX headers
|
||||
and libraries are not in default compiler/linker search paths.
|
||||
|
||||
* ``--with-ucx-libdir=DIR``:
|
||||
Look in directory for the UCX libraries. By default, Open MPI will
|
||||
look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases. This
|
||||
option is only needed for special configurations.
|
||||
|
||||
* ``--with-usnic``:
|
||||
Abort configure if Cisco usNIC support cannot be built.
|
||||
@@ -0,0 +1,16 @@
|
||||
.. This file is included by building-open-mpi.rst
|
||||
|
||||
OpenSHMEM functionality
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The following are command line options to set the default for various
|
||||
OpenSHMEM API behaviors that can be used with ``configure``:
|
||||
|
||||
* ``--disable-oshmem``:
|
||||
Disable building the OpenSHMEM implementation (by default, it is
|
||||
enabled).
|
||||
|
||||
* ``--disable-oshmem-fortran``:
|
||||
Disable building only the Fortran OpenSHMEM bindings. Please see
|
||||
the "Compiler Notes" section herein which contains further
|
||||
details on known issues with various Fortran compilers.
|
||||
@@ -0,0 +1,46 @@
|
||||
.. _label-building-ompi-cli-options-required-support-libraries:
|
||||
|
||||
CLI Options for required support libraries
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The following ``configure`` command line options are for Open MPI's
|
||||
:ref:`required support libraries
|
||||
<label-install-required-support-libraries>`
|
||||
|
||||
* ``--with-hwloc[=VALUE]``:
|
||||
* ``--with-libevent[=VALUE]``:
|
||||
* ``--with-pmix[=VALUE]``:
|
||||
* ``--with-prrte[=VALUE]``: These four options specify where to find
|
||||
the headers and libraries for the `Hwloc
|
||||
<https://www.open-mpi.org/projects/hwloc/>`_, `Libevent
|
||||
<https://libevent.org/>`_, `PMIx <https://openpmix.github.io/>`_,
|
||||
and `PRRTE <https://github.com/openpmix/prrte>`_ libraries,
|
||||
respectively. The following ``VALUE``\s are permitted:
|
||||
|
||||
* ``external``: Use an external installation (rely on default
|
||||
compiler and linker paths to find it). ``configure`` will abort
|
||||
if it cannot find suitable header files and libraries.
|
||||
* ``internal``: Use Open MPI's internal/bundled copy.
|
||||
* No value specified: Try the ``external`` behavior. If that fails,
|
||||
fall back to ``internal`` behavior. *This is the default behavior.*
|
||||
* ``DIR``: Specify the location of a specific installation to use.
|
||||
``configure`` will abort if it cannot find suitable header files
|
||||
and libraries under ``DIR``.
|
||||
|
||||
* ``--with-hwloc-libdir=LIBDIR``:
|
||||
* ``--with-libevent-libdir=LIBDIR``:
|
||||
* ``--with-prrte-libdir=LIBDIR``:
|
||||
* ``--with-pmix-libdir=LIBDIR``:
|
||||
:ref:`See the configure CLI
|
||||
options conventions <building-ompi-cli-options-conventions-label>`
|
||||
for a description of these four options.
|
||||
|
||||
* ``--with-valgrind[=DIR]``:
|
||||
Directory where the valgrind software is installed. If Open MPI
|
||||
finds Valgrind's header files, it will include additional support
|
||||
for Valgrind's memory-checking debugger.
|
||||
|
||||
Valgrind support is disabled by default in Open MPI. Enabling
|
||||
Valgrind support will eliminate a lot of false positives when
|
||||
running Valgrind on MPI applications. There is a minor performance
|
||||
penalty for enabling this option.
|
||||
@@ -0,0 +1,64 @@
|
||||
.. _building-ompi-cli-options-rpath-and-runpath-label:
|
||||
|
||||
Linker "rpath" and "runpath" functionality
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Open MPI and OpenSHMEM |ompi_ver| are both composed of multiple
|
||||
libraries that depend on each other.
|
||||
|
||||
When built from official distribution tarballs (from `the official
|
||||
Open MPI download site <https://www.open-mpi.org/software/ompi/>`_),
|
||||
Open MPI and OpenSHMEM are built with the following versions of the
|
||||
GNU Autotools:
|
||||
|
||||
* Autoconf |autoconf_min_version|
|
||||
* Automake |automake_min_version|
|
||||
* Libtool |libtool_min_version|
|
||||
|
||||
This set of GNU Autotools invokes the ``libtool`` executable to build
|
||||
Open MPI's libraries and executables with the `-rpath CLI option
|
||||
<https://www.gnu.org/software/libtool/manual/html_node/Link-mode.html#Link-mode>`_.
|
||||
|
||||
The behavior of ``libtool -rpath ...`` is, unfortunately, highly
|
||||
system-dependent. In conjunction with compiler-, linker-, and other
|
||||
system-level settings, the end result may be to effect "rpath"
|
||||
behavior, "runpath" behavior, or possibly even neither behavior.
|
||||
|
||||
Although the specific behavior of ``libtool -rpath ..`` is outside the
|
||||
scope of this documentation, you can run commands such as ``readelf -d
|
||||
...`` to find out with which behavior your Open MPI was built.
|
||||
|
||||
For example:
|
||||
|
||||
.. code::
|
||||
|
||||
shell$ ./configure --prefix=/opt/openmpi/ ...
|
||||
...
|
||||
shell$ make -j 32 all && make install
|
||||
...
|
||||
shell$ readelf -d /opt/openmpi/lib/libmpi.so | egrep -i 'rpath|runpath'
|
||||
0x000000000000001d (RUNPATH) Library runpath: [/opt/openmpi/lib]
|
||||
|
||||
The above output indicates that ``libmpi.so`` was built with "runpath"
|
||||
support, whereas output like this:
|
||||
|
||||
.. code::
|
||||
|
||||
shell$ readelf -d /opt/openmpi/lib/libmpi.so | egrep -i 'rpath|runpath'
|
||||
0x000000000000000f (RPATH) Library rpath: [/opt/openmpi/lib]
|
||||
|
||||
indicates that ``libmpi.so`` was built with "rpath" support.
|
||||
|
||||
.. note:: If you want to utilize additional compiler or linker flags
|
||||
(such as runpath flags) when building Open MPI/OpenSHMEM,
|
||||
you can :ref:`specify these flags on the configure command
|
||||
line <install-configure-compilers-and-flags-label>`.
|
||||
|
||||
For example, if invoking ``libtool -rpath ...`` on your
|
||||
system actually effects "rpath" behavior, and you wish to
|
||||
effect "runpath" behavior, you could set ``LDFLAGS`` when
|
||||
invoking ``configure``, like this:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure LDFLAGS=-Wl,--enable-new-dtags ...
|
||||
@@ -0,0 +1,57 @@
|
||||
.. This file is included by building-open-mpi.rst
|
||||
|
||||
Run-time system support
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The following are command line options for various runtime systems that
|
||||
can be used with ``configure``:
|
||||
|
||||
* ``--enable-mpirun-prefix-by-default``:
|
||||
This option forces the ``mpirun`` command to always behave as if
|
||||
``--prefix $prefix`` was present on the command line (where ``$prefix``
|
||||
is the value given to the ``--prefix`` option to configure). This
|
||||
prevents most ``rsh``/``ssh``-based users from needing to modify their
|
||||
shell startup files to set the ``PATH`` and/or ``LD_LIBRARY_PATH`` for
|
||||
Open MPI on remote nodes. Note, however, that such users may still
|
||||
desire to set ``PATH`` |mdash| perhaps even in their shell startup files |mdash|
|
||||
so that executables such as ``mpicc`` and ``mpirun`` can be found
|
||||
without needing to type long path names.
|
||||
|
||||
* ``--with-alps``:
|
||||
Force the building of for the Cray Alps run-time environment. If
|
||||
Alps support cannot be found, configure will abort.
|
||||
|
||||
* ``--with-lsf=DIR``:
|
||||
Specify the directory where the LSF libraries and header files are
|
||||
located. This option is generally only necessary if the LSF headers
|
||||
and libraries are not in default compiler/linker search paths.
|
||||
|
||||
LSF is a resource manager system, frequently used as a batch
|
||||
scheduler in HPC systems.
|
||||
|
||||
* ``--with-lsf-libdir=DIR``:
|
||||
Look in directory for the LSF libraries. By default, Open MPI will
|
||||
look in ``DIR/lib`` and ``DIR/lib64``, which covers most cases. This
|
||||
option is only needed for special configurations.
|
||||
|
||||
* ``--with-slurm``:
|
||||
Force the building of Slurm scheduler support.
|
||||
|
||||
* ``--with-sge``:
|
||||
Specify to build support for the Oracle Grid Engine (OGE) resource
|
||||
manager and/or the Open Grid Engine. OGE support is disabled by
|
||||
default; this option must be specified to build OMPI's OGE support.
|
||||
|
||||
The Oracle Grid Engine (OGE) and open Grid Engine packages are
|
||||
resource manager systems, frequently used as a batch scheduler in
|
||||
HPC systems. It used to be called the "Sun Grid Engine", which is
|
||||
why the option is still named ``--with-sge``.
|
||||
|
||||
* ``--with-tm=DIR``:
|
||||
Specify the directory where the TM libraries and header files are
|
||||
located. This option is generally only necessary if the TM headers
|
||||
and libraries are not in default compiler/linker search paths.
|
||||
|
||||
TM is the support library for the Torque and PBS Pro resource
|
||||
manager systems, both of which are frequently used as a batch
|
||||
scheduler in HPC systems.
|
||||
@@ -0,0 +1,13 @@
|
||||
``configure`` output summary
|
||||
============================
|
||||
|
||||
At the end of ``configure``, a summary of many configuration options
|
||||
are shown.
|
||||
|
||||
The Open MPI Team *highly* recommends examining the ``configure``
|
||||
summary to validate that Open MPI plans to build all the features that
|
||||
you expect it to build.
|
||||
|
||||
If the summary does not agree with what you think should be done, you
|
||||
can look earlier in ``configure``'s output to see why particular
|
||||
options and behaviors were selected.
|
||||
@@ -0,0 +1,33 @@
|
||||
.. _installing-custom-components-label:
|
||||
|
||||
Installing custom components
|
||||
============================
|
||||
|
||||
By default, Open MPI looks in two places for components at
|
||||
run-time (in order):
|
||||
|
||||
#. ``$prefix/lib/openmpi/``: This is the system-provided components
|
||||
directory, part of the installation tree of Open MPI itself.
|
||||
#. ``$HOME/.openmpi/components/``: This is where users can drop their
|
||||
own components that will automatically be "seen" by Open MPI at
|
||||
run-time. This is ideal for developmental, private, or otherwise
|
||||
unstable components.
|
||||
|
||||
Note that the directories and search ordering used for finding
|
||||
components in Open MPI is, itself, an MCA parameter. Setting the
|
||||
``mca_component_path`` changes this value (a colon-delimited list of
|
||||
directories).
|
||||
|
||||
Note also that components are only used on nodes where they are
|
||||
"visible". Hence, if your ``$prefix/lib/openmpi/`` is a directory on a
|
||||
local disk that is not shared via a network filesystem to other nodes
|
||||
where you run MPI jobs, then components that are installed to that
|
||||
directory will *only* be used by MPI jobs running on the local node.
|
||||
|
||||
More specifically: components have the same visibility as normal
|
||||
files. If you need a component to be available to all nodes where you
|
||||
run MPI jobs, then you need to ensure that it is visible on all nodes
|
||||
(typically either by installing it on all nodes for non-networked
|
||||
filesystem installs, or by installing them in a directory that is
|
||||
visible to all nodes via a networked filesystem). Open MPI does not
|
||||
automatically send components to remote nodes when MPI jobs are run.
|
||||
@@ -0,0 +1,16 @@
|
||||
Definitions
|
||||
===========
|
||||
|
||||
* **Source tree:** The tree where the Open MPI source code is located.
|
||||
It is typically the result of expanding an Open MPI distribution
|
||||
source code bundle, such as a tarball.
|
||||
* **Build tree:** The tree where Open MPI was built. It is always
|
||||
related to a specific source tree, but may actually be a different
|
||||
tree (:ref:`since Open MPI supports VPATH builds
|
||||
<building-open-mpi-vpath-label>`). Specifically, this is the tree
|
||||
where you invoked ``configure``, ``make``, etc. to build and install
|
||||
Open MPI.
|
||||
* **Installation tree:** The tree where Open MPI was installed. It is
|
||||
typically the "prefix" argument given to Open MPI's ``configure``
|
||||
script; it is the directory from which you run installed Open MPI
|
||||
executables.
|
||||
@@ -0,0 +1,41 @@
|
||||
|
||||
.. _building-open-mpi-downloading-label:
|
||||
|
||||
Downloading Open MPI
|
||||
====================
|
||||
|
||||
Open MPI is generally available two ways:
|
||||
|
||||
#. As source code.
|
||||
|
||||
* The best place to get an official Open MPI source code
|
||||
distribution is from `the main Open MPI web site
|
||||
<https://www.open-mpi.org/>`_.
|
||||
|
||||
* Downstream Open MPI packagers (e.g., Linux distributions)
|
||||
sometimes also provide source code distributions. They may
|
||||
include additional patches or modifications.
|
||||
|
||||
Consult your favorite downstream packager for more details.
|
||||
|
||||
.. caution:: Do **not** download an Open MPI source code tarball
|
||||
from GitHub.com. The tarballs automatically generated
|
||||
by GitHub.com are incomplete and will not build
|
||||
properly.
|
||||
|
||||
GitHub.com-generated tarballs are **not** official Open
|
||||
MPI releases.
|
||||
|
||||
#. As binary packages.
|
||||
|
||||
* The Open MPI community does not provide binary packages on `the
|
||||
main Open MPI web site <https://www.open-mpi.org/>`_.
|
||||
|
||||
* Various downstream packagers (e.g., Linux distributions,
|
||||
Homebrew, MacPorts, etc.) *do* provide pre-built, binary packages
|
||||
for Open MPI.
|
||||
|
||||
Consult your favorite downstream packager for more details.
|
||||
|
||||
The majority of the rest of this section deals with installing Open
|
||||
MPI from source.
|
||||
@@ -0,0 +1,24 @@
|
||||
Filesystem requirements
|
||||
=======================
|
||||
|
||||
.. _install-filesystem-timestamp-warning-label:
|
||||
|
||||
.. warning:: If you are building Open MPI on a network filesystem, the
|
||||
machine you on which you are building *must* be time-synchronized
|
||||
with the file server.
|
||||
|
||||
Specifically: Open MPI's build system *requires* accurate filesystem
|
||||
timestamps. If your ``make`` output shows that it ran GNU Automake,
|
||||
Autoconf, and/or Libtool, or includes warning about timestamps in the
|
||||
future, perhaps looking something like this::
|
||||
|
||||
Warning: File `Makefile.am' has modification time 3.6e+04 s in the future
|
||||
|
||||
**Know that this is not normal**, and you likely have an invalid
|
||||
build.
|
||||
|
||||
In this case, you should remove the Open MPI source directory and
|
||||
start over (e.g., by re-extracting the Open MPI tarball): either
|
||||
switch to build on a local filesystem, or ensure that the time on your
|
||||
build machine is synchronized with the time on your file server before
|
||||
building again.
|
||||
@@ -0,0 +1,24 @@
|
||||
.. _building-and-installing-section-label:
|
||||
|
||||
Building and installing Open MPI
|
||||
================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
quickstart
|
||||
downloading
|
||||
supported-systems
|
||||
definitions
|
||||
filesystem-requirements
|
||||
vpath-builds
|
||||
compilers-and-flags
|
||||
required-support-libraries
|
||||
configure-cli-options/index
|
||||
configure-output-summary
|
||||
make-targets
|
||||
installation-location
|
||||
custom-components
|
||||
updating
|
||||
packagers
|
||||
sysadmins
|
||||
@@ -0,0 +1,354 @@
|
||||
|
||||
.. _building-open-mpi-installation-location-label:
|
||||
|
||||
Installation location
|
||||
=====================
|
||||
|
||||
A common environment to run Open MPI is in a "Beowulf"-class or
|
||||
similar cluster (e.g., a bunch of 1U servers in a bunch of racks).
|
||||
Simply stated, Open MPI can run on a group of servers or workstations
|
||||
connected by a network. As mentioned in the
|
||||
:ref:`prerequisites section <running-prerequisites-label>` there are
|
||||
several caveats however (for example, you typically must have an
|
||||
account on all the machines, you can ``ssh`` between the
|
||||
nodes without using a password, etc.).
|
||||
|
||||
Regardless of whether Open MPI is installed on a shared / networked
|
||||
filesystem or independently on each node, it is usually easiest if
|
||||
Open MPI is available in the same filesystem path on every node.
|
||||
For example, if you install Open MPI to ``/opt/openmpi-|ompi_ver|`` on
|
||||
one node, ensure that it is available in ``/opt/openmpi-|ompi_ver|``
|
||||
on *all* nodes.
|
||||
|
||||
.. important:: For simplicity, the Open MPI team *strongly* recommends
|
||||
that you install Open MPI at the same path location on all nodes in
|
||||
your cluster. This *greatly* simplifies the user experience of
|
||||
running MPI jobs across multiple nodes in your cluster.
|
||||
|
||||
It is *possible* to install Open MPI in unique path locations in
|
||||
the different nodes in your cluster, but it is not *advisable*.
|
||||
|
||||
This raises the question for Open MPI system administrators: where to
|
||||
install the Open MPI binaries, header files, etc.? This discussion
|
||||
mainly addresses this question for homogeneous clusters (i.e., where
|
||||
all nodes and operating systems are the same), although elements of
|
||||
this discussion apply to heterogeneous clusters as well.
|
||||
|
||||
Filesystem types
|
||||
----------------
|
||||
|
||||
There are two common approaches.
|
||||
|
||||
Network filesystem
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Have a common filesystem, such as NFS, between all the machines to be
|
||||
used. Install Open MPI such that the installation directory is the
|
||||
*same value* on each node. This will *greatly* simplify user's shell
|
||||
startup scripts (e.g., ``.bashrc``, ``.cshrc``, ``.profile`` etc.)
|
||||
|mdash| the ``PATH`` can be set without checking which machine the
|
||||
user is on. It also simplifies the system administrator's job; when
|
||||
the time comes to patch or otherwise upgrade Open MPI, only one copy
|
||||
needs to be modified.
|
||||
|
||||
For example, consider a cluster of four machines: ``inky``,
|
||||
``blinky``, ``pinky``, and ``clyde``.
|
||||
|
||||
* Install Open MPI on ``inky``'s local hard drive in the directory
|
||||
``/opt/openmpi-VERSION``. The system administrator then mounts
|
||||
``inky:/opt/openmpi-VERSION`` on the remaining three machines, such
|
||||
that ``/opt/openmpi-VERSION`` on all machines is effectively "the
|
||||
same". That is, the following directories all contain the Open MPI
|
||||
installation:
|
||||
|
||||
.. code-block::
|
||||
|
||||
inky:/opt/openmpi-VERSION
|
||||
blinky:/opt/openmpi-VERSION
|
||||
pinky:/opt/openmpi-VERSION
|
||||
clyde:/opt/openmpi-VERSION
|
||||
|
||||
* Install Open MPI on ``inky``'s local hard drive in the directory
|
||||
``/usr/local/openmpi-VERSION``. The system administrator then
|
||||
mounts ``inky:/usr/local/openmpi-VERSION`` on *all four* machines in
|
||||
some other common location, such as ``/opt/openmpi-VERSION`` (a
|
||||
symbolic link can be installed on ``inky`` instead of a mount point
|
||||
for efficiency). This strategy is typically used for environments
|
||||
where one tree is NFS exported, but another tree is typically used
|
||||
for the location of actual installation. For example, the following
|
||||
directories all contain the Open MPI installation:
|
||||
|
||||
.. code-block::
|
||||
|
||||
inky:/opt/openmpi-VERSION
|
||||
blinky:/opt/openmpi-VERSION
|
||||
pinky:/opt/openmpi-VERSION
|
||||
clyde:/opt/openmpi-VERSION
|
||||
|
||||
Notice that there are the same four directories as the previous
|
||||
example, but on ``inky``, the directory is *actually* located in
|
||||
``/usr/local/openmpi-VERSION``.
|
||||
|
||||
There is a bit of a disadvantage in this approach; each of the remote
|
||||
nodes have to incur NFS (or whatever filesystem is used) delays to
|
||||
access the Open MPI directory tree. However, both the administration
|
||||
ease and low cost (relatively speaking) of using a networked file
|
||||
system usually greatly outweighs the cost. Indeed, once an MPI
|
||||
application is past MPI initialization, it doesn't use the Open MPI
|
||||
binaries very much.
|
||||
|
||||
Local filesystem
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
If you are concerned with networked filesystem costs of accessing the
|
||||
Open MPI binaries, you can install Open MPI on the local hard drive of
|
||||
each node in your system. Again, it is *highly* advisable to install
|
||||
Open MPI in the *same* directory on each node so that each user's
|
||||
``PATH`` can be set to the same value, regardless of the node that a
|
||||
user has logged on to.
|
||||
|
||||
This approach will save some network latency of accessing the Open MPI
|
||||
binaries, but is typically only used where users are very concerned
|
||||
about squeezing every single cycle out of their machines, or are
|
||||
running at extreme scale where a networked filesystem may get
|
||||
overwhelmed by filesystem requests for Open MPI binaries when running
|
||||
very large parallel jobs.
|
||||
|
||||
.. _building-open-mpi-install-overwrite-label:
|
||||
|
||||
Installing over a prior Open MPI installation
|
||||
---------------------------------------------
|
||||
|
||||
.. warning:: The Open MPI team does not recommend installing a new
|
||||
version of Open MPI over an existing / older installation of Open
|
||||
MPI.
|
||||
|
||||
In its default configuration, an Open MPI installation consists of
|
||||
several shared libraries, header files, executables, and plugins
|
||||
(dynamic shared objects |mdash| DSOs). These installation files act
|
||||
together as a single entity. The specific filenames and
|
||||
contents of these files are subject to change between different
|
||||
versions of Open MPI.
|
||||
|
||||
.. important:: Installing one version of Open MPI does *not* uninstall
|
||||
another version.
|
||||
|
||||
If you install a new version of Open MPI over an older version, this
|
||||
may not overwrite all the files from the older version. Hence, you
|
||||
may end up with an incompatible muddle of files from two different
|
||||
installations |mdash| which can cause problems.
|
||||
|
||||
See :ref:`updating Open MPI <building-open-mpi-updating-label>` for more
|
||||
information about updating or upgrading an installation of Open MPI.
|
||||
|
||||
Relocating an Open MPI installation
|
||||
-----------------------------------
|
||||
|
||||
It can be desirable to initially install Open MPI to one location
|
||||
(e.g., ``/path/to/openmpi``) and then later move it to another
|
||||
location (e.g., ``/opt/myproduct/bundled-openmpi-a.b.c``).
|
||||
|
||||
.. note:: Open MPI hard-codes some directory paths in its executables
|
||||
based on installation paths specified by the ``configure``
|
||||
script. For example, if you configure with an installation
|
||||
prefix of ``/opt/openmpi/``, Open MPI encodes in its
|
||||
executables that it should be able to find its help files in
|
||||
``/opt/openmpi/share/openmpi``.
|
||||
|
||||
The "installdirs" functionality in Open MPI lets you change any of
|
||||
these hard-coded directory paths at run time (*assuming* that you have
|
||||
already adjusted your ``PATH`` and/or ``LD_LIBRARY_PATH`` environment
|
||||
variables to the new location where Open MPI now resides).
|
||||
|
||||
There are three methods.
|
||||
|
||||
.. _install-location-opal-prefix:
|
||||
|
||||
Move an existing Open MPI installation to a new prefix
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Set the ``OPAL_PREFIX`` environment variable before launching Open
|
||||
MPI. For example, if Open MPI had initially been installed to
|
||||
``/opt/openmpi`` and the entire ``openmpi`` tree was later moved to
|
||||
``/home/openmpi``, setting ``OPAL_PREFIX`` to ``/home/openmpi`` will
|
||||
enable Open MPI to function properly.
|
||||
|
||||
.. note:: The ``OPAL_PREFIX`` mechanism relies on all installation
|
||||
directories being specified as relative to the ``prefix``
|
||||
directory specified during ``configure``.
|
||||
|
||||
For example, if Open MPI is configured the following way:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ ./configure --prefix=/opt/openmpi --libdir=/usr/lib ...
|
||||
|
||||
Then setting ``OPAL_PREFIX`` will not affect the run-time
|
||||
implications of ``libdir``, since ``/usr/lib`` is not
|
||||
specified as relative to ``/opt/openmpi``.
|
||||
|
||||
Instead of specifying absolute directories, you can make
|
||||
them relative to other ``configure``-recognized directories.
|
||||
For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
$ ./configure --prefix=/opt/openmpi --libdir='${exec_prefix}/x86_64/lib' ...
|
||||
|
||||
Note the additional shell quoting that is likely necessary
|
||||
to prevent shell variable expansion, and the additional
|
||||
``${}`` around ``exec_prefix`` that is necessary for Open MPI
|
||||
to recognize that it is a special name that needs to be
|
||||
expanded.
|
||||
|
||||
The directory names recognized by Open MPI are listed in the
|
||||
:ref:`Overriding individual directories
|
||||
<install-location-overriding-individual-directories>`
|
||||
section (below), without the ``OPAL_`` prefix, and in lower
|
||||
case. For example, the ``OPAL_SYSCONFDIR`` environment
|
||||
variable corresponds to ``${sysconfdir}``.
|
||||
|
||||
"Stage" an Open MPI installation in a temporary location
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
When *creating* self-contained installation packages, systems such as
|
||||
RPM install Open MPI into temporary locations. The package system
|
||||
then bundles up everything under the temporary location into a package
|
||||
that can be installed into its real location later. For example, when
|
||||
*creating* an RPM that will be installed to ``/opt/openmpi``, the RPM
|
||||
system will transparently prepend a "destination directory" (or
|
||||
"destdir") to the installation directory. As such, Open MPI will
|
||||
think that it is installed in ``/opt/openmpi``, but it is actually
|
||||
temporarily installed in (for example)
|
||||
``/var/rpm/build.1234/opt/openmpi``. If it is necessary to *use* Open
|
||||
MPI while it is installed in this staging area, the ``OPAL_DESTDIR``
|
||||
environment variable can be used; setting ``OPAL_DESTDIR`` to
|
||||
``/var/rpm/build.1234`` will automatically prefix every directory such
|
||||
that Open MPI can function properly.
|
||||
|
||||
.. _install-location-overriding-individual-directories:
|
||||
|
||||
Overriding individual directories
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Open MPI uses the GNU-specified directories (per Autoconf/Automake),
|
||||
and can be overridden by setting environment variables directly
|
||||
related to their common names. The list of environment variables that
|
||||
can be used is:
|
||||
|
||||
* ``OPAL_PREFIX``
|
||||
* ``OPAL_EXEC_PREFIX``
|
||||
* ``OPAL_BINDIR``
|
||||
* ``OPAL_SBINDIR``
|
||||
* ``OPAL_LIBEXECDIR``
|
||||
* ``OPAL_DATAROOTDIR``
|
||||
* ``OPAL_DATADIR``
|
||||
* ``OPAL_SYSCONFDIR``
|
||||
* ``OPAL_SHAREDSTATEDIR``
|
||||
* ``OPAL_LOCALSTATEDIR``
|
||||
* ``OPAL_LIBDIR``
|
||||
* ``OPAL_INCLUDEDIR``
|
||||
* ``OPAL_INFODIR``
|
||||
* ``OPAL_MANDIR``
|
||||
* ``OPAL_PKGDATADIR``
|
||||
* ``OPAL_PKGLIBDIR``
|
||||
* ``OPAL_PKGINCLUDEDIR``
|
||||
|
||||
Note that not all of the directories listed above are used by Open
|
||||
MPI; they are listed here in entirety for completeness.
|
||||
|
||||
Also note that several directories listed above are defined in terms
|
||||
of other directories. For example, the ``$bindir`` is defined by
|
||||
default as ``$prefix/bin``. Hence, overriding the ``$prefix`` (via
|
||||
``OPAL_PREFIX``) will automatically change the first part of the
|
||||
``$bindir`` (which is how method 1 described above works).
|
||||
Alternatively, ``OPAL_BINDIR`` can be set to an absolute value that
|
||||
ignores ``$prefix`` altogether.
|
||||
|
||||
.. _building-open-mpi-installation-location-multiple-copies-label:
|
||||
|
||||
Installing Multiple Copies of Open MPI
|
||||
--------------------------------------
|
||||
|
||||
Open MPI can handle a variety of different run-time environments
|
||||
(e.g., ssh, Slurm, PBS, etc.) and a variety of different
|
||||
interconnection networks (e.g., ethernet, InfiniBand, etc.)
|
||||
in a single installation. Specifically: because Open MPI is
|
||||
fundamentally powered by a component architecture, plug-ins for all
|
||||
these different run-time systems and interconnect networks can be
|
||||
installed in a single installation tree. The relevant plug-ins will
|
||||
only be used in the environments where they make sense.
|
||||
|
||||
Hence, there is no need to have one MPI installation for InfiniBand, one
|
||||
MPI installation for ethernet, one MPI installation for PBS, one MPI
|
||||
installation for ``ssh``, etc. Open MPI can handle all of these in a
|
||||
single installation.
|
||||
|
||||
However, there are some issues that Open MPI cannot solve. Binary
|
||||
compatibility between different compilers is such an issue and may require
|
||||
installation of multiple versions of Open MPI.
|
||||
|
||||
Let's examine this on a per-language basis (be sure see the big caveat at
|
||||
the end):
|
||||
|
||||
* *C:* Most C compilers are fairly compatible, such that if you compile
|
||||
Open MPI with one C library and link it to an application that was
|
||||
compiled with a different C compiler, everything should "just work."
|
||||
As such, a single installation of Open MPI should work for most C MPI
|
||||
applications.
|
||||
|
||||
* *C++:* The same is not necessarily true for C++. While Open MPI does not
|
||||
currently contain any C++ code (the MPI C++ bindings were removed in a prior
|
||||
release), and C++ compilers *should* produce ABI-equivalent code for C
|
||||
symbols, obscure problem can sometimes arise when mixing compilers from
|
||||
different suites. For example, if you compile Open MPI with the XYZ C/C++
|
||||
compiler, you may need to have the XYC C++ run-time libraries
|
||||
installed everywhere you want to run.
|
||||
|
||||
* *Fortran:* There are multiple issues with Fortran.
|
||||
|
||||
#. Fortran compilers do something called "symbol mangling," meaning that the
|
||||
back-end symbols may have slightly different names than their corresponding
|
||||
global variables, subroutines, and functions. There are 4 common name
|
||||
mangling schemes in use by Fortran compilers. On many systems (e.g.,
|
||||
Linux), Open MPI will automatically support all 4 schemes. As such, a
|
||||
single Open MPI installation *should* just work with multiple different
|
||||
Fortran compilers. However, on some systems, this is not possible (e.g.,
|
||||
OS X), and Open MPI will only support the name mangling scheme of the
|
||||
Fortran compiler that was identified during ``configure``.
|
||||
|
||||
#. That being said, there are two notable exceptions that do *not* work
|
||||
across Fortran compilers that are "different enough":
|
||||
|
||||
#. The C constants ``MPI_F_STATUS_IGNORE`` and ``MPI_F_STATUSES_IGNORE``
|
||||
will only compare properly to Fortran applications that were
|
||||
created with Fortran compilers that that use the same
|
||||
name-mangling scheme as the Fortran compiler with which Open MPI was
|
||||
configured.
|
||||
|
||||
#. Fortran compilers may have different values for the logical
|
||||
``.TRUE.`` constant. As such, any MPI function that uses the
|
||||
Fortran ``LOGICAL`` type may only get ``.TRUE.`` values back that
|
||||
correspond to the the ``.TRUE.`` value of the Fortran compiler with which
|
||||
Open MPI was configured.
|
||||
|
||||
#. Similar to C++, linking object files that Fortran language features such
|
||||
as modules and/or polymorphism from different
|
||||
Fortran compilers is not likely to work. The ``mpi`` and ``mpi_f08`` modules that
|
||||
Open MPI creates will likely only work with the Fortran compiler
|
||||
that was identified during ``configure`` (and used to build Open MPI).
|
||||
|
||||
The big caveat to all of this is that Open MPI will only work with
|
||||
different compilers *if all the datatype sizes are the same.* For
|
||||
example, even though Open MPI supports all 4 name mangling schemes,
|
||||
the size of the Fortran ``LOGICAL`` type may be 1 byte in some compilers
|
||||
and 4 bytes in others. This will likely cause Open MPI to perform
|
||||
unpredictably.
|
||||
|
||||
The bottom line is that Open MPI can support all manner of run-time
|
||||
systems and interconnects in a single installation, but supporting
|
||||
multiple compilers "sort of" works (i.e., is subject to trial and
|
||||
error) in some cases, and definitely does not work in other cases.
|
||||
There's unfortunately little that we can do about this |mdash| it's a
|
||||
compiler compatibility issue, and one that compiler authors have
|
||||
little incentive to resolve.
|
||||
@@ -0,0 +1,13 @@
|
||||
``make`` targets
|
||||
================
|
||||
|
||||
Open MPI supports all the ``make`` targets that are provided by GNU
|
||||
Automake, such as:
|
||||
|
||||
* ``all``: build the entire Open MPI package
|
||||
* ``install``: install Open MPI
|
||||
* ``uninstall``: remove all traces of Open MPI from the installation tree
|
||||
* ``clean``: clean out the build tree
|
||||
|
||||
Once Open MPI has been built and installed, it is safe to run ``make
|
||||
clean`` and/or remove the entire build tree.
|
||||
@@ -0,0 +1,318 @@
|
||||
.. _label-install-packagers:
|
||||
|
||||
Advice for packagers
|
||||
====================
|
||||
|
||||
.. _label-install-packagers-do-not-use-internal:
|
||||
|
||||
Do not use Open MPI's internal dependent libraries
|
||||
--------------------------------------------------
|
||||
|
||||
The Open MPI community **strongly** suggests that binary Open MPI
|
||||
packages should *not* include Hwloc, Libevent, PMIx, or PRRTE.
|
||||
:ref:`Although several of these libraries are required by Open MPI
|
||||
<label-install-required-support-libraries>` (and are therefore bundled
|
||||
in the Open MPI source code distribution for end-user convenience),
|
||||
binary Open MPI packages should limit themselves solely to Open MPI
|
||||
artifacts. Specifically: ensure to configure and build Open MPI
|
||||
against external installations of these required packages.
|
||||
|
||||
Packagers may therefore wish to configure Open MPI with something like
|
||||
the following:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Install Sphinx so that Open MPI can re-build its docs with the
|
||||
# installed PRRTE's docs
|
||||
|
||||
virtualalenv venv
|
||||
. ./venv/bin/activate
|
||||
pip install docs/requirements.txt
|
||||
|
||||
./configure --with-libevent=external --with-hwloc=external \
|
||||
--with-pmix=external --with-prrte=external ...
|
||||
|
||||
.. important:: Note the installation of the Sphinx tool so that Open
|
||||
MPI can re-build its documentation with the external
|
||||
PRRTE's documentation.
|
||||
|
||||
Failure to do this will mean Open MPI's documentation
|
||||
will be correct for the version of PRRTE that is
|
||||
bundled in the Open MPI distribution, but may not be
|
||||
entirely correct for the version of PRRTE that you are
|
||||
building against.
|
||||
|
||||
The ``external`` keywords will force Open MPI's ``configure`` to
|
||||
ignore all the bundled libraries and only look for external versions
|
||||
of these support libraries. This also has the benefit of causing
|
||||
``configure`` to fail if it cannot find the required support libraries
|
||||
outside of the Open MPI source tree |mdash| a good sanity check to
|
||||
ensure that your package is correctly relying on the
|
||||
independently-built and installed versions.
|
||||
|
||||
:ref:`See this section
|
||||
<label-building-ompi-cli-options-required-support-libraries>` for more
|
||||
information about the required support library ``--with-FOO`` command
|
||||
line options.
|
||||
|
||||
Have Sphinx installed
|
||||
---------------------
|
||||
|
||||
Since you should be (will be) installing Open MPI against an external
|
||||
PRRTE and PMIx, you should have `Sphinx
|
||||
<https://www.sphinx-doc.org/>`_ installed before running Open MPI's
|
||||
``configure`` script.
|
||||
|
||||
This will allow Open MPI to (re-)build its documentation according to
|
||||
the PMIx and PRRTE that you are building against.
|
||||
|
||||
To be clear: the Open MPI distribution tarball comes with pre-built
|
||||
documentation |mdash| rendered in HTML and nroff |mdash| that is
|
||||
suitable for the versions of PRRTE and PMIx that are bundled in that
|
||||
tarball.
|
||||
|
||||
However, if you are building Open MPI against not-bundled versions of
|
||||
PRRTE / PMIx (as all packagers should be), Open MPI needs to re-build
|
||||
its documentation with specific information from those external PRRTE
|
||||
/ PMIx installs. For that, you need to have Sphinx installed before
|
||||
running Open MPI's ``configure`` script.
|
||||
|
||||
|
||||
.. _label-install-packagers-dso-or-not:
|
||||
|
||||
Components ("plugins"): static or DSO?
|
||||
--------------------------------------
|
||||
|
||||
Open MPI contains a large number of components (sometimes called
|
||||
"plugins") to effect different types of functionality in MPI. For
|
||||
example, some components effect Open MPI's networking functionality:
|
||||
they may link against specialized libraries to provide
|
||||
highly-optimized network access.
|
||||
|
||||
Open MPI can build its components as Dynamic Shared Objects (DSOs) or
|
||||
statically included in core libraries (regardless of whether those
|
||||
libraries are built as shared or static libraries).
|
||||
|
||||
.. note:: As of Open MPI |ompi_ver|, ``configure``'s global default is
|
||||
to build all components as static (i.e., part of the Open
|
||||
MPI core libraries, not as DSOs). Prior to Open MPI v5.0.0,
|
||||
the global default behavior was to build most components as
|
||||
DSOs.
|
||||
|
||||
Why build components as DSOs?
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
There are advantages to building components as DSOs:
|
||||
|
||||
* Open MPI's core libraries |mdash| and therefore MPI applications
|
||||
|mdash| will have very few dependencies. For example, if you build
|
||||
Open MPI with support for a specific network stack, the libraries in
|
||||
that network stack will be dependencies of the DSOs, not Open MPI's
|
||||
core libraries (or MPI applications).
|
||||
|
||||
* Removing Open MPI functionality that you do not want is as simple as
|
||||
removing a DSO from ``$libdir/open-mpi``.
|
||||
|
||||
Why build components as part of Open MPI's core libraries?
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The biggest advantage to building the components as part of Open MPI's
|
||||
core libraries is when running at (very) large scales when Open MPI is
|
||||
installed on a network filesystem (vs. being installed on a local
|
||||
filesystem).
|
||||
|
||||
For example, consider launching a single MPI process on each of 1,000
|
||||
nodes. In this scenario, the following is accessed from the network
|
||||
filesystem:
|
||||
|
||||
#. The MPI application
|
||||
#. The core Open MPI libraries and their dependencies (e.g.,
|
||||
``libmpi``)
|
||||
|
||||
* Depending on your configuration, this is probably on the order of
|
||||
10-20 library files.
|
||||
|
||||
#. All DSO component files and their dependencies
|
||||
|
||||
* Depending on your configuration, this can be 200+ component
|
||||
files.
|
||||
|
||||
If all components are physically located in the libraries, then the
|
||||
third step loads zero DSO component files. When using a networked
|
||||
filesystem while launching at scale, this can translate to large
|
||||
performance savings.
|
||||
|
||||
.. note:: If not using a networked filesystem, or if not launching at
|
||||
scale, loading a large number of DSO files may not consume a
|
||||
noticeable amount of time during MPI process launch. Put
|
||||
simply: loading DSOs as indvidual files generally only
|
||||
matters when using a networked filesystem while launching at
|
||||
scale.
|
||||
|
||||
Direct controls for building components as DSOs or not
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Open MPI |ompi_ver| has two ``configure``-time defaults regarding the
|
||||
treatment of components that may be of interest to packagers:
|
||||
|
||||
#. Open MPI's libraries default to building as shared libraries
|
||||
(vs. static libraries). For example, on Linux, Open MPI will
|
||||
default to building ``libmpi.so`` (vs. ``libmpi.a``).
|
||||
|
||||
.. note:: See the descriptions of ``--disable-shared`` and
|
||||
``--enable-static`` :ref:`in this section
|
||||
<label-building-installation-cli-options>` for more
|
||||
details about how to change this default.
|
||||
|
||||
Also be sure to :ref:`see this warning about building
|
||||
static apps <label-building-fully-static-apps>`.
|
||||
|
||||
#. Open MPI will default to including its components in its libraries
|
||||
(as opposed to being compiled as dynamic shared objects, or DSOs).
|
||||
For example, ``libmpi.so`` on Linux systems will contain the UCX
|
||||
PML component, instead of the UCX PML being compiled into
|
||||
``mca_pml_ucx.so`` and dynamically opened at run time via
|
||||
``dlopen(3)``.
|
||||
|
||||
.. note:: See the descriptions of ``--enable-mca-dso`` and
|
||||
``--enable-mca-static`` :ref:`in this section
|
||||
<label-building-installation-cli-options>` for more
|
||||
details about how to change this defaults.
|
||||
|
||||
A side effect of these two defaults is that all the components
|
||||
included in the Open MPI libraries will bring their dependencies with
|
||||
them. For example (on Linux), if the XYZ PML component in the MPI
|
||||
layer requires ``libXYZ.so``, then these defaults mean that
|
||||
``libmpi.so`` will depend on ``libXYZ.so``. This dependency will
|
||||
likely be telegraphed into the Open MPI binary package that includes
|
||||
``libmpi.so``.
|
||||
|
||||
Conversely, if the XYZ PML component was built as a DSO, then |mdash|
|
||||
assuming no other parts of Open MPI require ``libXYZ.so`` |mdash|
|
||||
``libmpi.so`` would *not* be dependent on ``libXYZ.so``. Instead, the
|
||||
``mca_pml_xyz.so`` DSO would have the dependency upon ``libXYZ.so``.
|
||||
|
||||
Packagers can use these facts to potentially create multiple binary
|
||||
Open MPI packages, each with different dependencies by, for example,
|
||||
using ``--enable-mca-dso`` to selectively build some components as
|
||||
DSOs and leave the others included in their respective Open MPI
|
||||
libraries.
|
||||
|
||||
:ref:`See the section on building accelerator support
|
||||
<label-install-packagers-building-accelerator-support-as-dsos>` for a
|
||||
practical example where this can be useful.
|
||||
|
||||
.. _label-install-packagers-gnu-libtool-dependency-flattening:
|
||||
|
||||
GNU Libtool dependency flattening
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
When compiling Open MPI's components statically as part of Open MPI's
|
||||
core libraries, `GNU Libtool <https://www.gnu.org/software/libtool/>`_
|
||||
|mdash| which is used as part of Open MPI's build system |mdash| will
|
||||
attempt to "flatten" dependencies.
|
||||
|
||||
For example, the :ref:`ompi_info(1) <man1-ompi_info>` command links
|
||||
against the Open MPI core library ``libopen-pal``. This library will
|
||||
have dependencies on various HPC-class network stack libraries. For
|
||||
simplicity, the discussion below assumes that Open MPI was built with
|
||||
support for `Libfabric <https://libfabric.org/>`_ and `UCX
|
||||
<https://openucx.org/>`_, and therefore ``libopen-pal`` has direct
|
||||
dependencies on ``libfabric`` and ``libucx``.
|
||||
|
||||
In this scenario, GNU Libtool will automatically attempt to "flatten"
|
||||
these dependencies by linking :ref:`ompi_info(1) <man1-ompi_info>`
|
||||
directly to ``libfabric`` and ``libucx`` (vs. letting ``libopen-pal``
|
||||
pull the dependencies in at run time).
|
||||
|
||||
* In some environments (e.g., Ubuntu 22.04), the compiler and/or
|
||||
linker will automatically utilize the linker CLI flag
|
||||
``-Wl,--as-needed``, which will effectively cause these dependencies
|
||||
to *not* be flattened: :ref:`ompi_info(1) <man1-ompi_info>` will
|
||||
*not* have a direct dependencies on either ``libfabric`` or
|
||||
``libucx``.
|
||||
|
||||
* In other environments (e.g., Fedora 38), the compiler and linker
|
||||
will *not* utilize the ``-Wl,--as-needed`` linker CLI flag. As
|
||||
such, :ref:`ompi_info(1) <man1-ompi_info>` will show direct
|
||||
dependencies on ``libfabric`` and ``libucx``.
|
||||
|
||||
**Just to be clear:** these flattened dependencies *are not a
|
||||
problem*. Open MPI will function correctly with or without the
|
||||
flattened dependencies. There is no performance impact associated
|
||||
with having |mdash| or not having |mdash| the flattened dependencies.
|
||||
We mention this situation here in the documentation simply because it
|
||||
surprised some Open MPI downstream package managers to see that
|
||||
:ref:`ompi_info(1) <man1-ompi_info>` in Open MPI |ompi_ver| had more
|
||||
shared library dependencies than it did in prior Open MPI releases.
|
||||
|
||||
If packagers want :ref:`ompi_info(1) <man1-ompi_info>` to not have
|
||||
these flattened dependencies, use either of the following mechanisms:
|
||||
|
||||
#. Use ``--enable-mca-dso`` to force all components to be built as
|
||||
DSOs (this was actually the default behavior before Open MPI v5.0.0).
|
||||
|
||||
#. Add ``LDFLAGS=-Wl,--as-needed`` to the ``configure`` command line
|
||||
when building Open MPI.
|
||||
|
||||
.. note:: The Open MPI community specifically chose not to
|
||||
automatically utilize this linker flag for the following
|
||||
reasons:
|
||||
|
||||
#. Having the flattened dependencies does not cause any
|
||||
correctness or performance problems.
|
||||
#. There's multiple mechanisms (see above) for users or
|
||||
packagers to change this behavior, if desired.
|
||||
#. Certain environments have chosen to have |mdash| or
|
||||
not have |mdash| this flattened dependency behavior.
|
||||
It is not Open MPI's place to override these choices.
|
||||
#. In general, Open MPI's ``configure`` script only
|
||||
utilizes compiler and linker flags if they are
|
||||
*needed*. All other flags should be the user's /
|
||||
packager's choice.
|
||||
|
||||
.. _label-install-packagers-building-accelerator-support-as-dsos:
|
||||
|
||||
Building accelerator support as DSOs
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
If you are building a package that includes support for one or more
|
||||
accelerators, it may be desirable to build accelerator-related
|
||||
components as DSOs (see the :ref:`static or DSO?
|
||||
<label-install-packagers-dso-or-not>` section for details).
|
||||
|
||||
.. admonition:: Rationale
|
||||
:class: tip
|
||||
|
||||
Accelerator hardware is expensive, and may only be present on some
|
||||
compute nodes in an HPC cluster. Specifically: there may not be
|
||||
any accelerator hardware on "head" or compile nodes in an HPC
|
||||
cluster. As such, invoking Open MPI commands on a "head" node with
|
||||
an MPI that was built with static accelerator support but no
|
||||
accelerator hardware may fail to launch because of run-time linker
|
||||
issues (because the accelerator hardware support libraries are
|
||||
likely not present).
|
||||
|
||||
Building Open MPI's accelerator-related components as DSOs allows
|
||||
Open MPI to *try* opening the accelerator components, but proceed
|
||||
if those DSOs fail to open due to the lack of support libraries.
|
||||
|
||||
Use the ``--enable-mca-dso`` command line parameter to Open MPI's
|
||||
``configure`` command can allow packagers to build all
|
||||
accelerator-related components as DSO. For example:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
# Build all the accelerator-related components as DSOs (all other
|
||||
# components will default to being built in their respective
|
||||
# libraries)
|
||||
shell$ ./configure --enable-mca-dso=btl-smcuda,rcache-rgpusm,rcache-gpusm,accelerator
|
||||
|
||||
Per the example above, this allows packaging ``$libdir`` as part of
|
||||
the "main" Open MPI binary package, but then packaging
|
||||
``$libdir/openmpi/mca_accelerator_*.so`` and the other named
|
||||
components as sub-packages. These sub-packages may inherit
|
||||
dependencies on the CUDA and/or ROCM packages, for example. The
|
||||
"main" package can be installed on all nodes, and the
|
||||
accelerator-specific subpackage can be installed on only the nodes
|
||||
with accelerator hardware and support libraries.
|
||||
@@ -0,0 +1,101 @@
|
||||
.. _label-quickstart-building-open-mpi:
|
||||
|
||||
Quick start: Installing Open MPI
|
||||
================================
|
||||
|
||||
Although this section skips many details, it offers examples that will
|
||||
probably work in many environments.
|
||||
|
||||
.. caution:: Note that this section is a "Quick start" |mdash| it does
|
||||
not attempt to be comprehensive or describe how to build Open MPI
|
||||
in all supported environments. The examples below may therefore
|
||||
not work exactly as shown in your environment.
|
||||
|
||||
Please consult the other sections in this chapter for more details,
|
||||
if necessary.
|
||||
|
||||
.. important:: If you have checked out a *developer's copy* of Open MPI
|
||||
(i.e., you cloned from Git), you really need to read :doc:`the
|
||||
Developer's Guide </developers/index>` before attempting to build Open
|
||||
MPI. Really.
|
||||
|
||||
Binary packages
|
||||
---------------
|
||||
|
||||
Although the Open MPI community itself does not distribute binary
|
||||
packages for Open MPI, many downstream packagers do.
|
||||
|
||||
For example, many Linux distributions include Open MPI packages
|
||||
|mdash| even if they are not installed by default. You should consult
|
||||
the documentation and/or package list for your Linux distribution to
|
||||
see if you can use its built-in package system to install Open MPI.
|
||||
|
||||
The MacOS package managers `Homebrew <https://brew.sh/>`_ and
|
||||
`MacPorts <https://macports.org/>`_ both offer binary Open MPI
|
||||
packages:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# For Homebrew
|
||||
shell$ brew install openmpi
|
||||
|
||||
# For MacPorts
|
||||
shell$ port install openmpi
|
||||
|
||||
.. important:: Binary packages may or may not include support for
|
||||
features that are required on your platform (e.g., a
|
||||
specific networking stack). Or the binary packages
|
||||
available to you may be older / out of date. As such,
|
||||
it may be better to build and install Open MPI from a
|
||||
source tarball available from `the main Open MPI web
|
||||
site <https://www.open-mpi.org/>`_.
|
||||
|
||||
Building from source
|
||||
--------------------
|
||||
|
||||
Download the Open MPI source code from `the main Open MPI web site
|
||||
<https://www.open-mpi.org/>`_.
|
||||
|
||||
.. caution:: Do **not** download an Open MPI source code tarball from
|
||||
GitHub.com. The tarballs automatically generated by
|
||||
GitHub.com are incomplete and will not build properly.
|
||||
They are **not** official Open MPI releases.
|
||||
|
||||
Open MPI uses a traditional ``configure`` script paired with ``make``
|
||||
to build. Typical installs can be of the pattern:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ tar xf openmpi-<version>.tar.bz2
|
||||
shell$ cd openmpi-<version>
|
||||
shell$ ./configure --prefix=<path> [...options...] 2>&1 | tee config.out
|
||||
<... lots of output ...>
|
||||
|
||||
# Use an integer value of N for parallel builds
|
||||
shell$ make [-j N] all 2>&1 | tee make.out
|
||||
|
||||
# ...lots of output...
|
||||
|
||||
# Depending on the <prefix> chosen above, you may need root access
|
||||
# for the following:
|
||||
shell$ make install 2>&1 | tee install.out
|
||||
|
||||
# ...lots of output...
|
||||
|
||||
Note that VPATH builds are fully supported. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ tar xf openmpi-<version>.tar.bz2
|
||||
shell$ cd openmpi-<version>
|
||||
shell$ mkdir build
|
||||
shell$ cd build
|
||||
shell$ ../configure --prefix=<path> 2>&1 | tee config.out
|
||||
# ...etc.
|
||||
|
||||
The above patterns can be used in many environments.
|
||||
|
||||
Note that there are many, many configuration options available in the
|
||||
``./configure`` step. Some of them may be needed for your particular
|
||||
HPC network interconnect type and/or computing environment; see the
|
||||
rest of this chapter for descriptions of the available options.
|
||||
@@ -0,0 +1,494 @@
|
||||
.. _label-install-required-support-libraries:
|
||||
|
||||
Required support libraries
|
||||
==========================
|
||||
|
||||
|
||||
While Open MPI can be built with support for a wide variety of
|
||||
systems, a small set of support libraries are *required* in order to
|
||||
build Open MPI in *any* environment. Several of these packages are
|
||||
both fundamental to Open MPI's operation and not universally available
|
||||
in all environments. As such, these "fundamental" packages are both
|
||||
embedded in Open MPI's distribution tarballs and also directly
|
||||
incorporated into Open MPI's configure, build, and installation
|
||||
process.
|
||||
|
||||
:ref:`See below
|
||||
<required-support-libraries-configure-discovery-label>` for a
|
||||
description of how Open MPI chooses whether to use the embedded
|
||||
versions of these packages or versions already installed on your
|
||||
system.
|
||||
|
||||
* `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_
|
||||
|
||||
* This library is required; Open MPI will not build without it.
|
||||
* **Minimum version required:** |hwloc_min_version|
|
||||
|
||||
.. note:: While the minimum version of Hwloc will *work*, it may
|
||||
still have bugs and/or have less functionality as
|
||||
compared to later versions.
|
||||
|
||||
Other than the Hwloc restriction about v3.0.0 and beyond
|
||||
(see below), the Open MPI community generally recomends
|
||||
using the latest available version of Hwloc unless there
|
||||
is a specific reason not to.
|
||||
|
||||
* **Version embedded in Open MPI distribution:**
|
||||
|hwloc_embedded_version|
|
||||
|
||||
.. danger:: As of |ompi_ver|, Open MPI does not yet support the
|
||||
Hwloc v3.x series (which may not even be available at
|
||||
the time of Open MPI |ompi_ver|'s release). Hwloc v3.x
|
||||
is anticipated to break API and/or ABI compared to the
|
||||
Hwloc v2.x series.
|
||||
|
||||
Open MPI will refuse to build if it finds an external
|
||||
Hwloc installation that is >= v3.0.0 on the assumption
|
||||
that other HPC applications and/or libraries may be
|
||||
using it. Such a configuration could lead to obscure
|
||||
and potentially confusing run-time failures of Open MPI
|
||||
applications.
|
||||
|
||||
If Open MPI's ``configure`` script aborts because it
|
||||
finds an Hwloc installation that is >= v3.0.0, you can
|
||||
either ensure that Open MPI finds a < v3.0.0 Hwloc
|
||||
installation (e.g., by changing the order of paths in
|
||||
``LD_LIBRARY_PATH``), or force the use of Open MPI's
|
||||
bundled Hwloc via:
|
||||
|
||||
.. code::
|
||||
|
||||
shell$ ./configure --with-hwloc=internal ...
|
||||
|
||||
Regardless, *it is critically important* that if an MPI
|
||||
application |mdash| or any of its dependencies |mdash|
|
||||
uses Hwloc, it uses the *same* Hwloc with which Open MPI
|
||||
was compiled.
|
||||
|
||||
* `Libevent <https://libevent.org/>`_
|
||||
|
||||
* This library is required; Open MPI will not build without it.
|
||||
* **Minimum version required:** |event_min_version|
|
||||
* **Version embedded in Open MPI distribution:**
|
||||
|event_embedded_version|
|
||||
|
||||
.. note:: The Open MPI community has heavily tested Libevent
|
||||
|event_embedded_version|. Other versions should *work*,
|
||||
but |mdash| unlike Hwloc, OpenPMIx, and PRRTE |mdash|
|
||||
there is not much reason to upgrade to use a later
|
||||
version of Libevent.
|
||||
|
||||
* `OpenPMIx <https://docs.openpmix.org/>`_
|
||||
|
||||
* This library is required; Open MPI will not build without it.
|
||||
* **Minimum version required when building without PRRTE:**
|
||||
|pmix_min_version|
|
||||
|
||||
.. note:: While the minimum version of OpenPMIx will *work*, it
|
||||
may still have bugs and/or have less functionality as
|
||||
compared to later versions.
|
||||
|
||||
The Open MPI community generally recomends using the
|
||||
latest available version of OpenPMIx unless there is a
|
||||
specific reason not to.
|
||||
|
||||
.. note:: While `OpenPMIx <https://docs.openpmix.org/>`_ is the
|
||||
formal name of the software that implements the `PMIx
|
||||
<https://pmix.org/>`_ standard, the term "PMIx" is used
|
||||
extensively throughout this documentation to refer to
|
||||
the OpenPMIx software package.
|
||||
|
||||
* **Minimum version required when building with PRRTE:** `See the
|
||||
PRRTE project documentation <https://docs.prrte.org/>`_.
|
||||
* **Version embedded in Open MPI distribution:**
|
||||
|pmix_embedded_version|
|
||||
|
||||
* `PRRTE <https://github.com/openpmix/prrte>`_
|
||||
|
||||
* This library is optional in some environments. See below.
|
||||
* **Minimum version required:** |prte_min_version|
|
||||
|
||||
.. note:: While the minimum version of PRRTE will *work*, it may
|
||||
still have bugs and/or have less functionality as
|
||||
compared to later versions. For example, if you build
|
||||
and run with |prte_min_version|, you will not get a
|
||||
fully-populated ``mpirun(1)`` man page.
|
||||
|
||||
The Open MPI community generally recomends using the
|
||||
latest available version of PRRTE unless there is a
|
||||
specific reason not to.
|
||||
|
||||
* **Version embedded in Open MPI distribution:**
|
||||
|prte_embedded_version|
|
||||
|
||||
PRRTE provides Open MPI's full-featured ``mpirun`` / ``mpiexec`` MPI
|
||||
application launchers (the two commands are identical; they are
|
||||
symbolic links to the same executable).
|
||||
|
||||
.. warning:: If you are building the PRRTE that is embedded in the
|
||||
Open MPI |ompi_ver| distribution:
|
||||
|
||||
* If you are also building the PMIx that is embedded in
|
||||
the Open MPI |ompi_ver| distribution, that
|
||||
combination of packages is supported.
|
||||
|
||||
* If you are building against an external PMIx
|
||||
installation (i.e., a version of PMIx that is not
|
||||
embedded in the Open MPI |ompi_ver| distribution),
|
||||
you should check `the PRRTE project documentation
|
||||
<https://docs.prrte.org/>`_ to see what minimum
|
||||
version of PMIx is required.
|
||||
|
||||
* If your environment uses another MPI application launcher (e.g.,
|
||||
Slurm users can use the ``srun`` launcher to "direct launch" Open
|
||||
MPI applications), then the use of PRRTE is optional.
|
||||
* If your environment has no other MPI application launcher, then
|
||||
you need to install PRRTE and build Open MPI with PRRTE support.
|
||||
* Open MPI can use the copy of PRRTE embedded in its source code
|
||||
tree, or compile/link against an external PRRTE installation.
|
||||
:ref:`See this section for details about how to specify each
|
||||
method
|
||||
<label-building-ompi-cli-options-required-support-libraries>`.
|
||||
|
||||
.. note:: In general, the Open MPI community recommends using the most
|
||||
recent versions of both the :ref:`required support libraries
|
||||
<label-install-required-support-libraries>` and any other
|
||||
optional support libraries. This is because more recent
|
||||
versions typically tend to include bug fixes, sometimes
|
||||
affecting Open MPI functionality. As a specific example,
|
||||
there is a known issue with `Hardware Locality
|
||||
<https://www.open-mpi.org/projects/hwloc/>`_ releases older
|
||||
than v2.8.0 on systems with Intel Ponte Vecchio
|
||||
accelerators. If you run Open MPI on such systems, you need
|
||||
to use Hwloc v2.8.0 or newer, or you will experience
|
||||
undefined behavior. This effect is not unique to the
|
||||
Hardware Locality library; this is why the Open MPI
|
||||
community recommends using as recent as possible versions of
|
||||
*all* support libraries.
|
||||
|
||||
Library dependencies
|
||||
--------------------
|
||||
|
||||
These support libraries have dependencies upon each other:
|
||||
|
||||
.. The "source code" for this figure is simple a PPTX file by the same
|
||||
name in this same directory. If you ever need to edit this image,
|
||||
edit the PPTX, export it to PNG, and then trim the whitespace from
|
||||
the sides of the image.
|
||||
|
||||
.. figure:: required-support-libraries-dependency-graph.png
|
||||
:align: center
|
||||
|
||||
Open MPI required support library dependency graph.
|
||||
|
||||
The higher-level boxes depend on the lower-level boxes. Specifically:
|
||||
|
||||
* Open MPI depends on PRRTE, PMIx, Hwloc, and Libevent (i.e.,
|
||||
everything).
|
||||
* PRRTE depends on PMIx, Hwloc, and Libevent (i.e., everything except
|
||||
Open MPI).
|
||||
* PMIx depends on Hwloc and Libevent.
|
||||
* Hwloc does not depend on anything.
|
||||
* Libevent does not depend on anything.
|
||||
|
||||
At run time, it is critical that the run-time linker loads *exactly
|
||||
one copy* of each of these libraries.
|
||||
|
||||
.. note:: The required support libraries can have other dependencies,
|
||||
but for simplicity and relevance to building Open MPI,
|
||||
those other dependencies are not discussed here.
|
||||
|
||||
Potential problems
|
||||
------------------
|
||||
|
||||
Problems can (will) arise if multiple different copies of the above
|
||||
shared libraries are loaded into a single process. For example,
|
||||
consider if:
|
||||
|
||||
* Loading the Open MPI shared library causes the loading of Libevent
|
||||
shared library vA.B.C.
|
||||
* But then the subsequent loading of the PMIx shared library causes
|
||||
the loading of Libevent shared library vX.Y.Z.
|
||||
|
||||
Since there are now two different versions of the Libevent shared
|
||||
library loaded into the same process (yes, this can happen!),
|
||||
unpredictable behavior can (will) occur.
|
||||
|
||||
Many variations on this same basic erroneous scenario are possible.
|
||||
All of them are bad, and can be extremely difficult to diagnose.
|
||||
|
||||
Avoiding the problems
|
||||
---------------------
|
||||
|
||||
A simple way to avoid these problems is to configure your system such
|
||||
that it has exactly one copy of each of the required support libraries.
|
||||
|
||||
.. important:: If possible, use your OS / environment's package
|
||||
manager to install as many of these support libraries |mdash|
|
||||
including their development headers |mdash| as possible before
|
||||
invoking Open MPI's ``configure`` script.
|
||||
|
||||
Not all package managers provide all of the required support
|
||||
libraries. But even if your package manager installs |mdash| for
|
||||
example |mdash| only Libevent and Hwloc, that somewhat simplifies the
|
||||
final Open MPI configuration, and therefore avoids some potentially
|
||||
erroneous configurations.
|
||||
|
||||
.. _required-support-libraries-configure-discovery-label:
|
||||
|
||||
How ``configure`` finds the required libraries
|
||||
----------------------------------------------
|
||||
|
||||
In an attempt to strike a balance between end-user convenience and
|
||||
flexibility, Open MPI bundles these four required support libraries in
|
||||
its official distribution tarball.
|
||||
|
||||
Generally, if Open MPI cannot find a required support library, it will
|
||||
automatically configure, build, install, and use its bundled version
|
||||
as part of the main Open MPI configure, build, and installation
|
||||
process.
|
||||
|
||||
Put differently: Open MPI's ``configure`` script will examine the
|
||||
build machine and see if it can find each of the required support
|
||||
header files and libraries. If it cannot find them, it will attempt
|
||||
to fall back and use the corresponding bundled support library
|
||||
instead.
|
||||
|
||||
.. important:: Note, however, that ``configure`` is smart enough to
|
||||
understand the dependencies between the required support libraries.
|
||||
|
||||
Specifically: If ``configure`` finds the development headers and
|
||||
libraries for a given support library already installed on the
|
||||
system, then it will ignore both the corresponding bundled support
|
||||
library, *and it will also ignore all bundled support libraries
|
||||
that are below it in the dependency graph shown above.*
|
||||
|
||||
Build example 1
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
``configure`` finds the PRRTE development headers and libraries in
|
||||
``/usr/local``. This will cause the following to occur:
|
||||
|
||||
#. ``configure`` will ignore the PRRTE library that is bundled in the
|
||||
Open MPI source tree and will use the PRRTE that is already
|
||||
installed in ``/usr/local``.
|
||||
#. ``configure`` will also ignore the bundled PMIx, Hwloc, and
|
||||
Libevent libraries in the Open MPI source tree.
|
||||
|
||||
* If ``configure`` is unable to find header files and libraries for
|
||||
PMIx, Hwloc, and Libevent elsewhere on the build machine (i.e.,
|
||||
assumedly the same PMIx, Hwloc, and Libevent than the PRRTE in
|
||||
``/usr/local`` is using), this is an error: ``configure`` will
|
||||
abort, and therefore refuse to build Open MPI.
|
||||
|
||||
Build example 2
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
``configure`` does *not* find PRRTE on the build machine, but *does*
|
||||
find PMIx development headers and libraries in ``/opt/local``. This
|
||||
will cause the following to occur:
|
||||
|
||||
#. ``configure`` will set up to build the PRRTE library that is
|
||||
bundled in the Open MPI source tree.
|
||||
#. ``configure`` will ignore the PMIx library that is bundled in the
|
||||
Open MPI source tree and will use the PMIx that is already
|
||||
installed in ``/opt/local``.
|
||||
#. ``configure`` will also ignore the bundled Hwloc and Libevent
|
||||
libraries in the Open MPI source tree.
|
||||
|
||||
* If ``configure`` is unable to find header files and libraries for
|
||||
Hwloc and Libevent elsewhere on the build machine (i.e.,
|
||||
assumedly the same Hwloc and Libevent than the PMIx in
|
||||
``/opt/local`` is using), this is an error: ``configure`` will
|
||||
abort, and therefore refuse to build Open MPI.
|
||||
|
||||
Build example 3
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
``configure`` only finds the development headers and libraries for
|
||||
Libevent on the build machine. This will cause the following to
|
||||
occur:
|
||||
|
||||
#. ``configure`` will set up to build the PRRTE, PMIx, and Hwloc
|
||||
libraries that are bundled in the Open MPI source tree.
|
||||
#. ``configure`` will ignore the Libevent library that is bundled in
|
||||
the Open MPI source tree and will use the Libevent that is already
|
||||
installed.
|
||||
|
||||
Difficulties with C and Fortran
|
||||
-------------------------------
|
||||
|
||||
Sometimes you may see errors similar to the following when attempting
|
||||
to build Open MPI:
|
||||
|
||||
.. code-block::
|
||||
|
||||
...
|
||||
PPFC profile/pwin_unlock_f08.lo
|
||||
PPFC profile/pwin_unlock_all_f08.lo
|
||||
PPFC profile/pwin_wait_f08.lo
|
||||
FCLD libmpi_usempif08.la
|
||||
ld: library not found for -lhwloc
|
||||
collect2: error: ld returned 1 exit status
|
||||
make``2``: *** ``libmpi_usempif08.la`` Error 1
|
||||
|
||||
This error can happen when a number of factors occur together:
|
||||
|
||||
#. If Open MPI's ``configure`` script chooses to use an "external"
|
||||
installation of `hwloc <https://www.open-mpi.org/projects/hwloc/>`_
|
||||
and/or `Libevent <https://libevent.org/>`_ (i.e., outside of Open
|
||||
MPI's source tree).
|
||||
#. If Open MPI's ``configure`` script chooses C and Fortran compilers
|
||||
from different suites/installations.
|
||||
|
||||
Put simply: if the default search library search paths differ between
|
||||
the C and Fortran compiler suites, the C linker may find a
|
||||
system-installed ``libhwloc`` and/or ``libevent``, but the Fortran linker
|
||||
may not.
|
||||
|
||||
This may tend to happen more frequently starting with Open MPI v4.0.0
|
||||
on Mac OS because:
|
||||
|
||||
#. In v4.0.0, Open MPI's ``configure`` script was changed to "prefer"
|
||||
system-installed versions of hwloc and Libevent (vs. preferring the
|
||||
hwloc and Libevent that are bundled in the Open MPI distribution
|
||||
tarballs).
|
||||
#. In MacOS, it is common for `Homebrew <https://brew.sh/>`_ or
|
||||
`MacPorts <https://www.macports.org/>`_ to install:
|
||||
|
||||
* `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_
|
||||
* `Libevent <https://libevent.org/>`_
|
||||
|
||||
For example, as of July 2019, Homebrew:
|
||||
|
||||
* Installs hwloc v2.0.4 under ``/usr/local``
|
||||
* Installs the Gnu C and Fortran compiler suites v9.1.0 under
|
||||
``/usr/local``. *However*, the C compiler executable is named ``gcc-9``
|
||||
(not ``gcc``!), whereas the Fortran compiler executable is
|
||||
named ``gfortran``.
|
||||
|
||||
These factors, taken together, result in Open MPI's ``configure``
|
||||
script deciding the following:
|
||||
|
||||
* The C compiler is ``gcc`` (which is the MacOS-installed C
|
||||
compiler).
|
||||
* The Fortran compiler is ``gfortran`` (which is the
|
||||
Homebrew-installed Fortran compiler).
|
||||
* There is a suitable system-installed hwloc in ``/usr/local``, which
|
||||
can be found |mdash| by the C compiler/linker |mdash| without specifying any
|
||||
additional linker search paths.
|
||||
|
||||
The careful reader will realize that the C and Fortran compilers are
|
||||
from two entirely different installations. Indeed, their default
|
||||
library search paths are different:
|
||||
|
||||
* The MacOS-installed ``gcc`` will search ``/usr/local/lib`` by
|
||||
default.
|
||||
* The Homebrew-installed ``gfortran`` will *not* search
|
||||
``/usr/local/lib`` by default.
|
||||
|
||||
Hence, since the majority of Open MPI's source code base is in C, it
|
||||
compiles/links against hwloc successfully. But when Open MPI's
|
||||
Fortran code for the ``mpi_f08`` module is compiled and linked, the
|
||||
Homebrew-installed ``gfortran`` |mdash| which does not search
|
||||
``/usr/local/lib`` by default |mdash| cannot find ``libhwloc``, and the link
|
||||
fails.
|
||||
|
||||
There are a few different possible solutions to this issue:
|
||||
|
||||
#. The best solution is to always ensure that Open MPI uses a C and
|
||||
Fortran compiler from the same suite/installation. This will
|
||||
ensure that both compilers/linkers will use the same default
|
||||
library search paths, and all behavior should be consistent. For
|
||||
example, the following instructs Open MPI's ``configure`` script to
|
||||
use ``gcc-9`` for the C compiler, which (as of July 2019) is the
|
||||
Homebrew executable name for its installed C compiler:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure CC=gcc-9 ...
|
||||
|
||||
# You can be precise and specify an absolute path for the C
|
||||
# compiler, and/or also specify the Fortran compiler:
|
||||
shell$ ./configure CC=/usr/local/bin/gcc-9 FC=/usr/local/bin/gfortran ...
|
||||
|
||||
Note that this will likely cause ``configure`` to *not* find the
|
||||
Homebrew-installed hwloc, and instead fall back to using the
|
||||
bundled hwloc in the Open MPI source tree.
|
||||
|
||||
#. Alternatively, you can simply force ``configure`` to select the
|
||||
bundled versions of hwloc and libevent, which avoids the issue
|
||||
altogether:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure --with-hwloc=internal --with-libevent=internal ...
|
||||
|
||||
#. Finally, you can tell ``configure`` exactly where to find the
|
||||
external hwloc library. This can have some unintended
|
||||
consequences, however, because it will prefix both the C and
|
||||
Fortran linker's default search paths with ``/usr/local/lib``:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure --with-hwloc-libdir=/usr/local/lib ...
|
||||
|
||||
Overriding ``configure`` behavior
|
||||
---------------------------------
|
||||
|
||||
If ``configure``'s default searching behavior is not sufficient for
|
||||
your environment, you can use :ref:`command line options to override
|
||||
its default behavior
|
||||
<label-building-ompi-cli-options-required-support-libraries>`.
|
||||
|
||||
For example, if PMIx and/or PRRTE are installed such that the default
|
||||
header file and linker search paths will not find them, you can
|
||||
provide command line options telling Open MPI's ``configure`` where to
|
||||
search. Here's an example ``configure`` invocation where PMIx and
|
||||
PRRTE have both been installed to ``/opt/open-mpi-stuff``:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
./configure --prefix=$HOME/openmpi-install \
|
||||
--with-pmix=/opt/open-mpi-stuff \
|
||||
--with-prrte=/opt/open-mpi-stuff ...
|
||||
|
||||
As another example, if you do not have root-level privileges to use
|
||||
the OS / environment package manager, and if you have a simple MPI
|
||||
application (e.g., that has no external library dependencies), you may
|
||||
wish to configure Open MPI something like this:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
./configure --prefix=$HOME/openmpi-install \
|
||||
--with-libevent=internal --with-hwloc=internal \
|
||||
--with-pmix=internal --with-prrte=internal ...
|
||||
|
||||
The ``internal`` keywords force ``configure`` to use all four bundled
|
||||
versions of the required libraries.
|
||||
|
||||
.. danger:: Be very, very careful when overriding ``configure``'s
|
||||
default search behavior for these libraries. Remember the critical
|
||||
requirement: that Open MPI infrastructure and applications load
|
||||
*exactly one copy* of each support library. For simplicity, it may
|
||||
be desirable to ensure to use exactly the support libraries that
|
||||
Open MPI was compiled and built against.
|
||||
|
||||
For example, using the Open MPI installed from the sample
|
||||
``configure`` line (above), you may want to prefix your run-time
|
||||
linker search path (e.g., ``LD_LIBRARY_PATH`` on Linux) with
|
||||
``$HOME/openmpi-install/lib``. This will ensure that linker finds
|
||||
the four support libraries from your Open MPI installation tree,
|
||||
even if other copies of the same support libraries are present
|
||||
elsewhere on your system.
|
||||
|
||||
(Strong) Advice for packagers
|
||||
-----------------------------
|
||||
|
||||
If you are an Open MPI packager, we **strongly** suggest that your
|
||||
Open MPI package should not include Hwloc, Libevent, PMIx, or PRRTE.
|
||||
Instead, it should depend on external, independently-built versions of
|
||||
these packages.
|
||||
|
||||
See the :ref:`Advice for packagers <label-install-packagers>` section
|
||||
for more details.
|
||||
@@ -0,0 +1,67 @@
|
||||
Supported Systems
|
||||
=================
|
||||
|
||||
Operating systems, hardware, and network requirements for this version of
|
||||
Open MPI.
|
||||
|
||||
Supported Operating Systems
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
We primarily develop Open MPI on Linux and MacOS.
|
||||
|
||||
Other operating systems are supported, however. The exact list of
|
||||
operating systems supported has changed over time (e.g., native
|
||||
Microsoft Windows support was added in v1.3.3, and although it was
|
||||
removed prior to v1.8, is still supported through Cygwin). :ref:`See
|
||||
the Platform Notes section <platform-notes-section-label>` for a
|
||||
listing of the OSes that that version supports.
|
||||
|
||||
Open MPI is fairly POSIX-neutral, so it will run without *too* many
|
||||
modifications on most POSIX-like systems. Hence, if we haven't listed
|
||||
your favorite operating system here, it may not be difficult to get
|
||||
Open MPI to compile and run properly. The biggest obstacle is
|
||||
typically the assembly language, but that's fairly modular and we're
|
||||
happy to provide information about how to port it to new platforms.
|
||||
|
||||
It should be noted that we are quite open to accepting patches for
|
||||
operating systems that we do not currently support. If we do not have
|
||||
systems to test these on, we probably will only claim to
|
||||
"unofficially" support those systems.
|
||||
|
||||
Supported Hardware Platforms
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Essentially all the common platforms that the operating
|
||||
systems listed in the previous section support.
|
||||
|
||||
For example, Linux runs on a *wide* variety of platforms, and we
|
||||
certainly don't claim to test all of them. Open MPI includes
|
||||
Linux-compiler-based assembly for support of Intel, AMD, ARM, and
|
||||
PowerPC chips, for example.
|
||||
|
||||
|
||||
Supported Network Interconnects
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:ref:`See the Platform Notes section <platform-notes-section-label>`
|
||||
for a list of networks supported in this specific release of Open MPI.
|
||||
|
||||
The set of commonly-available HPC-class network interconnects has
|
||||
evolved and changed over time.
|
||||
|
||||
Reflecting that evolution, each release of Open MPI supports a
|
||||
specific set of such network interconnects. You will need to check
|
||||
the documentation of the version of your Open MPI installation to see
|
||||
which interconnects it supports. A general rule of thumb is that a
|
||||
given Open MPI version tends to support the popular HPC-class
|
||||
interconnects at the time of its release.
|
||||
|
||||
This, unfortunately, does mean that Open MPI removes support for
|
||||
networks that are no longer commonly-used in HPC environments. If you
|
||||
still have one of these older interconnects, not all new version of
|
||||
Open MPI may support your interconnect |mdash| sorry! This simply reflects
|
||||
the reality of limited development, testing, and maintenance
|
||||
resources.
|
||||
|
||||
That being said, :doc:`contributions are always welcome!
|
||||
</contributing>`.
|
||||
@@ -0,0 +1,171 @@
|
||||
Advice for System Administrators
|
||||
================================
|
||||
|
||||
Several members of the Open MPI team have strong system
|
||||
administrator backgrounds; we recognize the value of having software
|
||||
that is friendly to system administrators. Here are some of the reasons
|
||||
that Open MPI is attractive for system administrators:
|
||||
|
||||
* Simple, standards-based installation
|
||||
* Reduction of the number of MPI installations
|
||||
* Ability to set system-level and user-level parameters
|
||||
* Scriptable information sources about the Open MPI installation
|
||||
|
||||
See the rest of this section for more details.
|
||||
|
||||
Setting Global MCA Parameters
|
||||
-----------------------------
|
||||
|
||||
MCA parameters are a way to tweak Open MPI's behavior at
|
||||
run-time. For example, MCA parameters can specify:
|
||||
|
||||
* Which interconnect networks to use
|
||||
* Which interconnect networks *not* to use
|
||||
* The size difference between eager sends and rendezvous protocol sends
|
||||
* How many registered buffers to pre-pin (e.g., for InfiniBand)
|
||||
* The size of the pre-pinned registered buffers
|
||||
* ...etc.
|
||||
|
||||
It can be quite valuable for a system administrator to play with such
|
||||
values a bit and find an "optimal" setting for a particular
|
||||
operating environment. These values can then be set in a global text
|
||||
file that all users will, by default, inherit when they run Open MPI
|
||||
jobs.
|
||||
|
||||
For example, say that you have a cluster with 2 ethernet networks |mdash|
|
||||
one for NFS and other system-level operations, and one for MPI jobs.
|
||||
The system administrator can tell Open MPI to not use the NFS TCP
|
||||
network at a system level, such that when users invoke ``mpirun`` or
|
||||
``mpiexec`` to launch their jobs, they will automatically only be using
|
||||
the network meant for MPI jobs.
|
||||
|
||||
:ref:`See how to set MCA params
|
||||
<label-running-setting-mca-param-values>` for information on how to
|
||||
set global MCA parameters.
|
||||
|
||||
Setting MCA Parameters for a Global Open MPI installation
|
||||
---------------------------------------------------------
|
||||
|
||||
Which global MCA parameters to set depends on both your specific
|
||||
cluster setup and the applications that typically run there.
|
||||
|
||||
The best thing to do is to use the :ref:`ompi_info(1) <man1-ompi_info>` command
|
||||
to see what parameters are available and relevant to you. Specifically,
|
||||
``ompi_info --all`` can be used to show all the parameters that are available
|
||||
for each plug-in. Two common places that system administrators like
|
||||
to tweak are:
|
||||
|
||||
* *Only allow specific networks:* If you have a cluster with a
|
||||
high-speed interconnect (such as InfiniBand) and a
|
||||
low-speed ethernet network (e.g., 1Gpbps). The high-speed network is
|
||||
intended for MPI jobs;
|
||||
the control network is intended for NFS and other
|
||||
administrative-level tasks. In this case, you can simply turn off Open
|
||||
MPI's TCP support. The ``btl`` framework contains Open MPI's network
|
||||
support; in this case, you want to disable the ``tcp`` plug-in. You can
|
||||
do this by adding the following line in the file
|
||||
``$prefix/etc/openmpi-mca-params.conf``:
|
||||
|
||||
.. code-block::
|
||||
|
||||
btl = ^tcp
|
||||
|
||||
This tells Open MPI to load all BTL components *except* ``tcp``.
|
||||
|
||||
Consider another example: your cluster has two TCP networks, one for
|
||||
NFS and administration-level jobs, and another for MPI jobs. You can
|
||||
tell Open MPI to ignore the TCP network used by NFS by adding the
|
||||
following line in the file ``$prefix/etc/openmpi-mca-params.conf``:
|
||||
|
||||
.. code-block::
|
||||
|
||||
btl_tcp_if_exclude = lo,eth0
|
||||
|
||||
The value of this parameter is the device names to exclude. In this
|
||||
case, we're excluding ``lo`` (localhost, because Open MPI has its own
|
||||
internal loopback device) and ``eth0``.
|
||||
|
||||
* *Tune the parameters for specific networks:* Each network plug-in
|
||||
has a variety of different tunable parameters. Use the ``ompi_info``
|
||||
command to see what is available. You show *all* available parameters
|
||||
with:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ompi_info --param all all
|
||||
|
||||
.. note:: Starting with Open MPI v1.8, ``ompi_info`` categorizes its
|
||||
parameters in *levels*, as defined by the MPI_T interface from
|
||||
the MPI standard. You will need to specify ``--level 9`` (or
|
||||
``--all``) to show *all* MCA parameters. `See this blog entry
|
||||
<https://blogs.cisco.com/performance/open-mpi-and-the-mpi-3-mpi_t-interface>`_
|
||||
for further information.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ompi_info --level 9
|
||||
# or
|
||||
shell$ ompi_info --all
|
||||
|
||||
Beware: there are *many* variables available. You can limit the
|
||||
output by showing all the parameters in a specific framework or in a
|
||||
specific plug-in with the command line parameters:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ompi_info --param btl all --level 9
|
||||
|
||||
Shows all the parameters of all BTL components, and:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ompi_info --param btl tcp --level 9
|
||||
|
||||
Shows all the parameters of just the ``tcp`` BTL component.
|
||||
|
||||
Adding a new plugin to a global Open MPI installation
|
||||
-----------------------------------------------------
|
||||
|
||||
If you add a new component (such as support for a new network) to a global
|
||||
Open MPI installation, Open MPI will
|
||||
simply open the new plugin at run-time |mdash| your applications do not need
|
||||
to be recompiled or re-linked.
|
||||
|
||||
:ref:`See this section <installing-custom-components-label>` for more
|
||||
details.
|
||||
|
||||
Upgrading network hardware with a global Open MPI installation
|
||||
--------------------------------------------------------------
|
||||
|
||||
If you upgrade your network hardware and your installation of Open MPI uses
|
||||
shared libraries, you simply need to
|
||||
recompile the Open MPI components that support that network and
|
||||
re-install them.
|
||||
|
||||
More specifically, Open MPI shifts the dependency on the underlying
|
||||
network away from the MPI applications and to the Open MPI plug-ins.
|
||||
This is a major advantage over many other MPI implementations.
|
||||
MPI applications will simply open the new plugin when they run.
|
||||
|
||||
User customization of a global Open MPI installation
|
||||
----------------------------------------------------
|
||||
|
||||
It is typically sufficient for a single Open MPI
|
||||
installation (or perhaps a small number of Open MPI installations,
|
||||
depending on compiler interoperability,
|
||||
:ref:`see installing multiple copies of Open MPI
|
||||
<building-open-mpi-installation-location-multiple-copies-label>` for
|
||||
more information) to serve an entire parallel
|
||||
operating environment.
|
||||
|
||||
However, a system-wide Open MPI installation can be customized on a
|
||||
per-user basis in two important ways:
|
||||
|
||||
* *Per-user MCA parameters:* Each user can
|
||||
:ref:`set their own set of MCA parameters, <label-running-setting-mca-param-values>`
|
||||
potentially overriding system-wide defaults.
|
||||
* *Per-user plug-ins:* Users can install their own Open MPI
|
||||
plug-ins under ``$HOME/.openmpi/components``. In this way, developers can
|
||||
experiment with new components without destabilizing Open MPI for the rest
|
||||
of the users on the system. Or power users can download 3rd party components
|
||||
(perhaps even research-quality components) without affecting other users.
|
||||
@@ -0,0 +1,39 @@
|
||||
|
||||
.. _building-open-mpi-updating-label:
|
||||
|
||||
Updating or upgrading an Open MPI installation
|
||||
==============================================
|
||||
|
||||
The Open MPI team recommends one of the following methods for
|
||||
upgrading your Open MPI installation:
|
||||
|
||||
* Install newer versions of Open MPI into a different directory. For
|
||||
example, install into ``/opt/openmpi-a.b.c`` and
|
||||
``/opt/openmpi-x.y.z`` for versions a.b.c and x.y.z, respectively.
|
||||
* Completely uninstall the old version of Open MPI before installing
|
||||
the new version. The ``make uninstall`` process from Open MPI a.b.c
|
||||
build tree should completely uninstall that version from the
|
||||
installation tree, making it safe to install a new version (e.g.,
|
||||
version x.y.z) into the same installation tree.
|
||||
* Remove the old installation directory entirely and then install the
|
||||
new version. For example ``rm -rf /opt/openmpi`` *(assuming that
|
||||
there is nothing else of value in this tree!)* The installation of
|
||||
Open MPI x.y.z will safely re-create the ``/opt/openmpi`` tree.
|
||||
This method is preferable if you no longer have the source and build
|
||||
trees to Open MPI a.b.c available from which to ``make
|
||||
uninstall``.
|
||||
* Go into the Open MPI a.b.c installation directory and manually
|
||||
remove all old Open MPI files. Then install Open MPI x.y.z into the
|
||||
same installation directory. This can be a somewhat painful,
|
||||
annoying, and error-prone process. *We do not recommend it.*
|
||||
Indeed, if you no longer have access to the original Open MPI a.b.c
|
||||
source and build trees, it may be far simpler to download Open MPI
|
||||
version a.b.c again from the Open MPI web site, configure it with
|
||||
the same installation prefix, and then run ``make uninstall``. Or
|
||||
use one of the other methods, above.
|
||||
|
||||
Depending on the version of Open MPI that applications were initially compiled
|
||||
against and the target version of Open MPI to which you upgraded, users may
|
||||
need to recompile their applications.
|
||||
See the :doc:`section on Open MPI's version numbering scheme </version-numbering>`
|
||||
for more information.
|
||||
@@ -0,0 +1,16 @@
|
||||
.. _building-open-mpi-vpath-label:
|
||||
|
||||
VPATH builds
|
||||
============
|
||||
|
||||
VPATH build are fully supported. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ tar xf openmpi-VERSION.tar.bz2
|
||||
shell$ cd openmpi-VERSION
|
||||
shell$ mkdir build
|
||||
shell$ cd build
|
||||
shell$ ../configure [...options...]
|
||||
<... lots of output ...>
|
||||
shell$ make -j 4
|
||||
@@ -0,0 +1,293 @@
|
||||
Launching with Grid Engine
|
||||
==========================
|
||||
|
||||
Open MPI supports the family of run-time schedulers including the Sun
|
||||
Grid Engine (SGE), Oracle Grid Engine (OGE), Grid Engine (GE), Son of
|
||||
Grid Engine, and others.
|
||||
|
||||
This documentation will collectively refer to all of them as "Grid
|
||||
Engine", unless a referring to a specific flavor of the Grid Engine
|
||||
family.
|
||||
|
||||
Verify Grid Engine support
|
||||
--------------------------
|
||||
|
||||
.. important:: To build Grid Engine support in Open MPI, you will need
|
||||
to explicitly request the SGE support with the ``--with-sge``
|
||||
command line switch to Open MPI's ``configure`` script.
|
||||
|
||||
To verify if support for Grid Engine is configured into your Open MPI
|
||||
installation, run ``prte_info`` as shown below and look for
|
||||
``gridengine``.
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ prte_info | grep gridengine
|
||||
MCA ras: gridengine (MCA v2.0, API v2.0, Component v1.3)
|
||||
|
||||
.. note:: PRRTE is the software layer that provides run-time
|
||||
environment support to Open MPI. Open MPI typically hides most
|
||||
PMIx and PRRTE details from the end user, but this is one place
|
||||
that Open MPI is unable to hide the fact that PRRTE provides this
|
||||
functionality, not Open MPI. Hence, users need to use the
|
||||
``prte_info`` command to check for Grid Engine support (not
|
||||
``ompi_info``).
|
||||
|
||||
Launching
|
||||
---------
|
||||
|
||||
When Grid Engine support is included, Open MPI will automatically
|
||||
detect when it is running inside SGE and will just "do the Right
|
||||
Thing."
|
||||
|
||||
Specifically, if you execute an ``mpirun`` command in a Grid Engine
|
||||
job, it will automatically use the Grid Engine mechanisms to launch
|
||||
and kill processes. There is no need to specify what nodes to run on
|
||||
|mdash| Open MPI will obtain this information directly from Grid
|
||||
Engine and default to a number of processes equal to the slot count
|
||||
specified. For example, this will run 4 MPI processes on the nodes
|
||||
that were allocated by Grid Engine:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Get the environment variables for Grid Engine
|
||||
|
||||
# (Assuming Grid Engine is installed at /opt/sge and $Grid
|
||||
# Engine_CELL is 'default' in your environment)
|
||||
shell$ . /opt/sge/default/common/settings.sh
|
||||
|
||||
# Allocate an Grid Engine interactive job with 4 slots from a
|
||||
# parallel environment (PE) named 'ompi' and run a 4-process Open
|
||||
# MPI job
|
||||
shell$ qrsh -pe ompi 4 -b y mpirun -n 4 mpi-hello-world
|
||||
|
||||
There are also other ways to submit jobs under Grid Engine:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Submit a batch job with the 'mpirun' command embedded in a script
|
||||
shell$ qsub -pe ompi 4 my_mpirun_job.csh
|
||||
|
||||
# Submit a Grid Engine and OMPI job and mpirun in one line
|
||||
shell$ qrsh -V -pe ompi 4 mpirun hostname
|
||||
|
||||
# Use qstat(1) to show the status of Grid Engine jobs and queues
|
||||
shell$ qstat -f
|
||||
|
||||
In reference to the setup, be sure you have a Parallel Environment
|
||||
(PE) defined for submitting parallel jobs. You don't have to name your
|
||||
PE "ompi". The following example shows a PE named "ompi" that would
|
||||
look like:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ qconf -sp ompi
|
||||
pe_name ompi
|
||||
slots 99999
|
||||
user_lists NONE
|
||||
xuser_lists NONE
|
||||
start_proc_args NONE
|
||||
stop_proc_args NONE
|
||||
allocation_rule $fill_up
|
||||
control_slaves TRUE
|
||||
job_is_first_task FALSE
|
||||
urgency_slots min
|
||||
accounting_summary FALSE
|
||||
qsort_args NONE
|
||||
|
||||
.. note:: ``qsort_args`` is necessary with the Son of Grid Engine
|
||||
distribution, version 8.1.1 and later, and probably only applicable
|
||||
to it.
|
||||
|
||||
.. note:: For very old versions of Sun Grid Engine, omit
|
||||
``accounting_summary`` too.
|
||||
|
||||
You may want to alter other parameters, but the important one is
|
||||
``control_slaves``, specifying that the environment has "tight
|
||||
integration". Note also the lack of a start or stop procedure. The
|
||||
tight integration means that mpirun automatically picks up the slot
|
||||
count to use as a default in place of the ``-n`` argument, picks up a
|
||||
host file, spawns remote processes via ``qrsh`` so that Grid Engine
|
||||
can control and monitor them, and creates and destroys a per-job
|
||||
temporary directory (``$TMPDIR``), in which Open MPI's directory will
|
||||
be created (by default).
|
||||
|
||||
Be sure the queue will make use of the PE that you specified:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ qconf -sq all.q
|
||||
[...snipped...]
|
||||
pe_list make cre ompi
|
||||
[...snipped...]
|
||||
|
||||
To determine whether the Grid Engine parallel job is successfully
|
||||
launched to the remote nodes, you can pass in the MCA parameter
|
||||
``--mca plm_base_verbose 1`` to ``mpirun``.
|
||||
|
||||
This will add in a ``-verbose`` flag to the ``qrsh -inherit`` command
|
||||
that is used to send parallel tasks to the remote Grid Engine
|
||||
execution hosts. It will show whether the connections to the remote
|
||||
hosts are established successfully or not.
|
||||
|
||||
.. error:: TODO is this site still live? Doesn't look like it.. Jeff
|
||||
emailed Dave Love on 31 Dec 2021 to ask if this is still the
|
||||
correct URL.
|
||||
|
||||
Update March 2022: it doesn't look like this web site is good any
|
||||
more. Perhaps use https://github.com/grisu48/gridengine instead...?
|
||||
|
||||
Various Grid Engine documentation with pointers to more is available
|
||||
at `the Son of GridEngine site <http://arc.liv.ac.uk/sge/>`_, and
|
||||
configuration instructions can be found at `the Son of GridEngine
|
||||
configuration how-to site
|
||||
<http://arc.liv.ac.uk/SGE/howto/sge-configs.html>`_.
|
||||
|
||||
Grid Engine tight integration support of the ``qsub -notify`` flag
|
||||
------------------------------------------------------------------
|
||||
|
||||
If you are running SGE 6.2 Update 3 or later, then the ``-notify``
|
||||
flag is supported. If you are running earlier versions, then the
|
||||
``-notify`` flag will not work and using it will cause the job to be
|
||||
killed.
|
||||
|
||||
To use ``-notify``, one has to be careful. First, let us review what
|
||||
``-notify`` does. Here is an excerpt from the qsub man page for the
|
||||
``-notify`` flag.
|
||||
|
||||
The ``-notify`` flag, when set causes Sun Grid Engine to send
|
||||
warning signals to a running job prior to sending the signals
|
||||
themselves. If a SIGSTOP is pending, the job will receive a SIGUSR1
|
||||
several seconds before the SIGSTOP. If a SIGKILL is pending, the
|
||||
job will receive a SIGUSR2 several seconds before the SIGKILL. The
|
||||
amount of time delay is controlled by the notify parameter in each
|
||||
queue configuration.
|
||||
|
||||
Let us assume the reason you want to use the ``-notify`` flag is to
|
||||
get the SIGUSR1 signal prior to getting the SIGTSTP signal. Something
|
||||
like this batch script can be used:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
#! /bin/bash
|
||||
#$ -S /bin/bash
|
||||
#$ -V
|
||||
#$ -cwd
|
||||
#$ -N Job1
|
||||
#$ -pe ompi 16
|
||||
#$ -j y
|
||||
#$ -l h_rt=00:20:00
|
||||
mpirun -n 16 -mca orte_forward_job_control 1 mpi-hello-world
|
||||
|
||||
.. error:: Ralph: Does ``orte_forward_job_control`` still exist?
|
||||
|
||||
However, one has to make one of two changes to this script for things
|
||||
to work properly. By default, a SIGUSR1 signal will kill a shell
|
||||
script. So we have to make sure that does not happen. Here is one way
|
||||
to handle it:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
#! /bin/bash
|
||||
#$ -S /bin/bash
|
||||
#$ -V
|
||||
#$ -cwd
|
||||
#$ -N Job1
|
||||
#$ -pe ompi 16
|
||||
#$ -j y
|
||||
#$ -l h_rt=00:20:00
|
||||
exec mpirun -n 16 -mca orte_forward_job_control 1 mpi-hello-world
|
||||
|
||||
Alternatively, one can catch the signals in the script instead of doing
|
||||
an exec on the mpirun:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
#! /bin/bash
|
||||
#$ -S /bin/bash
|
||||
#$ -V
|
||||
#$ -cwd
|
||||
#$ -N Job1
|
||||
#$ -pe ompi 16
|
||||
#$ -j y
|
||||
#$ -l h_rt=00:20:00
|
||||
|
||||
function sigusr1handler()
|
||||
{
|
||||
echo "SIGUSR1 caught by shell script" 1>&2
|
||||
}
|
||||
|
||||
function sigusr2handler()
|
||||
{
|
||||
echo "SIGUSR2 caught by shell script" 1>&2
|
||||
}
|
||||
|
||||
trap sigusr1handler SIGUSR1
|
||||
trap sigusr2handler SIGUSR2
|
||||
|
||||
mpirun -n 16 -mca orte_forward_job_control 1 mpi-hello-world
|
||||
|
||||
Grid Engine job suspend / resume support
|
||||
----------------------------------------
|
||||
|
||||
To suspend the job, you send a SIGTSTP (not SIGSTOP) signal to
|
||||
``mpirun``. ``mpirun`` will catch this signal and forward it to the
|
||||
``mpi-hello-world`` as a SIGSTOP signal. To resume the job, you send
|
||||
a SIGCONT signal to ``mpirun`` which will be caught and forwarded to
|
||||
the ``mpi-hello-world``.
|
||||
|
||||
By default, this feature is not enabled. This means that both the
|
||||
SIGTSTP and SIGCONT signals will simply be consumed by the ``mpirun``
|
||||
process. To have them forwarded, you have to run the job with ``--mca
|
||||
orte_forward_job_control 1``. Here is an example on Solaris:
|
||||
|
||||
.. error:: TODO Ralph: does ``orte_forward_job_control`` still exist?
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -mca orte_forward_job_control 1 -n 2 mpi-hello-world
|
||||
|
||||
In another window, we suspend and continue the job:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ prstat -p 15301,15303,15305
|
||||
PID USERNAME SIZE RSS STATE PRI NICE TIME CPU PROCESS/NLWP
|
||||
15305 rolfv 158M 22M cpu1 0 0 0:00:21 5.9% mpi-hello-world/1
|
||||
15303 rolfv 158M 22M cpu2 0 0 0:00:21 5.9% mpi-hello-world/1
|
||||
15301 rolfv 8128K 5144K sleep 59 0 0:00:00 0.0% mpirun/1
|
||||
|
||||
shell$ kill -TSTP 15301
|
||||
shell$ prstat -p 15301,15303,15305
|
||||
PID USERNAME SIZE RSS STATE PRI NICE TIME CPU PROCESS/NLWP
|
||||
15303 rolfv 158M 22M stop 30 0 0:01:44 21% mpi-hello-world/1
|
||||
15305 rolfv 158M 22M stop 20 0 0:01:44 21% mpi-hello-world/1
|
||||
15301 rolfv 8128K 5144K sleep 59 0 0:00:00 0.0% mpirun/1
|
||||
|
||||
shell$ kill -CONT 15301
|
||||
shell$ prstat -p 15301,15303,15305
|
||||
PID USERNAME SIZE RSS STATE PRI NICE TIME CPU PROCESS/NLWP
|
||||
15305 rolfv 158M 22M cpu1 0 0 0:02:06 17% mpi-hello-world/1
|
||||
15303 rolfv 158M 22M cpu3 0 0 0:02:06 17% mpi-hello-world/1
|
||||
15301 rolfv 8128K 5144K sleep 59 0 0:00:00 0.0% mpirun/1
|
||||
|
||||
Note that all this does is stop the ``mpi-hello-world`` processes. It
|
||||
does not, for example, free any pinned memory when the job is in the
|
||||
suspended state.
|
||||
|
||||
To get this to work under the Grid Engine environment, you have to
|
||||
change the ``suspend_method`` entry in the queue. It has to be set to
|
||||
SIGTSTP. Here is an example of what a queue should look like.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ qconf -sq all.q
|
||||
qname all.q
|
||||
[...snipped...]
|
||||
starter_method NONE
|
||||
suspend_method SIGTSTP
|
||||
resume_method NONE
|
||||
|
||||
Note that if you need to suspend other types of jobs with SIGSTOP
|
||||
(instead of SIGTSTP) in this queue then you need to provide a script
|
||||
that can implement the correct signals for each job type.
|
||||
@@ -0,0 +1,51 @@
|
||||
.. _label-running-mpi-applications:
|
||||
|
||||
Launching MPI applications
|
||||
==========================
|
||||
|
||||
Open MPI can launch MPI processes in a wide variety of environments,
|
||||
but they can generally be broken down into two categories:
|
||||
|
||||
#. Scheduled environments: these are systems where a resource manager
|
||||
and/or scheduler are used to control access to the compute nodes.
|
||||
Popular resource managers include Slurm, PBS/Pro/Torque, and LSF.
|
||||
#. Non-scheduled environments: these are systems where resource
|
||||
managers are not used. Launches are typically local (e.g., on a
|
||||
single laptop or workstation) or via ``ssh`` (e.g., across a small
|
||||
number of nodes).
|
||||
|
||||
Similar to many MPI implementations, Open MPI provides the commands
|
||||
:ref:`mpirun(1) <man1-mpirun>` and :ref:`mpiexec(1) <man1-mpiexec>` to
|
||||
launch MPI jobs. This section deals with using these commands.
|
||||
|
||||
Note, however, that in Open MPI, :ref:`mpirun(1) <man1-mpirun>` and
|
||||
:ref:`mpiexec(1) <man1-mpiexec>` are exactly identical. Specifically,
|
||||
they are symbolic links to a common back-end launcher command.
|
||||
|
||||
.. note:: The name of the back-end launcher command has changed over
|
||||
time (it used to be ``orterun``, it is now ``prte``). This
|
||||
back-end name is largely irrelevant to the user.
|
||||
|
||||
The rest of this section usually refers only to :ref:`mpirun(1)
|
||||
<man1-mpirun>`, even though the same discussions also apply to
|
||||
:ref:`mpiexec(1) <man1-mpiexec>` (because they are both, in fact, the
|
||||
same command).
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
quickstart
|
||||
prerequisites
|
||||
pmix-and-prrte
|
||||
scheduling
|
||||
|
||||
localhost
|
||||
ssh
|
||||
slurm
|
||||
lsf
|
||||
tm
|
||||
gridengine
|
||||
|
||||
unusual
|
||||
troubleshooting
|
||||
@@ -0,0 +1,115 @@
|
||||
Launching only on the local node
|
||||
================================
|
||||
|
||||
It is common to develop MPI applications on a single workstation or
|
||||
laptop, and then move to a larger parallel / HPC environment once the
|
||||
MPI application is ready.
|
||||
|
||||
Open MPI supports running multi-process MPI jobs on a single machine.
|
||||
In such cases, you can simply avoid listing a hostfile or remote
|
||||
hosts, and simply list a number of MPI processes to launch. For
|
||||
example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -n 6 mpi-hello-world
|
||||
Hello world, I am 0 of 6 (running on my-laptop))
|
||||
Hello world, I am 1 of 6 (running on my-laptop)
|
||||
...
|
||||
Hello world, I am 5 of 6 (running on my-laptop)
|
||||
|
||||
If you do not specify the ``-n`` option, ``mpirun`` will default to
|
||||
launching as many MPI processes as there are processor cores (not
|
||||
hyperthreads) on the machine.
|
||||
|
||||
MPI communication
|
||||
-----------------
|
||||
|
||||
When running on a single machine, Open MPI will most likely use the
|
||||
``ob1`` PML and the following BTLs for MPI communication between
|
||||
peers:
|
||||
|
||||
* ``self``: used for sending and receiving loopback MPI messages
|
||||
|mdash| where the source and destination MPI process are the same.
|
||||
* ``sm``: used for sending and receiving MPI messages where the source
|
||||
and destination MPI processes can share memory (e.g., via SYSV or
|
||||
POSIX shared memory mechanisms).
|
||||
|
||||
Shared memory MPI communication
|
||||
-------------------------------
|
||||
|
||||
.. error:: TODO This should really be moved to the networking section.
|
||||
|
||||
The ``sm`` BTL supports two modes of shared memory communication:
|
||||
|
||||
#. **Two-copy:** Otherwise known as "copy-in / copy-out", this mode is
|
||||
where the sender copies data into shared memory and the receiver
|
||||
copies the data out.
|
||||
|
||||
This mechanism is always available.
|
||||
|
||||
#. **Single copy:** In this mode, the sender or receiver makes a
|
||||
single copy of the message data from the source buffer in one
|
||||
process to the destination buffer in another process. Open MPI
|
||||
supports three flavors of shared memory single-copy transfers:
|
||||
|
||||
* `Linux KNEM <https://knem.gitlabpages.inria.fr/>`_. This is a
|
||||
standalone Linux kernel module, made specifically for HPC and MPI
|
||||
libraries to enable high-performance single-copy message
|
||||
transfers.
|
||||
|
||||
Open MPI must be able to find the KNEM header files in order to
|
||||
build support for KNEM.
|
||||
|
||||
* `Linux XPMEM <https://github.com/hjelmn/xpmem>`_. Similar to
|
||||
KNEM, this is a standalone Linux kernel module, made specifically
|
||||
for HPC and MPI libraries to enable high-performance single-copy
|
||||
message transfers. It is derived from the Cray XPMEM system.
|
||||
|
||||
Open MPI must be able to find the XPMEM header files in order to
|
||||
build support for XPMEM.
|
||||
|
||||
* Linux Cross-Memory Attach (CMA). This mechanism is built-in to
|
||||
modern versions of the Linux kernel. Although more performance
|
||||
than the two-copy shared memory transfer mechanism, CMA is the
|
||||
lowest performance of the single-copy mechanisms. However, CMA
|
||||
is likely the most widely available because it is enabled by
|
||||
default in several modern Linux distributions.
|
||||
|
||||
Open MPI must be built on a Linux system with a recent enough
|
||||
Glibc and kernel version in order to build support for Linux CMA.
|
||||
|
||||
Which mechanism is used at run time depends both on how Open MPI was
|
||||
built and how your system is configured. You can check to see which
|
||||
single-copy mechanisms Open MPI was built with via two mechanisms:
|
||||
|
||||
#. At the end of running ``configure``, Open MPI emits a list of
|
||||
transports for which it found relevant header files and libraries
|
||||
such that it will be able to build support for them. You might see
|
||||
lines like this, for example:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
Shared memory/copy in+copy out: yes
|
||||
Shared memory/Linux CMA: yes
|
||||
Shared memory/Linux KNEM: no
|
||||
Shared memory/XPMEM: no
|
||||
|
||||
The above output indicates that Open MPI will be built with 2-copy
|
||||
(as mentioned above, 2-copy is *always* available) and with Linux
|
||||
CMA support. KNEM and XPMEM support will *not* be built.
|
||||
|
||||
#. After Open MPI is installed, the ``ompi_info`` command can show
|
||||
which ``smsc`` (shared memory single copy) components are
|
||||
available:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
shell$ ompi_info | grep smsc
|
||||
MCA smsc: cma (MCA v2.1.0, API v1.0.0, Component v5.1.0)
|
||||
|
||||
This Open MPI installation only supports the Linux CMA single-copy
|
||||
mechanism.
|
||||
|
||||
.. note:: As implied by the SMSC component names, none of them are
|
||||
supported on macOS. macOS users will use the two-copy mechanism.
|
||||
@@ -0,0 +1,58 @@
|
||||
Launching with LSF
|
||||
==================
|
||||
|
||||
Open MPI supports the LSF resource manager.
|
||||
|
||||
Verify LSF support
|
||||
------------------
|
||||
|
||||
The ``prte_info`` command can be used to determine whether or not an
|
||||
installed Open MPI includes LSF support:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ prte_info | grep lsf
|
||||
|
||||
If the Open MPI installation includes support for LSF, you
|
||||
should see a line similar to that below. Note the MCA version
|
||||
information varies depending on which version of Open MPI is
|
||||
installed.
|
||||
|
||||
.. code-block::
|
||||
|
||||
MCA ras: lsf (MCA v2.1.0, API v2.0.0, Component v3.0.0)
|
||||
|
||||
.. note:: PRRTE is the software layer that provides run-time
|
||||
environment support to Open MPI. Open MPI typically hides most
|
||||
PMIx and PRRTE details from the end user, but this is one place
|
||||
that Open MPI is unable to hide the fact that PRRTE provides this
|
||||
functionality, not Open MPI. Hence, users need to use the
|
||||
``prte_info`` command to check for LSF support (not
|
||||
``ompi_info``).
|
||||
|
||||
Launching
|
||||
---------
|
||||
|
||||
When properly configured, Open MPI obtains both the list of hosts and
|
||||
how many processes to start on each host from LSF directly. Hence, it
|
||||
is unnecessary to specify the ``--hostfile``, ``--host``, or ``-n``
|
||||
options to ``mpirun``. Open MPI will use LSF-native mechanisms
|
||||
to launch and kill processes (``ssh`` is not required).
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Allocate a job using 4 nodes with 2 processors per node and run the job on the nodes allocated by LSF
|
||||
shell$ bsub -n 8 -R "span[ptile=2]" "mpirun mpi-hello-world"
|
||||
|
||||
|
||||
This will run the MPI processes on the nodes that were allocated by
|
||||
LSF. Or, if submitting a script:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my_script.sh
|
||||
#!/bin/sh
|
||||
mpirun mpi-hello-world
|
||||
shell$ bsub -n 8 -R "span[ptile=2]" < my_script.sh
|
||||
@@ -0,0 +1,68 @@
|
||||
.. _label-running-role-of-pmix-and-prte:
|
||||
|
||||
The role of PMIx and PRRTE
|
||||
==========================
|
||||
|
||||
Prior versions of Open MPI were layered on top of the Open
|
||||
Run-Time Environment (ORTE). ORTE originally started as a small
|
||||
portion of the Open MPI code base, but over time, ORTE effectively
|
||||
spun off into its own sub-project. ORTE ultimately evolved into the
|
||||
`Process Management Interface Exascale (PMIx) standard and
|
||||
corresponding OpenPMIx software project <https://openpmix.github.io/>`_.
|
||||
The OpenPMIx project then evolved its own `PMIx Reference Run-Time
|
||||
Environment (PRRTE) <https://github.com/openpmix/prrte>`_ project.
|
||||
PRRTE has effectively replaced ORTE in the Open MPI implementation.
|
||||
|
||||
Open MPI uses both of these external packages for its run-time system support.
|
||||
|
||||
Both PMIx and PRRTE have many configure- and run-time options. Open
|
||||
MPI attempts to hide most of these details from the end user, and
|
||||
instead present a unified "everything is Open MPI" interface. Open
|
||||
MPI will translate configuration directives to PMIx and PRRTE as
|
||||
relevant, hiding such minutia from the end-user.
|
||||
|
||||
This is an intentional design decision on the part of the Open MPI
|
||||
developer community: HPC and MPI are complicated enough. We do not
|
||||
want to burden the average end user with needing to understand which
|
||||
abstractions and configuration options belong to Open MPI vs. PMIx
|
||||
vs. PRRTE.
|
||||
|
||||
Advanced users can peek into the PMIx and PRRTE internals and tweak
|
||||
additional configuration settings if necessary, but we hope that that
|
||||
will rarely be necessary.
|
||||
|
||||
PMIx
|
||||
----
|
||||
|
||||
The `Process Management Interface for Exascale (PMIx)
|
||||
<https://pmix.org>`_ package is used by Open MPI for the management,
|
||||
communication, and coordination of MPI processes with a back-end
|
||||
run-time system.
|
||||
|
||||
The "back-end run-time system" may range from a low-infrastructure
|
||||
system that simply uses ``ssh`` to remotely execute commands (with no
|
||||
other infrastructure) to an environment with a full-featured resource
|
||||
manager and scheduler such as Slurm, PBS/Pro/Torque, or LSF.
|
||||
|
||||
PMIx presents a unified API that hides many of the complexities of
|
||||
communication with these back-end run-time environments. Open MPI
|
||||
uses the PMIx API to discover, communicate, and coordinate with any
|
||||
supported back-end run-time system without needing to know the
|
||||
intimiate details of that system.
|
||||
|
||||
PRRTE
|
||||
-----
|
||||
|
||||
The `PMIx Reference Runtime Environment
|
||||
<https://github.com/openpmix/prrte>`_ is, as its name implies, a
|
||||
reference run-time environment that utilizes the PMIx API. It mainly
|
||||
provides run-time environment infrastructure for environments that do
|
||||
not natively have them. In practical terms, this typically means
|
||||
providing infrastructure for non-scheduled environments that have no
|
||||
concept of distributed scheduling, file staging, remote stdout/stderr
|
||||
redirection, and only have ``ssh`` to execute commands on remote
|
||||
nodes.
|
||||
|
||||
Open MPI uses PRRTE to deal with the practical issues of the back-end
|
||||
run-time environment such as launching, monitoring, killing, and
|
||||
reaping remote processes.
|
||||
@@ -0,0 +1,256 @@
|
||||
.. _running-prerequisites-label:
|
||||
|
||||
Prerequisites
|
||||
=============
|
||||
|
||||
Successful launch of Open MPI jobs requires the ability to
|
||||
find Open MPI's executables and shared libraries on all nodes at run
|
||||
time.
|
||||
|
||||
In general, Open MPI's ``mpicc`` sets the paths to these with the ``runpath`` linker
|
||||
option when an application is compiled and linked.
|
||||
If the Open MPI executables and libraries can be found via in system-default
|
||||
search paths or the ``runpath`` embedded in an application (i.e., without the
|
||||
user needing to set or modify ``PATH`` or ``LD_LIBRARY_PATH``), then
|
||||
nothing additional needs to be done. However, if the Open MPI binaries
|
||||
are not found, the instructions below may be used to locate them.
|
||||
|
||||
If the ``runpath`` embedded in an application is not sufficient to locate
|
||||
the Open MPI binaries, then the ``PATH`` and ``LD_LIBRARY_PATH`` can be used
|
||||
to locate them.
|
||||
In general, Open MPI requires that its executables are in your
|
||||
``PATH`` on every node on which you will run and if Open MPI was
|
||||
compiled as dynamic libraries (which is the default), the directory
|
||||
where its libraries are located must be in your ``LD_LIBRARY_PATH`` on
|
||||
every node.
|
||||
For example:
|
||||
|
||||
* If Open MPI is installed in ``/usr/bin`` and ``/usr/lib``), that is
|
||||
usually sufficient, and the user does not need to do anything extra.
|
||||
* If Open MPI is installed in a location that is not searched by
|
||||
default, users may need to add ``$prefix/bin`` to their ``PATH`` and
|
||||
``$libdir`` (which defaults to ``$prefix/lib``) to their
|
||||
``LD_LIBRARY_PATH``.
|
||||
|
||||
.. caution:: In scheduled environments, ensuring Open MPI's
|
||||
executables and libraries can be found on the node that
|
||||
executes :ref:`mpirun(1) <man1-mpirun>` may be
|
||||
sufficient.
|
||||
|
||||
In non-scheduled environments, users may need to set
|
||||
the ``PATH`` and ``LD_LIBRARY_PATH`` environment
|
||||
variables in their shell setup files (e.g.,
|
||||
``$HOME/.bashrc``) so that non-interactive
|
||||
``ssh``-based logins will be able to find the Open MPI
|
||||
executables and libraries.
|
||||
|
||||
For example, if Open MPI was installed with a prefix of
|
||||
``/opt/openmpi``, then the following should be in your
|
||||
``PATH`` and ``LD_LIBRARY_PATH``
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Environment variable
|
||||
- Value to add
|
||||
|
||||
* - ``PATH``
|
||||
- ``/opt/openmpi/bin``
|
||||
|
||||
* - ``LD_LIBRARY_PATH``
|
||||
- ``/opt/openmpi/lib``
|
||||
|
||||
Depending on your environment, you may need to set these
|
||||
values in your shell startup files (e.g., ``.bashrc``,
|
||||
``.cshrc``, etc.).
|
||||
|
||||
Additionally, Open MPI requires that jobs can be started on remote
|
||||
nodes without any input from the keyboard. For example, if using
|
||||
``ssh`` as the remote agent, you must have your environment setup to
|
||||
allow execution on remote nodes without entering a password or
|
||||
passphrase.
|
||||
|
||||
Adding Open MPI to ``PATH`` and ``LD_LIBRARY_PATH``
|
||||
---------------------------------------------------
|
||||
|
||||
Open MPI *must* be able to find its executables in your ``PATH``
|
||||
on every node (if Open MPI was compiled as dynamic libraries, then its
|
||||
library path must appear in ``LD_LIBRARY_PATH`` as well). As such, your
|
||||
configuration/initialization files need to add Open MPI to your ``PATH``
|
||||
/ ``LD_LIBRARY_PATH`` properly.
|
||||
|
||||
How to do this may be highly dependent upon your local configuration;
|
||||
you may need to consult with your local system administrator. Some
|
||||
system administrators take care of these details for you, some don't.
|
||||
Some common examples are included below, however.
|
||||
|
||||
You must have at least a minimum understanding of how your shell works
|
||||
to get Open MPI in your ``PATH`` / ``LD_LIBRARY_PATH`` properly. Note
|
||||
that Open MPI must be added to your ``PATH`` and ``LD_LIBRARY_PATH``
|
||||
in the following situations:
|
||||
|
||||
#. When you login to an interactive shell
|
||||
|
||||
If your interactive login environment is not configured properly,
|
||||
executables like ``mpicc`` will not be found, and it is typically
|
||||
obvious what is wrong. The Open MPI executable directory can
|
||||
manually be added to the ``PATH``, or the user's startup files can
|
||||
be modified such that the Open MPI executables are added to the
|
||||
``PATH`` every login. This latter approach is preferred.
|
||||
|
||||
All shells have some kind of script file that is executed at login
|
||||
time to set things like ``PATH`` and ``LD_LIBRARY_PATH`` and
|
||||
perform other environmental setup tasks. This startup file is the
|
||||
one that needs to be edited to add Open MPI to the ``PATH`` and
|
||||
``LD_LIBRARY_PATH``. Consult the manual page for your shell for
|
||||
specific details (some shells are picky about the permissions of
|
||||
the startup file, for example). The table below lists some common
|
||||
shells and the startup files that they read/execute upon login:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 10 90
|
||||
|
||||
* - Shell
|
||||
- Interactive login startup files
|
||||
|
||||
* - ``bash``
|
||||
- ``.bash_profile`` if it exists, or ``.bash_login`` if it
|
||||
exists, or ``.profile`` if it exists
|
||||
|
||||
(in that order). Note that some Linux distributions
|
||||
automatically come with
|
||||
|
||||
``.bash_profile`` scripts for users that automatically
|
||||
execute ``.bashrc`` as well.
|
||||
|
||||
Consult the ``bash(1)`` man page for more information.
|
||||
|
||||
* - ``zsh``
|
||||
- ``.zshrc`` followed by ``.zshenv``
|
||||
|
||||
* - ``sh`` (or Bash
|
||||
|
||||
named ``sh``)
|
||||
- ``.profile``
|
||||
|
||||
* - ``csh``
|
||||
- ``.cshrc`` followed by ``.login``
|
||||
|
||||
* - ``tcsh``
|
||||
- ``.tcshrc`` if it exists, ``.cshrc`` if it does not, followed by
|
||||
``.login``
|
||||
|
||||
#. When you login to non-interactive shells on remote nodes
|
||||
|
||||
If your non-interactive remote environment is not configured
|
||||
properly, executables like :ref:`mpirun(1) <man1-mpirun>` will not function properly,
|
||||
and it can be somewhat confusing to figure out.
|
||||
|
||||
The startup files in question here are the ones that are
|
||||
automatically executed for a non-interactive login on a remote node
|
||||
(e.g., ``ssh othernode ps``). Note that not all shells support
|
||||
this, and that some shells use different files for this than listed
|
||||
for interactive logins. Some shells will supersede non-interactive
|
||||
login startup files with files for interactive logins. That is,
|
||||
running non-interactive login startup file *may* automatically
|
||||
invoke interactive login startup file. The following table lists
|
||||
some common shells and the startup file that is automatically
|
||||
executed, either by Open MPI or by the shell itself:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
:widths: 10 90
|
||||
|
||||
* - Shell
|
||||
- Non-interactive login startup files
|
||||
|
||||
* - ``bash``
|
||||
- ``.bashrc`` if it exists
|
||||
|
||||
* - ``zsh``
|
||||
- ``.zshrc`` followed by ``.zshenv``
|
||||
|
||||
* - ``sh`` (or Bash
|
||||
|
||||
named ``sh``)
|
||||
- This shell does not execute any file automatically,
|
||||
|
||||
so Open MPI will execute the ``.profile`` script
|
||||
|
||||
before invoking Open MPI executables on remote nodes
|
||||
|
||||
* - ``csh``
|
||||
- ``.cshrc``
|
||||
|
||||
* - ``tcsh``
|
||||
- ``.tcshrc`` if it exists, ``.cshrc`` if it does not
|
||||
|
||||
|
||||
Using the ``--prefix`` option with mpirun
|
||||
-----------------------------------------
|
||||
|
||||
If users are unable to add the relevant directories to ``PATH`` and
|
||||
``LD_LIBRARY_PATH``, the :ref:`mpirun(1) <man1-mpirun>` ``--prefix``
|
||||
option *may* be sufficient.
|
||||
|
||||
There are some situations where you cannot modify the ``PATH`` or
|
||||
``LD_LIBRARY_PATH`` |mdash| e.g., some ISV applications prefer to hide
|
||||
all parallelism from the user, and therefore do not want to make the
|
||||
user modify their shell startup files. Another case is where you want
|
||||
a single user to be able to launch multiple MPI jobs simultaneously,
|
||||
each with a different MPI implementation. Hence, setting shell
|
||||
startup files to point to one MPI implementation would be problematic.
|
||||
|
||||
In such cases, you have two options:
|
||||
|
||||
#. Use the :ref:`mpirun(1) <man1-mpirun>` ``--prefix`` command line
|
||||
option (described below).
|
||||
#. Modify the wrapper compilers to include directives to include
|
||||
run-time search locations for the Open MPI libraries.
|
||||
|
||||
:ref:`mpirun(1) <man1-mpirun>`'s ``--prefix`` command line option takes as an argument the
|
||||
top-level directory where Open MPI was installed. While relative
|
||||
directory names are possible, they can become ambiguous depending on
|
||||
the job launcher used; using absolute directory names is strongly
|
||||
recommended.
|
||||
|
||||
For example, say that Open MPI was installed into
|
||||
``/opt/openmpi-VERSION``. You would use the ``--prefix`` option
|
||||
thusly:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun --prefix /opt/openmpi-VERSION -n 4 a.out
|
||||
|
||||
This will prefix the ``PATH`` and ``LD_LIBRARY_PATH`` on both the
|
||||
local and remote hosts with ``/opt/openmpi-VERSION/bin`` and
|
||||
``/opt/openmpi-VERSION/lib``, respectively. This is *usually*
|
||||
unnecessary when using resource managers to launch jobs (e.g., Slurm,
|
||||
Torque, etc.) because they tend to copy the entire local environment
|
||||
|mdash| to include the ``PATH`` and ``LD_LIBRARY_PATH`` |mdash| to
|
||||
remote nodes before execution. As such, if ``PATH`` and
|
||||
``LD_LIBRARY_PATH`` are set properly on the local node, the resource
|
||||
manager will automatically propagate those values out to remote nodes.
|
||||
The ``--prefix`` option is therefore usually most useful in
|
||||
``ssh``-based environments (or similar).
|
||||
|
||||
It is possible to make this the default behavior by passing to
|
||||
``configure`` the flag ``--enable-mpirun-prefix-by-default``. This
|
||||
will make :ref:`mpirun(1) <man1-mpirun>` behave exactly the same as
|
||||
``mpirun --prefix $prefix ...``, where ``$prefix`` is the value given
|
||||
to ``--prefix`` in ``configure``.
|
||||
|
||||
Finally, note that specifying the absolute pathname to :ref:`mpirun(1)
|
||||
<man1-mpirun>` is equivalent to using the ``--prefix`` argument. For
|
||||
example, the following is equivalent to the above command line that
|
||||
uses ``--prefix``:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ /opt/openmpi-VERSION/bin/mpirun -n 4 a.out
|
||||
|
||||
.. error:: TODO Josh H points out that we might also want to mention
|
||||
``OMPIHOME`` for PRRTE's ``.ini`` file here. Leaving this
|
||||
as a future to-do item, since PRRTE's ``.ini`` file support
|
||||
does not exist yet.
|
||||
@@ -0,0 +1,252 @@
|
||||
.. _label-quickstart-launching-apps:
|
||||
|
||||
Quick start: Launching MPI applications
|
||||
=======================================
|
||||
|
||||
Although this section skips many details, it offers examples that will
|
||||
probably work in many environments.
|
||||
|
||||
.. caution:: Note that this section is a "Quick start" |mdash| it does
|
||||
not attempt to be comprehensive or describe how to build Open MPI
|
||||
in all supported environments. The examples below may therefore
|
||||
not work exactly as shown in your environment.
|
||||
|
||||
Please consult the other sections in this chapter for more details,
|
||||
if necessary.
|
||||
|
||||
Using ``mpirun`` to launch applications
|
||||
---------------------------------------
|
||||
|
||||
Open MPI supports both :ref:`mpirun(1) <man1-mpirun>` and
|
||||
:ref:`mpiexec(1) <man1-mpiexec>` (they are exactly equivalent) to
|
||||
launch MPI applications. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -n 2 mpi-hello-world
|
||||
# or
|
||||
shell$ mpiexec -n 2 mpi-hello-world
|
||||
# or
|
||||
shell$ mpiexec -n 1 mpi-hello-world : -n 1 mpi-hello-world
|
||||
|
||||
are all equivalent. For simplicity, the rest of this documentation
|
||||
will simply refer to ``mpirun``.
|
||||
|
||||
Other ``mpirun`` options
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
``mpirun`` supports the ``--help`` option which provides a usage
|
||||
message and a summary of the options that it supports. It should be
|
||||
considered the definitive list of what options are provided.
|
||||
|
||||
Several notable options are:
|
||||
|
||||
* ``--hostfile``: Specify a hostfile for launchers (such as the
|
||||
``rsh`` launcher) that need to be told on which hosts to start
|
||||
parallel applications. Note that for compatibility with other MPI
|
||||
implementations, *--machinefile* is a synonym for ``--hostfile``.
|
||||
* ``--host``: Specify a host or list of hosts to run on (see
|
||||
:ref:`this FAQ entry for more details
|
||||
<running-scheduling-host-option-label>`).
|
||||
* ``-n``: Indicate the number of processes to start.
|
||||
* ``--mca``: Set MCA parameters (see :ref:`how to set MCA params
|
||||
<label-running-setting-mca-param-values>` for more details).
|
||||
* ``--wdir DIRECTORY``: Set the working directory of the started
|
||||
applications. If not supplied, the current working directory is
|
||||
assumed (or ``$HOME``, if the current working directory does not
|
||||
exist on all nodes).
|
||||
* ``-x ENV_VARIABLE_NAME``: The name of an environment variable to
|
||||
export to the parallel application. The ``-x`` option can be
|
||||
specified multiple times to export multiple environment variables to
|
||||
the parallel application.
|
||||
|
||||
Note that the :ref:`mpirun(1) <man1-mpirun>` command supports a
|
||||
*large* number of options. Be sure to see the :ref:`mpirun(1)
|
||||
<man1-mpirun>` man page for much more information.
|
||||
|
||||
Launching on a single host
|
||||
--------------------------
|
||||
|
||||
It is common to develop MPI applications on a single laptop or
|
||||
workstation. In such simple "single program, multiple data (SPMD)" cases,
|
||||
use :ref:`mpirun(1) <man1-mpirun>` and
|
||||
specify how many MPI processes you want to launch via the ``-n``
|
||||
option:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun -n 6 mpi-hello-world
|
||||
Hello world, I am 0 of 6 (running on my-laptop))
|
||||
Hello world, I am 1 of 6 (running on my-laptop)
|
||||
...
|
||||
Hello world, I am 5 of 6 (running on my-laptop)
|
||||
|
||||
This starts a six-process parallel application, running six copies
|
||||
of the executable named ``mpi-hello-world``.
|
||||
|
||||
If you do not specify the ``-n`` option, :ref:`mpirun(1)
|
||||
<man1-mpirun>` will default to launching as many MPI processes as
|
||||
there are processor cores (not hyperthreads) on the machine.
|
||||
|
||||
Launching in a non-scheduled environments (via ``ssh``)
|
||||
-------------------------------------------------------
|
||||
|
||||
In general, Open MPI requires the following to launch and run MPI
|
||||
applications:
|
||||
|
||||
#. You must be able to login to remote nodes non-interactively (e.g.,
|
||||
without entering a password or passphrase).
|
||||
#. Open MPI's executables must be findable (e.g., in your ``PATH``).
|
||||
#. Open MPI's libraries must be findable (e.g., in your
|
||||
``LD_LIBRARY_PATH``).
|
||||
|
||||
:ref:`mpirun(1) <man1-mpirun>` accepts a ``--hostfile`` option (and its
|
||||
synonym, the ``--machinefile`` option) to specify a hostfile containing one
|
||||
hostname per line:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my-hostfile.txt
|
||||
node1.example.com
|
||||
node2.example.com
|
||||
node3.example.com slots=2
|
||||
node4.example.com slots=10
|
||||
|
||||
The optional ``slots`` attribute tells Open MPI the *maximum* number
|
||||
of processes that can be allocated to that node. If ``slots`` is not
|
||||
provided, Open MPI |mdash| by default |mdash| uses the number of
|
||||
processor cores (not hyperthreads) on that node.
|
||||
|
||||
Assuming that each of the 4 nodes in `my-hostfile.txt` have 16 cores:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun --hostfile my-hostfile.txt mpi-hello-world
|
||||
Hello world, I am 0 of 44 (running on node1.example.com)
|
||||
Hello world, I am 1 of 44 (running on node1.example.com)
|
||||
...
|
||||
Hello world, I am 15 of 44 (running on node1.example.com)
|
||||
Hello world, I am 16 of 44 (running on node2.example.com)
|
||||
Hello world, I am 17 of 44 (running on node2.example.com)
|
||||
...
|
||||
Hello world, I am 31 of 44 (running on node2.example.com)
|
||||
Hello world, I am 32 of 44 (running on node3.example.com)
|
||||
Hello world, I am 33 of 44 (running on node3.example.com)
|
||||
Hello world, I am 34 of 44 (running on node4.example.com)
|
||||
...
|
||||
Hello world, I am 43 of 44 (running on node4.example.com)
|
||||
|
||||
You can see the breakdown of how many processes Open MPI launched on
|
||||
each node:
|
||||
|
||||
* node1: 16, because no ``slots`` was specified
|
||||
* node2: 16, because no ``slots`` was specified
|
||||
* node3: 2, because ``slots=2`` was specified
|
||||
* node2: 10, because ``slots=10`` was specified
|
||||
|
||||
Note, however, that not all environments require a hostfile. For
|
||||
example, Open MPI will automatically detect when it is running in
|
||||
batch / scheduled environments (such as Slurm, PBS/Torque, SGE,
|
||||
LoadLeveler), and will use host information provided by those systems.
|
||||
|
||||
Also note that if using a launcher that requires a hostfile and no
|
||||
hostfile is specified, all processes are launched on the local host.
|
||||
|
||||
Launching in scheduled environments
|
||||
-----------------------------------
|
||||
|
||||
In scheduled environments (e.g., in a Slurm job, or PBS/Pro, or LSF,
|
||||
or any other schedule), the user tells the scheduler how many MPI
|
||||
processes to launch, and the scheduler decides which hosts to use.
|
||||
The scheduler then passes both pieces of information (the number of
|
||||
processes and the hosts to use) to Open MPI.
|
||||
|
||||
There are two ways to launch in a scheduled environment. Nominally,
|
||||
they both achieve the same thing: they launch MPI processes. Them
|
||||
main user-observable difference between the two methods is that
|
||||
:ref:`mpirun(1) <man1-mpirun>` has many more features than scheduler
|
||||
direct launchers.
|
||||
|
||||
Using Open MPI's :ref:`mpirun(1) <man1-mpirun>`
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. note:: Technically, Open MPI's :ref:`mpirun(1) <man1-mpirun>` is a
|
||||
thin layer around the PRRTE ``prun``. Hence, most of the
|
||||
functionality described here is really about ``prun``. For
|
||||
simplicity, however, this docmentation will describe
|
||||
everything in terms of ``mpirun``.
|
||||
|
||||
When using the full-featured :ref:`mpirun(1) <man1-mpirun>` in a
|
||||
scheduled environment, there is no need to specify a hostfile or
|
||||
number of MPI processes to launch. :ref:`mpirun(1) <man1-mpirun>`
|
||||
will receive this information directly from the scheduler. Hence, if
|
||||
you want to launch an MPI job that completely "fills" your scheduled
|
||||
allocation (i.e., one MPI process for each slot in the scheduled
|
||||
allocation), you can simply:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Write a script that runs your MPI application
|
||||
shell$ cat my-slurm-script.sh
|
||||
#!/bin/sh
|
||||
# There is no need to specify -n or --hostfile because that
|
||||
# information will automatically be provided by Slurm.
|
||||
mpirun mpi-hello-world
|
||||
|
||||
You then submit the ``my-slurm-script.sh`` script to Slurm for
|
||||
execution:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Use -n to indicate how many MPI processes you want to run.
|
||||
# Slurm will pick the specific hosts which will be used.
|
||||
shell$ sbatch -n 40 my-slurm-script.sh
|
||||
Submitted batch job 1234
|
||||
shell$
|
||||
|
||||
After Slurm job 1234 completes, you can look at the output file to see
|
||||
what happened:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat slurm-1234.out
|
||||
Hello world, I am 0 of 40 (running on node37.example.com)
|
||||
Hello world, I am 1 of 40 (running on node37.example.com)
|
||||
Hello world, I am 2 of 40 (running on node37.example.com)
|
||||
...
|
||||
Hello world, I am 39 of 40 (running on node19.example.com)
|
||||
|
||||
Note that the Slurm scheduler picked the hosts on which the processes
|
||||
ran.
|
||||
|
||||
The above example shows that simply invoking ``mpirun
|
||||
mpi-hello-world`` |mdash| with no other CLI options |mdash| obtains
|
||||
the number of processes to run and hosts to use from the scheduler.
|
||||
|
||||
:ref:`mpirun(1) <man1-mpirun>` has many more features not described in
|
||||
this Quick Start section. For example, while uncommon in scheduled
|
||||
environments, you can use ``-n`` and/or ``--hostfile`` to launch in
|
||||
subsets of the overall scheduler allocation. See the :ref:`mpirun(1)
|
||||
<man1-mpirun>` man page for more details.
|
||||
|
||||
Using the scheduler to "direct launch" (without :ref:`mpirun(1) <man1-mpirun>`)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Some schedulers (such as Slurm) have the ability to "direct launch"
|
||||
MPI processes without using Open MPI's :ref:`mpirun(1) <man1-mpirun>`.
|
||||
For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ srun -n 40 mpi-hello-world
|
||||
Hello world, I am 0 of 40 (running on node14.example.com)
|
||||
Hello world, I am 1 of 40 (running on node14.example.com)
|
||||
Hello world, I am 2 of 40 (running on node14.example.com)
|
||||
...
|
||||
Hello world, I am 39 of 40 (running on node203.example.com)
|
||||
shell$
|
||||
|
||||
Similar to the prior example, this example launches 40 copies of
|
||||
``mpi-hello-world``, but it does so via the Slurm ``srun`` command
|
||||
without using :ref:`mpirun(1) <man1-mpirun>`.
|
||||
|
||||
@@ -0,0 +1,551 @@
|
||||
Scheduling processes across hosts
|
||||
=================================
|
||||
|
||||
Open MPI provides many options for scheduling application processes across
|
||||
hosts, including oversubscribing processes to processors. This section
|
||||
describes how to define that mapping.
|
||||
|
||||
Scheduling overview
|
||||
-------------------
|
||||
|
||||
If you are not oversubscribing your hosts
|
||||
(i.e., trying to run more processes than slots available on that
|
||||
host), scheduling is pretty simple and occurs either on a by-slot or
|
||||
by-node round robin schedule. If you're oversubscribing, the issue
|
||||
gets much more complicated |mdash| keep reading.
|
||||
|
||||
The more complete answer is: Open MPI schedules processes to nodes by
|
||||
asking two questions from each application on the ``mpirun`` command
|
||||
line:
|
||||
|
||||
#. *How many* processes should be launched?
|
||||
#. *Where* should those processes be launched?
|
||||
|
||||
The "how many" question is directly answered with the ``-n`` switch
|
||||
to ``mpirun``. If ``-n`` is not specified on the ``mpirun`` command
|
||||
line, its value is the sum of the slots on all the nodes.
|
||||
|
||||
The "where" question is a little more complicated, and depends on
|
||||
three factors:
|
||||
|
||||
#. The final node list (e.g., after ``-hostname`` / ``--host``
|
||||
exclusionary or inclusionary processing)
|
||||
#. The scheduling policy (which applies to all applications in a
|
||||
single job)
|
||||
#. The default and maximum number of slots on each host
|
||||
|
||||
Open MPI currently supports two scheduling policies: by slot and by
|
||||
node:
|
||||
|
||||
#. *By slot:* This is the default scheduling policy, but can also be
|
||||
explicitly requested by using either the ``--map-by slot`` option
|
||||
to ``mpirun`` or by setting the MCA parameter
|
||||
``rmaps_default_mapping_policy`` to the string ``slot``.
|
||||
|
||||
In this mode, Open MPI will schedule processes on a node until all
|
||||
of its default slots are exhausted before proceeding to the next
|
||||
node. In MPI terms, this means that Open MPI tries to maximize the
|
||||
number of adjacent ranks in ``MPI_COMM_WORLD`` on the same host
|
||||
without oversubscribing that host.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ cat my-hosts
|
||||
node0 slots=2 max_slots=20
|
||||
node1 slots=2 max_slots=20
|
||||
shell$ mpirun --hostfile my-hosts -n 8 --map-by slot hello | sort
|
||||
Hello World I am rank 0 of 8 running on node0
|
||||
Hello World I am rank 1 of 8 running on node0
|
||||
Hello World I am rank 2 of 8 running on node1
|
||||
Hello World I am rank 3 of 8 running on node1
|
||||
Hello World I am rank 4 of 8 running on node0
|
||||
Hello World I am rank 5 of 8 running on node0
|
||||
Hello World I am rank 6 of 8 running on node1
|
||||
Hello World I am rank 7 of 8 running on node1
|
||||
|
||||
#. *By node:* This policy can be requested either by using the
|
||||
``--map-by node`` option to ``mpirun`` or by setting the MCA parameter
|
||||
``rmaps_default_mapping_policy`` to the string "node".
|
||||
|
||||
In this mode, Open MPI will schedule a single process on each node
|
||||
in a round-robin fashion (looping back to the beginning of the node
|
||||
list as necessary) until all processes have been scheduled. Nodes
|
||||
are skipped once their default slot counts are exhausted.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ cat my-hosts
|
||||
node0 slots=2 max_slots=20
|
||||
node1 slots=2 max_slots=20
|
||||
shell$ mpirun --hostname my-hosts -n 8 --map-by node hello | sort
|
||||
Hello World I am rank 0 of 8 running on node0
|
||||
Hello World I am rank 1 of 8 running on node1
|
||||
Hello World I am rank 2 of 8 running on node0
|
||||
Hello World I am rank 3 of 8 running on node1
|
||||
Hello World I am rank 4 of 8 running on node0
|
||||
Hello World I am rank 5 of 8 running on node1
|
||||
Hello World I am rank 6 of 8 running on node0
|
||||
Hello World I am rank 7 of 8 running on node1
|
||||
|
||||
In both policies, if the default slot count is exhausted on all nodes
|
||||
while there are still processes to be scheduled, Open MPI will trigger
|
||||
an oversubscription condition.
|
||||
|
||||
If ``:OVERSUBSCRIBE`` is added as a modifier to the ``--map-by``
|
||||
option (e.g., ``mpirun --map-by node:OVERSUBSCRIBE ...`` |mdash| :ref:`see
|
||||
this section <running-mpi-apps-oversubscribing-label>` for more
|
||||
details), Open MPI will continue to loop through the list of nodes
|
||||
again and try to schedule one more process to each node until all
|
||||
processes are scheduled. Nodes are skipped in this process if their
|
||||
maximum slot count is exhausted. If the maximum slot count is
|
||||
exhausted on all nodes while there are still processes to be
|
||||
scheduled, Open MPI will abort without launching any processes.
|
||||
|
||||
If ``:OVERSUBSCRIBE`` is *not* specified and an oversubscription
|
||||
condition occurs, Open MPI will abort without launching any processes.
|
||||
|
||||
.. _running-scheduling-hostfile-option-label:
|
||||
|
||||
Scheduling with the --hostfile option
|
||||
-------------------------------------
|
||||
|
||||
The ``--hostfile`` option to ``mpirun`` takes a filename that lists
|
||||
hosts on which to launch MPI processes.
|
||||
|
||||
.. important:: The hosts listed in a hostfile have *nothing* to do
|
||||
with which network interfaces are used for MPI
|
||||
communication. They are *only* used to specify on
|
||||
which hosts to launch MPI processes.
|
||||
|
||||
Hostfiles are simple text files with hosts specified, one per line.
|
||||
Each host can also specify a default and maximum number of *slots* to
|
||||
be used on that host (i.e., the maximum number of processes that will
|
||||
be launched on that node). Comments are also supported, and blank
|
||||
lines are ignored. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
# This is an example hostfile. Comments begin with #.
|
||||
#
|
||||
# Since no slots are specified, the number of slots defaults to the
|
||||
# number of processor cores available on the machine.
|
||||
foo.example.com
|
||||
|
||||
# We want to allow launching a maximum of 2 processes on this host
|
||||
# (e.g., potentially because it has two processor cores):
|
||||
bar.example.com slots=2
|
||||
|
||||
Slots are discussed in much more detail :ref:`in this section
|
||||
<running-scheduling-slots-label>`.
|
||||
|
||||
Hostfiles works in two different ways:
|
||||
|
||||
#. *Exclusionary:* If a list of hosts to run on has been provided by
|
||||
another source (e.g., by a hostfile or a batch scheduler such as
|
||||
Slurm, PBS/Torque, SGE, etc.), the hosts provided by the hostfile
|
||||
must be in the already-provided host list. If the
|
||||
hostfile-specified nodes are *not* in the already-provided host
|
||||
list, ``mpirun`` will abort without launching anything.
|
||||
|
||||
In this case, hostfiles act like an exclusionary filter |mdash|
|
||||
they limit the scope of where processes will be scheduled from the
|
||||
original list of hosts to produce a final list of hosts.
|
||||
|
||||
For example, say that a scheduler job contains hosts ``node01``
|
||||
through ``node04``. If you run:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ cat my_hosts
|
||||
node03
|
||||
shell$ mpirun -n 1 --hostfile my_hosts hostname
|
||||
|
||||
This will run a single copy of ``hostname`` on the host ``node03``.
|
||||
|
||||
However, presuming your job was allocated only to ``node03`` and
|
||||
you run the following:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ cat my_hosts
|
||||
node17
|
||||
shell$ mpirun -n 1 --hostfile my_hosts hostname
|
||||
|
||||
This is an error (because ``node17`` is not allocated to your job),
|
||||
and ``mpirun`` will abort.
|
||||
|
||||
Finally, note that in exclusionary mode, processes will *only* be
|
||||
executed on the hostfile-specified hosts, If this ends up causing
|
||||
an oversubscription situation, ``mpirun`` will abort by default.
|
||||
|
||||
#. *Inclusionary:* If a list of hosts has *not* been provided by
|
||||
another source, then the hosts provided by the ``--hostfile``
|
||||
option will be used as the original and final host list.
|
||||
|
||||
In this case, ``--hostfile`` acts as an inclusionary agent; all
|
||||
``--hostfile``-supplied hosts become available for scheduling
|
||||
processes. For example (assume that you are *not* in a scheduling
|
||||
environment where a list of nodes is being transparently supplied):
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ cat my_hosts
|
||||
node01.example.com slots=1
|
||||
node02.example.com slots=1
|
||||
node03.example.com slots=1
|
||||
shell$ mpirun -n 3 --hostfile my_hosts hostname
|
||||
|
||||
This will launch a single copy of ``hostname`` on the hosts
|
||||
``node01.example.com``, ``node02.example.com``, and
|
||||
``node03.example.com``.
|
||||
|
||||
Note, too, that ``--hostfile`` is essentially a per-application switch.
|
||||
Hence, if you specify multiple applications (as in an MPMD job),
|
||||
``--hostfile`` can be specified multiple times:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ cat hostfile_1
|
||||
node01.example.com
|
||||
shell$ cat hostfile_2
|
||||
node02.example.com
|
||||
shell$ mpirun -n 1 --hostfile hostfile_1 hostname : -n 1 --hostfile hostfile_2 uptime
|
||||
node01.example.com
|
||||
06:11:45 up 1 day, 2:32, 0 users, load average: 21.65, 20.85, 19.84
|
||||
|
||||
Notice that ``hostname`` was launched on ``node01.example.com`` and
|
||||
``uptime`` was launched on ``node02.example.com``.
|
||||
|
||||
.. _running-scheduling-host-option-label:
|
||||
|
||||
Scheduling with the --host option
|
||||
---------------------------------
|
||||
|
||||
The ``--host`` option to ``mpirun`` takes a comma-delimited list of
|
||||
hosts on which to run. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun -n 3 --host a,b,c hostname
|
||||
|
||||
Will launch *one* copy of ``hostname`` on each of hosts ``a``, ``b``,
|
||||
and ``c``. Specifically: each host defaults to 1 slot, unless
|
||||
specified by the ``:N`` suffix. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun --host a,b:2,c:3 hostname
|
||||
|
||||
Will launch one copy of ``hostname`` on ``a``, two copies of
|
||||
``hostname`` on ``b``, and three copies of ``hostname`` and ``c``.
|
||||
|
||||
Slots are discussed in much more detail :ref:`in this section
|
||||
<running-scheduling-slots-label>`.
|
||||
|
||||
.. important:: The hosts specified by the ``--host`` option have
|
||||
*nothing* to do with which network interfaces are used
|
||||
for MPI communication. They are *only* used to specify
|
||||
on which hosts to launch MPI processes.
|
||||
|
||||
``--host`` works in two different ways:
|
||||
|
||||
#. *Exclusionary:* If a list of hosts to run on has been provided by
|
||||
another source (e.g., by a hostfile or a batch scheduler such as
|
||||
Slurm, PBS/Torque, SGE, etc.), the hosts provided by the ``--host``
|
||||
option must be in the already-provided host list. If the
|
||||
``--host``-specified nodes are *not* in the already-provided host
|
||||
list, ``mpirun`` will abort without launching anything.
|
||||
|
||||
In this case, the ``--host`` option acts like an exclusionary
|
||||
filter |mdash| it limits the scope of where processes will be
|
||||
scheduled from the original list of hosts to produce a final list
|
||||
of hosts.
|
||||
|
||||
For example, say that the hostfile ``my_hosts`` contains the hosts
|
||||
``node1`` through ``node4``. If you run:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun -n 1 --hostfile my_hosts --host node3 hostname
|
||||
|
||||
This will run a single copy of ``hostname`` on the host ``node3``.
|
||||
However, if you run:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun -n 1 --hostfile my_hosts --host node17 hostname
|
||||
|
||||
This is an error (because ``node17`` is not listed in
|
||||
``my_hosts``); ``mpirun`` will abort.
|
||||
|
||||
Finally, note that in exclusionary mode, processes will *only* be
|
||||
executed on the ``--host``-specified hosts. If this ends up
|
||||
causing an oversubscription situation, ``mpirun`` will abort by
|
||||
default.
|
||||
|
||||
#. *Inclusionary:* If a list of hosts has *not* been provided by
|
||||
another source, then the hosts provided by the ``--host`` option
|
||||
will be used as the original and final host list.
|
||||
|
||||
In this case, ``--host`` acts as an inclusionary agent; all
|
||||
``--host``-supplied hosts become available for scheduling
|
||||
processes. For example (assume that you are *not* in a scheduling
|
||||
environment where a list of nodes is being transparently supplied):
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun -n 3 --host a,b,c hostname
|
||||
|
||||
This will launch a single copy of ``hostname`` on the hosts ``a``,
|
||||
``b``, and ``c``.
|
||||
|
||||
Note, too, that ``--host`` is essentially a per-application switch.
|
||||
Hence, if you specify multiple applications (as in an MPMD job),
|
||||
``--host`` can be specified multiple times:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun -n 1 --host a hostname : -n 1 --host b uptime
|
||||
|
||||
This will launch ``hostname`` on host ``a`` and ``uptime`` on host ``b``.
|
||||
|
||||
.. _running-scheduling-slots-label:
|
||||
|
||||
Process Slots
|
||||
-------------
|
||||
|
||||
*Slots* are Open MPI's representation of how many processes can be
|
||||
launched on a given host.
|
||||
|
||||
Open MPI maintains the number of slots for each host in a given
|
||||
parallel job, and |mdash| by default |mdash| will not let you launch
|
||||
more processes on a host than it has slots.
|
||||
|
||||
.. important:: It is common to set the number of slots on a host to be
|
||||
less than or equal to the number of processor cores on
|
||||
that host.
|
||||
|
||||
**But it is important to realize that Open MPI's concept
|
||||
of slots is actually unrelated to the number of
|
||||
physical processor cores on a host.**
|
||||
|
||||
Specifically: the number of slots on a host can be less
|
||||
than, equal to, or more than the number of processor
|
||||
cores on a host.
|
||||
|
||||
If you wish to run more processes on a host than it has slots,
|
||||
:ref:`see the section on oversubscription
|
||||
<running-mpi-apps-oversubscribing-label>`.
|
||||
|
||||
Calculating the number of slots
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The number of slots on a host depends on a few factors:
|
||||
|
||||
#. If the host is specified by a job scheduler (e.g., Slurm,
|
||||
PBS/Torque, etc.), the job scheduler specifies the number of slots
|
||||
for that host.
|
||||
|
||||
#. If the host is specified in a hostfile:
|
||||
|
||||
#. If the ``slots`` parameter is specified, that value is used for
|
||||
the number of slots on that host.
|
||||
#. Otherwise:
|
||||
|
||||
#. If ``--map-by :HWTCPUS`` was specified, the number of slots
|
||||
defaults to the number of hardware threads on that host.
|
||||
#. Otherwise, the number of slots defaults to the number of
|
||||
processor cores on that host.
|
||||
|
||||
#. If the host is specified via the ``--host`` command line option:
|
||||
|
||||
#. If the ``:N`` suffix is specified, ``N`` is used for the number
|
||||
of slots on that host.
|
||||
#. Otherwise, the number of slots defaults to 1.
|
||||
#. If the same host name is specified multiple times, the slots
|
||||
value for that host is increased by ``N`` if ``:N`` is
|
||||
specified, or increased by 1 if ``:N`` is not specified.
|
||||
|
||||
.. caution:: The exact scheme used to determine the number of slots
|
||||
has varied between different major versions of Open MPI.
|
||||
The scheme described above is relevant for Open MPI
|
||||
|ompi_series|.
|
||||
|
||||
Max slot counts, however, are rarely specified by schedulers. The max
|
||||
slot count for each node will default to "infinite" if it is not
|
||||
provided (meaning that Open MPI will oversubscribe the node if you ask
|
||||
it to |mdash| see more on oversubscribing in :ref:`this section
|
||||
<running-mpi-apps-oversubscribing-label>`).
|
||||
|
||||
Here are some examples, all from unscheduled environments:
|
||||
|
||||
#. Use a hostfile and specify the ``slots`` parameter.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my-hostfile
|
||||
node01.example.come slots=4
|
||||
shell$ mpirun --hostfile my-hostfile hostname
|
||||
node01
|
||||
node01
|
||||
node01
|
||||
node01
|
||||
|
||||
This launched 4 processes because ``slots=4`` was specified in the
|
||||
hostfile.
|
||||
|
||||
#. Use a hostfile and do *not* specify the ``slots`` parameter (assume
|
||||
that ``node01.example.com`` has 2 processor cores):
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my-hostfile
|
||||
node01.example.come
|
||||
shell$ mpirun --hostfile my-hostfile hostname
|
||||
node01
|
||||
node01
|
||||
|
||||
This launched 2 processes because ``slots`` was not specified, and
|
||||
``node02`` has 2 processor cores.
|
||||
|
||||
#. Use ``--host``:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun --host node01.example.com hostname
|
||||
node01
|
||||
|
||||
This launched 1 processes because ``--host`` with no ``:N`` suffix
|
||||
increments the slot count for that host by 1.
|
||||
|
||||
#. Use ``--host`` with a ``:N`` suffix:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun --host node01.example.com:2 hostname
|
||||
node01
|
||||
node01
|
||||
|
||||
This launched 2 processes because ``:2`` was specified on the
|
||||
command line.
|
||||
|
||||
#. Use ``--host`` with a ``:N`` suffix, and mention the host multiple times:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ mpirun --host node01.example.com:2,node01.example.com hostname
|
||||
node01
|
||||
node01
|
||||
node01
|
||||
|
||||
This launched 3 processes because ``:2`` was specified on the
|
||||
command line, and then ``node01.example.com`` was specified an
|
||||
additional time, incrementing the slot count for that host to 3.
|
||||
|
||||
.. _running-mpi-apps-oversubscribing-label:
|
||||
|
||||
Oversubscribing nodes
|
||||
---------------------
|
||||
|
||||
Running more MPI processes than processors are available on a node is called
|
||||
*oversubscribing* the node. Open MPI can oversubscribe nodes,
|
||||
but it very much matters *how* you do it.
|
||||
|
||||
Specifically: it is critical that Open MPI *knows* that you are
|
||||
oversubscribing the node, or **severe** performance degradation can
|
||||
result.
|
||||
|
||||
.. important:: Here is a good general rule to follow: **never specify
|
||||
a number of slots that is more than the available
|
||||
number of processors.**
|
||||
|
||||
For example, if you want to run 4 processes on a host with 2 processor
|
||||
cores, then indicate that you only have 2 slots but want to run 4
|
||||
processes. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# In a hostfile, the number of slots will default to the number of
|
||||
# processor cores on the host
|
||||
shell$ cat my-hostfile
|
||||
localhost
|
||||
shell$ mpirun -n 4 --hostfile my-hostfile a.out
|
||||
|
||||
Specifically: we strongly suggest that you do **NOT** have a hostfile
|
||||
that contains ``slots=4`` (because there are only two available
|
||||
processor cores).
|
||||
|
||||
That being said, the above command will fail, because you are trying
|
||||
to run 4 processes but there are only 2 slots available. You must
|
||||
specifically tell Open MPI that it is ok to oversubscribe via
|
||||
``--map-by :OVERSUBSCRIBE``:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my-hostfile
|
||||
# For the purposes of this example, explicitly tell Open MPI
|
||||
# that we have 2 slots on the host.
|
||||
localhost slots=2
|
||||
shell$ mpirun -n 4 --hostfile my-hostfile --map-by :OVERSUBSCRIBE a.out
|
||||
|
||||
The reason you should tell Open MPI whether you're oversubscribing or
|
||||
not (i.e., never specify a ``slots`` value more than the number of
|
||||
processor cores available) is because Open MPI basically runs its
|
||||
message passing progression engine in two modes: *aggressive* and
|
||||
*degraded*.
|
||||
|
||||
#. *Degraded:* When Open MPI thinks that it is in an oversubscribed
|
||||
mode (i.e., more processes are running than there are processor
|
||||
cores available), MPI processes will automatically run in
|
||||
*degraded* mode and frequently yield the processor to its peers,
|
||||
thereby allowing all processes to make progress.
|
||||
|
||||
.. note:: Be sure to see :ref:`this FAQ entry
|
||||
<tuning-using-paffinity-label>` that describes how
|
||||
degraded mode affects processor and memory
|
||||
affinity.
|
||||
|
||||
#. *Aggressive:* When Open MPI thinks that it is in an exactly- or
|
||||
under-subscribed mode (i.e., the number of running processes is
|
||||
equal to or less than the number of available processor cores), MPI
|
||||
processes will automatically run in *aggressive* mode, meaning that
|
||||
they will never voluntarily give up the processor to other
|
||||
processes. With some network transports, this means that Open MPI
|
||||
will spin in tight loops attempting to make message passing
|
||||
progress, effectively causing other processes to not get any CPU
|
||||
cycles (and therefore never make any progress).
|
||||
|
||||
For example, on a node with a two processor cores:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ cat my-hostfile
|
||||
localhost slots=4
|
||||
shell$ mpirun -n 4 --hostfile my-hostfile a.out
|
||||
|
||||
This would cause all 4 MPI processes to run in *aggressive* mode
|
||||
because Open MPI thinks that there are 4 available processor cores to
|
||||
use. This is actually a lie (there are only 2 processor core |mdash|
|
||||
not 4), and can cause extremely bad performance.
|
||||
|
||||
Forcing aggressive or degraded performance mode
|
||||
-----------------------------------------------
|
||||
|
||||
The MCA parameter ``mpi_yield_when_idle`` controls whether an MPI
|
||||
process runs in Aggressive or Degraded performance mode. Setting it
|
||||
to 0 forces Aggressive mode; setting it to 1 forces Degraded mode (see
|
||||
:ref:`this FAQ entry <label-running-setting-mca-param-values>` to see how
|
||||
to set MCA parameters).
|
||||
|
||||
Note that this value *only* affects the behavior of MPI processes when
|
||||
they are blocking in MPI library calls. It does not affect behavior
|
||||
of non-MPI processes, nor does it affect the behavior of a process
|
||||
that is not inside an MPI library call.
|
||||
|
||||
Open MPI normally sets this parameter automatically and
|
||||
users are cautioned against setting this parameter unless you are
|
||||
really, absolutely, positively sure of what you are doing.
|
||||
@@ -0,0 +1,122 @@
|
||||
Launching with Slurm
|
||||
====================
|
||||
|
||||
Open MPI supports two modes of launching parallel MPI jobs under
|
||||
Slurm:
|
||||
|
||||
#. Using Open MPI's full-features ``mpirun`` launcher.
|
||||
#. Using Slurm's "direct launch" capability.
|
||||
|
||||
Unless there is a strong reason to use ``srun`` for direct launch, the
|
||||
Open MPI team recommends using ``mpirun`` for launching under Slurm jobs.
|
||||
|
||||
.. note:: In versions of Open MPI prior to 5.0.x, using ``srun`` for
|
||||
direct launch could be faster than using ``mpirun``. **This is no
|
||||
longer true.**
|
||||
|
||||
Using ``mpirun``
|
||||
----------------
|
||||
|
||||
When ``mpirun`` is launched in a Slurm job, ``mpirun`` will
|
||||
automatically utilize the Slurm infrastructure for launching and
|
||||
controlling the individual MPI processes.
|
||||
Hence, it is unnecessary to specify the ``--hostfile``,
|
||||
``--host``, or ``-n`` options to ``mpirun``.
|
||||
|
||||
.. note:: Using ``mpirun`` is the recommended method for launching Open
|
||||
MPI jobs in Slurm jobs.
|
||||
|
||||
``mpirun``'s Slurm support should always be available, regardless
|
||||
of how Open MPI or Slurm was installed.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Allocate a Slurm job with 4 slots
|
||||
shell$ salloc -n 4
|
||||
salloc: Granted job allocation 1234
|
||||
|
||||
# Now run an Open MPI job on all the slots allocated by Slurm
|
||||
shell$ mpirun mpi-hello-world
|
||||
|
||||
This will run the 4 MPI processes on the node(s) that were allocated
|
||||
by Slurm.
|
||||
|
||||
Or, if submitting a script:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my_script.sh
|
||||
#!/bin/sh
|
||||
mpirun mpi-hello-world
|
||||
shell$ sbatch -n 4 my_script.sh
|
||||
srun: jobid 1235 submitted
|
||||
shell$
|
||||
|
||||
Similar to the ``salloc`` case, no command line options specifying
|
||||
number of MPI processes were necessary, since Open MPI will obtain
|
||||
that information directly from Slurm at run time.
|
||||
|
||||
Using Slurm's "direct launch" functionality
|
||||
-------------------------------------------
|
||||
|
||||
Assuming that Slurm was configured with its PMIx plugin, you can use
|
||||
``srun`` to "direct launch" Open MPI applications without the use of
|
||||
Open MPI's ``mpirun`` command.
|
||||
|
||||
First, you must ensure that Slurm was built and installed with PMIx
|
||||
support. This can determined as shown below:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ srun --mpi=list
|
||||
MPI plugin types are...
|
||||
none
|
||||
pmi2
|
||||
pmix
|
||||
specific pmix plugin versions available: pmix_v4
|
||||
|
||||
The output from ``srun`` may vary somewhat depending on the version of Slurm installed.
|
||||
If PMIx is not present in the output, then you will not be able to use srun
|
||||
to launch Open MPI applications.
|
||||
|
||||
.. note:: PMI-2 is not supported in Open MPI 5.0.0 and later releases.
|
||||
|
||||
Provided the Slurm installation includes the PMIx plugin, Open MPI applications
|
||||
can then be launched directly via the ``srun`` command. For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ srun -N 4 --mpi=pmix mpi-hello-world
|
||||
|
||||
Or you can use ``sbatch`` with a script:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my_script.sh
|
||||
#!/bin/sh
|
||||
srun --mpi=pmix mpi-hello-world
|
||||
shell$ sbatch -N 4 my_script.sh
|
||||
srun: jobid 1235 submitted
|
||||
shell$
|
||||
|
||||
Similar using ``mpirun`` inside of an ``sbatch`` batch script, no
|
||||
``srun`` command line options specifying number of processes were
|
||||
necessary, because ``sbatch`` set all the relevant Slurm-level
|
||||
parameters about number of processes, cores, partition, etc.
|
||||
|
||||
Slurm 20.11
|
||||
-----------
|
||||
|
||||
There were some changes in Slurm behavior that were introduced in
|
||||
Slurm 20.11.0 and subsequently reverted out in Slurm 20.11.3.
|
||||
|
||||
SchedMD (the makers of Slurm) strongly suggest that all Open MPI users
|
||||
avoid using Slurm versions 20.11.0 through 20.11.2.
|
||||
|
||||
Indeed, you will likely run into problems using just about any version
|
||||
of Open MPI these problematic Slurm releases.
|
||||
|
||||
.. important:: Please either downgrade to an older version or upgrade
|
||||
to a newer version of Slurm.
|
||||
@@ -0,0 +1,240 @@
|
||||
Launching with SSH
|
||||
==================
|
||||
|
||||
When launching Open MPI jobs in a non-scheduled environment, ``ssh``
|
||||
is typically used to launch commands on remote nodes. As listed in
|
||||
the :doc:`quick start section </launching-apps/quickstart>`,
|
||||
successfully launching MPI applications with ``ssh`` requires the
|
||||
following:
|
||||
|
||||
#. You must be able to non-interactively login |mdash| without
|
||||
entering a password or passphrase |mdash| to all remote nodes from
|
||||
all remotes nodes.
|
||||
#. Open MPI's executables must be findable (e.g., in your ``PATH``).
|
||||
#. Open MPI's libraries must be findable (e.g., in your
|
||||
``LD_LIBRARY_PATH``).
|
||||
|
||||
Specifying the hosts for an MPI job
|
||||
-----------------------------------
|
||||
|
||||
There are three mechanisms for specifying the hosts that an MPI job will run on:
|
||||
|
||||
#. The ``--hostfile`` option to :ref:`mpirun(1) <man1-mpirun>`.
|
||||
|
||||
Use this option to specify a list of hosts on which to run. Note
|
||||
that for compatibility with other MPI implementations,
|
||||
``--machinefile`` is a synonym for ``--hostfile``.
|
||||
See :ref:`this section <running-scheduling-hostfile-option-label>` for more
|
||||
information about the ``--hostfile`` option.
|
||||
|
||||
#. The ``--host`` option to :ref:`mpirun(1) <man1-mpirun>`.
|
||||
|
||||
This option can be used to specify a list of hosts on which to run
|
||||
on the command line.
|
||||
See :ref:`this section <running-scheduling-host-option-label>` for more
|
||||
information about the ``--host`` option.
|
||||
|
||||
#. Running in a scheduled environment.
|
||||
|
||||
If you are running in a scheduled environment (e.g., in a Slurm,
|
||||
Torque, or LSF job), Open MPI will automatically get the lists of
|
||||
hosts from the scheduler. See the next subsections for details about
|
||||
launching MPI jobs in supported scheduled environements.
|
||||
|
||||
.. important:: The specification of hosts using any of the above
|
||||
methods has nothing to do with the network interfaces
|
||||
that are used for MPI traffic. The list of hosts is
|
||||
*only* used for specifying which hosts on which to
|
||||
launch MPI processes.
|
||||
|
||||
Non-interactive ``ssh`` logins
|
||||
------------------------------
|
||||
|
||||
SSH keys must be setup such that the following can be executed without
|
||||
being prompted for password or passphrase:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ssh othernode echo hello
|
||||
hello
|
||||
shell$
|
||||
|
||||
Consult instructions and tutorials from around the internet to learn
|
||||
how to setup SSH keys. Try Google search terms like "passwordless
|
||||
SSH" or "SSH key authentication".
|
||||
|
||||
For simplicity, it may be desirable to configure your SSH keys
|
||||
without passphrases. This adds some risk, however (e.g., if your SSH
|
||||
keys are compromised). But it simplifies your SSH setup because you
|
||||
will not need to use ``ssh-agent``. Evaluate the risk level you are
|
||||
comfortable with.
|
||||
|
||||
.. important:: Open MPI uses a tree-based pattern to launch processes
|
||||
on remote nodes. This means that Open MPI must be able to
|
||||
non-interactively login |mdash| without being prompted for password
|
||||
or passphrase |mdash| *to any node* in the host list *from any
|
||||
node* in the host list.
|
||||
|
||||
It may *not* be sufficient to only setup an SSH key from the node
|
||||
where you are invoking :ref:`mpirun(1) <man1-mpirun>` to all other
|
||||
nodes.
|
||||
|
||||
If you have a shared ``$HOME`` filesystem between your nodes, you can
|
||||
setup a single SSH key that is used to login to all nodes.
|
||||
|
||||
Finding Open MPI executables and libraries
|
||||
------------------------------------------
|
||||
|
||||
Once Open MPI is able to use ``ssh`` to invoke executables on a remote
|
||||
node, it must be able to find its helper executables and shared
|
||||
libraries on that remote node.
|
||||
|
||||
If Open MPI is installed in a system-level folder (e.g., in
|
||||
``/usr/bin``), Open MPI will likely be able to find its executables
|
||||
and libraries on the remote node with no additional assistance.
|
||||
|
||||
If, however, Open MPI is installed into a path that is not searched by
|
||||
default, you will need to provide assistance so that Open MPI can find
|
||||
its executables and libraries.
|
||||
|
||||
.. important:: For simplicity, it is *strongly* recommended that you
|
||||
install Open MPI in the same location on all nodes in your job.
|
||||
See the :doc:`Installation location section
|
||||
</installing-open-mpi/installation-location>` for more details.
|
||||
|
||||
You can do this in one of two ways.
|
||||
|
||||
Use "prefix" behavior
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. note:: "Prefix" behavior is only available with :ref:`mpirun(1)
|
||||
<man1-mpirun>`; it is not available via resource manager direct
|
||||
launch mechanisms. However, this section is about using ``ssh`` to
|
||||
launch MPI jobs, which means that there is no resource manager, and
|
||||
therefore there is no direct launch mechanism available.
|
||||
|
||||
When "prefix" behavior is enabled, Open MPI will automatically set the
|
||||
``$PATH`` and ``$LD_LIBRARY_PATH`` on remote nodes before executing
|
||||
remote commands.
|
||||
|
||||
.. important:: Open MPI assumes that the installation ``prefix``,
|
||||
``bindir``, and ``libdir`` are the same on the remote node as they
|
||||
are on the local node. If they are not, *then you should not use
|
||||
the "prefix" behavior.*
|
||||
|
||||
You can enable "prefix" behavior in one of three ways:
|
||||
|
||||
#. Use an absolute path name to invoke :ref:`mpirun(1) <man1-mpirun>`.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ $HOME/my-openmpi/bin/mpirun --hostfile my-hostfile.txt mpi-hello-world
|
||||
|
||||
Simply using the absolute path name to :ref:`mpirun(1)
|
||||
<man1-mpirun>` tells Open MPI to enable "prefix" mode.
|
||||
|
||||
|
||||
#. Use the ``--prefix`` option to :ref:`mpirun(1) <man1-mpirun>`.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ $HOME/my-openmpi/bin/mpirun --hostfile my-hostfile.txt \
|
||||
--prefix $HOME/my-openmpi \
|
||||
mpi-hello-world
|
||||
|
||||
The ``-prefix`` option takes a single argument: the prefix path to
|
||||
use for the bindir and libdir on the remote node.
|
||||
|
||||
#. Configure Open MPI with ``--enable-mpirun-prefix-by-default``.
|
||||
|
||||
If Open MPI is built this way, :ref:`mpirun(1) <man1-mpirun>` will
|
||||
always enable "prefix" behavior.
|
||||
|
||||
Set the ``PATH`` and ``LD_LIBRARY_PATH`` in your shell startup files
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Consider the case where Open MPI was configured with:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ ./configure --prefix=$HOME/my-openmpi ...
|
||||
|
||||
In this cause, Open MPI will be installed into ``$HOME/my-openmpi``.
|
||||
This path is almost certainly not in any system-default search paths,
|
||||
so it must be added to the ``$PATH`` and ``$LD_LIBRARY_PATH``
|
||||
environment variables.
|
||||
|
||||
Specifically: the goal is that the following non-interactive commands
|
||||
must be able to execute without error:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# First, ensure that this command returns the correct ompi_info
|
||||
# instance (i.e., $HOME/my-openmpi/bin/ompi_info).
|
||||
shell$ ssh remotenode which ompi_info
|
||||
/home/myusername/my-openmpi/bin/ompi_info
|
||||
|
||||
# Next, ensure that you can run that ompi_info command without
|
||||
# error
|
||||
shell$ ssh remotenode ompi_info
|
||||
|
||||
# ... lots of output ...
|
||||
|
||||
Ensure that you do not see any errors about libraries that cannot be
|
||||
found.
|
||||
|
||||
All shells have some kind of script file that is executed at login
|
||||
time perform environmental setup tasks. This startup file is the one
|
||||
that needs to be edited to:
|
||||
|
||||
#. Add Open MPI's executable path (which is likely ``$prefix/bin``, or
|
||||
``$HOME/my-openmpi/bin`` in this example) to the ``$PATH``
|
||||
environment variable.
|
||||
#. Add Open MPI's library path (which is likely ``$prefix/lib``, or
|
||||
``$HOME/my-openmpi/lib`` in this example) to the
|
||||
``$LD_LIBRARY_PATH`` environment variable.
|
||||
|
||||
You probably want to add Open MPI's libraries to the *front* of
|
||||
``$PATH`` and ``$LD_LIBRARY_PATH`` to ensure that this Open MPI
|
||||
installation's files are found *first*.
|
||||
|
||||
Consult the manual page for your shell for specific details (some
|
||||
shells are picky about the permissions of the startup file, for
|
||||
example). The list below contains some common shells and the startup
|
||||
files that they read/execute upon login:
|
||||
|
||||
.. list-table::
|
||||
:header-rows: 1
|
||||
|
||||
* - Shell
|
||||
- Non-interactive login
|
||||
- Interactive login
|
||||
|
||||
* - ``bash`` or ``zsh``
|
||||
- ``$HOME/.bashrc`` if it exists.
|
||||
- #. ``$HOME/.bash_profile`` if it exists, or
|
||||
#. ``$HOME/.bash_login`` if it exists, or
|
||||
#. ``$HOME/.profile`` if it exists (in that order).
|
||||
|
||||
Note that some Linux distributions automatically come
|
||||
with ``$HOME/.bash_profile`` scripts for users that
|
||||
automatically execute ``$HOME/.bashrc`` as well. Consult the
|
||||
bash man page for more information.
|
||||
|
||||
* - ``sh``
|
||||
- This shell does not execute any file automatically, so Open MPI
|
||||
will execute the ``$HOME/.profile`` script before invoking Open
|
||||
MPI executables on remote nodes
|
||||
- ``$HOME/.profile``
|
||||
|
||||
* - ``csh``
|
||||
- ``$HOME/.cshrc``
|
||||
- ``$HOME/.cshrc`` followed by ``$HOME/.login``
|
||||
|
||||
* - ``tcsh``
|
||||
- #. ``$HOME/.tcshrc`` if it exists, or
|
||||
#. ``$HOME/.cshrc`` if it does not
|
||||
- #. ``$HOME/.tcshrc`` if it exists, or
|
||||
#. ``$HOME/.cshrc`` if it does not
|
||||
|
||||
Afterwards, execute ``$HOME/.login``
|
||||
@@ -0,0 +1,72 @@
|
||||
Launching with PBS / Torque
|
||||
===========================
|
||||
|
||||
Open MPI supports PBS, PBS Pro, Torque, and other related resource
|
||||
managers.
|
||||
|
||||
Verify PBS/Torque support
|
||||
-------------------------
|
||||
|
||||
The ``prte_info`` command can be used to determine whether or not an
|
||||
installed Open MPI includes Torque/PBS Pro support:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ prte_info | grep ras
|
||||
|
||||
If the Open MPI installation includes support for PBS/Torque, you
|
||||
should see a line similar to that below. Note the MCA version
|
||||
information varies depending on which version of Open MPI is
|
||||
installed.
|
||||
|
||||
.. code-block::
|
||||
|
||||
MCA ras: tm (MCA v2.1.0, API v2.0.0, Component v3.0.0)
|
||||
|
||||
.. note:: PRRTE is the software layer that provides run-time
|
||||
environment support to Open MPI. Open MPI typically hides most
|
||||
PMIx and PRRTE details from the end user, but this is one place
|
||||
that Open MPI is unable to hide the fact that PRRTE provides this
|
||||
functionality, not Open MPI. Hence, users need to use the
|
||||
``prte_info`` command to check for PBS/Torque support (not
|
||||
``ompi_info``).
|
||||
|
||||
Launching
|
||||
---------
|
||||
|
||||
When properly configured, Open MPI obtains both the list of hosts and
|
||||
how many processes to start on each host from Torque / PBS Pro
|
||||
directly. Hence, it is unnecessary to specify the ``--hostfile``,
|
||||
``--host``, or ``-n`` options to ``mpirun``. Open MPI will use
|
||||
PBS/Torque-native mechanisms to launch and kill processes (``ssh`` is
|
||||
not required).
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Allocate a PBS job with 4 nodes
|
||||
shell$ qsub -I -lnodes=4
|
||||
|
||||
# Now run an Open MPI job on all the nodes allocated by PBS/Torque
|
||||
shell$ mpirun mpi-hello-world
|
||||
|
||||
This will run the MPI processes on the nodes that were allocated by
|
||||
PBS/Torque. Or, if submitting a script:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
shell$ cat my_script.sh
|
||||
#!/bin/sh
|
||||
mpirun mpi-hello-world
|
||||
shell$ qsub -l nodes=4 my_script.sh
|
||||
|
||||
.. warning:: Do not modify ``$PBS_NODEFILE``!
|
||||
|
||||
We've had reports from some sites that system administrators modify
|
||||
the ``$PBS_NODEFILE`` in each job according to local policies.
|
||||
This will currently cause Open MPI to behave in an unpredictable
|
||||
fashion. As long as no new hosts are added to the hostfile, it
|
||||
*usually* means that Open MPI will incorrectly map processes to
|
||||
hosts, but in some cases it can cause Open MPI to fail to launch
|
||||
processes altogether.
|
||||
@@ -0,0 +1,267 @@
|
||||
Troubleshooting
|
||||
===============
|
||||
|
||||
Launching MPI jobs can be a complex process that involves many moving parts.
|
||||
This section attempts to provide solutions to some of the most common
|
||||
problems users encounter.
|
||||
|
||||
Messages about missing symbols
|
||||
------------------------------
|
||||
|
||||
Open MPI loads a lot of plugins (sometimes called "components" or
|
||||
"modules") at run time. Sometimes a plugin can fail to load because it
|
||||
can't resolve all the symbols that it needs. There are a few reasons
|
||||
why this can happen.
|
||||
|
||||
* The plugin is for a different version of Open MPI. :ref:`See this
|
||||
section <building-open-mpi-install-overwrite-label>` for an
|
||||
explanation of how Open MPI might try to open the "wrong" plugins.
|
||||
* An application is trying to manually dynamically open ``libmpi`` in
|
||||
a private symbol space. For example, if an application is not
|
||||
linked against ``libmpi``, but rather calls something like this:
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
/* This is a Linux example |mdash| the issue is similar/the same on other
|
||||
operating systems */
|
||||
handle = dlopen("libmpi.so", RTLD_NOW | RTLD_LOCAL);
|
||||
|
||||
This is due to some deep run-time linker voodoo |mdash| it is
|
||||
discussed towards the end of `this post to the Open MPI developer's
|
||||
list
|
||||
<https://www.mail-archive.com/devel@lists.open-mpi.org/msg07981.html>`_.
|
||||
Briefly, the issue is this:
|
||||
|
||||
#. The dynamic library ``libmpi`` is opened in a "local" symbol
|
||||
space.
|
||||
#. ``MPI_INIT`` is invoked, which tries to open Open MPI's plugins.
|
||||
#. Open MPI's plugins rely on symbols in ``libmpi`` (and other Open
|
||||
MPI support libraries); these symbols must be resolved when the
|
||||
plugin is loaded.
|
||||
#. However, since ``libmpi`` was opened in a "local" symbol space,
|
||||
its symbols are not available to the plugins that it opens.
|
||||
#. Hence, the plugin fails to load because it can't resolve all of
|
||||
its symbols, and displays a warning message to that effect.
|
||||
|
||||
The ultimate fix for this issue is a bit bigger than Open MPI,
|
||||
unfortunately |mdash| it's a POSIX issue (as briefly described in the
|
||||
devel mailing list posting, above).
|
||||
|
||||
However, there are several common workarounds:
|
||||
|
||||
* Dynamically open ``libmpi`` in a public / global symbol scope
|
||||
|mdash| not a private / local scope. This will enable
|
||||
``libmpi``'s symbols to be available for resolution when Open MPI
|
||||
dynamically opens its plugins.
|
||||
* If ``libmpi`` is opened as part of some underlying framework where
|
||||
it is not possible to change the private / local scope to a public
|
||||
/ global scope, then dynamically open ``libmpi`` in a public /
|
||||
global scope before invoking the underlying framework. This
|
||||
sounds a little gross (and it is), but at least the run-time
|
||||
linker is smart enough to not load ``libmpi`` twice |mdash| but it
|
||||
does keeps ``libmpi`` in a public scope.
|
||||
* Use the ``--disable-dlopen`` or ``--disable-mca-dso`` options to
|
||||
Open MPI's ``configure`` script (see this TODO NONEXISTENT FAQ entry
|
||||
for more details on these
|
||||
options). These options slurp all of Open MPI's plugins up in to
|
||||
``libmpi`` |mdash| meaning that the plugins physically reside in
|
||||
``libmpi`` and will not be dynamically opened at run time.
|
||||
* Build Open MPI as a static library by configuring Open MPI with
|
||||
``--disable-shared`` and ``--enable-static``. This has the same
|
||||
effect as ``--disable-dlopen``, but it also makes ``libmpi.a`` (as
|
||||
opposed to a shared library).
|
||||
|
||||
Errors about missing libraries
|
||||
------------------------------
|
||||
|
||||
When building Open MPI with the compilers that have libraries in
|
||||
non-default search path locations, you may see errors about those
|
||||
compiler's support libraries when trying to launch MPI applications if
|
||||
their corresponding environments were not setup properly.
|
||||
|
||||
For example, you may see warnings similar to the following:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# With the Intel compiler suite
|
||||
shell$ mpirun -n 1 --host node1.example.com mpi_hello
|
||||
prted: error while loading shared libraries: libimf.so: cannot open shared object file: No such file or directory
|
||||
--------------------------------------------------------------------------
|
||||
A daemon (pid 11893) died unexpectedly with status 127 while
|
||||
attempting to launch so we are aborting.
|
||||
...more error messages...
|
||||
|
||||
# With the PGI compiler suite
|
||||
shell$ mpirun -n 1 --host node1.example.com mpi_hello
|
||||
prted: error while loading shared libraries: libpgcc.so: cannot open shared object file: No such file or directory
|
||||
...more error messages...
|
||||
|
||||
# With the PathScale compiler suite
|
||||
shell$ mpirun -n 1 --host node1.example.com mpi_hello
|
||||
prted: error while loading shared libraries: libmv.so: cannot open shared object file: No such file or directory
|
||||
...more error messages...
|
||||
|
||||
Specifically, Open MPI first attempts to launch a "helper" daemon
|
||||
``prted`` on ``node1.example.com``, but it failed because one of
|
||||
``prted``'s dependent libraries was not able to be found. The
|
||||
libraries shown above (``libimf.so``, ``libpgcc.so``, and
|
||||
``libmv.so``) are specific to their compiler suites (Intel, PGI, and
|
||||
PathScale, respectively). As such, it is likely that the user did not
|
||||
setup the compiler library in their environment properly on this node.
|
||||
|
||||
Double check that you have setup the appropriate compiler environment
|
||||
on the target node, for both interactive and non-interactive logins.
|
||||
|
||||
.. note:: It is a common error to ensure that the compiler environment
|
||||
is setup properly for *interactive* logins, but not for
|
||||
*non-interactive* logins.
|
||||
|
||||
Here's an example of a user-compiled MPI application working fine
|
||||
locally, but failing when invoked non-interactively on a remote node:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Compile a trivial MPI application
|
||||
head_node$ cd $HOME
|
||||
head_node$ mpicc mpi_hello.c -o mpi_hello
|
||||
|
||||
# Run it locally; it works fine
|
||||
head_node$ ./mpi_hello
|
||||
Hello world, I am 0 of 1.
|
||||
|
||||
# Run it remotely interactively; it works fine
|
||||
head_node$ ssh node2.example.com
|
||||
|
||||
Welcome to node2.
|
||||
node2$ ./mpi_hello
|
||||
Hello world, I am 0 of 1.
|
||||
node2$ exit
|
||||
|
||||
# Run it remotely *NON*-interactively; it fails
|
||||
head_node$ ssh node2.example.com $HOME/mpi_hello
|
||||
mpi_hello: error while loading shared libraries: libimf.so: cannot open shared object file: No such file or directory
|
||||
|
||||
In cases like this, check your shell script startup files and verify
|
||||
that the appropriate compiler environment is setup properly for
|
||||
non-interactive logins.
|
||||
|
||||
Problems when running across multiple hosts
|
||||
-------------------------------------------
|
||||
|
||||
When you are able to run MPI jobs on a single host, but fail to run
|
||||
them across multiple hosts, try the following:
|
||||
|
||||
#. Ensure that your launcher is able to launch across multiple hosts.
|
||||
For example, if you are using ``ssh``, try to ``ssh`` to each
|
||||
remote host and ensure that you are not prompted for a password.
|
||||
For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ssh remotehost hostname
|
||||
remotehost
|
||||
|
||||
If you are unable to launch across multiple hosts, check that your
|
||||
SSH keys are setup properly. Or, if you are running in a managed
|
||||
environment, such as in a Slurm, Torque, or other job launcher,
|
||||
check that you have reserved enough hosts, are running in an
|
||||
allocated job, etc.
|
||||
|
||||
#. Ensure that your ``PATH`` and ``LD_LIBRARY_PATH`` are set correctly
|
||||
on each remote host on which you are trying to run. For example,
|
||||
with ``ssh``:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ ssh remotehost env | grep -i path
|
||||
PATH=...path on the remote host...
|
||||
LD_LIBRARY_PATH=...LD library path on the remote host...
|
||||
|
||||
If your ``PATH`` or ``LD_LIBRARY_PATH`` are not set properly, see
|
||||
:ref:`this section <running-prerequisites-label>` for
|
||||
the correct values. Keep in mind that it is fine to have multiple
|
||||
Open MPI installations installed on a machine; the *first* Open MPI
|
||||
installation found by ``PATH`` and ``LD_LIBARY_PATH`` is the one
|
||||
that matters.
|
||||
|
||||
#. Run a simple, non-MPI job across multiple hosts. This verifies
|
||||
that the Open MPI run-time system is functioning properly across
|
||||
multiple hosts. For example, try running the ``hostname`` command:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun --host remotehost hostname
|
||||
remotehost
|
||||
shell$ mpirun --host remotehost,otherhost hostname
|
||||
remotehost
|
||||
otherhost
|
||||
|
||||
If you are unable to run non-MPI jobs across multiple hosts, check
|
||||
for common problems such as:
|
||||
|
||||
#. Check your non-interactive shell setup on each remote host to
|
||||
ensure that it is setting up the ``PATH`` and
|
||||
``LD_LIBRARY_PATH`` properly.
|
||||
#. Check that Open MPI is finding and launching the correct
|
||||
version of Open MPI on the remote hosts.
|
||||
#. Ensure that you have firewalling disabled between hosts (Open
|
||||
MPI opens random TCP and sometimes random UDP ports between
|
||||
hosts in a single MPI job).
|
||||
#. Try running with the ``plm_base_verbose`` MCA parameter at level
|
||||
10, which will enable extra debugging output to see how Open MPI
|
||||
launches on remote hosts. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
mpirun --mca plm_base_verbose 10 --host remotehost hostname``
|
||||
|
||||
#. Now run a simple MPI job across multiple hosts that does not
|
||||
involve MPI communications. The ``hello_c`` program in the
|
||||
``examples`` directory in the Open MPI distribution is a good
|
||||
choice. This verifies that the MPI subsystem is able to initialize
|
||||
and terminate properly. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun --host remotehost,otherhost hello_c
|
||||
Hello, world, I am 0 of 1, (Open MPI VERSION, package: Open MPI jsquyres@example.com Distribution, ident: VERSION, DATE)
|
||||
Hello, world, I am 1 of 1, (Open MPI VERSION, package: Open MPI jsquyres@example.com Distribution, ident: VERSION, DATE)
|
||||
|
||||
If you are unable to run simple, non-communication MPI jobs, this
|
||||
can indicate that your Open MPI installation is unable to
|
||||
initialize properly on remote hosts. Double check your
|
||||
non-interactive login setup on remote hosts.
|
||||
|
||||
#. Now run a simple MPI job across multiple hosts that does does some
|
||||
simple MPI communications. The ``ring_c`` program in the
|
||||
``examples`` directory in the Open MPI distribution is a good
|
||||
choice. This verifies that the MPI subsystem is able to pass MPI
|
||||
traffic across your network. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun --host remotehost,otherhost ring_c
|
||||
Process 0 sending 10 to 0, tag 201 (1 processes in ring)
|
||||
Process 0 sent to 0
|
||||
Process 0 decremented value: 9
|
||||
Process 0 decremented value: 8
|
||||
Process 0 decremented value: 7
|
||||
Process 0 decremented value: 6
|
||||
Process 0 decremented value: 5
|
||||
Process 0 decremented value: 4
|
||||
Process 0 decremented value: 3
|
||||
Process 0 decremented value: 2
|
||||
Process 0 decremented value: 1
|
||||
Process 0 decremented value: 0
|
||||
Process 0 exiting
|
||||
|
||||
If you are unable to run simple MPI jobs across multiple hosts,
|
||||
this may indicate a problem with the network(s) that Open MPI is
|
||||
trying to use for MPI communications. Try limiting the networks
|
||||
that it uses, and/or exploring levels 1 through 3 MCA parameters
|
||||
for the communications module that you are using. For example, if
|
||||
you're using the TCP BTL, see the output of:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --level 3 --param btl tcp
|
||||
@@ -0,0 +1,164 @@
|
||||
Unusual jobs
|
||||
============
|
||||
|
||||
Open MPI can run many types of applications, including non-MPI programs.
|
||||
This section describes some of the less common kinds of programs that can
|
||||
be executed.
|
||||
|
||||
|
||||
Running non-MPI programs with :ref:`mpirun(1) <man1-mpirun>`
|
||||
------------------------------------------------------------
|
||||
|
||||
Non-MPI programs can be launched with :ref:`mpirun(1) <man1-mpirun>`,
|
||||
for example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun -n 2 --host a,b uptime
|
||||
|
||||
This will launch a copy of the Unix command ``uptime`` on the hosts ``a``
|
||||
and ``b``.
|
||||
|
||||
:ref:`mpirun(1) <man1-mpirun>` works equally well for MPI and non-MPI
|
||||
applications.
|
||||
|
||||
Running GUI applications
|
||||
------------------------
|
||||
|
||||
Running GUI applications depends on your local setup and may require additional
|
||||
setup.
|
||||
|
||||
You will need to have graphics forwarding (e.g., X11
|
||||
forwarding) enabled from the remote processes to the display where you
|
||||
want output to appear. In a secure environment, you can simply allow
|
||||
all X requests to be shown on the target display and set the
|
||||
``DISPLAY`` environment variable in all MPI processes' environments to
|
||||
the target display, perhaps something like this:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ hostname
|
||||
my_desktop.secure-cluster.example.com
|
||||
shell$ xhost +
|
||||
shell$ mpirun -n 4 -x DISPLAY=my_desktop.secure-cluster.example.com a.out
|
||||
|
||||
However, this technique is not generally suitable for unsecure
|
||||
environments (because it allows anyone to read and write to your
|
||||
display). A slightly more secure way is to only allow X connections
|
||||
from the nodes where your application will be running:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ hostname
|
||||
my_desktop.secure-cluster.example.com
|
||||
shell$ xhost +compute1 +compute2 +compute3 +compute4
|
||||
compute1 being added to access control list
|
||||
compute2 being added to access control list
|
||||
compute3 being added to access control list
|
||||
compute4 being added to access control list
|
||||
shell$ mpirun -n 4 -x DISPLAY=my_desktop.secure-cluster.example.com a.out
|
||||
|
||||
(assuming that the four nodes you are running on are ``compute1``
|
||||
through ``compute4``).
|
||||
|
||||
Other methods are available, but they involve sophisticated X
|
||||
forwarding through :ref:`mpirun(1) <man1-mpirun>` and are generally
|
||||
more complicated than desirable.
|
||||
|
||||
Running curses-based applications
|
||||
---------------------------------
|
||||
|
||||
Open MPI provides fairly sophisticated stdin / stdout / stderr
|
||||
forwarding. However, it does not work well with curses, ncurses,
|
||||
readline, or other sophisticated I/O packages that generally require
|
||||
direct control of the terminal.
|
||||
|
||||
Every application and I/O library is different |mdash| you should try to
|
||||
see if yours is supported. But chances are that it won't work.
|
||||
|
||||
Launching an MPMD MPI job
|
||||
-------------------------
|
||||
|
||||
Open MPI supports multiple program, multiple data (MPMD) style launches,
|
||||
either from the command line or from a file. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun -n 2 a.out : -n 2 b.out
|
||||
|
||||
This will launch a single parallel application, but the first two
|
||||
processes will be instances of the ``a.out`` executable, and the
|
||||
second two processes will be instances of the ``b.out`` executable.
|
||||
In MPI terms, this will be a single ``MPI_COMM_WORLD``, but the
|
||||
``a.out`` processes will be ranks 0 and 1 in ``MPI_COMM_WORLD``, while
|
||||
the ``b.out`` processes will be ranks 2 and 3 in ``MPI_COMM_WORLD``.
|
||||
|
||||
:ref:`mpirun(1) <man1-mpirun>` can also accept a parallel application
|
||||
specified in a file instead of on the command line. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
shell$ mpirun --app my_appfile
|
||||
|
||||
where the file ``my_appfile`` contains the following:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
# Comments are supported; comments begin with #
|
||||
# Application context files specify each sub-application in the
|
||||
# parallel job, one per line. The first sub-application is the 2
|
||||
# a.out processes:
|
||||
-n 2 a.out
|
||||
# The second sub-application is the 2 b.out processes:
|
||||
-n 2 b.out
|
||||
|
||||
This will result in the same behavior as running ``a.out`` and ``b.out``
|
||||
from the command line.
|
||||
|
||||
Connecting independent MPI applications
|
||||
---------------------------------------
|
||||
|
||||
In certain environments, Open MPI supports connecting multiple,
|
||||
independent MPI applications using mechanism defined in the MPI
|
||||
specification such as ``MPI_Comm_connect() / MPI_Comm_accept()`` and
|
||||
publishing connection information using ``MPI_Publish_name() /
|
||||
MPI_Lookup_name()``. These mechanisms require a centralized service
|
||||
to exchange contact information across multiple jobs.
|
||||
|
||||
Beginning with Open MPI v5.0.0 this can be achieved by starting an
|
||||
instance of the prte server with the ``report-uri`` option to
|
||||
display the contact information of the prte server. This information
|
||||
can then be used for launching subsequent MPI applications.
|
||||
|
||||
The following commands show an example for launching two MPI jobs
|
||||
that will connect to each other at runtime using the MPI-2 based
|
||||
functionality.
|
||||
|
||||
|
||||
Step 1: start the standalone prte server
|
||||
|
||||
.. code-block::
|
||||
|
||||
user@myhost:~/ompi-install/bin$ ./prte --report-uri <filename>
|
||||
DVM ready
|
||||
|
||||
Step 2: Launch the first MPI application providing the uri of the
|
||||
prte server
|
||||
|
||||
.. code-block::
|
||||
|
||||
user@myhost:~/app1-dir$ mpiexec --dvm file:<filename> -np 4 ./mpi_app_1
|
||||
|
||||
Step 3: Launch the second MPI application providing the uri of the
|
||||
prte server again
|
||||
|
||||
.. code-block::
|
||||
|
||||
user@myhost:~/app2-dir$ mpiexec --dvm file:<filename> -np 4 ./mpi_app_2
|
||||
|
||||
|
||||
In case the prte server has been started as a system server using the
|
||||
``--system-server`` argument (e.g. the nodes used by the MPI
|
||||
applications are not shared by multiple jobs), the sequence can be
|
||||
simplified by using ``mpiexec --dvm system`` or ``mpiexec --dvm
|
||||
system-first`` instead of the uri of the prte server.
|
||||
@@ -0,0 +1,55 @@
|
||||
License
|
||||
=========
|
||||
|
||||
Open MPI license
|
||||
----------------
|
||||
|
||||
The Open MPI license is as follows:
|
||||
|
||||
.. literalinclude:: ompi.txt
|
||||
:language: text
|
||||
|
||||
Additional licenses
|
||||
-------------------
|
||||
|
||||
Open MPI includes code from the MPICH project.
|
||||
|
||||
.. literalinclude:: mpich.txt
|
||||
:language: text
|
||||
|
||||
Redistributed code
|
||||
------------------
|
||||
|
||||
The Open MPI distribution tarball includes the following other open
|
||||
source code bases in their entirety.
|
||||
|
||||
Libevent
|
||||
^^^^^^^^
|
||||
|
||||
.. literalinclude:: libevent.txt
|
||||
:language: text
|
||||
|
||||
Hwloc
|
||||
^^^^^
|
||||
|
||||
.. literalinclude:: hwloc.txt
|
||||
:language: text
|
||||
|
||||
PMIx
|
||||
^^^^
|
||||
|
||||
.. literalinclude:: pmix.txt
|
||||
:language: text
|
||||
|
||||
PRRTE
|
||||
^^^^^
|
||||
|
||||
.. literalinclude:: prrte.txt
|
||||
:language: text
|
||||
|
||||
Treematch
|
||||
^^^^^^^^^
|
||||
|
||||
.. literalinclude:: treematch.txt
|
||||
:language: text
|
||||
:encoding: latin-1
|
||||
@@ -0,0 +1,22 @@
|
||||
Open MPI manual pages
|
||||
=====================
|
||||
|
||||
All of the content in this section is also installed as individual
|
||||
manual ("man") pages in an Open MPI installation. If your man page
|
||||
search path includes the location where Open MPI installed its man
|
||||
pages (which defaults to ``$prefix/share/man``), you can invoke
|
||||
commands similar to the following to see the same content that you see
|
||||
in these HTML pages:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ man MPI_Send
|
||||
|
||||
Open MPI man pages, by section:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
man1/index
|
||||
man3/index
|
||||
man7/index
|
||||
@@ -0,0 +1,11 @@
|
||||
Commands (section 1)
|
||||
====================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
ompi-wrapper-compiler.1.rst
|
||||
mpirun.1.rst
|
||||
mpisync.1.rst
|
||||
ompi_info.1.rst
|
||||
opal_wrapper.1.rst
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,57 @@
|
||||
.. _man1-mpisync:
|
||||
|
||||
|
||||
mpisync
|
||||
=======
|
||||
|
||||
.. include_body
|
||||
|
||||
mpisync |mdash| Open MPI timing tools
|
||||
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
``mpisync [options]``
|
||||
|
||||
``mpirun_prof [options]``
|
||||
|
||||
``ompi_timing_post [<timing-file>] [<processed-file>]``
|
||||
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
``mpisync``: determines clock offsets relative to Head Node Process
|
||||
(HNP). It accepts the following options:
|
||||
|
||||
* ``-o``, ``--output``: The name of output file where offsets related
|
||||
to HNP will be written
|
||||
|
||||
* ``-h``, ``--help``: Print help information
|
||||
|
||||
``ompi_timing_post`` takes the timing output file as input parameter.
|
||||
The events are sorted by the timestamps. Next, the timestamps are
|
||||
replaced with time offsets relative to the ``first`` : ``previous``
|
||||
event.
|
||||
|
||||
``mpirun_prof`` is a wrapper around :ref:`mpirun <man1-mpirun>` that
|
||||
performs clock synchronisation and post-processing of the timing
|
||||
output file.
|
||||
|
||||
|
||||
NOTES
|
||||
-----
|
||||
|
||||
The mpisync code was derived from MPIPerf project:
|
||||
http://mpiperf.cpct.sibsutis.ru/index.php/Main/Documentation
|
||||
|
||||
|
||||
FILES
|
||||
-----
|
||||
|
||||
The output file has following format:
|
||||
|
||||
.. code::
|
||||
|
||||
<hostname> <round-trip-time> <offset-from-hnp>
|
||||
@@ -0,0 +1,258 @@
|
||||
.. _man1-mpicc:
|
||||
.. _man1-mpic++:
|
||||
.. _man1-mpicxx:
|
||||
.. _man1-mpifort:
|
||||
.. _man1-mpijavac:
|
||||
|
||||
Open MPI Wrapper Compilers
|
||||
==========================
|
||||
|
||||
.. include_body
|
||||
|
||||
mpicc, mpic++, mpicxx, mpifort, mpijavac |mdash| Open MPI wrapper compilers
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
``mpicc [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
``mpic++ [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
``mpicxx [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
``mpifort [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
``mpijavac [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
The following deprecated commands are also available |mdash| but
|
||||
``mpifort`` should be used instead:
|
||||
|
||||
``mpif77 [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
``mpif90 [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
On case-sensitive filesystems, the following command will also be
|
||||
available:
|
||||
|
||||
``mpiCC [--showme | --showme:compile | --showme:link] ...``
|
||||
|
||||
|
||||
OPTIONS
|
||||
-------
|
||||
|
||||
The options below apply to all of the wrapper compilers:
|
||||
|
||||
* ``--showme``: This option comes in several different variants (see
|
||||
below). None of the variants invokes the underlying compiler; they
|
||||
all provide information on how the underlying compiler would have
|
||||
been invoked had ``--showme`` not been used. The basic ``--showme``
|
||||
option outputs the command line that would be executed to compile
|
||||
the program.
|
||||
|
||||
.. note:: If a non-filename argument is passed on the command line,
|
||||
the ``--showme`` option will *not* display any additional
|
||||
flags. For example, both ``"mpicc --showme`` and
|
||||
``mpicc --showme my_source.c`` will show all the
|
||||
wrapper-supplied flags. But ``mpicc
|
||||
--showme -v`` will only show the underlying compiler name
|
||||
and ``-v``.
|
||||
|
||||
* ``--showme:compile``: Output the compiler flags that would have been
|
||||
supplied to the underlying compiler.
|
||||
|
||||
* ``--showme:link``: Output the linker flags that would have been
|
||||
supplied to the underlying compiler.
|
||||
|
||||
* ``--showme:command``: Outputs the underlying compiler
|
||||
command (which may be one or more tokens).
|
||||
|
||||
* ``--showme:incdirs``: Outputs a space-delimited (but otherwise
|
||||
undecorated) list of directories that the wrapper compiler would
|
||||
have provided to the underlying compiler to indicate
|
||||
where relevant header files are located.
|
||||
|
||||
* ``--showme:libdirs``: Outputs a space-delimited (but otherwise
|
||||
undecorated) list of directories that the wrapper compiler would
|
||||
have provided to the underlying linker to indicate where relevant
|
||||
libraries are located.
|
||||
|
||||
* ``--showme:libs`` Outputs a space-delimited (but otherwise
|
||||
undecorated) list of library names that the wrapper compiler would
|
||||
have used to link an application. For example: ``mpi open-pal
|
||||
util``.
|
||||
|
||||
* ``--showme:version``: Outputs the version number of Open MPI.
|
||||
|
||||
* ``--showme:help``: Output a brief usage help message.
|
||||
|
||||
See the man page for your underlying compiler for other options that
|
||||
can be passed through mpicc.
|
||||
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
Conceptually, the role of these commands is quite simple:
|
||||
transparently add relevant compiler and linker flags to the user's
|
||||
command line that are necessary to compile / link Open MPI programs,
|
||||
and then invoke the underlying compiler to actually perform the
|
||||
command.
|
||||
|
||||
As such, these commands are frequently referred to as "wrapper"
|
||||
compilers because they do not actually compile or link applications
|
||||
themselves; they only add in command line flags and invoke the
|
||||
back-end compiler.
|
||||
|
||||
Background
|
||||
----------
|
||||
|
||||
Open MPI provides wrapper compilers for several languages:
|
||||
|
||||
* ``mpicc``: C
|
||||
|
||||
* ``mpic++``, ``mpicxx`` (and on systems with case-sensitive file
|
||||
systems, ``mpiCC``): C++
|
||||
|
||||
.. note:: ``mpic++``, ``mpicxx``, and ``mpiCC`` all invoke the same
|
||||
underlying C++ compiler with the same options. All are
|
||||
provided as compatibility with other MPI implementations.
|
||||
|
||||
* ``mpifort`` (and its legacy/deprecated aliaes ``mpif77`` and
|
||||
``mpif90``): Fortran
|
||||
|
||||
* ``mpijavac``: Java
|
||||
|
||||
The wrapper compilers for each of the languages are identical; they
|
||||
can be use interchangeably. The different names are provided solely
|
||||
for backwards compatibility.
|
||||
|
||||
|
||||
Fortran Notes
|
||||
-------------
|
||||
|
||||
The Fortran wrapper compiler for MPI (``mpifort``, and its
|
||||
legacy/deprecated names ``mpif77`` and ``mpif90``) can compile and
|
||||
link MPI applications that use any/all of the MPI Fortran bindings:
|
||||
``mpif.h``, the ``mpi`` module, and the ``mpi_f08`` module (assuming
|
||||
Open MPI was installed with support for each of these Fortran
|
||||
bindings). Specifically: it is no longer necessary to use different
|
||||
wrapper compilers for applications that use ``mpif.h``
|
||||
vs. applications that use the ``mpi`` module |mdash| just use ``mpifort``
|
||||
for all Fortran MPI applications.
|
||||
|
||||
Note, however, that the Fortran compiler may require additional
|
||||
command-line options to enforce a specific Fortran dialect. For
|
||||
example, in some versions of the IBM XLF compiler, if ``xlf90`` is the
|
||||
underlying Fortran compiler, ``-qfixed`` may be necessary to compile
|
||||
fixed-format Fortran source files.
|
||||
|
||||
Finally, note that ``mpifort`` will be inoperative and will return an
|
||||
error on use if Fortran support was not built into the MPI layer.
|
||||
|
||||
Overview
|
||||
--------
|
||||
|
||||
``mpicc`` is a convenience wrappers for the underlying C compiler.
|
||||
Translation of an Open MPI program requires the linkage of the Open
|
||||
MPI-specific libraries which may not reside in one of the standard
|
||||
search directories of ``ld(1)``. It also often requires the inclusion
|
||||
of header files what may also not be found in a standard location.
|
||||
|
||||
``mpicc`` passes its arguments to the underlying C compiler along with
|
||||
the ``-I``, ``-L`` and ``-l`` options required by Open MPI programs.
|
||||
|
||||
The same is true for all the other language wrapper compilers.
|
||||
|
||||
The Open MPI Team *strongly* encourages using the wrapper compilers
|
||||
instead of attempting to link to the Open MPI libraries manually. This
|
||||
allows the specific implementation of Open MPI to change without
|
||||
forcing changes to linker directives in users' Makefiles. Indeed, the
|
||||
specific set of flags and libraries used by the wrapper compilers
|
||||
depends on how Open MPI was configured and built; the values can change
|
||||
between different installations of the same version of Open MPI.
|
||||
|
||||
Indeed, since the wrappers are simply thin shells on top of an
|
||||
underlying compiler, there are very, very few compelling reasons *not*
|
||||
to use Open MPI's wrapper compilers. When it is not possible to use
|
||||
the wrappers directly, the ``--showme:compile`` and ``--showme:link``
|
||||
options should be used to determine what flags the wrappers would have
|
||||
used. For example:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
shell$ cc -c file1.c `mpicc --showme:compile`
|
||||
|
||||
shell$ cc -c file2.c `mpicc --showme:compile`
|
||||
|
||||
shell$ cc file1.o file2.o `mpicc --showme:link` -o my_mpi_program
|
||||
|
||||
|
||||
NOTES
|
||||
-----
|
||||
|
||||
It is possible to make the wrapper compilers multi-lib aware. That is,
|
||||
the libraries and includes specified may differ based on the compiler
|
||||
flags specified (for example, with the GNU compilers on Linux, a
|
||||
different library path may be used if ``-m32`` is seen versus ``-m64``
|
||||
being seen). This is not the default behavior in a standard build, but
|
||||
can be activated (for example, in a binary package providing both 32
|
||||
and 64 bit support). `More information can be found here
|
||||
<https://github.com/open-mpi/ompi/wiki/compilerwrapper3264>`_.
|
||||
|
||||
|
||||
.. _man1-ompi-wrapper-compiler-files:
|
||||
|
||||
FILES
|
||||
-----
|
||||
|
||||
The strings that the wrapper compilers insert into the command line
|
||||
before invoking the underlying compiler are stored in a text file
|
||||
created by Open MPI and installed to
|
||||
``$pkgdata/NAME-wrapper-data.txt``, where:
|
||||
|
||||
* ``$pkgdata`` is typically ``$prefix/share/openmpi``
|
||||
* ``$prefix`` is the top installation directory of Open MPI
|
||||
* ``NAME`` is the name of the wrapper compiler (e.g.,
|
||||
``$pkgdata/mpicc-wrapper-data.txt``)
|
||||
|
||||
It is rarely necessary to edit these files, but they can be examined to
|
||||
gain insight into what flags the wrappers are placing on the command
|
||||
line.
|
||||
|
||||
|
||||
ENVIRONMENT VARIABLES
|
||||
---------------------
|
||||
|
||||
By default, the wrappers use the compilers that were selected when
|
||||
Open MPI was configured. These compilers were either found
|
||||
automatically by Open MPI's "configure" script, or were selected by
|
||||
the user in the ``CC``, ``CXX``, and/or ``FC`` environment variables
|
||||
before ``configure`` was invoked. Additionally, other arguments specific
|
||||
to the compiler may have been selected by configure.
|
||||
|
||||
These values can be selectively overridden by either editing the text
|
||||
files containing this configuration information (see the :ref:`FILES
|
||||
<man1-ompi-wrapper-compiler-files>` section), or by setting selected
|
||||
environment variables of the form ``ompi_value``.
|
||||
|
||||
Valid value names are:
|
||||
|
||||
* ``CPPFLAGS``: Flags added when invoking the preprocessor (C or C++)
|
||||
|
||||
* ``LDFLAGS``: Flags added when invoking the linker (C, C++, or
|
||||
Fortran)
|
||||
|
||||
* ``LIBS``: Libraries added when invoking the linker (C, C++, or
|
||||
Fortran)
|
||||
|
||||
* ``CC``: C compiler
|
||||
|
||||
* ``CFLAGS``: C compiler flags
|
||||
|
||||
* ``CXX``: C++ compiler
|
||||
|
||||
* ``CXXFLAGS``: C++ compiler flags
|
||||
|
||||
* ``FC``: Fortran compiler
|
||||
|
||||
* ``FCFLAGS``: Fortran compiler flags
|
||||
@@ -0,0 +1,227 @@
|
||||
.. _man1-ompi_info:
|
||||
|
||||
|
||||
ompi_info
|
||||
=========
|
||||
|
||||
.. include_body
|
||||
|
||||
ompi_info |mdash| Display information about the Open MPI installation
|
||||
|
||||
|
||||
SYNOPSIS
|
||||
--------
|
||||
|
||||
``ompi_info [options]``
|
||||
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
``ompi_info`` provides detailed information about the Open MPI
|
||||
installation. It can be useful for at least three common scenarios:
|
||||
|
||||
#. Checking local configuration and seeing how Open MPI was installed.
|
||||
|
||||
#. Submitting bug reports / help requests to the Open MPI community
|
||||
(see :doc:`Getting help </getting-help>`).
|
||||
|
||||
#. Seeing a list of installed Open MPI plugins and querying what MCA
|
||||
parameters they support.
|
||||
|
||||
.. note:: ``ompi_info`` defaults to only showing a few MCA parameters
|
||||
by default (i.e., level 1 parameters). Use the ``--level``
|
||||
option to enable showing more options (see the :ref:`LEVELS
|
||||
<man1-ompi_info-levels>` section for more information).
|
||||
|
||||
|
||||
OPTIONS
|
||||
-------
|
||||
|
||||
``ompi_info`` accepts the following options:
|
||||
|
||||
* ``-a``, ``--all``: Show all configuration options and MCA
|
||||
parameters. Also changes the default MCA parameter level to 9,
|
||||
unless ``--level`` is also specified.
|
||||
|
||||
* ``--arch``: Show architecture on which Open MPI was compiled.
|
||||
|
||||
* ``-c``, ``--config``: Show configuration options
|
||||
|
||||
* ``-gmca``, ``--gmca <param> <value>``: Pass global MCA parameters
|
||||
that are applicable to all contexts.
|
||||
|
||||
* ``-h``, ``--help``: Shows help / usage message.
|
||||
|
||||
* ``--hostname``: Show the hostname on which Open MPI was configured
|
||||
and built.
|
||||
|
||||
* ``--internal``: Show internal MCA parameters (not meant to be
|
||||
modified by users).
|
||||
|
||||
* ``--level <level>``: Show only variables with at most this level
|
||||
(1-9). The default is 1 unless ``--all`` is specified without
|
||||
``--level``, in which case the default is 9. See the :ref:`LEVELS
|
||||
<man1-ompi_info-levels>` section for more information.
|
||||
|
||||
* ``-mca``, ``--mca <param> <value>``: Pass context-specific MCA
|
||||
parameters; they are considered global if ``--gmca`` is not used and
|
||||
only one context is specified.
|
||||
|
||||
* ``--param <type> <component>``: Show MCA parameters. The first
|
||||
parameter is the type of the component to display; the second
|
||||
parameter is the specific component to display (or the keyword
|
||||
``all``, meaning "display all components of this type").
|
||||
|
||||
* ``-t``, ``--type``: Show MCA parameters of the type specified in the
|
||||
parameter. Accepts the following parameters: ``unsigned_int``,
|
||||
``unsigned_long``, ``unsigned_long_long``, ``size_t``, ``string``,
|
||||
``version_string``, ``bool``, ``double``. By default level is 1
|
||||
unless it is specified with ``--level``.
|
||||
|
||||
* ``--parsable``: When used in conjunction with other parameters, the
|
||||
output is displayed in a machine-parsable format ``--parseable``
|
||||
Synonym for ``--parsable``.
|
||||
|
||||
* ``--path <type>``: Show paths that Open MPI was configured
|
||||
with. Accepts the following parameters: ``prefix``, ``bindir``,
|
||||
``libdir``, ``incdir``, ``pkglibdir``, ``sysconfdir``.
|
||||
|
||||
* ``--pretty``: When used in conjunction with other parameters, the output is
|
||||
displayed in "prettyprint" format (default)
|
||||
|
||||
* ``--selected-only``: Show only variables from selected components.
|
||||
|
||||
* ``-V``, ``--version``: Show version of Open MPI.
|
||||
|
||||
.. _man1-ompi_info-levels:
|
||||
|
||||
LEVELS
|
||||
------
|
||||
|
||||
Open MPI has many, many run-time tunable parameters (called "MCA
|
||||
parameters"), and usually only a handful of them are useful to a given
|
||||
user.
|
||||
|
||||
As such, Open MPI has divided these parameters up into nine distinct
|
||||
levels, broken down into three categories, each with three
|
||||
sub-categories.
|
||||
|
||||
Note that since each MCA parameter is accessible through the MPI_T
|
||||
control variable API (introduced in MPI-3.0), these levels exactly
|
||||
correspond to the nine MPI_T cvar levels.
|
||||
|
||||
The three categories are:
|
||||
|
||||
#. **End user**: Generally, these are parameters that are required for
|
||||
correctness, meaning that a user may need to set these just to get
|
||||
their MPI application to run correctly. For example, BTL
|
||||
``if_include`` and ``if_exclude`` parameters fit into this
|
||||
category.
|
||||
|
||||
#. **Application tuner**: Generally, these are parameters that can be
|
||||
used to tweak MPI application performance. This even includes
|
||||
parameters that control resource exhaustion levels (e.g., number of
|
||||
free list entries, size of buffers, etc.), and could be considered
|
||||
"correctness" parameters if they're set too low. But, really |mdash|
|
||||
they're tuning parameters.
|
||||
|
||||
#. **Open MPI developer**: Parameters in this category either don't
|
||||
fit in the other two, or are specifically intended for debugging /
|
||||
development of Open MPI itself.
|
||||
|
||||
And within each category, there are three sub-categories:
|
||||
|
||||
#. **Basic**: This sub-category is for parameters that everyone in
|
||||
this category will want to see |mdash| even less-advanced end users,
|
||||
application tuners, and new OMPI developers.
|
||||
|
||||
#. **Detailed**: This sub-category is for parameters that are
|
||||
generally useful, but users probably won't need to change them
|
||||
often.
|
||||
|
||||
#. **All**: This sub-category is for all other parameters. Such
|
||||
parameters are likely fairly esoteric.
|
||||
|
||||
Combining the categories and sub-categories, here's how Open MPI
|
||||
defines all nine levels:
|
||||
|
||||
#. Basic information of interest to end users.
|
||||
#. Detailed information of interest to end users.
|
||||
#. All remaining information of interest to end users.
|
||||
#. Basic information required for application tuners.
|
||||
#. Detailed information required for application tuners.
|
||||
#. All remaining information required for application tuners.
|
||||
#. Basic information for Open MPI implementors.
|
||||
#. Detailed information for Open MPI implementors.
|
||||
#. All remaining information for Open MPI implementors.
|
||||
|
||||
By default, ``ompi_info`` only shows level 1 MCA parameters. To see
|
||||
more MCA parameters, use the ``--level`` command line option.
|
||||
|
||||
|
||||
EXAMPLES
|
||||
--------
|
||||
|
||||
Show the default output of options and listing of installed
|
||||
components in a human-readable / prettyprint format:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info
|
||||
|
||||
Show the default output of options and listing of installed components
|
||||
in a machine-parsable format:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --parsable
|
||||
|
||||
Show the level 1 MCA parameters of the "tcp" BTL component in a
|
||||
human-readable / prettyprint format:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --param btl tcp
|
||||
|
||||
Show the level 1 through level 6 MCA parameters of the "tcp" BTL
|
||||
component in a human-readable / prettyprint format:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --param btl tcp --level 6
|
||||
|
||||
Show the level 1 MCA parameters of the "tcp" BTL component in a
|
||||
machine-parsable format:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --param btl tcp --parsable
|
||||
|
||||
Show the level 1 through level 3 MCA parameters of string type in a
|
||||
human-readable / prettyprint format:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --type string --pretty-print --level 3
|
||||
|
||||
Show the "bindir" that Open MPI was configured with:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --path bindir
|
||||
|
||||
Show the version of Open MPI version numbers in a prettyprint format:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --version
|
||||
|
||||
Show *all* information about the Open MPI installation, including all
|
||||
components that can be found, all the MCA parameters that they support
|
||||
(i.e., levels 1 through 9), versions of Open MPI and the components,
|
||||
etc.:
|
||||
|
||||
.. code-block::
|
||||
|
||||
ompi_info --all
|
||||
@@ -0,0 +1,33 @@
|
||||
.. _man1-opal_wrapper:
|
||||
|
||||
|
||||
opal_wrapper
|
||||
============
|
||||
|
||||
.. include_body
|
||||
|
||||
opal_wrapper |mdash| Back-end Open MPI wrapper command
|
||||
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
``opal_wrapper`` is not meant to be called directly by end users. It
|
||||
is automatically invoked as the back-end by the Open MPI wrapper
|
||||
commands such as: ``mpicc``, ``mpicxx``, and ``mpifort``
|
||||
(and its legacy/deprecated aliases ``mpif77`` and ``mpif90``).
|
||||
|
||||
Some Open MPI installations may have additional wrapper commands,
|
||||
and/or have renamed the wrapper compilers listed above to avoid
|
||||
executable name conflicts with other MPI implementations. Hence, you
|
||||
may also have wrapper compilers installed including the following
|
||||
names: ``mpifort.openmpi`` (and the legacy/deprecated aliases
|
||||
``mpif90.openmpi`` and ``mpif77.openmpi``), ``mpicxx.openmpi``,
|
||||
``mpicc.openmpi``.
|
||||
|
||||
.. seealso::
|
||||
The following may exist depending on your particular Open MPI
|
||||
installation:
|
||||
:ref:`mpicc(1) <man1-mpicc>`,
|
||||
:ref:`mpicxx(1) <man1-mpicxx>`,
|
||||
:ref:`mpifort(1) <man1-mpifort>`
|
||||
@@ -0,0 +1,47 @@
|
||||
Almost all MPI routines return an error value; C routines as the return result
|
||||
of the function and Fortran routines in the last argument.
|
||||
|
||||
Before the error value is returned, the current MPI error handler associated
|
||||
with the communication object (e.g., communicator, window, file) is called.
|
||||
If no communication object is associated with the MPI call, then the call is
|
||||
considered attached to MPI_COMM_SELF and will call the associated MPI error
|
||||
handler. When MPI_COMM_SELF is not initialized (i.e., before
|
||||
:ref:`MPI_Init`/:ref:`MPI_Init_thread`, after :ref:`MPI_Finalize`, or when using the Sessions
|
||||
Model exclusively) the error raises the initial error handler. The initial
|
||||
error handler can be changed by calling :ref:`MPI_Comm_set_errhandler` on
|
||||
MPI_COMM_SELF when using the World model, or the mpi_initial_errhandler CLI
|
||||
argument to mpiexec or info key to :ref:`MPI_Comm_spawn`/:ref:`MPI_Comm_spawn_multiple`.
|
||||
If no other appropriate error handler has been set, then the MPI_ERRORS_RETURN
|
||||
error handler is called for MPI I/O functions and the MPI_ERRORS_ABORT error
|
||||
handler is called for all other MPI functions.
|
||||
|
||||
Open MPI includes three predefined error handlers that can be used:
|
||||
|
||||
* ``MPI_ERRORS_ARE_FATAL``
|
||||
Causes the program to abort all connected MPI processes.
|
||||
* ``MPI_ERRORS_ABORT``
|
||||
An error handler that can be invoked on a communicator,
|
||||
window, file, or session. When called on a communicator, it
|
||||
acts as if :ref:`MPI_Abort` was called on that communicator. If
|
||||
called on a window or file, acts as if :ref:`MPI_Abort` was called
|
||||
on a communicator containing the group of processes in the
|
||||
corresponding window or file. If called on a session,
|
||||
aborts only the local process.
|
||||
* ``MPI_ERRORS_RETURN``
|
||||
Returns an error code to the application.
|
||||
|
||||
MPI applications can also implement their own error handlers by calling:
|
||||
|
||||
* :ref:`MPI_Comm_create_errhandler` then :ref:`MPI_Comm_set_errhandler`
|
||||
* :ref:`MPI_File_create_errhandler` then :ref:`MPI_File_set_errhandler`
|
||||
* :ref:`MPI_Session_create_errhandler` then :ref:`MPI_Session_set_errhandler` or at :ref:`MPI_Session_init`
|
||||
* :ref:`MPI_Win_create_errhandler` then :ref:`MPI_Win_set_errhandler`
|
||||
|
||||
Note that MPI does not guarantee that an MPI program can continue past
|
||||
an error.
|
||||
|
||||
See the :ref:`MPI man page <open-mpi>` for a full list of :ref:`MPI error codes <open-mpi-errors>`.
|
||||
|
||||
See the Error Handling section of the MPI-|mpi_standard_version| standard for
|
||||
more information.
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
.. _mpix_comm_ack_failed:
|
||||
|
||||
MPIX_Comm_ack_failed
|
||||
====================
|
||||
.. include_body
|
||||
|
||||
:ref:`MPIX_Comm_get_failed` - acknowledge failed processes in a communicator.
|
||||
|
||||
This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
C Syntax
|
||||
^^^^^^^^
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#include <mpi.h>
|
||||
#include <mpi-ext.h>
|
||||
|
||||
int MPIX_Comm_ack_failed(MPI_Comm comm, int num_to_ack, int *num_acked)
|
||||
|
||||
Fortran Syntax
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE MPI
|
||||
USE MPI_EXT
|
||||
! or the older form: INCLUDE 'mpif.h'
|
||||
|
||||
MPIX_COMM_ACK_FAILED(COMM, NUM_TO_ACK, NUM_ACKED, IERROR)
|
||||
INTEGER COMM, NUM_TO_ACK, NUM_ACKED, IERROR
|
||||
|
||||
Fortran 2008 Syntax
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE mpi_f08
|
||||
USE mpi_ext_f08
|
||||
|
||||
MPIX_Comm_ack_failed(comm, num_to_ack, num_acked, ierror)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
INTEGER, INTENT(IN) :: num_to_ack
|
||||
INTEGER, INTENT(OUT) :: num_acked
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
INPUT PARAMETERS
|
||||
----------------
|
||||
* ``comm``: Communicator (handle).
|
||||
* ``num_to_ack``: maximum number of process failures to acknowledge in *comm* (integer)
|
||||
|
||||
OUTPUT PARAMETERS
|
||||
-----------------
|
||||
* ``num_acked``: number of acknowledged failures in *comm* (integer).
|
||||
* ``ierror``: Fortran only: Error status (integer).
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
his local operation gives the users a way to **acknowledge**
|
||||
locally notified failures on *comm*. The operation acknowledges the first
|
||||
*num_to_ack* process failures on *comm*, that is, it acknowledges the
|
||||
failure of members with a rank lower than *num_to_ack* in the group that
|
||||
would be produced by a concurrent call to :ref:`MPIX_Comm_get_failed` on
|
||||
the same *comm*.
|
||||
|
||||
The operation also sets the value of *num_acked* to the current number of
|
||||
acknowledged process failures in *comm*, that is, a process failure has been
|
||||
acknowledged on *comm* if and only if the rank of the process is lower than
|
||||
*num_acked* in the group that would be produced by a subsequent call to
|
||||
:ref:`MPIX_Comm_get_failed` on the same *comm*.
|
||||
|
||||
*num_acked* can be larger than *num_to_ack* when process failures have been
|
||||
acknowledged in a prior call to :ref:`MPIX_Comm_ack_failed`.
|
||||
|
||||
EFFECT OF ACKNOWLEDGING FAILURES
|
||||
--------------------------------
|
||||
|
||||
After an MPI process failure is acknowledged on *comm*, unmatched
|
||||
MPI_ANY_SOURCE receive operations on the same *comm* that would have raised
|
||||
an error of class MPIX_ERR_PROC_FAILED_PENDING proceed without further raising
|
||||
errors due to this acknowledged failure.
|
||||
|
||||
Also, :ref:`MPIX_Comm_agree` on the same *comm* will not raise an error of
|
||||
class MPI_ERR_PROC_FAILED due to this acknowledged failure.
|
||||
|
||||
USAGE PATTERNS
|
||||
--------------
|
||||
|
||||
One may query, without side effect, for the number of currently aknowledged
|
||||
process failures *comm* by supplying 0 in *num_to_ack*.
|
||||
|
||||
Conversely, one may unconditionally acknowledge all currently known process
|
||||
failures in *comm* by supplying the size of the group of *comm* in *num_to_ack*.
|
||||
|
||||
Note that the number of acknowledged processes, as returned in *num_acked*,
|
||||
can be smaller or larger than the value supplied in *num_to_ack*; It is
|
||||
however never larger than the size of the group returned by a subsequent call
|
||||
to :ref:`MPIX_Comm_get_failed`.
|
||||
|
||||
EFFECT ON COLLECTIVE OPERATIONS
|
||||
-------------------------------
|
||||
|
||||
Calling :ref:`MPIX_Comm_ack_failed` on a communicator with failed MPI
|
||||
processes has no effect on collective operations (except for :ref:`MPIX_Comm_agree`).
|
||||
If a collective operation would raise an error due to the communicator
|
||||
containing a failed process it will continue to raise an error even after
|
||||
the failure has been acknowledged. In order to use collective operations
|
||||
between MPI processes of a communicator that contains failed MPI processes,
|
||||
users should create a new communicator (e.g., by calling :ref:`MPIX_Comm_shrink`).
|
||||
|
||||
WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
|
||||
------------------------------------------
|
||||
|
||||
When the communicator is an inter-communicator, the failures of members
|
||||
in both the local and the remote groups of *comm* are acknowledged.
|
||||
|
||||
ERRORS
|
||||
------
|
||||
|
||||
.. include:: ./ERRORS.rst
|
||||
|
||||
.. seealso::
|
||||
* :ref:`MPIX_Comm_get_failed`
|
||||
* :ref:`MPIX_Comm_agree`
|
||||
@@ -0,0 +1,175 @@
|
||||
.. _mpix_comm_agree:
|
||||
|
||||
MPIX_Comm_agree
|
||||
===============
|
||||
.. include_body
|
||||
|
||||
:ref:`MPIX_Comm_agree`, :ref:`MPIX_Comm_iagree` - Agree on a flag value
|
||||
from all live processes and distributes the result back to all live
|
||||
processes, even after process failures.
|
||||
|
||||
This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
C Syntax
|
||||
^^^^^^^^
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#include <mpi.h>
|
||||
#include <mpi-ext.h>
|
||||
|
||||
int MPIX_Comm_agree(MPI_Comm comm, int *flag)
|
||||
|
||||
int MPIX_Comm_iagree(MPI_Comm comm, int *flag, MPI_Request *request)
|
||||
|
||||
|
||||
Fortran Syntax
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE MPI
|
||||
USE MPI_EXT
|
||||
! or the older form: INCLUDE 'mpif.h'
|
||||
|
||||
MPIX_COMM_AGREE(COMM, FLAG, IERROR)
|
||||
INTEGER COMM, FLAG, IERROR
|
||||
|
||||
MPIX_COMM_IAGREE(COMM, FLAG, REQUEST, IERROR)
|
||||
INTEGER COMM, FLAG, REQUEST, IERROR
|
||||
|
||||
|
||||
Fortran 2008 Syntax
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE mpi_f08
|
||||
USE mpi_ext_f08
|
||||
|
||||
MPIX_Comm_agree(comm, flag, ierror)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
INTEGER, INTENT(INOUT) :: flag
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
MPIX_COMM_IAGREE(COMM, FLAG, REQUEST, IERROR)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
INTEGER, INTENT(INOUT), ASYNCHRONOUS :: flag
|
||||
TYPE(MPI_Request), INTENT(OUT) :: request
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
INPUT PARAMETERS
|
||||
----------------
|
||||
* ``comm``: Communicator (handle).
|
||||
* ``flag``: Binary flags (integer).
|
||||
|
||||
OUTPUT PARAMETERS
|
||||
-----------------
|
||||
* ``flag``: Reduced binary flags (integer).
|
||||
* ``request``: Request (handle, non-blocking only).
|
||||
* ``ierror``: Fortran only: Error status (integer).
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
This collective communication agrees on the integer value *flag* and
|
||||
(implicitly) on the group of failed processes in *comm*.
|
||||
|
||||
On completion, all non-failed MPI processes have agreed to set the
|
||||
output integer value of *flag* to the result of a *bitwise AND*
|
||||
operation over the contributed input values of *flag*.
|
||||
|
||||
:ref:`MPIX_Comm_iagree` is the non-blocking variant of :ref:`MPIX_Comm_agree`.
|
||||
|
||||
PROCESS FAILURES
|
||||
----------------
|
||||
|
||||
When an MPI process fails before contributing to the agree operation,
|
||||
the *flag* is computed ignoring its contribution, and the operation
|
||||
raises an error of class MPIX_ERR_PROC_FAILED.
|
||||
|
||||
When an error of class MPIX_ERR_PROC_FAILED is raised, it is consistently
|
||||
raised at all MPI processes in the group(s) of *comm*.
|
||||
|
||||
After :ref:`MPIX_Comm_agree` raised an error of class MPIX_ERR_PROC_FAILED,
|
||||
the group produced by a subsequent call to :ref:`MPIX_Comm_get_failed` on
|
||||
*comm* contains every MPI process that didn't contribute to the
|
||||
computation of *flag*.
|
||||
|
||||
WHEN THE COMMUNICATOR CONTAINS ACKNOWLEDGED FAILURES
|
||||
----------------------------------------------------
|
||||
|
||||
If **all** MPI processes in the group of *comm* have acknowledged the failure
|
||||
of an MPI process (using :ref:`MPIX_Comm_ack_failed`) prior to the call to
|
||||
:ref:`MPIX_Comm_agree` (or :ref:`MPIX_Comm_iagree`), the MPIX_ERR_PROC_FAILED
|
||||
error is not raised when the output value of *flag* ignores the
|
||||
contribution of that failed process. Note that this is an uniform property:
|
||||
if a non-contributing process is found to be not-acknowledged at any live
|
||||
process in *comm*, all processes raise an error of class MPIX_ERR_PROC_FAILED.
|
||||
|
||||
**Example 1:** Using a combination of :ref:`MPIX_Comm_ack_failed` and
|
||||
:ref:`MPIX_Comm_agree` users can propagate and synchronize the knowledge
|
||||
of failures across all MPI processes in *comm*.
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
Comm_get_failed_consistent(MPI_Comm c, MPI_Group * g) {
|
||||
int rc; int T=1;
|
||||
int size; int num_acked;
|
||||
MPI_Group gf;
|
||||
int ranges[3] = {0, 0, 1};
|
||||
|
||||
MPI_Comm_size(c, &size);
|
||||
|
||||
do {
|
||||
/* this routine is not pure: calling MPI_Comm_ack_failed
|
||||
* affects the state of the communicator c */
|
||||
MPIX_Comm_ack_failed(c, size, &num_acked);
|
||||
/* we simply ignore the T value in this example */
|
||||
rc = MPIX_Comm_agree(c, &T);
|
||||
} while( rc != MPI_SUCCESS );
|
||||
/* after this loop, MPIX_Comm_agree has returned MPI_SUCCESS at
|
||||
* all processes, so all processes have Acknowledged the same set of
|
||||
* failures. Let's get that set of failures in the g group. */
|
||||
if( 0 == num_acked ) {
|
||||
*g = MPI_GROUP_EMPTY;
|
||||
}
|
||||
else {
|
||||
MPIX_Comm_get_failed(c, &gf);
|
||||
ranges[1] = num_acked - 1;
|
||||
MPI_Group_range_incl(gf, 1, ranges, g);
|
||||
MPI_Group_free(&gf);
|
||||
}
|
||||
}
|
||||
|
||||
WHEN THE COMMUNICATOR IS REVOKED
|
||||
--------------------------------
|
||||
|
||||
This function never raises an error of class MPIX_ERR_REVOKED.
|
||||
The defined semantics of :ref:`MPIX_Comm_agree` are maintained when *comm*
|
||||
is revoked, or when the group of *comm* contains failed MPI processes.
|
||||
In particular, :ref:`MPIX_Comm_agree` is a collective operation, even
|
||||
when *comm* is revoked.
|
||||
|
||||
WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
|
||||
------------------------------------------
|
||||
|
||||
When the communicator is an inter-communicator, the value of *flag* is
|
||||
a *bitwise AND* operation over the values contributed by the remote
|
||||
group.
|
||||
|
||||
When an error of class MPIX_ERR_PROC_FAILED is raised, it is consistently
|
||||
raised at all MPI processes in the group(s) of *comm*, that is, both
|
||||
the local and remote groups of the inter-communicator.
|
||||
|
||||
ERRORS
|
||||
------
|
||||
|
||||
.. include:: ./ERRORS.rst
|
||||
|
||||
.. seealso::
|
||||
* :ref:`MPIX_Comm_is_revoked`
|
||||
* :ref:`MPIX_Comm_ack_failed`
|
||||
@@ -0,0 +1,98 @@
|
||||
.. _mpix_comm_get_failed:
|
||||
|
||||
MPIX_Comm_get_failed
|
||||
====================
|
||||
.. include_body
|
||||
|
||||
:ref:`MPIX_Comm_get_failed` - Obtain a group that lists failed processes
|
||||
in a communicator.
|
||||
|
||||
This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
C Syntax
|
||||
^^^^^^^^
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#include <mpi.h>
|
||||
#include <mpi-ext.h>
|
||||
|
||||
int MPIX_Comm_get_failed(MPI_Comm comm, MPI_Group *failedgrp)
|
||||
|
||||
Fortran Syntax
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE MPI
|
||||
USE MPI_EXT
|
||||
! or the older form: INCLUDE 'mpif.h'
|
||||
|
||||
MPIX_COMM_GET_FAILED(COMM, FAILEDGRP, IERROR)
|
||||
INTEGER COMM, FAILEDGRP, IERROR
|
||||
|
||||
Fortran 2008 Syntax
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE mpi_f08
|
||||
USE mpi_ext_f08
|
||||
|
||||
MPIX_Comm_get_failed(comm, failedgrp, ierror)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
TYPE(MPI_Group), INTENT(OUT) :: failedgrp
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
INPUT PARAMETERS
|
||||
----------------
|
||||
* ``comm``: Communicator (handle).
|
||||
|
||||
OUTPUT PARAMETERS
|
||||
-----------------
|
||||
* ``failedgrp``: Group (handle).
|
||||
* ``ierror``: Fortran only: Error status (integer).
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
This local operation returns the group *failedgrp* of processes from the
|
||||
communicator *comm* that are locally known to have failed.
|
||||
The *failedgrp* can be empty, that is, equal to MPI_GROUP_EMPTY.
|
||||
|
||||
For any two groups obtained from calls to that routine at the same MPI
|
||||
process, with the same *comm*, the intersection of the largest group with
|
||||
the smallest group is MPI_IDENT to the smallest group, that is, the same
|
||||
processes have the same ranks in the two groups, up to the size of the
|
||||
smallest group.
|
||||
|
||||
PROCESS FAILURES
|
||||
----------------
|
||||
|
||||
MPI makes no assumption about asynchronous progress of the failure detection.
|
||||
A valid MPI implementation may choose to update the group of locally known
|
||||
failed MPI processes only when it enters a function that must raise a fault
|
||||
tolerance error.
|
||||
|
||||
It is possible that only the calling MPI process has detected the reported
|
||||
failure. If global knowledge is necessary, MPI processes detecting failures
|
||||
should call :ref:`MPIX_Comm_revoke` to enforce an error at other ranks.
|
||||
|
||||
WHEN COMMUNICATOR IS AN INTER-COMMUNICATOR
|
||||
------------------------------------------
|
||||
|
||||
When the communicator is an inter-communicator, the value of *failedgrp*
|
||||
contains the members known to have failed in both the local and the remote
|
||||
groups of *comm*.
|
||||
|
||||
ERRORS
|
||||
------
|
||||
|
||||
.. include:: ./ERRORS.rst
|
||||
|
||||
.. seealso::
|
||||
* :ref:`MPIX_Comm_revoke`
|
||||
* :ref:`MPIX_Comm_ack_failed`
|
||||
@@ -0,0 +1,8 @@
|
||||
.. _mpix_comm_iagree:
|
||||
|
||||
MPIX_Comm_iagree
|
||||
================
|
||||
.. include_body
|
||||
|
||||
.. include:: ../man3/MPIX_Comm_agree.3.rst
|
||||
:start-after: .. include_body
|
||||
@@ -0,0 +1,115 @@
|
||||
.. _mpix_comm_is_revoked:
|
||||
|
||||
MPIX_Comm_is_revoked
|
||||
====================
|
||||
.. include_body
|
||||
|
||||
:ref:`MPIX_Comm_is_revoked` - Test if a communicator is revoked.
|
||||
|
||||
This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
C Syntax
|
||||
^^^^^^^^
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#include <mpi.h>
|
||||
#include <mpi-ext.h>
|
||||
|
||||
int MPIX_Comm_is_revoked(MPI_Comm comm, int *flag)
|
||||
|
||||
Fortran Syntax
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE MPI
|
||||
USE MPI_EXT
|
||||
! or the older form: INCLUDE 'mpif.h'
|
||||
|
||||
MPIX_COMM_IS_REVOKED(COMM, FLAG, IERROR)
|
||||
INTEGER COMM, IERROR
|
||||
LOGICAL FLAG
|
||||
|
||||
Fortran 2008 Syntax
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE mpi_f08
|
||||
USE mpi_ext_f08
|
||||
|
||||
MPIX_Comm_is_revoked(comm, flag, ierror)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
LOGICAL, INTENT(OUT) :: flag
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
INPUT PARAMETERS
|
||||
----------------
|
||||
* ``comm``: Communicator (handle).
|
||||
|
||||
OUTPUT PARAMETERS
|
||||
-----------------
|
||||
* ``flag``: *true* if the communicator is revoked.
|
||||
* ``ierror``: Fortran only: Error status (integer).
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
Returns *flag = true* if the communicator associated with the handle
|
||||
*comm* is revoked at the calling process. It returns *flag = false*
|
||||
otherwise. The operation is local.
|
||||
|
||||
HOW CAN A COMMUNICATOR BECOME REVOKED
|
||||
-------------------------------------
|
||||
|
||||
A communicator can become revoked when
|
||||
|
||||
1. the user calls the :ref:`MPIX_Comm_revoke` procedure on *comm* to revoke
|
||||
the communicator at the local process;
|
||||
2. an MPI operation raised the error class MPIX_ERR_REVOKED because
|
||||
another process called the :ref:`MPIX_Comm_revoke` procedure on *comm*;
|
||||
3. the communicator has the info key *mpi_error_range* set to *group* or
|
||||
*universe*, in which case the failure of any process in, respectively,
|
||||
the group of *comm* or the MPI universe caused an operation to raise
|
||||
an error of class MPIX_ERR_REVOKED.
|
||||
|
||||
REVOKE PROPAGATION AND ORDERING
|
||||
-------------------------------
|
||||
|
||||
Note that in a multithreaded application, a thread calling
|
||||
:ref:`MPIX_Comm_is_revoked` may return *flag = true* before the operation
|
||||
that raises the first exception of class MPIX_ERR_REVOKED has completed
|
||||
in a concurrent thread.
|
||||
|
||||
EFFECT OF A COMMUNICATOR REVOCATION
|
||||
-----------------------------------
|
||||
|
||||
Once a communicator has been revoked at an MPI process, all subsequent
|
||||
non-local operations on that communicator (with some exceptions listed
|
||||
below), are considered local and must complete by raising an error of
|
||||
class MPIX_ERR_REVOKED at that MPI process.
|
||||
|
||||
OPERATIONS ON A REVOKED COMMUNICATOR
|
||||
------------------------------------
|
||||
|
||||
The following operations never raise an error of class MPIX_ERR_REVOKED,
|
||||
and complete with their normal semantics on a revoked communicator.
|
||||
|
||||
* :ref:`MPIX_Comm_agree`
|
||||
* :ref:`MPIX_Comm_iagree`
|
||||
* :ref:`MPIX_Comm_shrink`
|
||||
* :ref:`MPIX_Comm_ishrink`
|
||||
|
||||
ERRORS
|
||||
------
|
||||
|
||||
.. include:: ./ERRORS.rst
|
||||
|
||||
.. seealso::
|
||||
* :ref:`MPIX_Comm_revoke`
|
||||
* :ref:`MPIX_Comm_agree`
|
||||
* :ref:`MPIX_Comm_shrink`
|
||||
@@ -0,0 +1,8 @@
|
||||
.. _mpix_comm_ishrink:
|
||||
|
||||
MPIX_Comm_ishrink
|
||||
=================
|
||||
.. include_body
|
||||
|
||||
.. include:: ../man3/MPIX_Comm_shrink.3.rst
|
||||
:start-after: .. include_body
|
||||
@@ -0,0 +1,92 @@
|
||||
.. _mpix_comm_revoke:
|
||||
|
||||
MPIX_Comm_revoke
|
||||
================
|
||||
.. include_body
|
||||
|
||||
:ref:`MPIX_Comm_revoke` - Revoke a communicator, causing errors to be
|
||||
raised, at all ranks, for non-local operations on the communicator.
|
||||
|
||||
This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
C Syntax
|
||||
^^^^^^^^
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#include <mpi.h>
|
||||
#include <mpi-ext.h>
|
||||
|
||||
int MPIX_Comm_revoke(MPI_Comm comm)
|
||||
|
||||
Fortran Syntax
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE MPI
|
||||
USE MPI_EXT
|
||||
! or the older form: INCLUDE 'mpif.h'
|
||||
|
||||
MPIX_COMM_REVOKE(COMM, IERROR)
|
||||
INTEGER COMM, IERROR
|
||||
|
||||
Fortran 2008 Syntax
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE mpi_f08
|
||||
USE mpi_ext_f08
|
||||
|
||||
MPIX_Comm_revoke(comm, ierror)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
INPUT PARAMETERS
|
||||
----------------
|
||||
* ``comm``: Communicator (handle).
|
||||
|
||||
OUTPUT PARAMETERS
|
||||
-----------------
|
||||
* ``ierror``: Fortran only: Error status (integer).
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
This function marks the communicator *comm* as revoked at all MPI processes
|
||||
in the groups (local and remote) associated with the communicator *comm*.
|
||||
This function is not collective and therefore does not have a matching call
|
||||
on remote MPI processes.
|
||||
|
||||
The documentation for :ref:`MPIX_Comm_is_revoked` details the conditions
|
||||
for when a communicator becomes revoked locally, and what semantics apply
|
||||
on a revoked communicator. In summary, when a communicator is revoked,
|
||||
non-local operation raise an exception of class MPIX_ERR_REVOKED,
|
||||
except for select fault-tolerant operations.
|
||||
|
||||
PROPAGATION OF THE REVOKED STATE AND ORDERING
|
||||
---------------------------------------------
|
||||
|
||||
The implementation propagates the revoked state in a fault-tolerant manner;
|
||||
thus, the communicator becomes revoked at all non-failed MPI processes
|
||||
belonging to *comm* despite failed processes.
|
||||
|
||||
There is no particular ordering between the revocation call at another
|
||||
process and the completion of operations at a local process, for example,
|
||||
a receive operation can raise an error of class MPIX_ERR_REVOKED, even if
|
||||
the send operation procedure is called before the revoke procedure at the
|
||||
sender.
|
||||
|
||||
ERRORS
|
||||
------
|
||||
|
||||
.. include:: ./ERRORS.rst
|
||||
|
||||
.. seealso::
|
||||
* :ref:`MPIX_Comm_is_revoked`
|
||||
* :ref:`MPIX_Comm_agree`
|
||||
* :ref:`MPIX_Comm_shrink`
|
||||
@@ -0,0 +1,117 @@
|
||||
.. _mpix_comm_shrink:
|
||||
|
||||
MPIX_Comm_shrink
|
||||
================
|
||||
.. include_body
|
||||
|
||||
:ref:`MPIX_Comm_shrink`, :ref:`MPIX_Comm_ishrink` - Create a new communicator
|
||||
that includes all processes from the parent communicator that have not failed.
|
||||
|
||||
This is part of the User Level Fault Mitigation :ref:`ULFM extension <ulfm-label>`.
|
||||
|
||||
SYNTAX
|
||||
------
|
||||
|
||||
C Syntax
|
||||
^^^^^^^^
|
||||
|
||||
.. code-block:: c
|
||||
|
||||
#include <mpi.h>
|
||||
#include <mpi-ext.h>
|
||||
|
||||
int MPIX_Comm_shrink(MPI_Comm comm, MPI_Comm *newcomm)
|
||||
|
||||
int MPIX_Comm_ishrink(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request)
|
||||
|
||||
|
||||
Fortran Syntax
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE MPI
|
||||
USE MPI_EXT
|
||||
! or the older form: INCLUDE 'mpif.h'
|
||||
|
||||
MPIX_COMM_SHRINK(COMM, NEWCOMM, IERROR)
|
||||
INTEGER COMM, NEWCOMM, IERROR
|
||||
|
||||
MPIX_COMM_ISHRINK(COMM, NEWCOMM, REQUEST, IERROR)
|
||||
INTEGER COMM, NEWCOMM, REQUEST, IERROR
|
||||
|
||||
|
||||
Fortran 2008 Syntax
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. code-block:: fortran
|
||||
|
||||
USE mpi_f08
|
||||
USE mpi_ext_f08
|
||||
|
||||
MPIX_Comm_shrink(comm, newcomm, ierror)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
TYPE(MPI_Comm), INTENT(OUT) :: newcomm
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
MPIX_Comm_ishrink(comm, newcomm, request, ierror)
|
||||
TYPE(MPI_Comm), INTENT(IN) :: comm
|
||||
TYPE(MPI_Comm), INTENT(OUT), ASYNCHRONOUS :: newcomm
|
||||
TYPE(MPI_Request), INTENT(OUT) :: request
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
|
||||
INPUT PARAMETERS
|
||||
----------------
|
||||
* ``comm``: Communicator (handle).
|
||||
|
||||
OUTPUT PARAMETERS
|
||||
-----------------
|
||||
* ``newcomm``: Communicator (handle).
|
||||
* ``request``: Request (handle, non-blocking only).
|
||||
* ``ierror``: Fortran only: Error status (integer).
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
||||
This collective operation creates a new intra- or intercommunicator
|
||||
*newcomm* from the intra- or intercommunicator *comm*, respectively, by
|
||||
excluding the group of failed MPI processes as shrinkd upon during the
|
||||
operation.
|
||||
|
||||
The groups of *newcomm* must include every MPI process that returns from
|
||||
:ref:`MPIX_Comm_shrink`, and it must exclude every MPI process whose failure
|
||||
caused an operation on *comm* to raise an MPI error of class
|
||||
MPIX_ERR_PROC_FAILED or MPIX_ERR_PROC_FAILED_PENDING at a member of the
|
||||
groups of *newcomm*, before that member initiated the shrink operation.
|
||||
|
||||
Said otherwise, this procedure is semantically equivalent to an
|
||||
:ref:`MPI_Comm_split` operation that would succeed despite failures, where
|
||||
members of the groups of *newcomm* participate with the same color and a key
|
||||
equal to their rank in *comm*.
|
||||
|
||||
:ref:`MPIX_Comm_ishrink` is the non-blocking variant of :ref:`MPIX_Comm_shrink`.
|
||||
Note that, as with :ref:`MPI_Comm_idup`, it is erroneous to use *newcomm*
|
||||
before *request* has completed.
|
||||
|
||||
WHEN THE COMMUNICATOR IS REVOKED OR CONTAINS FAILED PROCESSES
|
||||
-------------------------------------------------------------
|
||||
|
||||
This function never raises an error of classes MPIX_ERR_REVOKED or
|
||||
MPIX_ERR_PROC_FAILED. The defined semantics of :ref:`MPIX_Comm_shrink` and
|
||||
:ref:`MPIX_Comm_ishrink` are maintained when *comm* is revoked, or when the
|
||||
group of *comm* contains failed MPI processes. In particular,
|
||||
:ref:`MPIX_Comm_shrink` and :ref:`MPIX_Comm_ishrink` are collective operations,
|
||||
even when *comm* is revoked.
|
||||
|
||||
The implementation will strive to detect all failures during the shrink
|
||||
operation, but in certain circumpstances, the group of *newcomm* may still
|
||||
contain failed MPI processes, whose failure will be detected in subsequent
|
||||
MPI operations on *newcomm*.
|
||||
|
||||
ERRORS
|
||||
------
|
||||
|
||||
.. include:: ./ERRORS.rst
|
||||
|
||||
.. seealso::
|
||||
* :ref:`MPIX_Comm_is_revoked`
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user