Merge pull request open-mpi#7740 from abouteiller/export/ulfm-to-ompi5

Export/ulfm to ompi5

Author: bosilca

README.FT.ULFM.md

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2021 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -139,6 +139,9 @@ AC_DEFUN([OMPI_SETUP_PRRTE_INTERNAL], [
 
     # add the extra libs
     internal_prrte_args="$internal_prrte_args --with-prte-extra-lib=\"$internal_prrte_libs\" --with-prte-extra-ltlib=\"$internal_prrte_libs\""
+    AS_IF([test "$with_ft" != "no"],
+          [internal_prrte_args="--enable-prte-ft $internal_prrte_args"],
+          [])
 
     # Pass all our compiler/linker flags to PRRTE, so that it
     # picks up how to build an internal HWLOC, libevent, and PMIx, plus

config/ompi_setup_prrte.m4

@@ -1,4 +1,8 @@
 dnl
+dnl Copyright (c) 2004-2020 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2009-2012 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2013-2014 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
@@ -12,11 +16,15 @@ dnl
 #
 # --with-ft=TYPE
 #  TYPE:
+#    - mpi (synonym for 'ulfm')
 #    - LAM (synonym for 'cr' currently)
 #    - cr
 #  /* General FT sections */
 #  #if OPAL_ENABLE_FT == 0 /* FT Disabled globaly */
 #  #if OPAL_ENABLE_FT == 1 /* FT Enabled globaly */
+#  /* ULFM Specific sections */
+#  #if OPAL_ENABLE_FT_MPI == 0 /* FT ULFM Disabled */
+#  #if OPAL_ENABLE_FT_MPI == 1 /* FT ULFM Enabled */
 #  /* CR Specific sections */
 #  #if OPAL_ENABLE_FT_CR == 0 /* FT Ckpt/Restart Disabled */
 #  #if OPAL_ENABLE_FT_CR == 1 /* FT Ckpt/Restart Enabled */
@@ -28,14 +36,13 @@ AC_DEFUN([OPAL_SETUP_FT_BANNER],[
 ])
 
 AC_DEFUN([OPAL_SETUP_FT_OPTIONS],[
-    AC_REQUIRE([OPAL_SETUP_FT_BANNER])
     # define a variable that tells us that these options were enabled
     opal_setup_ft_options="yes"
     AC_ARG_WITH(ft,
                 [AC_HELP_STRING([--with-ft=TYPE],
-                [Specify the type of fault tolerance to enable. Options: LAM (LAM/MPI-like), cr (Checkpoint/Restart), (default: disabled)])],
-                [opal_want_ft=1],
-                [opal_want_ft=0])
+                [Specify the type of fault tolerance to enable. Options: mpi (ULFM), LAM (LAM/MPI-like), cr (Checkpoint/Restart) (default: mpi)])],
+                [],
+                [with_ft=auto]) # If not specified act as if --with-ft=mpi, but make external prte support failure only if hard requested
 
     #
     # Checkpoint/restart enabled debugging
@@ -60,12 +67,15 @@ AC_DEFUN([OPAL_SETUP_FT_OPTIONS],[
 ])
 
 AC_DEFUN([OPAL_SETUP_FT],[
+    AC_REQUIRE([OPAL_SETUP_FT_BANNER])
     if test "$opal_setup_ft_options" = "yes"; then
         AC_MSG_CHECKING([if want fault tolerance])
     fi
-    if test "x$with_ft" != "x" || test "$opal_want_ft" = "1"; then
+
+    if test x"$with_ft" != "xno"; then
         opal_want_ft=1
         opal_want_ft_cr=0
+        opal_want_ft_mpi=0
         opal_want_ft_type=none
 
         as_save_IFS=$IFS
@@ -74,8 +84,18 @@ AC_DEFUN([OPAL_SETUP_FT],[
             IFS=$as_save_IFS
 
             # Default value
-            if test "$opt" = "" || test "$opt" = "yes"; then
-                opal_want_ft_cr=1
+            if test "$opt" = "auto"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "yes"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ULFM"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ulfm"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "MPI"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "mpi"; then
+                opal_want_ft_mpi=1
             elif test "$opt" = "LAM"; then
                 opal_want_ft_cr=1
             elif test "$opt" = "lam"; then
@@ -89,28 +109,57 @@ AC_DEFUN([OPAL_SETUP_FT],[
                 AC_MSG_ERROR([Cannot continue])
             fi
         done
-        if test "$opal_want_ft_cr" = 1; then
+        if test "$opal_want_ft_mpi" = 1; then
+            opal_want_ft_type="mpi"
+        elif test "$opal_want_ft_cr" = 1; then
             opal_want_ft_type="cr"
         fi
 
+        # If we use external PRTE, does it support FT?
+        AS_IF([test "$internal_prrte_build" = "0" -a "$opal_want_ft_type" != "none"], [
+            AS_IF([prte_info | $GREP "Resilience support: yes"], [], [
+                AS_IF([test "$with_ft" != auto], [
+                    AC_MSG_ERROR([Requested enabling fault-tolerance and using external launcher, but external PRTE doesn't support resilience; you can either use the internal PRTE, recompile the external PRTE with fault-tolerance, or disable fault-tolerance. ABORTING.])
+                ], [
+                    AC_MSG_WARN([**************************************************])
+                    AC_MSG_WARN([*** Requested external PRTE which doesn't have   *])
+                    AC_MSG_WARN([*** Resilience compiled-in.                      *])
+                    AC_MSG_WARN([*** To enable Open MPI Fault-Tolerance, either   *])
+                    AC_MSG_WARN([***   use the internal PRTE, or                  *])
+                    AC_MSG_WARN([***   compile the external PRTE with resilience  *])
+                    AC_MSG_WARN([*** DISABLING FAULT TOLERANCE SUPPORT.           *])
+                    AC_MSG_WARN([**************************************************])
+                    opal_want_ft_mpi=0
+                    opal_want_ft_cr=0
+                    opal_want_ft_type="none"
+                ])
+            ])
+        ])
         AC_MSG_RESULT([Enabled $opal_want_ft_type (Specified $with_ft)])
-        AC_MSG_WARN([**************************************************])
-        AC_MSG_WARN([*** Fault Tolerance Integration into Open MPI is *])
-        AC_MSG_WARN([*** a research quality implementation, and care  *])
-        AC_MSG_WARN([*** should be used when choosing to enable it.   *])
-        AC_MSG_WARN([**************************************************])
+        AS_IF([test "$opal_want_ft_type" != "none"], [
+            AC_MSG_WARN([**************************************************])
+            AC_MSG_WARN([*** Fault Tolerance Integration into Open MPI is *])
+            AC_MSG_WARN([*** compiled-in, but off by default. Use mpiexec *])
+            AC_MSG_WARN([*** and MCA parameters to turn it on.            *])
+            AC_MSG_WARN([*** Not all components support fault tolerance.  *])
+            AC_MSG_WARN([**************************************************])
+        ])
     else
         opal_want_ft=0
+        opal_want_ft_mpi=0
         opal_want_ft_cr=0
         if test "$opal_setup_ft_options" = "yes"; then
             AC_MSG_RESULT([Disabled fault tolerance])
         fi
     fi
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT], [$opal_want_ft],
                        [Enable fault tolerance general components and logic])
+    AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_MPI], [$opal_want_ft_mpi],
+                       [Enable fault tolerance MPI ULFM components and logic])
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_CR], [$opal_want_ft_cr],
                        [Enable fault tolerance checkpoint/restart components and logic])
     AM_CONDITIONAL(WANT_FT, test "$opal_want_ft" = "1")
+    AM_CONDITIONAL(WANT_FT_MPI, test "$opal_want_ft_mpi" = "1")
     AM_CONDITIONAL(WANT_FT_CR,  test "$opal_want_ft_cr" = "1")
 
     if test "$opal_setup_ft_options" = "yes"; then
@@ -175,4 +224,5 @@ AC_DEFUN([OPAL_SETUP_FT],[
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_THREAD], [$opal_want_ft_thread],
                        [Enable fault tolerance thread in Open PAL])
     AM_CONDITIONAL(WANT_FT_THREAD, test "$opal_want_ft_thread" = "1")
+    OPAL_SUMMARY_ADD([[Miscellaneous]],[[Fault Tolerance support]],[unnecessary], [$opal_want_ft_type])
 ])

config/opal_setup_ft.m4

@@ -1116,29 +1116,11 @@ AC_INCLUDES_DEFAULT
 # checkpoint results
 AC_CACHE_SAVE
 
-###########################################################
+##################################
 #        Fault Tolerance
-#
-# The FT code in the OMPI trunk is currently broken. We don't
-# have an active maintainer for it at this time, and it isn't
-# clear if/when we will return to it. We have therefore removed
-# the configure options supporting it until such time as it
-# can be fixed.
-#
-# However, we recognize that there are researchers who use this
-# option on their independent branches. In such cases, simply
-# uncomment the line below to render the FT configure options
-# visible again
-#
-###########################################################
-dnl OPAL_SETUP_FT_OPTIONS
-###########################################################
-# The following line is always required as it contains the
-# AC_DEFINE and AM_CONDITIONAL calls that set variables used
-# throughout the build system. If the above line is commented
-# out, then those variables will be set to "off". Otherwise,
-# they are controlled by the options
-OPAL_SETUP_FT
+# Part1: must happen before prte
+##################################
+OPAL_SETUP_FT_OPTIONS
 
 ##################################
 # 3rd-party packages not called ROMIO
@@ -1160,6 +1142,12 @@ AC_SUBST(OPAL_3RDPARTY_DIST_SUBDIRS)
 AC_SUBST(OPAL_3RDPARTY_EXTRA_DIST)
 AC_SUBST(OPAL_3RDPARTY_DISTCLEAN_DIRS)
 
+##################################
+#        Fault Tolerance
+# Part2: must happen after prte
+##################################
+OPAL_SETUP_FT
+
 ##################################
 # MCA
 ##################################

configure.ac

@@ -27,6 +27,10 @@
 amca_paramdir = $(AMCA_PARAM_SETS_DIR)
 dist_amca_param_DATA = amca-param-sets/example.conf
 
+if WANT_FT_MPI
+dist_amca_param_DATA += amca-param-sets/ft-mpi
+endif # WANT_FT_MPI
+
 if WANT_FT_CR
 dist_amca_param_DATA += \
 	amca-param-sets/ft-enable-cr \

contrib/Makefile.am

@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2020     The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# An Aggregate MCA Parameter Set to setup an environment that can support
+# User-Level Failure Mitigation (ULFM) fault tolerance (must also be 
+# compiled in with --with-ft=mpi).
+#
+# Usage:
+#   shell$ mpirun --tune ft-mpi ./app
+#
+
+mpi_ft_enable=true
+
+# Since failures are expected, reduce the verbosity of the transport errors
+btl_base_warn_peer_error=false
+
+#
+# Performance tuning parameters (default shown)
+# By default the PRTE failure detector is used (see README.ULFM.md)
+#mpi_ft_detector=false
+#mpi_ft_detector_thread=false
+#mpi_ft_detector_rdma_heartbeat=false
+#mpi_ft_detector_period=3.
+#mpi_ft_detector_timeout=10.
+#
+
+
+#
+# Select only ULFM ready components
+# disabling non-tested and known broken components in FT-MPI builds
+#
+
+#
+# The following frameworks/components are TESTED
+# They handle faults amd should be prefered when running with FT.
+#   pml     ob1
+#   btl     tcp, self, sm(+xpmem,+cma), ugni, uct
+#   coll    base/basic, tuned, ftagree, libnbc
+pml=ob1
+threads=pthreads
+
+#
+# The following frameworks/components are UNTESTED, but **may** work.
+# They should run without faults, and **may** work with faults.
+# You may try and report if successfull.
+#   btl     ofi, portals4, smcuda, usnic, sm(+knem)
+#   coll    inter, sm, sync, cuda, monitoring
+#   pml     monitoring, v/vprotocol
+# We will disable only the components for which good components are known to exist.
+btl=^usnic
+# older versions of xpmem generate bus errors when the other end is dead.
+#btl_sm_single_copy_mechanism=cma
+
+
+#
+# The following frameworks/components are UNTESTED, and probably won't work.
+# They should run without faults, and will probably crash/deadlock after a fault.
+# You may try at your own risk.
+#   coll    hcoll, portals4
+#   topo    (all)
+#   osc     (all)
+#   io      (all)
+#   fcoll   (all)
+#   fbtl    (all)
+# We will disable only the components for which good components are known to exist.
+# Other untested components are selectable but will issue a runtime warning at
+# initiation if FT is enabled.
+coll=^hcoll,portals4
+
+#
+# The following frameworks/components are NOT WORKING. Do not enable these with FT.
+#   mtl     (all)
+#   pml     cm, crcpw, ucx
+mtl=^ofi,portals4,psm2
+# allready enforced by pml=ob1 above
+#pml=^cm,crcpw,ucx
+# allready enforced by threads=pthreads above
+#threads=^argobots,qthreads
+# There is a bug in libevent with the "select" backend that causes an infinite loop
+# when an unplanned disconnect happens. Use something else, or bail.
+opal_event_include=epoll,devpoll,kqueue,evport,poll
+

contrib/amca-param-sets/ft-mpi

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -52,6 +52,9 @@
  *
  * MPI_WTIME_IS_GLOBAL is set to 0 (a conservative answer).
  *
+ * MPI_FT is set to 0 or 1 (according to OPAL_ENABLE_FT_MPI and
+ * ompi_ftmpi_enabled)
+ *
  * MPI_APPNUM is set as the result of a GPR subscription.
  *
  * MPI_LASTUSEDCODE is set to an initial value and is reset every time
@@ -123,7 +126,9 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_DISP_UNIT)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_CREATE_FLAVOR)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL))) {
+        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL)) ||
+        OMPI_SUCCESS != (ret = create_comm(MPI_FT, false)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
+        0) {
         return ret;
     }
 
@@ -133,6 +138,14 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = set_f(MPI_HOST, MPI_PROC_NULL)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_IO, MPI_ANY_SOURCE)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_WTIME_IS_GLOBAL, 0)) ||
+#if OPAL_ENABLE_FT_MPI
+        /* Although we always define the key to ease fortran integration,
+         * lets not set a default value to the attribute if we do not 
+         * have fault tolerance built in. */
+        OMPI_SUCCESS != (ret = set_f(MPI_FT, ompi_ftmpi_enabled)) ||
+#else
+        OMPI_SUCCESS != (ret = set_f(MPI_FT, false)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = set_f(MPI_LASTUSEDCODE,
                                      ompi_mpi_errcode_lastused))) {
         return ret;
@@ -161,6 +174,7 @@ int ompi_attr_free_predefined(void)
         OMPI_SUCCESS != (ret = free_comm(MPI_APPNUM)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_LASTUSEDCODE)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_UNIVERSE_SIZE)) ||
+        OMPI_SUCCESS != (ret = free_comm(MPI_FT)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_BASE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_DISP_UNIT)) ||

ompi/attribute/attribute_predefined.c

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -34,3 +34,8 @@ lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
 	communicator/comm_cid.c \
 	communicator/comm_request.c
 
+if WANT_FT_MPI
+lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+	communicator/ft/comm_ft.c communicator/ft/comm_ft_reliable_bcast.c communicator/ft/comm_ft_propagator.c communicator/ft/comm_ft_detector.c communicator/ft/comm_ft_revoke.c
+endif # WANT_FT_MPI
+