Merge pull request #7800 from abouteiller/mpi-next/errors_abort

MPI4: Add ERRORS_ABORT infrastructure

Author: awlauria

contrib/check_unnecessary_headers.sh

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -186,7 +186,7 @@ SEARCH_HEADER[5]="ompi/datatype/convertor.h OMPI_COMM_INTER OMPI_COMM_CART OMPI_
 SEARCH_HEADER[6]="ompi/datatype/datatype.h MPI_Datatype DT_MAX_PREDEFINED DT_FLAG_ MAX_DT_COMPONENT_COUNT opal_ddt_count_t dt_type_desc_t ompi_datatype_t ompi_predefined_datatype_t ompi_ddt_init ompi_ddt_finalize ompi_ddt_create_ ompi_ddt_duplicate ompi_ddt_is_predefined ompi_ddt_create_from_packed_description"
 SEARCH_HEADER[7]="ompi/datatype/datatype_internal.h DDT_DUMP_STACK DT_ ddt_elem_id_description ddt_elem_desc ddt_elem_desc_t ddt_loop_desc ddt_loop_desc_t ddt_endloop_desc ddt_endloop_desc_t dt_elem_desc CREATE_LOOP_START CREATE_LOOP_END CREATE_ELEM ompi_complex_float_t ompi_complex_double_t ompi_complex_long_double_t ompi_ddt_basicDatatypes BASIC_DDT_FROM_ELEM ompi_ddt_default_convertors_init ompi_ddt_default_convertors_fini SAVE_STACK PUSH_STACK ompi_ddt_safeguard_pointer_debug_breakpoint OMPI_DDT_SAFEGUARD_POINTER GET_FIRST_NON_LOOP UPDATE_INTERNAL_COUNTERS ompi_ddt_print_args"
 SEARCH_HEADER[8]="ompi/errhandler/errhandler.h OMPI_ERRHANDLER_LANG_ ompi_errhandler_lang_t OMPI_ERRHANDLER_TYPE_ ompi_errhandler_type_t ompi_errhandler_t ompi_predefined_errhandler_t ompi_mpi_errhandler_null OMPI_ERRHANDLER_CHECK OMPI_ERRHANDLER_RETURN ompi_errhandler_init ompi_errhandler_finalize OMPI_ERRHANDLER_INVOKE ompi_errhandler_invoke ompi_errhandler_request_invoke ompi_errhandler_create ompi_errhandler_is_intrinsic ompi_errhandler_fortran_handler_fn_t OMPI_ERR_INIT_FINALIZE MPI_Errhandler"
-SEARCH_HEADER[9]="ompi/errhandler/errhandler_predefined.h ompi_mpi_errors_are_fatal_ ompi_mpi_errors_return_ ompi_mpi_errors_throw_exceptions"
+SEARCH_HEADER[9]="ompi/errhandler/errhandler_predefined.h ompi_mpi_errors_are_fatal_ ompi_mpi_errors_return_ ompi_mpi_errors_abort_ ompi_mpi_errors_throw_exceptions"
 ###
 SEARCH_HEADER[10]="ompi/file/file.h OMPI_FILE_ISCLOSED OMPI_FILE_HIDDEN ompi_file_t ompi_predefined_file_t ompi_mpi_file_null ompi_file_f_to_c_table ompi_file_init ompi_file_open ompi_file_set_name ompi_file_close ompi_file_finalize ompi_file_invalid MPI_File MPI_FILE_NULL ompi_mpi_cxx_file_errhandler_invoke" # THE LAST ONE WAS FOR THE CXX INTERFACE
 SEARCH_HEADER[11]="ompi/group/group.h ompi_group_sporadic_list_t ompi_group_sporadic_data_t ompi_group_strided_data_t ompi_group_bitmap_data_t ompi_group_t ompi_predefined_group_t OMPI_GROUP_ ompi_group_f_to_c_table ompi_mpi_group_null ompi_group_allocate ompi_group_increment_proc_count ompi_group_decrement_proc_count ompi_group_size ompi_group_rank ompi_set_group_rank ompi_group_translate_ranks ompi_group_free ompi_group_get_proc_ptr ompi_group_calc_ ompi_group_peer_lookup ompi_group_div_ceil MPI_Group"

ompi/debuggers/MPI_Handles_interface.txt

@@ -314,8 +314,8 @@ C++: MPI::Errhandler
 
 MPI allows applications to define their own error handlers.  The
 default error handler is to abort the MPI job.  Error handlers can be
-attached to communicators, files, and windows.  There are 3 predefined
-error handlers (MPI_ERRORS_ARE_FATAL, MPI_ERRORS_RETURN,
+attached to communicators, files, and windows.  There are 4 predefined
+error handlers (MPI_ERRORS_ARE_FATAL, MPI_ERRORS_RETURN, MPI_ERRORS_ABORT,
 MPI::ERRORS_THROW_EXCEPTIONS), and applications can create their own
 error handlers.
 

ompi/debuggers/ompi_mpihandles_dll.c

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
@@ -384,6 +384,8 @@ int mpidbg_init_per_process(mqs_process *process,
         int i = 0;
         fill_map(image, "MPI_ERRORS_ARE_FATAL", "ompi_mpi_errors_are_fatal",
                  &mpidbg_errhandler_name_map[i++]);
+        fill_map(image, "MPI_ERRORS_ABORT", "ompi_mpi_errors_abort",
+                 &mpidbg_errhandler_name_map[i++]);
         fill_map(image, "MPI_ERRORS_RETURN", "ompi_mpi_errors_return",
                  &mpidbg_errhandler_name_map[i++]);
         fill_map(image, "MPI_ERRHANDLER_NULL", "ompi_mpi_errhandler_null",

ompi/errhandler/errcode.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -85,6 +85,7 @@ static ompi_mpi_errcode_t ompi_err_not_same;
 static ompi_mpi_errcode_t ompi_err_no_space;
 static ompi_mpi_errcode_t ompi_err_no_such_file;
 static ompi_mpi_errcode_t ompi_err_port;
+static ompi_mpi_errcode_t ompi_err_proc_aborted;
 static ompi_mpi_errcode_t ompi_err_quota;
 static ompi_mpi_errcode_t ompi_err_read_only;
 static ompi_mpi_errcode_t ompi_err_rma_conflict;
@@ -186,6 +187,7 @@ int ompi_mpi_errcode_init (void)
     CONSTRUCT_ERRCODE( ompi_err_no_space, MPI_ERR_NO_SPACE, "MPI_ERR_NO_SPACE: no space left on device" );
     CONSTRUCT_ERRCODE( ompi_err_no_such_file, MPI_ERR_NO_SUCH_FILE, "MPI_ERR_NO_SUCH_FILE: no such file or directory" );
     CONSTRUCT_ERRCODE( ompi_err_port, MPI_ERR_PORT, "MPI_ERR_PORT: invalid port" );
+    CONSTRUCT_ERRCODE( ompi_err_proc_aborted, MPI_ERR_PROC_ABORTED, "MPI_ERR_PROC_ABORTED: operation failed because a remote peer has aborted" );
     CONSTRUCT_ERRCODE( ompi_err_quota, MPI_ERR_QUOTA, "MPI_ERR_QUOTA: out of quota" );
     CONSTRUCT_ERRCODE( ompi_err_read_only, MPI_ERR_READ_ONLY, "MPI_ERR_READ_ONLY: file is read only" );
     CONSTRUCT_ERRCODE( ompi_err_rma_conflict, MPI_ERR_RMA_CONFLICT, "MPI_ERR_RMA_CONFLICT: rma conflict during operation" );
@@ -282,6 +284,7 @@ int ompi_mpi_errcode_finalize(void)
     OBJ_DESTRUCT(&ompi_err_no_space);
     OBJ_DESTRUCT(&ompi_err_no_such_file);
     OBJ_DESTRUCT(&ompi_err_port);
+    OBJ_DESTRUCT(&ompi_err_proc_aborted);
     OBJ_DESTRUCT(&ompi_err_quota);
     OBJ_DESTRUCT(&ompi_err_read_only);
     OBJ_DESTRUCT(&ompi_err_rma_conflict);

ompi/errhandler/errhandler.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -68,6 +68,9 @@ ompi_predefined_errhandler_t *ompi_mpi_errhandler_null_addr =
 ompi_predefined_errhandler_t ompi_mpi_errors_are_fatal = {{{0}}};
 ompi_predefined_errhandler_t *ompi_mpi_errors_are_fatal_addr =
     &ompi_mpi_errors_are_fatal;
+ompi_predefined_errhandler_t ompi_mpi_errors_abort = {{{0}}};
+ompi_predefined_errhandler_t *ompi_mpi_errors_abort_addr =
+    &ompi_mpi_errors_abort;
 ompi_predefined_errhandler_t ompi_mpi_errors_return = {{{0}}};
 ompi_predefined_errhandler_t *ompi_mpi_errors_return_addr =
     &ompi_mpi_errors_return;
@@ -127,6 +130,19 @@ int ompi_errhandler_init(void)
   opal_string_copy(ompi_mpi_errors_return.eh.eh_name, "MPI_ERRORS_RETURN",
                    sizeof(ompi_mpi_errors_return.eh.eh_name));
 
+  OBJ_CONSTRUCT( &ompi_mpi_errors_abort.eh, ompi_errhandler_t );
+  if( ompi_mpi_errors_abort.eh.eh_f_to_c_index != OMPI_ERRORS_ABORT_FORTRAN )
+      return OMPI_ERROR;
+  ompi_mpi_errors_abort.eh.eh_mpi_object_type = OMPI_ERRHANDLER_TYPE_PREDEFINED;
+  ompi_mpi_errors_abort.eh.eh_lang = OMPI_ERRHANDLER_LANG_C;
+  ompi_mpi_errors_abort.eh.eh_comm_fn = ompi_mpi_errors_abort_comm_handler;
+  ompi_mpi_errors_abort.eh.eh_file_fn = ompi_mpi_errors_abort_file_handler;
+  ompi_mpi_errors_abort.eh.eh_win_fn  = ompi_mpi_errors_abort_win_handler ;
+  ompi_mpi_errors_abort.eh.eh_fort_fn = NULL;
+  opal_string_copy(ompi_mpi_errors_abort.eh.eh_name,
+                   "MPI_ERRORS_ABORT",
+                   sizeof(ompi_mpi_errors_abort.eh.eh_name));
+
   /* If we're going to use C++, functions will be fixed up during
      MPI::Init.  Note that it is proper to use ERRHANDLER_LANG_C here;
      the dispatch function is in C (although in libmpi_cxx); the

ompi/errhandler/errhandler.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -49,7 +49,8 @@ BEGIN_C_DECLS
 enum {
   OMPI_ERRHANDLER_NULL_FORTRAN = 0,
   OMPI_ERRORS_ARE_FATAL_FORTRAN,
-  OMPI_ERRORS_RETURN_FORTRAN
+  OMPI_ERRORS_RETURN_FORTRAN,
+  OMPI_ERRORS_ABORT_FORTRAN,
 };
 
 
@@ -167,6 +168,12 @@ OMPI_DECLSPEC extern ompi_predefined_errhandler_t *ompi_mpi_errors_are_fatal_add
 OMPI_DECLSPEC extern ompi_predefined_errhandler_t ompi_mpi_errors_return;
 OMPI_DECLSPEC extern ompi_predefined_errhandler_t *ompi_mpi_errors_return_addr;
 
+/*
+ * Global variable for MPI_ERRORS_ABORT (_addr flavor is for F03 bindings)
+ */
+OMPI_DECLSPEC extern ompi_predefined_errhandler_t ompi_mpi_errors_abort;
+OMPI_DECLSPEC extern ompi_predefined_errhandler_t *ompi_mpi_errors_abort_addr;
+
 /**
  * Global variable for MPI::ERRORS_THROW_EXCEPTIONS.  Will abort if
  * MPI_INIT wasn't called as MPI::INIT (_addr flavor is for F03 bindings)

ompi/errhandler/errhandler_invoke.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -41,10 +41,10 @@ int ompi_errhandler_invoke(ompi_errhandler_t *errhandler, void *mpi_object,
     ompi_win_t *win;
     ompi_file_t *file;
 
-    /* If we got no errorhandler, then just invoke errors_abort */
+    /* If we got no errorhandler, then just invoke errors_are_fatal */
     if (NULL == errhandler) {
         ompi_mpi_errors_are_fatal_comm_handler(NULL, NULL, message);
-	return err_code;
+        return err_code;
     }
 
     /* Figure out what kind of errhandler it is, figure out if it's

ompi/errhandler/errhandler_predefined.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
+ * Copyright (c) 2004-2020 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -47,7 +47,7 @@
 /*
  * Local functions
  */
-static void backend_fatal(char *type, struct ompi_communicator_t *comm,
+static void backend_abort(int fatal, char *type, struct ompi_communicator_t *comm,
                           char *name, int *error_code, va_list arglist);
 static void out(char *str, char *arg);
 
@@ -68,7 +68,7 @@ void ompi_mpi_errors_are_fatal_comm_handler(struct ompi_communicator_t **comm,
       name = NULL;
       abort_comm = NULL;
   }
-  backend_fatal("communicator", abort_comm, name, error_code, arglist);
+  backend_abort(true, "communicator", abort_comm, name, error_code, arglist);
   va_end(arglist);
 }
 
@@ -89,7 +89,7 @@ void ompi_mpi_errors_are_fatal_file_handler(struct ompi_file_t **file,
       name = NULL;
       abort_comm = NULL;
   }
-  backend_fatal("file", abort_comm, name, error_code, arglist);
+  backend_abort(true, "file", abort_comm, name, error_code, arglist);
   va_end(arglist);
 }
 
@@ -108,7 +108,67 @@ void ompi_mpi_errors_are_fatal_win_handler(struct ompi_win_t **win,
   } else {
       name = NULL;
   }
-  backend_fatal("win", abort_comm, name, error_code, arglist);
+  backend_abort(true, "win", abort_comm, name, error_code, arglist);
+  va_end(arglist);
+}
+
+void ompi_mpi_errors_abort_comm_handler(struct ompi_communicator_t **comm,
+                                            int *error_code, ...)
+{
+  char *name;
+  struct ompi_communicator_t *abort_comm;
+  va_list arglist;
+
+  va_start(arglist, error_code);
+
+  if ( (NULL != comm) && (NULL != *comm) ) {
+      name = (*comm)->c_name;
+      abort_comm = *comm;
+  } else {
+      name = NULL;
+      abort_comm = NULL;
+  }
+  backend_abort(false, "communicator", abort_comm, name, error_code, arglist);
+  va_end(arglist);
+}
+
+
+void ompi_mpi_errors_abort_file_handler(struct ompi_file_t **file,
+                                            int *error_code, ...)
+{
+  char *name;
+  struct ompi_communicator_t *abort_comm;
+  va_list arglist;
+
+  va_start(arglist, error_code);
+
+  if (NULL != file) {
+      name = (*file)->f_filename;
+      abort_comm = (*file)->f_comm;
+  } else {
+      name = NULL;
+      abort_comm = NULL;
+  }
+  backend_abort(false, "file", abort_comm, name, error_code, arglist);
+  va_end(arglist);
+}
+
+
+void ompi_mpi_errors_abort_win_handler(struct ompi_win_t **win,
+                                           int *error_code, ...)
+{
+  char *name;
+  struct ompi_communicator_t *abort_comm = NULL;
+  va_list arglist;
+
+  va_start(arglist, error_code);
+
+  if (NULL != win) {
+      name = (*win)->w_name;
+  } else {
+      name = NULL;
+  }
+  backend_abort(false, "win", abort_comm, name, error_code, arglist);
   va_end(arglist);
 }
 
@@ -175,7 +235,7 @@ static void out(char *str, char *arg)
  * there's no need to handle the pre-MPI_INIT and post-MPI_FINALIZE
  * errors here.
  */
-static void backend_fatal_aggregate(char *type,
+static void backend_abort_aggregate(int fatal, char *type,
                                     struct ompi_communicator_t *comm,
                                     char *name, int *error_code,
                                     va_list arglist)
@@ -199,7 +259,7 @@ static void backend_fatal_aggregate(char *type,
                  ompi_process_info.nodename,
                  (int) ompi_process_info.pid) == -1) {
         prefix = NULL;
-        // non-fatal, we could still go on to give useful information here...
+        // non-abort, we could still go on to give useful information here...
         opal_output(0, "%s", "Could not write node and PID to prefix");
         opal_output(0, "Node: %s", ompi_process_info.nodename);
         opal_output(0, "PID: %d", (int) ompi_process_info.pid);
@@ -224,7 +284,7 @@ static void backend_fatal_aggregate(char *type,
 
     if (NULL != name) {
         opal_show_help("help-mpi-errors.txt",
-                       "mpi_errors_are_fatal",
+                       fatal? "mpi_errors_are_fatal": "mpi_errors_abort",
                        false,
                        usable_prefix,
                        (NULL == arg) ? "" : "in",
@@ -267,15 +327,15 @@ static void backend_fatal_aggregate(char *type,
 
 /*
  * Note that this function has to handle pre-MPI_INIT and
- * post-MPI_FINALIZE errors, which backend_fatal_aggregate() does not
+ * post-MPI_FINALIZE errors, which backend_abort_aggregate() does not
  * have to handle.
  *
  * This function also intentionally does not call malloc(), just in
  * case we're being called due to some kind of stack/memory error --
  * we *might* be able to get a message out if we're not further
  * corrupting the stack by calling malloc()...
  */
-static void backend_fatal_no_aggregate(char *type,
+static void backend_abort_no_aggregate(int fatal, char *type,
                                        struct ompi_communicator_t *comm,
                                        char *name, int *error_code,
                                        va_list arglist)
@@ -303,7 +363,7 @@ static void backend_fatal_no_aggregate(char *type,
                 "*** Unfortunately, no further information is available on *which* MPI\n"
                 "*** function was invoked, sorry.  :-(\n", NULL);
         }
-        out("*** Your MPI job will now abort.\n", NULL);
+        if(fatal) out("*** Your MPI job will now abort.\n", NULL);
     } else if (state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
         if (NULL != arg) {
             out("*** The %s() function was called after MPI_FINALIZE was invoked.\n"
@@ -314,7 +374,7 @@ static void backend_fatal_no_aggregate(char *type,
                 "*** Unfortunately, no further information is available on *which* MPI\n"
                 "*** function was invoked, sorry.  :-(\n", NULL);
         }
-        out("*** Your MPI job will now abort.\n", NULL);
+        if(fatal) out("*** Your MPI job will now abort.\n", NULL);
     }
 
     else {
@@ -365,23 +425,30 @@ static void backend_fatal_no_aggregate(char *type,
                 out("*** Error code: %d (no associated error message)\n", intbuf);
             }
         }
-        /* out("*** MPI_ERRORS_ARE_FATAL: your MPI job will now abort\n", NULL); */
-        out("*** MPI_ERRORS_ARE_FATAL (processes in this %s will now abort,\n", type);
-        out("***    and potentially your MPI job)\n", NULL);
-
+        /* out("*** MPI_ERRORS_ABORT: your MPI job will now abort\n", NULL); */
+        if(fatal) {
+            out("*** MPI_ERRORS_ARE_FATAL (processes in this %s will now abort,\n", type);
+            out("***    and MPI will try to terminate your MPI job as well)\n", NULL);
+        }
+        else {
+            out("*** MPI_ERRORS_ABORT (processes in this %s will now abort,\n", type);
+            out("***    and potentially the rest of your MPI job)\n", NULL);
+        }
     }
     va_end(arglist);
 }
 
-static void backend_fatal(char *type, struct ompi_communicator_t *comm,
+static void backend_abort(int fatal, char *type, struct ompi_communicator_t *comm,
                           char *name, int *error_code,
                           va_list arglist)
 {
+    int err = MPI_ERR_UNKNOWN;
+
     /* We only want aggregation while the rte is initialized */
     if (ompi_rte_initialized) {
-        backend_fatal_aggregate(type, comm, name, error_code, arglist);
+        backend_abort_aggregate(fatal, type, comm, name, error_code, arglist);
     } else {
-        backend_fatal_no_aggregate(type, comm, name, error_code, arglist);
+        backend_abort_no_aggregate(fatal, type, comm, name, error_code, arglist);
     }
 
     /* In most instances the communicator will be valid. If not, we are either early in
@@ -392,9 +459,9 @@ static void backend_fatal(char *type, struct ompi_communicator_t *comm,
         comm = &ompi_mpi_comm_self.comm;
     }
 
-    if (NULL != error_code) {
-        ompi_mpi_abort(comm, *error_code);
-    } else {
-        ompi_mpi_abort(comm, 1);
-    }
+    if (NULL != error_code)
+        err = *error_code;
+
+    /* Call abort without a specified comm to force RTE Job termination */
+    ompi_mpi_abort(fatal? NULL: comm, err);
 }