MPI-4: Issue warning when MPI_Cancel called for non blocking send request, MPI_Isend

 * Issue a deprecation warning when MPI_Cancel is called referencing
   a request created by a point to point non-blocking send, such as MPI_Isend.
 * The warning is issued only when MPI version is >= 4
 * This change only applies to the ob1 and UCX components since we don't have
   systems to test anything else.
 * The message is controlled on a per-task basis by a new MCA option,
   `--mca pml_base_warn_dep_cancel_send {always | once | never}`

Signed-off-by: David Wootton <dwootton@us.ibm.com>
Signed-off-by: Joshua Hursey <jhursey@us.ibm.com>

Author: David Wootton

ompi/mca/pml/base/pml_base_frame.c

@@ -15,7 +15,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -42,6 +42,9 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/base.h"
 #include "ompi/mca/pml/base/pml_base_request.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 /*
  * The following file was created by configure.  It contains extern
@@ -100,12 +103,28 @@ char *ompi_pml_base_bsend_allocator_name = NULL;
 static char *ompi_pml_base_wrapper = NULL;
 #endif
 
+#if MPI_VERSION >= 4
+#define OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE
+int ompi_pml_base_warn_dep_cancel_send_level = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT;
+mca_base_var_enum_value_t ompi_pml_base_warn_dep_cancel_send_values[] = {
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS, .string = "always"},
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE,   .string = "once"},
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER,  .string = "never"},
+    {0, NULL}
+};
+#endif
+
 static int mca_pml_base_register(mca_base_register_flag_t flags)
 {
 #if !MCA_ompi_pml_DIRECT_CALL
     int var_id;
 #endif
 
+#if MPI_VERSION >= 4
+    mca_base_var_enum_t *ompi_pml_base_warn_dep_cancel_send_enum = NULL;
+    int rc;
+#endif
+
     ompi_pml_base_bsend_allocator_name = "basic";
     (void) mca_base_var_register("ompi", "pml", "base", "bsend_allocator", NULL,
                                  MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
@@ -124,6 +143,21 @@ static int mca_pml_base_register(mca_base_register_flag_t flags)
     (void) mca_base_var_register_synonym(var_id, "ompi", "pml", NULL, "wrapper", 0);
 #endif
 
+#if MPI_VERSION >= 4
+    mca_base_var_enum_create("pml_base_deprecate_warnings", ompi_pml_base_warn_dep_cancel_send_values,
+                             &ompi_pml_base_warn_dep_cancel_send_enum);
+    rc = mca_base_var_register("ompi", "pml", "base", "warn_dep_cancel_send",
+                               "How often to issue warnings for deprecated cancelation of send requests",
+                               MCA_BASE_VAR_TYPE_INT, ompi_pml_base_warn_dep_cancel_send_enum, 0, 0,
+                               OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                               &ompi_pml_base_warn_dep_cancel_send_level);
+    OBJ_RELEASE(ompi_pml_base_warn_dep_cancel_send_enum);
+    if (OPAL_ERR_VALUE_OUT_OF_BOUNDS == rc) {
+        ompi_pml_base_warn_dep_cancel_send_level = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT;
+        opal_output(0, "pml:base:register: Warning invalid deprecation warning value specified. Using default: %d",
+                    ompi_pml_base_warn_dep_cancel_send_level);
+    }
+#endif
     return OMPI_SUCCESS;
 }
 

ompi/mca/pml/base/pml_base_sendreq.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,6 +20,9 @@
 #include <string.h>
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/pml_base_sendreq.h"
+#if MPI_VERSION >= 4
+#include "opal/sys/atomic.h"
+#endif
 
 static void mca_pml_base_send_request_construct(mca_pml_base_send_request_t* req);
 static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req);
@@ -47,3 +51,17 @@ static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req)
      */
 }
 
+#if MPI_VERSION >= 4
+int mca_pml_cancel_send_callback(struct ompi_request_t *request, int flag)
+{
+    static opal_atomic_int32_t send_deprecate_count = 0;
+    int32_t val;
+
+    val = opal_atomic_add_fetch_32(&send_deprecate_count, 1);
+    if ( (OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE   == ompi_pml_base_warn_dep_cancel_send_level && (1 == val)) ||
+         (OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS == ompi_pml_base_warn_dep_cancel_send_level) ) {
+        opal_output(0, "WARNING: Calling MPI_Cancel for a request created by a non-blocking send is deprecated.");
+    }
+    return OMPI_SUCCESS;
+}
+#endif

ompi/mca/pml/base/pml_base_sendreq.h

@@ -16,6 +16,7 @@
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -67,6 +68,19 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
  * is done in the send request start routine.
  */
 
+#if MPI_VERSION >= 4
+/*
+ * Deprecation of MPI_Cancel for non blocking sends in MPI 4.0 and later
+ * is handled by adding a callback for the cancel request.
+ */
+#define MCA_PML_BASE_SEND_CANCEL_CALLBACK(req)                              \
+    if (NULL == (req)->req_base.req_ompi.req_cancel) {                      \
+        (req)->req_base.req_ompi.req_cancel = mca_pml_cancel_send_callback; \
+    }
+#else
+#define MCA_PML_BASE_SEND_CANCEL_CALLBACK(req)
+#endif
+
 #define MCA_PML_BASE_SEND_REQUEST_INIT( request,                          \
                                         addr,                             \
                                         count,                            \
@@ -95,6 +109,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
       (request)->req_base.req_pml_complete = false;                       \
       (request)->req_base.req_free_called = false;                        \
       (request)->req_base.req_ompi.req_status._cancelled = 0;             \
+      MCA_PML_BASE_SEND_CANCEL_CALLBACK(request)                          \
       (request)->req_bytes_packed = 0;                                    \
                                                                           \
       /* initialize datatype convertor for this request */                \
@@ -153,6 +168,21 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
     } while (0)
 
 
+#if MPI_VERSION >= 4
+enum {
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS = 1,
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE   = 2,
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER  = 3,
+};
+OMPI_DECLSPEC extern int ompi_pml_base_warn_dep_cancel_send_level;
+
+/**
+ * Cancellation callback used to issue a deprecation warning on
+ * nonblocking sends (per MPI 4.0).
+ */
+OMPI_DECLSPEC extern int mca_pml_cancel_send_callback(struct ompi_request_t *request, int flag);
+#endif
+
 END_C_DECLS
 
 #endif

ompi/mca/pml/ob1/pml_ob1_isend.c

@@ -15,6 +15,7 @@
  * Copyright (c) 2014-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +30,9 @@
 #include "pml_ob1_recvreq.h"
 #include "ompi/peruse/peruse-internal.h"
 #include "ompi/runtime/ompi_spc.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 /**
  * Single usage request. As we allow recursive calls (as an
@@ -181,6 +185,12 @@ int mca_pml_ob1_isend(const void *buf,
             /* NTH: it is legal to return ompi_request_empty since the only valid
              * field in a send completion status is whether or not the send was
              * cancelled (which it can't be at this point anyway). */
+#if MPI_VERSION >= 4
+            if (OPAL_UNLIKELY(OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER != ompi_pml_base_warn_dep_cancel_send_level)) {
+                *request = &ompi_request_empty_send;
+                return OMPI_SUCCESS;
+            }
+#endif
             *request = &ompi_request_empty;
             return OMPI_SUCCESS;
         }

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -21,6 +21,7 @@
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -127,6 +128,10 @@ static int mca_pml_ob1_send_request_free(struct ompi_request_t** request)
 
 static int mca_pml_ob1_send_request_cancel(struct ompi_request_t* request, int complete)
 {
+#if MPI_VERSION >= 4
+    mca_pml_cancel_send_callback(request, complete);
+#endif
+
 #if OPAL_ENABLE_FT_MPI
     ompi_communicator_t* comm = request->req_mpi_object.comm;
     mca_pml_ob1_send_request_t* pml_req = (mca_pml_ob1_send_request_t*)request;

ompi/request/request.c

@@ -20,6 +20,7 @@
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,11 +35,17 @@
 #include "ompi/request/request.h"
 #include "ompi/request/request_default.h"
 #include "ompi/constants.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 opal_pointer_array_t             ompi_request_f_to_c_table = {{0}};
 ompi_predefined_request_t        ompi_request_null = {{{{{0}}}}};
 ompi_predefined_request_t        *ompi_request_null_addr = &ompi_request_null;
 ompi_request_t                   ompi_request_empty = {{{{0}}}};
+#if MPI_VERSION >= 4
+ompi_request_t                   ompi_request_empty_send = {{{{0}}}};
+#endif
 ompi_status_public_t             ompi_status_empty = {0};
 ompi_request_fns_t               ompi_request_functions = {
     ompi_request_default_test,
@@ -179,6 +186,36 @@ int ompi_request_init(void)
         return OMPI_ERR_REQUEST;
     }
 
+#if MPI_VERSION >= 4
+    /*
+     * This is a copy of the ompi_request_empty object where the only difference
+     * is that the req_cancel callback is set to a callback which issues a
+     * message that calling MPI_Cancel on a non-blocking send request is
+     * deprecated.
+     */
+    OBJ_CONSTRUCT(&ompi_request_empty_send, ompi_request_t);
+    ompi_request_empty_send.req_type = OMPI_REQUEST_NULL;
+    ompi_request_empty_send.req_status.MPI_SOURCE = MPI_PROC_NULL;
+    ompi_request_empty_send.req_status.MPI_TAG = MPI_ANY_TAG;
+    ompi_request_empty_send.req_status.MPI_ERROR = MPI_SUCCESS;
+    ompi_request_empty_send.req_status._ucount = 0;
+    ompi_request_empty_send.req_status._cancelled = 0;
+
+    ompi_request_empty_send.req_complete = REQUEST_COMPLETED;
+    ompi_request_empty_send.req_state = OMPI_REQUEST_ACTIVE;
+    ompi_request_empty_send.req_persistent = false;
+    ompi_request_empty_send.req_f_to_c_index =
+        opal_pointer_array_add(&ompi_request_f_to_c_table, &ompi_request_empty_send);
+    ompi_request_empty_send.req_start = NULL; /* should not be called */
+    ompi_request_empty_send.req_free = ompi_request_empty_free;
+    ompi_request_empty_send.req_cancel = mca_pml_cancel_send_callback;
+    ompi_request_empty_send.req_mpi_object.comm = &ompi_mpi_comm_world.comm;
+
+    if (2 != ompi_request_empty_send.req_f_to_c_index) {
+        return OMPI_ERR_REQUEST;
+    }
+#endif
+
     ompi_status_empty.MPI_SOURCE = MPI_ANY_SOURCE;
     ompi_status_empty.MPI_TAG = MPI_ANY_TAG;
     ompi_status_empty.MPI_ERROR = MPI_SUCCESS;

ompi/request/request.h

@@ -18,6 +18,7 @@
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -383,6 +384,9 @@ OMPI_DECLSPEC extern ompi_predefined_request_t        *ompi_request_null_addr;
 OMPI_DECLSPEC extern ompi_request_t         ompi_request_empty;
 OMPI_DECLSPEC extern ompi_status_public_t   ompi_status_empty;
 OMPI_DECLSPEC extern ompi_request_fns_t     ompi_request_functions;
+#if MPI_VERSION >= 4
+OMPI_DECLSPEC extern ompi_request_t         ompi_request_empty_send;
+#endif
 
 /**
  * Initialize the MPI_Request subsystem; invoked during MPI_INIT.