Merge pull request #10627 from jjhursey/patch-dep-send

jjhursey · web-flow · commit 5bc215a06b0f · 2022-08-08T08:37:45.000-05:00
MPI-4: Issue warning when MPI_Cancel called for non blocking send request, MPI_Isend
diff --git a/ompi/mca/pml/base/pml_base_frame.c b/ompi/mca/pml/base/pml_base_frame.c
@@ -15,7 +15,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -42,6 +42,9 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/base.h"
 #include "ompi/mca/pml/base/pml_base_request.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 /*
  * The following file was created by configure.  It contains extern
@@ -100,12 +103,28 @@ char *ompi_pml_base_bsend_allocator_name = NULL;
 static char *ompi_pml_base_wrapper = NULL;
 #endif
 
+#if MPI_VERSION >= 4
+#define OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE
+int ompi_pml_base_warn_dep_cancel_send_level = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT;
+mca_base_var_enum_value_t ompi_pml_base_warn_dep_cancel_send_values[] = {
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS, .string = "always"},
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE,   .string = "once"},
+    {.value = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER,  .string = "never"},
+    {0, NULL}
+};
+#endif
+
 static int mca_pml_base_register(mca_base_register_flag_t flags)
 {
 #if !MCA_ompi_pml_DIRECT_CALL
     int var_id;
 #endif
 
+#if MPI_VERSION >= 4
+    mca_base_var_enum_t *ompi_pml_base_warn_dep_cancel_send_enum = NULL;
+    int rc;
+#endif
+
     ompi_pml_base_bsend_allocator_name = "basic";
     (void) mca_base_var_register("ompi", "pml", "base", "bsend_allocator", NULL,
                                  MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
@@ -124,6 +143,21 @@ static int mca_pml_base_register(mca_base_register_flag_t flags)
     (void) mca_base_var_register_synonym(var_id, "ompi", "pml", NULL, "wrapper", 0);
 #endif
 
+#if MPI_VERSION >= 4
+    mca_base_var_enum_create("pml_base_deprecate_warnings", ompi_pml_base_warn_dep_cancel_send_values,
+                             &ompi_pml_base_warn_dep_cancel_send_enum);
+    rc = mca_base_var_register("ompi", "pml", "base", "warn_dep_cancel_send",
+                               "How often to issue warnings for deprecated cancelation of send requests",
+                               MCA_BASE_VAR_TYPE_INT, ompi_pml_base_warn_dep_cancel_send_enum, 0, 0,
+                               OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                               &ompi_pml_base_warn_dep_cancel_send_level);
+    OBJ_RELEASE(ompi_pml_base_warn_dep_cancel_send_enum);
+    if (OPAL_ERR_VALUE_OUT_OF_BOUNDS == rc) {
+        ompi_pml_base_warn_dep_cancel_send_level = OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_DEFAULT;
+        opal_output(0, "pml:base:register: Warning invalid deprecation warning value specified. Using default: %d",
+                    ompi_pml_base_warn_dep_cancel_send_level);
+    }
+#endif
     return OMPI_SUCCESS;
 }
 
diff --git a/ompi/mca/pml/base/pml_base_sendreq.c b/ompi/mca/pml/base/pml_base_sendreq.c
@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -19,6 +20,9 @@
 #include <string.h>
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/pml_base_sendreq.h"
+#if MPI_VERSION >= 4
+#include "opal/sys/atomic.h"
+#endif
 
 static void mca_pml_base_send_request_construct(mca_pml_base_send_request_t* req);
 static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req);
@@ -47,3 +51,17 @@ static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req)
      */
 }
 
+#if MPI_VERSION >= 4
+int mca_pml_cancel_send_callback(struct ompi_request_t *request, int flag)
+{
+    static opal_atomic_int32_t send_deprecate_count = 0;
+    int32_t val;
+
+    val = opal_atomic_add_fetch_32(&send_deprecate_count, 1);
+    if ( (OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE   == ompi_pml_base_warn_dep_cancel_send_level && (1 == val)) ||
+         (OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS == ompi_pml_base_warn_dep_cancel_send_level) ) {
+        opal_output(0, "WARNING: Calling MPI_Cancel for a request created by a non-blocking send is deprecated.");
+    }
+    return OMPI_SUCCESS;
+}
+#endif
diff --git a/ompi/mca/pml/base/pml_base_sendreq.h b/ompi/mca/pml/base/pml_base_sendreq.h
@@ -16,6 +16,7 @@
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2017      Intel, Inc. All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -67,6 +68,19 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
  * is done in the send request start routine.
  */
 
+#if MPI_VERSION >= 4
+/*
+ * Deprecation of MPI_Cancel for non blocking sends in MPI 4.0 and later
+ * is handled by adding a callback for the cancel request.
+ */
+#define MCA_PML_BASE_SEND_CANCEL_CALLBACK(req)                              \
+    if (NULL == (req)->req_base.req_ompi.req_cancel) {                      \
+        (req)->req_base.req_ompi.req_cancel = mca_pml_cancel_send_callback; \
+    }
+#else
+#define MCA_PML_BASE_SEND_CANCEL_CALLBACK(req)
+#endif
+
 #define MCA_PML_BASE_SEND_REQUEST_INIT( request,                          \
                                         addr,                             \
                                         count,                            \
@@ -95,6 +109,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
       (request)->req_base.req_pml_complete = false;                       \
       (request)->req_base.req_free_called = false;                        \
       (request)->req_base.req_ompi.req_status._cancelled = 0;             \
+      MCA_PML_BASE_SEND_CANCEL_CALLBACK(request)                          \
       (request)->req_bytes_packed = 0;                                    \
                                                                           \
       /* initialize datatype convertor for this request */                \
@@ -153,6 +168,21 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION( mca_pml_base_send_request_t );
     } while (0)
 
 
+#if MPI_VERSION >= 4
+enum {
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ALWAYS = 1,
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_ONCE   = 2,
+      OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER  = 3,
+};
+OMPI_DECLSPEC extern int ompi_pml_base_warn_dep_cancel_send_level;
+
+/**
+ * Cancellation callback used to issue a deprecation warning on
+ * nonblocking sends (per MPI 4.0).
+ */
+OMPI_DECLSPEC extern int mca_pml_cancel_send_callback(struct ompi_request_t *request, int flag);
+#endif
+
 END_C_DECLS
 
 #endif
diff --git a/ompi/mca/pml/ob1/pml_ob1_isend.c b/ompi/mca/pml/ob1/pml_ob1_isend.c
@@ -15,6 +15,7 @@
  * Copyright (c) 2014-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +30,9 @@
 #include "pml_ob1_recvreq.h"
 #include "ompi/peruse/peruse-internal.h"
 #include "ompi/runtime/ompi_spc.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 /**
  * Single usage request. As we allow recursive calls (as an
@@ -181,6 +185,12 @@ int mca_pml_ob1_isend(const void *buf,
             /* NTH: it is legal to return ompi_request_empty since the only valid
              * field in a send completion status is whether or not the send was
              * cancelled (which it can't be at this point anyway). */
+#if MPI_VERSION >= 4
+            if (OPAL_UNLIKELY(OMPI_PML_BASE_WARN_DEP_CANCEL_SEND_NEVER != ompi_pml_base_warn_dep_cancel_send_level)) {
+                *request = &ompi_request_empty_send;
+                return OMPI_SUCCESS;
+            }
+#endif
             *request = &ompi_request_empty;
             return OMPI_SUCCESS;
         }
diff --git a/ompi/mca/pml/ob1/pml_ob1_sendreq.c b/ompi/mca/pml/ob1/pml_ob1_sendreq.c
@@ -21,6 +21,7 @@
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -127,6 +128,10 @@ static int mca_pml_ob1_send_request_free(struct ompi_request_t** request)
 
 static int mca_pml_ob1_send_request_cancel(struct ompi_request_t* request, int complete)
 {
+#if MPI_VERSION >= 4
+    mca_pml_cancel_send_callback(request, complete);
+#endif
+
 #if OPAL_ENABLE_FT_MPI
     ompi_communicator_t* comm = request->req_mpi_object.comm;
     mca_pml_ob1_send_request_t* pml_req = (mca_pml_ob1_send_request_t*)request;
diff --git a/ompi/mca/pml/ucx/pml_ucx.c b/ompi/mca/pml/ucx/pml_ucx.c
@@ -5,7 +5,7 @@
  *                         reserved.
  * Copyright (c) 2018-2019 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2018 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
@@ -358,6 +358,9 @@ int mca_pml_ucx_init(int enable_mpi_threads)
     /* Create a completed request to be returned from isend */
     OBJ_CONSTRUCT(&ompi_pml_ucx.completed_send_req, ompi_request_t);
     mca_pml_ucx_completed_request_init(&ompi_pml_ucx.completed_send_req);
+#if MPI_VERSION >= 4
+    ompi_pml_ucx.completed_send_req.req_cancel = mca_pml_cancel_send_callback;
+#endif
 
     opal_progress_register(mca_pml_ucx_progress);
 
@@ -924,6 +927,11 @@ int mca_pml_ucx_isend(const void *buf, size_t count, ompi_datatype_t *datatype,
     } else if (!UCS_PTR_IS_ERR(req)) {
         PML_UCX_VERBOSE(8, "got request %p", (void*)req);
         req->req_mpi_object.comm = comm;
+#if MPI_VERSION >= 4
+        if (OPAL_LIKELY(mca_pml_ucx_request_cancel == req->req_cancel)) {
+            req->req_cancel      = mca_pml_ucx_request_cancel_send;
+        }
+#endif
         *request                 = req;
         return OMPI_SUCCESS;
     } else {
diff --git a/ompi/mca/pml/ucx/pml_ucx_request.c b/ompi/mca/pml/ucx/pml_ucx_request.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2016      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -14,6 +15,7 @@
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "ompi/message/message.h"
 #include "ompi/runtime/ompi_spc.h"
+#include "ompi/request/request.h"
 #include <inttypes.h>
 
 
@@ -29,12 +31,20 @@ static int mca_pml_ucx_request_free(ompi_request_t **rptr)
     return OMPI_SUCCESS;
 }
 
-static int mca_pml_ucx_request_cancel(ompi_request_t *req, int flag)
+int mca_pml_ucx_request_cancel(ompi_request_t *req, int flag)
 {
     ucp_request_cancel(ompi_pml_ucx.ucp_worker, req);
     return OMPI_SUCCESS;
 }
 
+#if MPI_VERSION >= 4
+int mca_pml_ucx_request_cancel_send(ompi_request_t *req, int flag)
+{
+    mca_pml_cancel_send_callback(req, flag);
+    return mca_pml_ucx_request_cancel(req, flag);
+}
+#endif
+
 __opal_attribute_always_inline__ static inline void
 mca_pml_ucx_send_completion_internal(void *request, ucs_status_t status)
 {
diff --git a/ompi/mca/pml/ucx/pml_ucx_request.h b/ompi/mca/pml/ucx/pml_ucx_request.h
@@ -3,6 +3,7 @@
  * Copyright (c) 2016-2021 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -15,6 +16,9 @@
 
 #include "pml_ucx.h"
 #include "pml_ucx_datatype.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 
 enum {
@@ -145,6 +149,11 @@ void mca_pml_ucx_request_init(void *request);
 
 void mca_pml_ucx_request_cleanup(void *request);
 
+int mca_pml_ucx_request_cancel(ompi_request_t *req, int flag);
+#if MPI_VERSION >= 4
+int mca_pml_ucx_request_cancel_send(ompi_request_t *req, int flag);
+#endif
+
 
 static inline void mca_pml_ucx_request_reset(ompi_request_t *req)
 {
diff --git a/ompi/request/request.c b/ompi/request/request.c
@@ -20,6 +20,7 @@
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,11 +35,17 @@
 #include "ompi/request/request.h"
 #include "ompi/request/request_default.h"
 #include "ompi/constants.h"
+#if MPI_VERSION >= 4
+#include "ompi/mca/pml/base/pml_base_sendreq.h"
+#endif
 
 opal_pointer_array_t             ompi_request_f_to_c_table = {{0}};
 ompi_predefined_request_t        ompi_request_null = {{{{{0}}}}};
 ompi_predefined_request_t        *ompi_request_null_addr = &ompi_request_null;
 ompi_request_t                   ompi_request_empty = {{{{0}}}};
+#if MPI_VERSION >= 4
+ompi_request_t                   ompi_request_empty_send = {{{{0}}}};
+#endif
 ompi_status_public_t             ompi_status_empty = {0};
 ompi_request_fns_t               ompi_request_functions = {
     ompi_request_default_test,
@@ -179,6 +186,36 @@ int ompi_request_init(void)
         return OMPI_ERR_REQUEST;
     }
 
+#if MPI_VERSION >= 4
+    /*
+     * This is a copy of the ompi_request_empty object where the only difference
+     * is that the req_cancel callback is set to a callback which issues a
+     * message that calling MPI_Cancel on a non-blocking send request is
+     * deprecated.
+     */
+    OBJ_CONSTRUCT(&ompi_request_empty_send, ompi_request_t);
+    ompi_request_empty_send.req_type = OMPI_REQUEST_NULL;
+    ompi_request_empty_send.req_status.MPI_SOURCE = MPI_PROC_NULL;
+    ompi_request_empty_send.req_status.MPI_TAG = MPI_ANY_TAG;
+    ompi_request_empty_send.req_status.MPI_ERROR = MPI_SUCCESS;
+    ompi_request_empty_send.req_status._ucount = 0;
+    ompi_request_empty_send.req_status._cancelled = 0;
+
+    ompi_request_empty_send.req_complete = REQUEST_COMPLETED;
+    ompi_request_empty_send.req_state = OMPI_REQUEST_ACTIVE;
+    ompi_request_empty_send.req_persistent = false;
+    ompi_request_empty_send.req_f_to_c_index =
+        opal_pointer_array_add(&ompi_request_f_to_c_table, &ompi_request_empty_send);
+    ompi_request_empty_send.req_start = NULL; /* should not be called */
+    ompi_request_empty_send.req_free = ompi_request_empty_free;
+    ompi_request_empty_send.req_cancel = mca_pml_cancel_send_callback;
+    ompi_request_empty_send.req_mpi_object.comm = &ompi_mpi_comm_world.comm;
+
+    if (2 != ompi_request_empty_send.req_f_to_c_index) {
+        return OMPI_ERR_REQUEST;
+    }
+#endif
+
     ompi_status_empty.MPI_SOURCE = MPI_ANY_SOURCE;
     ompi_status_empty.MPI_TAG = MPI_ANY_TAG;
     ompi_status_empty.MPI_ERROR = MPI_SUCCESS;
diff --git a/ompi/request/request.h b/ompi/request/request.h
@@ -18,6 +18,7 @@
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2018      Triad National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -383,6 +384,9 @@ OMPI_DECLSPEC extern ompi_predefined_request_t        *ompi_request_null_addr;
 OMPI_DECLSPEC extern ompi_request_t         ompi_request_empty;
 OMPI_DECLSPEC extern ompi_status_public_t   ompi_status_empty;
 OMPI_DECLSPEC extern ompi_request_fns_t     ompi_request_functions;
+#if MPI_VERSION >= 4
+OMPI_DECLSPEC extern ompi_request_t         ompi_request_empty_send;
+#endif
 
 /**
  * Initialize the MPI_Request subsystem; invoked during MPI_INIT.
