Merge pull request open-mpi#9101 from jsquyres/pr/ofi-fixes

OFI and usNIC fixes

Author: jsquyres

ompi/mca/mtl/ofi/mtl_ofi.h

@@ -6,7 +6,7 @@
  *                         reserved.
  * Copyright (c) 2018-2020 Amazon.com, Inc. or its affiliates. All rights
  *                         reserved.
- *
+ * Copyright (c) 2021      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -68,16 +68,16 @@ extern int ompi_mtl_ofi_del_procs(struct mca_mtl_base_module_t *mtl,
 int ompi_mtl_ofi_progress_no_inline(void);
 
 #if OPAL_HAVE_THREAD_LOCAL
-extern opal_thread_local int per_thread_ctx;
-extern opal_thread_local struct fi_cq_tagged_entry wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
+extern opal_thread_local int ompi_mtl_ofi_per_thread_ctx;
+extern opal_thread_local struct fi_cq_tagged_entry ompi_mtl_ofi_wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
 #endif
 
 /* Set OFI context for operations which generate completion events */
 __opal_attribute_always_inline__ static inline void
 set_thread_context(int ctxt)
 {
 #if OPAL_HAVE_THREAD_LOCAL
-    per_thread_ctx = ctxt;
+    ompi_mtl_ofi_per_thread_ctx = ctxt;
     return;
 #endif
 }
@@ -87,7 +87,7 @@ __opal_attribute_always_inline__ static inline void
 get_thread_context(int *ctxt)
 {
 #if OPAL_HAVE_THREAD_LOCAL
-    *ctxt = per_thread_ctx;
+    *ctxt = ompi_mtl_ofi_per_thread_ctx;
 #endif
     return;
 }
@@ -106,24 +106,24 @@ ompi_mtl_ofi_context_progress(int ctxt_id)
     struct fi_cq_err_entry error = { 0 };
     ssize_t ret;
 #if !OPAL_HAVE_THREAD_LOCAL
-    struct fi_cq_tagged_entry wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
+    struct fi_cq_tagged_entry ompi_mtl_ofi_wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
 #endif
 
     /**
      * Read the work completions from the CQ.
      * From the completion's op_context, we get the associated OFI request.
      * Call the request's callback.
      */
-    ret = fi_cq_read(ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq, (void *)&wc,
+    ret = fi_cq_read(ompi_mtl_ofi.ofi_ctxt[ctxt_id].cq, (void *)&ompi_mtl_ofi_wc,
                      ompi_mtl_ofi.ofi_progress_event_count);
     if (ret > 0) {
         count+= ret;
         events_read = ret;
         for (i = 0; i < events_read; i++) {
-            if (NULL != wc[i].op_context) {
-                ofi_req = TO_OFI_REQ(wc[i].op_context);
+            if (NULL != ompi_mtl_ofi_wc[i].op_context) {
+                ofi_req = TO_OFI_REQ(ompi_mtl_ofi_wc[i].op_context);
                 assert(ofi_req);
-                ret = ofi_req->event_callback(&wc[i], ofi_req);
+                ret = ofi_req->event_callback(&ompi_mtl_ofi_wc[i], ofi_req);
                 if (OMPI_SUCCESS != ret) {
                     opal_output(0, "%s:%d: Error returned by request event callback: %zd.\n"
                                    "*** The Open MPI OFI MTL is aborting the MPI job (via exit(3)).\n",

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved
  *
- * Copyright (c) 2014-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2014-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
@@ -40,8 +40,8 @@ static int av_type;
 static int ofi_tag_mode;
 
 #if OPAL_HAVE_THREAD_LOCAL
-    opal_thread_local int per_thread_ctx;
-    opal_thread_local struct fi_cq_tagged_entry wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
+    opal_thread_local int ompi_mtl_ofi_per_thread_ctx;
+    opal_thread_local struct fi_cq_tagged_entry ompi_mtl_ofi_wc[MTL_OFI_MAX_PROG_EVENT_COUNT];
 #endif
 
 /*
@@ -374,8 +374,8 @@ select_ofi_provider(struct fi_info *providers,
 }
 
 static void
-ompi_mtl_ofi_define_tag_mode(int ofi_tag_mode, int *bits_for_cid) {
-    switch (ofi_tag_mode) {
+ompi_mtl_ofi_define_tag_mode(int ofi_tag_mode_arg, int *bits_for_cid) {
+    switch (ofi_tag_mode_arg) {
         case MTL_OFI_TAG_1:
             *bits_for_cid = (int) MTL_OFI_CID_BIT_COUNT_1;
             ompi_mtl_ofi.base.mtl_max_tag = (int)((1ULL << (MTL_OFI_TAG_BIT_COUNT_1 - 1)) - 1);

opal/mca/btl/usnic/btl_usnic_cagent.c

@@ -47,7 +47,7 @@ static opal_event_t ipc_event;
 static struct timeval ack_timeout;
 static opal_list_t udp_port_listeners;
 static opal_list_t ipc_listeners;
-static volatile int ipc_accepts = 0;
+static volatile uint32_t ipc_accepts = 0;
 /* JMS The pings_pending and ping_results should probably both be hash
    tables for more efficient lookups */
 static opal_list_t pings_pending;

opal/mca/btl/usnic/btl_usnic_stats.c

@@ -380,7 +380,7 @@ static void setup_mpit_pvars_enum(void)
 
     /* Free the strings (mca_base_var_enum_create() strdup()'ed them
        into private storage, so we don't need them any more) */
-    for (int i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
+    for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
         free((char *) devices[i].string);
     }
     free(devices);

opal/mca/common/ofi/common_ofi.c

@@ -4,7 +4,7 @@
  *                         reserved.
  * Copyright (c) 2020-2021 Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2020-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -31,7 +31,7 @@ OPAL_DECLSPEC opal_common_ofi_module_t opal_common_ofi = {.prov_include = NULL,
                                                           .registered = 0,
                                                           .verbose = 0};
 
-static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream";
+static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream,usnic";
 static opal_mutex_t opal_common_ofi_mutex = OPAL_MUTEX_STATIC_INIT;
 
 OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item)