btl/ofi: Add cache bypass mechanism

Libfabric has its own registration cache which can cause
some conflicts, particularly with cuda buffers due to their
delayed invalidation strategy involving buffer ids.

Added a cache bypass flag for providers known to use
a registration cache.

Signed-off-by: William Zhang <wilzhang@amazon.com>

Author: wckzhang

opal/mca/btl/ofi/btl_ofi.h

@@ -140,6 +140,9 @@ struct mca_btl_ofi_module_t {
 
     /** registration cache */
     mca_rcache_base_module_t *rcache;
+    /* If the underlying OFI provider has its own cache, we want to bypass
+     * rcache registration */
+    bool bypass_cache;
 };
 typedef struct mca_btl_ofi_module_t mca_btl_ofi_module_t;
 

opal/mca/btl/ofi/btl_ofi_component.c

@@ -575,6 +575,7 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     module->outstanding_rdma = 0;
     module->use_virt_addr = false;
     module->use_fi_mr_bind = false;
+    module->bypass_cache = false;
 
     if (ofi_info->domain_attr->mr_mode == FI_MR_BASIC
         || ofi_info->domain_attr->mr_mode & FI_MR_VIRT_ADDR) {
@@ -585,6 +586,13 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
         module->use_fi_mr_bind = true;
     }
 
+    /* Currently there is no API to query whether the libfabric provider
+     * uses an underlying registration cache. For now, just check for known
+     * providers that use registration caching. */
+    if (!strncasecmp(info->fabric_attr->prov_name, "efa", 3)) {
+        module->bypass_cache = true;
+    }
+
     /* create endpoint list */
     OBJ_CONSTRUCT(&module->endpoints, opal_list_t);
     OBJ_CONSTRUCT(&module->module_lock, opal_mutex_t);

opal/mca/btl/ofi/btl_ofi_module.c

@@ -194,8 +194,12 @@ mca_btl_ofi_register_mem(struct mca_btl_base_module_t *btl,
     mca_btl_ofi_reg_t *reg;
     int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
     int rc;
+    uint32_t cache_flags = 0;
+    if (ofi_module->bypass_cache) {
+	   cache_flags |= MCA_RCACHE_FLAGS_CACHE_BYPASS;
+    }
 
-    rc = ofi_module->rcache->rcache_register(ofi_module->rcache, base, size, 0, access_flags,
+    rc = ofi_module->rcache->rcache_register(ofi_module->rcache, base, size, cache_flags, access_flags,
                                              (mca_rcache_base_registration_t **) &reg);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         return NULL;