io/ompio and fbtl: introduce a function to check for support of atomic ops

edgargabriel · edgargabriel · commit cb638c221529 · 2021-03-06T09:03:50.000-06:00
this is a new function in the fbtl framework. A default implementation is
provided in the fbtl/base directory (which simply returns false), but
an fbtl component can provide its own implementation. As of right now,
the ime and pvfs2 components use the default from base, posix provides
its own check_atomicity function.

Signed-off-by: Edgar Gabriel &lt;egabriel@central.uh.edu&gt;
diff --git a/ompi/mca/fbtl/base/base.h b/ompi/mca/fbtl/base/base.h
@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -50,6 +50,8 @@ OMPI_DECLSPEC int mca_fbtl_base_find_available(bool enable_progress_threads,
 OMPI_DECLSPEC int mca_fbtl_base_init_file (struct ompio_file_t *file);
 
 OMPI_DECLSPEC int mca_fbtl_base_get_param (struct ompio_file_t *file, int keyval);
+
+OMPI_DECLSPEC bool mca_fbtl_base_check_atomicity (struct ompio_file_t *file);
 /*
  * Globals
  */
diff --git a/ompi/mca/fbtl/base/fbtl_base_file_select.c b/ompi/mca/fbtl/base/fbtl_base_file_select.c
@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2011 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -259,3 +259,13 @@ int mca_fbtl_base_file_select (struct ompio_file_t *file,
 
     return err;
 }
+
+
+bool mca_fbtl_base_check_atomicity (struct ompio_file_t *file)
+{
+    /* by default, return false. An fbtl can overwrite this setting
+    ** if they have support for atomic operations through locks or other
+    ** measures.
+    */
+    return false;
+}
diff --git a/ompi/mca/fbtl/fbtl.h b/ompi/mca/fbtl/fbtl.h
@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2014 University of Houston. All rights reserved.
+ * Copyright (c) 2008-2021 University of Houston. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018      Research Organization for Information Science
@@ -135,6 +135,10 @@ typedef bool (*mca_fbtl_base_module_progress_fn_t)
 
 typedef void (*mca_fbtl_base_module_request_free_fn_t)
     ( struct mca_ompio_request_t *request);
+
+typedef bool (*mca_fbtl_base_module_check_atomicity_fn_t)
+    (struct ompio_file_t *file);
+
 /*
  * ***********************************************************************
  * ***************************  module structure *************************
@@ -150,12 +154,13 @@ struct mca_fbtl_base_module_1_0_0_t {
     mca_fbtl_base_module_finalize_1_0_0_fn_t fbtl_module_finalize;
 
     /* FBTL function pointers */
-    mca_fbtl_base_module_preadv_fn_t        fbtl_preadv;
-    mca_fbtl_base_module_ipreadv_fn_t       fbtl_ipreadv;
-    mca_fbtl_base_module_pwritev_fn_t       fbtl_pwritev;
-    mca_fbtl_base_module_ipwritev_fn_t      fbtl_ipwritev;
-    mca_fbtl_base_module_progress_fn_t      fbtl_progress;
-    mca_fbtl_base_module_request_free_fn_t  fbtl_request_free;
+    mca_fbtl_base_module_preadv_fn_t          fbtl_preadv;
+    mca_fbtl_base_module_ipreadv_fn_t         fbtl_ipreadv;
+    mca_fbtl_base_module_pwritev_fn_t         fbtl_pwritev;
+    mca_fbtl_base_module_ipwritev_fn_t        fbtl_ipwritev;
+    mca_fbtl_base_module_progress_fn_t        fbtl_progress;
+    mca_fbtl_base_module_request_free_fn_t    fbtl_request_free;
+    mca_fbtl_base_module_check_atomicity_fn_t fbtl_check_atomicity;
 };
 typedef struct mca_fbtl_base_module_1_0_0_t mca_fbtl_base_module_1_0_0_t;
 typedef mca_fbtl_base_module_1_0_0_t mca_fbtl_base_module_t;
diff --git a/ompi/mca/fbtl/ime/fbtl_ime.c b/ompi/mca/fbtl/ime/fbtl_ime.c
@@ -11,6 +11,7 @@
 #include "mpi.h"
 
 #include "ompi/mca/fbtl/fbtl.h"
+#include "ompi/mca/fbtl/base/base.h"
 #include "ompi/mca/fbtl/ime/fbtl_ime.h"
 
 /*
@@ -26,8 +27,8 @@ static mca_fbtl_base_module_1_0_0_t ime =  {
     mca_fbtl_ime_pwritev,         /* blocking write */
     mca_fbtl_ime_ipwritev,        /* non-blocking write */
     mca_fbtl_ime_progress,        /* module specific progress */
-    mca_fbtl_ime_request_free     /* free module specific data items on the request */
-};
+    mca_fbtl_ime_request_free,     /* free module specific data items on the request */
+    mca_fbtl_base_check_atomicity  /* check whether atomicity is supported on this fs */};
 /*
  * *******************************************************************
  * ************************* structure ends **************************
@@ -179,4 +180,4 @@ void mca_fbtl_ime_complete_cb  (struct ime_aiocb *aiocb, int err, ssize_t bytes)
 {
     ssize_t *req_status = (ssize_t *) aiocb->user_context;
     *req_status = err == 0 ? bytes : FBTL_IME_REQ_ERROR;
-}
+}
diff --git a/ompi/mca/fbtl/posix/fbtl_posix.c b/ompi/mca/fbtl/posix/fbtl_posix.c
@@ -58,12 +58,13 @@ static mca_fbtl_base_module_1_0_0_t posix =  {
 #if defined (FBTL_POSIX_HAVE_AIO)
     mca_fbtl_posix_ipwritev,        /* non-blocking write */
     mca_fbtl_posix_progress,        /* module specific progress */
-    mca_fbtl_posix_request_free     /* free module specific data items on the request */
+    mca_fbtl_posix_request_free,    /* free module specific data items on the request */
 #else
     NULL,                           /* non-blocking write */
     NULL,                           /* module specific progress */
-    NULL                            /* free module specific data items on the request */
+    NULL,                           /* free module specific data items on the request */
 #endif
+    mca_fbtl_posix_check_atomicity  /* check whether atomicity is supported on this fs */
 };
 /*
  * *******************************************************************
@@ -291,3 +292,27 @@ void mca_fbtl_posix_request_free ( mca_ompio_request_t *req)
 #endif
   return;
 }
+
+bool mca_fbtl_posix_check_atomicity ( ompio_file_t *file)
+{    
+    struct flock lock;
+    
+    lock.l_type   = F_WRLCK;
+    lock.l_whence = SEEK_SET;
+    lock.l_start  = 0;
+    lock.l_len    = 0;
+    lock.l_pid    = 0;
+    
+    if (fcntl(file->fd, F_GETLK, &lock) < 0)
+    {
+#ifdef VERBOSE
+        printf("Failed to get lock info for '%s': %s\n", filename, strerror(errno));
+#endif
+        return false;
+    }
+
+#ifdef VERBOSE
+    printf("Lock would have worked, l_type=%d\n", (int)lock.l_type);
+#endif
+    return true;
+}
diff --git a/ompi/mca/fbtl/posix/fbtl_posix.h b/ompi/mca/fbtl/posix/fbtl_posix.h
@@ -64,6 +64,7 @@ ssize_t mca_fbtl_posix_ipwritev (ompio_file_t *file,
 
 bool mca_fbtl_posix_progress     ( mca_ompio_request_t *req);
 void mca_fbtl_posix_request_free ( mca_ompio_request_t *req);
+bool mca_fbtl_posix_check_atomicity ( ompio_file_t *file);
 
 int mca_fbtl_posix_lock ( struct flock *lock, ompio_file_t *fh, int op, 
                           OMPI_MPI_OFFSET_TYPE iov_offset, off_t len, int flags,
diff --git a/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.c b/ompi/mca/fbtl/pvfs2/fbtl_pvfs2.c
@@ -28,6 +28,7 @@
 #include "ompi_config.h"
 #include "mpi.h"
 #include "ompi/mca/fbtl/fbtl.h"
+#include "ompi/mca/fbtl/base/base.h"
 #include "ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h"
 
 /*
@@ -43,7 +44,8 @@ static mca_fbtl_base_module_1_0_0_t pvfs2 =  {
     mca_fbtl_pvfs2_pwritev,         /* blocking write */
     NULL,                           /* non-blocking write */
     NULL,                           /* module specific progress */
-    NULL                            /* free module specific data items on the request */
+    NULL,                           /* free module specific data items on the request */
+    mca_fbtl_base_check_atomicity   /* check whether atomicity is supported on this fs */
 };
 /*
  * *******************************************************************
diff --git a/ompi/mca/io/ompio/io_ompio_file_open.c b/ompi/mca/io/ompio/io_ompio_file_open.c
@@ -311,7 +311,7 @@ int mca_io_ompio_file_get_type_extent (ompi_file_t *fh,
 int mca_io_ompio_file_set_atomicity (ompi_file_t *fh,
                                      int flag)
 {
-    int tmp;
+    int tmp, ret = OMPI_SUCCESS;
     mca_common_ompio_data_t *data;
 
     data = (mca_common_ompio_data_t *) fh->f_io_selected_data;
@@ -335,10 +335,22 @@ int mca_io_ompio_file_set_atomicity (ompi_file_t *fh,
         return OMPI_ERROR;
     }
 
-    data->ompio_fh.f_atomicity = flag;
+    bool result;
+    if ( flag ) {
+        result = data->ompio_fh.f_fbtl->fbtl_check_atomicity(&data->ompio_fh);
+        if ( result ) {
+            data->ompio_fh.f_atomicity = flag;
+        }
+        else {
+            ret = MPI_ERR_IO;
+        }
+    }        
+    else {
+        data->ompio_fh.f_atomicity = flag;
+    }
     OPAL_THREAD_UNLOCK(&fh->f_lock);
 
-    return OMPI_SUCCESS;
+    return ret;
 }
 
 int mca_io_ompio_file_get_atomicity (ompi_file_t *fh,
