accelerator: update API

- extend the accelerator API by three more functions
- add a new constant for default accelerator
- minor bug fixes in the existing files/version
- update the create_event interface
- implementation of the new IPC functions in the rocm and null component
- add stubs for new functions to cuda and ze components

Signed-off-by: Edgar Gabriel <Edgar.Gabriel@amd.com>

Author: edgargabriel

ompi/mca/pml/ob1/pml_ob1_accelerator.c

@@ -223,7 +223,7 @@ int mca_pml_ob1_accelerator_init(void)
 
     /* Create the events since they can be reused. */
     for (i = 0; i < accelerator_event_max; i++) {
-        result = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_dtoh_array[i]);
+        result = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_dtoh_array[i], false);
         if (OPAL_SUCCESS != result) {
             opal_output_verbose(1, mca_pml_ob1_output, "Accelerator create event failed.");
             rc = OPAL_ERROR;
@@ -256,7 +256,7 @@ int mca_pml_ob1_accelerator_init(void)
 
     /* Create the events since they can be reused. */
     for (i = 0; i < accelerator_event_max; i++) {
-        result = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_htod_array[i]);
+        result = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_htod_array[i], false);
         if (OPAL_SUCCESS != result) {
             opal_output_verbose(1, mca_pml_ob1_output, "Accelerator create event failed.");
             rc = OPAL_ERROR;

opal/include/opal/Makefile.am

@@ -30,7 +30,7 @@ headers += \
 	opal/frameworks.h \
 	opal/opal_portable_platform.h \
 	opal/opal_portable_platform_real.h \
-	opal/opal_cuda.h
+	opal/opal_gpu.h
 
 nodist_headers += \
 	opal/version.h

opal/mca/accelerator/accelerator.h

@@ -4,6 +4,8 @@
  *                         reserved.
  * Copyright (c)           Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
+ * Copyright (c) 2023      Advanced Micro Devices, Inc. All Rights reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -110,19 +112,28 @@ struct opal_accelerator_stream_t {
     void *stream;
 };
 typedef struct opal_accelerator_stream_t opal_accelerator_stream_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_stream_t);
+
+/* Constant indicating the default/zero stream */
+#define MCA_ACCELERATOR_STREAM_DEFAULT (opal_accelerator_stream_t *)0x00000002
 
 #define IPC_MAX_HANDLE_SIZE 64
 struct opal_accelerator_ipc_handle_t {
+    opal_object_t super;
     size_t size;
     uint8_t handle[IPC_MAX_HANDLE_SIZE];
+    void* dev_ptr;
 };
 typedef struct opal_accelerator_ipc_handle_t opal_accelerator_ipc_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ipc_handle_t);
 
 struct opal_accelerator_ipc_event_handle_t {
+    opal_object_t super;
     size_t size;
     uint8_t handle[IPC_MAX_HANDLE_SIZE];
 };
 typedef struct opal_accelerator_ipc_event_handle_t opal_accelerator_ipc_event_handle_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ipc_event_handle_t);
 
 struct opal_accelerator_pci_attr_t {
     uint16_t domain_id;
@@ -132,7 +143,6 @@ struct opal_accelerator_pci_attr_t {
 };
 typedef struct opal_accelerator_pci_attr_t opal_accelerator_pci_attr_t;
 
-OBJ_CLASS_DECLARATION(opal_accelerator_stream_t);
 
 struct opal_accelerator_event_t {
     opal_object_t super;
@@ -180,14 +190,15 @@ typedef int (*opal_accelerator_base_module_create_stream_fn_t)(
  * corresponding stream. This function will allocate memory for the object.
  * To release the memory, call OBJ_RELEASE(*event);
  *
- * @param[IN] dev_id         Associated device for the event or
+ * @param[IN]  dev_id        Associated device for the event or
  *                           MCA_ACCELERATOR_NO_DEVICE_ID
- * @param[IN] event          Event to create
+ * @param[OUT] event         Event to create
+ * @param[IN]  enable_ipc    support inter-process tracking of the event 
  *
  * @return                   OPAL_SUCCESS or error status on failure.
  */
 typedef int (*opal_accelerator_base_module_create_event_fn_t)(
-    int dev_id, opal_accelerator_event_t **event);
+      int dev_id, opal_accelerator_event_t **event, bool enable_ipc);
 
 /**
  * Records an event on a stream. An event recorded on the stream is
@@ -219,6 +230,19 @@ typedef int (*opal_accelerator_base_module_record_event_fn_t)(
 typedef int (*opal_accelerator_base_module_query_event_fn_t)(
     int dev_id, opal_accelerator_event_t *event);
 
+/**
+ * Make a stream wait on an event
+ *
+ * @param[IN] dev_id         Associated device for the event or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN] event          Event to wait on
+ * @param[IN] stream         Stream to wait
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_wait_event_fn_t)(
+   int dev_id, opal_accelerator_event_t *event,  opal_accelerator_stream_t *stream);
+
 /**
  * Copies memory asynchronously from src to dest. Memory of dest and src
  * may not overlap. Optionally can specify the transfer type to
@@ -342,8 +366,10 @@ typedef int (*opal_accelerator_base_module_get_address_range_fn_t)(
  *
  * opal_accelerator_base_module_get_ipc_handle_fn_t()
  * opal_accelerator_base_module_open_ipc_handle_fn_t()
+ * opal_accelerator_base_module_import_ipc_event_handle_fn_t()
  * opal_accelerator_base_module_get_ipc_event_handle_fn_t()
  * opal_accelerator_base_module_open_ipc_event_handle_fn_t()
+ * opal_accelerator_base_module_import_ipc_event_handle_fn_t()
  *
  * must be implemented.
  *
@@ -354,6 +380,8 @@ typedef bool (*opal_accelerator_base_module_is_ipc_enabled_fn_t)(void);
 
 /**
  * Gets an IPC memory handle for an existing device memory allocation.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
  *
  * @param[IN]  dev_id        Associated device for the IPC memory handle or
  *                           MCA_ACCELERATOR_NO_DEVICE_ID
@@ -366,31 +394,67 @@ typedef bool (*opal_accelerator_base_module_is_ipc_enabled_fn_t)(void);
 typedef int (*opal_accelerator_base_module_get_ipc_handle_fn_t)(
     int dev_id, void *dev_ptr, opal_accelerator_ipc_handle_t *handle);
 
+/**
+ * Creates an opal_accelerator_ipc_handle object given the 64byte IPC handle,
+ * which was created using module_get_ipc_handle_fn on another process.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
+ *
+ * @param[IN]  dev_id        Associated device for the IPC memory handle or
+ *                           MCA_ACCELERATOR_NO_DEVICE_ID
+ * @param[IN]  ipc_handle    64 byte IPC handle transfered from another process
+ * @param[OUT] handle        Pointer to IPC handle object
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ *
+ */
+typedef int (*opal_accelerator_base_module_import_ipc_handle_fn_t)(
+    int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE], opal_accelerator_ipc_handle_t *handle);
+
 /**
  * Opens an IPC memory handle from another process and returns
  * a device pointer usable in the local process.
  *
  * @param[IN]  dev_id        Associated device for the IPC memory handle or
  *                           MCA_ACCELERATOR_NO_DEVICE_ID
- * @param[IN]  handle        IPC handle object from another process
+ * @param[IN]  handle        IPC handle created using the module_create_ipc_handle_fn
  * @param[OUT] dev_ptr       Returned device pointer
  *
- * @return                   OPAL_SUCCESS or error status on failure
+ * @return                   OPAL_SUCCESS on success,
+ *                           OPAL_ERR_WOULD_BLOCK if the memory region is already mapped
+ *                           or error status on other failures
  */
 typedef int (*opal_accelerator_base_module_open_ipc_handle_fn_t)(
     int dev_id, opal_accelerator_ipc_handle_t *handle, void **dev_ptr);
 
 /**
  * Gets an IPC event handle for an event created by opal_accelerator_base_module_create_event_fn_t.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
  *
- * @param[IN]  event         Event created previously
- * @param[OUT] handle        Pointer to IPC event handle object
+ * @param[IN] event          Event created previously
+ * @param[IN] handle         Pointer to IPC event handle object
  *
  * @return                   OPAL_SUCCESS or error status on failure
  */
 typedef int (*opal_accelerator_base_module_get_ipc_event_handle_fn_t)(
     opal_accelerator_event_t *event, opal_accelerator_ipc_event_handle_t *handle);
 
+/**
+ * Creates an opal_accelerator_ipc_event_handle object using the 64 byte IPC event handle
+ * which was created using module_get_ipc_event_handle_fn on another process.
+ * This interface assumes that the object has been declared statically,
+ * hence one has to call OBJ_DESTRUCT(handle) on it.
+ *
+ * @param[IN] event          Event created previously
+ * @param[IN] ipc_handle     64 byte IPC handle object
+ * @param[OUT] handle        Pointer to IPC event handle object
+ *
+ * @return                   OPAL_SUCCESS or error status on failure
+ */
+typedef int (*opal_accelerator_base_module_import_ipc_event_handle_fn_t)(
+     uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE], opal_accelerator_ipc_event_handle_t *handle);
+
 /**
  * Opens an IPC event handle from another process opened by
  * opal_accelerator_base_module_get_ipc_event_handle_fn_t.
@@ -490,6 +554,7 @@ typedef struct {
     opal_accelerator_base_module_create_event_fn_t create_event;
     opal_accelerator_base_module_record_event_fn_t record_event;
     opal_accelerator_base_module_query_event_fn_t query_event;
+    opal_accelerator_base_module_wait_event_fn_t wait_event;
 
     opal_accelerator_base_module_memcpy_async_fn_t mem_copy_async;
     opal_accelerator_base_module_memcpy_fn_t mem_copy;
@@ -501,8 +566,10 @@ typedef struct {
 
     opal_accelerator_base_module_is_ipc_enabled_fn_t is_ipc_enabled;
     opal_accelerator_base_module_get_ipc_handle_fn_t get_ipc_handle;
+    opal_accelerator_base_module_import_ipc_handle_fn_t import_ipc_handle;
     opal_accelerator_base_module_open_ipc_handle_fn_t open_ipc_handle;
     opal_accelerator_base_module_get_ipc_event_handle_fn_t get_ipc_event_handle;
+    opal_accelerator_base_module_import_ipc_event_handle_fn_t import_ipc_event_handle;
     opal_accelerator_base_module_open_ipc_event_handle_fn_t open_ipc_event_handle;
 
     opal_accelerator_base_module_host_register_fn_t host_register;

opal/mca/accelerator/base/accelerator_base_frame.c

@@ -57,6 +57,19 @@ OBJ_CLASS_INSTANCE(
     NULL,
     NULL);
 
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_ipc_handle_t,
+    opal_object_t,
+    NULL,
+    NULL);
+
+OBJ_CLASS_INSTANCE(
+    opal_accelerator_ipc_event_handle_t,
+    opal_object_t,
+    NULL,
+    NULL);
+
+
 MCA_BASE_FRAMEWORK_DECLARE(opal, accelerator, "OPAL Accelerator Framework",
                            opal_accelerator_base_frame_register, opal_accelerator_base_frame_open,
                            opal_accelerator_base_frame_close, mca_accelerator_base_static_components,

opal/mca/accelerator/cuda/accelerator_cuda.c

@@ -26,9 +26,10 @@
 static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *flags);
 static int accelerator_cuda_create_stream(int dev_id, opal_accelerator_stream_t **stream);
 
-static int accelerator_cuda_create_event(int dev_id, opal_accelerator_event_t **event);
+static int accelerator_cuda_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc);
 static int accelerator_cuda_record_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
 static int accelerator_cuda_query_event(int dev_id, opal_accelerator_event_t *event);
+static int accelerator_cuda_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream);
 
 static int accelerator_cuda_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
                                   opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type);
@@ -44,10 +45,14 @@ static int accelerator_cuda_get_address_range(int dev_id, const void *ptr, void
 static bool accelerator_cuda_is_ipc_enabled(void);
 static int accelerator_cuda_get_ipc_handle(int dev_id, void *dev_ptr,
                                            opal_accelerator_ipc_handle_t *handle);
+static int accelerator_cuda_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                              opal_accelerator_ipc_handle_t *handle);
 static int accelerator_cuda_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
                                             void **dev_ptr);
 static int accelerator_cuda_get_ipc_event_handle(opal_accelerator_event_t *event,
                                                  opal_accelerator_ipc_event_handle_t *handle);
+static int accelerator_cuda_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                    opal_accelerator_ipc_event_handle_t *handle);
 static int accelerator_cuda_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
                                                   opal_accelerator_event_t *event);
 
@@ -60,6 +65,8 @@ static int accelerator_cuda_device_can_access_peer( int *access, int dev1, int d
 
 static int accelerator_cuda_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id);
 
+#define GET_STREAM(_stream) (_stream == MCA_ACCELERATOR_STREAM_DEFAULT ? 0 : *((CUstream *)_stream->stream))
+
 opal_accelerator_base_module_t opal_accelerator_cuda_module =
 {
     accelerator_cuda_check_addr,
@@ -69,6 +76,7 @@ opal_accelerator_base_module_t opal_accelerator_cuda_module =
     accelerator_cuda_create_event,
     accelerator_cuda_record_event,
     accelerator_cuda_query_event,
+    accelerator_cuda_wait_event,
 
     accelerator_cuda_memcpy_async,
     accelerator_cuda_memcpy,
@@ -79,8 +87,10 @@ opal_accelerator_base_module_t opal_accelerator_cuda_module =
 
     accelerator_cuda_is_ipc_enabled,
     accelerator_cuda_get_ipc_handle,
+    accelerator_cuda_import_ipc_handle,
     accelerator_cuda_open_ipc_handle,
     accelerator_cuda_get_ipc_event_handle,
+    accelerator_cuda_import_ipc_event_handle,
     accelerator_cuda_open_ipc_event_handle,
 
     accelerator_cuda_host_register,
@@ -260,7 +270,8 @@ static void opal_accelerator_cuda_stream_destruct(opal_accelerator_cuda_stream_t
 {
     CUresult result;
 
-    if (NULL != stream->base.stream) {
+    if (MCA_ACCELERATOR_STREAM_DEFAULT != (opal_accelerator_stream_t *)stream &&
+        NULL != stream->base.stream) {
         result = cuStreamDestroy(*(CUstream *)stream->base.stream);
         if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
             opal_show_help("help-accelerator-cuda.txt", "cuStreamDestroy failed", true,
@@ -276,7 +287,7 @@ OBJ_CLASS_INSTANCE(
     NULL,
     opal_accelerator_cuda_stream_destruct);
 
-static int accelerator_cuda_create_event(int dev_id, opal_accelerator_event_t **event)
+static int accelerator_cuda_create_event(int dev_id, opal_accelerator_event_t **event, bool enable_ipc)
 {
     CUresult result;
     int delayed_init = opal_accelerator_cuda_delayed_init();
@@ -294,7 +305,8 @@ static int accelerator_cuda_create_event(int dev_id, opal_accelerator_event_t **
         OBJ_RELEASE(*event);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
-    result = cuEventCreate((*event)->event, CU_EVENT_DISABLE_TIMING);
+    result = cuEventCreate((*event)->event, enable_ipc ? CU_EVENT_DISABLE_TIMING|CU_EVENT_INTERPROCESS :
+			   CU_EVENT_DISABLE_TIMING);
     if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
         opal_show_help("help-accelerator-cuda.txt", "cuEventCreate failed", true,
                        OPAL_PROC_MY_HOSTNAME, result);
@@ -328,11 +340,13 @@ static int accelerator_cuda_record_event(int dev_id, opal_accelerator_event_t *e
 {
     CUresult result;
 
-    if (NULL == stream || NULL == event) {
+    if ((MCA_ACCELERATOR_STREAM_DEFAULT != stream &&
+        (NULL == stream || NULL == stream->stream)) ||
+        NULL == event) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    result = cuEventRecord(*(CUevent *)event->event, *(CUstream *)stream->stream);
+    result = cuEventRecord(*(CUevent *)event->event, GET_STREAM(stream));
     if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
         opal_show_help("help-accelerator-cuda.txt", "cuEventRecord failed", true,
                        OPAL_PROC_MY_HOSTNAME, result);
@@ -369,6 +383,10 @@ static int accelerator_cuda_query_event(int dev_id, opal_accelerator_event_t *ev
             }
     }
 }
+static int accelerator_cuda_wait_event(int dev_id, opal_accelerator_event_t *event, opal_accelerator_stream_t *stream)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
 
 static int accelerator_cuda_memcpy_async(int dest_dev_id, int src_dev_id, void *dest, const void *src, size_t size,
                                   opal_accelerator_stream_t *stream, opal_accelerator_transfer_type_t type)
@@ -380,11 +398,12 @@ static int accelerator_cuda_memcpy_async(int dest_dev_id, int src_dev_id, void *
         return delayed_init;
     }
 
-    if (NULL == stream || NULL == dest || NULL == src || size <= 0) {
+    if ((MCA_ACCELERATOR_STREAM_DEFAULT != stream && NULL == stream) ||
+        NULL == dest || NULL == src || size <= 0) {
         return OPAL_ERR_BAD_PARAM;
     }
 
-    result = cuMemcpyAsync((CUdeviceptr) dest, (CUdeviceptr) src, size, *(CUstream *)stream->stream);
+    result = cuMemcpyAsync((CUdeviceptr) dest, (CUdeviceptr) src, size, GET_STREAM(stream));
     if (OPAL_UNLIKELY(CUDA_SUCCESS != result)) {
         opal_show_help("help-accelerator-cuda.txt", "cuMemcpyAsync failed", true, dest, src,
                        size, result);
@@ -547,6 +566,12 @@ static int accelerator_cuda_get_ipc_handle(int dev_id, void *dev_ptr,
     return OPAL_ERR_NOT_IMPLEMENTED;
 }
 
+static int accelerator_cuda_import_ipc_handle(int dev_id, uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                              opal_accelerator_ipc_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
 static int accelerator_cuda_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
                                             void **dev_ptr)
 {
@@ -559,6 +584,12 @@ static int accelerator_cuda_get_ipc_event_handle(opal_accelerator_event_t *event
     return OPAL_ERR_NOT_IMPLEMENTED;
 }
 
+static int accelerator_cuda_import_ipc_event_handle(uint8_t ipc_handle[IPC_MAX_HANDLE_SIZE],
+                                                    opal_accelerator_ipc_event_handle_t *handle)
+{
+    return OPAL_ERR_NOT_IMPLEMENTED;
+}
+
 static int accelerator_cuda_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
                                                   opal_accelerator_event_t *event)
 {