Skip to content

Commit ab56e6f

Browse files
edgargabrieledgargabriel
committed
common/ompio: make individual read operations work.
Signed-off-by: Edgar Gabriel <egabriel@central.uh.edu>
1 parent f6b3a0a commit ab56e6f

File tree

3 files changed

+66
-38
lines changed

3 files changed

+66
-38
lines changed

‎ompi/mca/common/ompio/common_ompio_buffer.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,24 @@
4040
_decoded_iov->iov_len = _max_data; \
4141
_iov_count=1;}
4242

43+
#define OMPIO_PREPARE_READ_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
44+
OBJ_CONSTRUCT( _convertor, opal_convertor_t); \
45+
opal_convertor_copy_and_prepare_for_recv ( _fh->f_file_convertor, &(_datatype->super), _count, _buf, 0, _convertor ); \
46+
opal_convertor_get_packed_size( _convertor, &_max_data ); \
47+
_tbuf = mca_common_ompio_alloc_buf (_fh, _max_data); \
48+
if ( NULL == _tbuf ) { \
49+
opal_output(1, "common_ompio: error allocating memory\n"); \
50+
return OMPI_ERR_OUT_OF_RESOURCE; \
51+
} \
52+
_decoded_iov = (struct iovec *) malloc ( sizeof ( struct iovec )); \
53+
if ( NULL == _decoded_iov ) { \
54+
opal_output(1, "common_ompio: could not allocate memory.\n"); \
55+
return OMPI_ERR_OUT_OF_RESOURCE; \
56+
} \
57+
_decoded_iov->iov_base = _tbuf; \
58+
_decoded_iov->iov_len = _max_data; \
59+
_iov_count=1;}
60+
4361
#if OPAL_CUDA_SUPPORT
4462
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf,
4563
int *is_gpu, int *is_managed);

‎ompi/mca/common/ompio/common_ompio_file_read.c‎

Lines changed: 45 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -88,17 +88,33 @@ int mca_common_ompio_file_read (ompio_file_t *fh,
8888
return ret;
8989
}
9090

91-
91+
bool need_to_copy = false;
92+
opal_convertor_t convertor;
9293
#if OPAL_CUDA_SUPPORT
9394
int is_gpu, is_managed;
94-
opal_convertor_t convertor;
9595
mca_common_ompio_check_gpu_buf ( fh, buf, &is_gpu, &is_managed);
9696
if ( is_gpu && !is_managed ) {
97+
need_to_copy = true;
98+
}
99+
#endif
100+
101+
if ( !( fh->f_flags & OMPIO_DATAREP_NATIVE ) &&
102+
!(datatype == &ompi_mpi_byte.dt ||
103+
datatype == &ompi_mpi_char.dt )) {
104+
/* only need to copy if any of these conditions are given:
105+
1. buffer is an unmanaged CUDA buffer (checked above).
106+
2. Datarepresentation is anything other than 'native' and
107+
3. datatype is not byte or char (i.e it does require some actual
108+
work to be done e.g. for external32.
109+
*/
110+
need_to_copy = true;
111+
}
112+
113+
if ( need_to_copy ) {
97114
char *tbuf=NULL;
98115

99-
OMPIO_CUDA_PREPARE_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);
100-
101-
}
116+
OMPIO_PREPARE_READ_BUF(fh,buf,count,datatype,tbuf,&convertor,max_data,decoded_iov,iov_count);
117+
}
102118
else {
103119
mca_common_ompio_decode_datatype (fh,
104120
datatype,
@@ -109,16 +125,7 @@ int mca_common_ompio_file_read (ompio_file_t *fh,
109125
&decoded_iov,
110126
&iov_count);
111127
}
112-
#else
113-
mca_common_ompio_decode_datatype (fh,
114-
datatype,
115-
count,
116-
buf,
117-
&max_data,
118-
fh->f_mem_convertor,
119-
&decoded_iov,
120-
&iov_count);
121-
#endif
128+
122129
if ( 0 < max_data && 0 == fh->f_iov_count ) {
123130
if ( MPI_STATUS_IGNORE != status ) {
124131
status->_ucount = 0;
@@ -170,15 +177,14 @@ int mca_common_ompio_file_read (ompio_file_t *fh,
170177
}
171178
}
172179

173-
#if OPAL_CUDA_SUPPORT
174-
if ( is_gpu && !is_managed ) {
180+
if ( need_to_copy ) {
175181
size_t pos=0;
176182

177183
opal_convertor_unpack (&convertor, decoded_iov, &iov_count, &pos );
178184
opal_convertor_cleanup (&convertor);
179185
mca_common_ompio_release_buf (fh, decoded_iov->iov_base);
180186
}
181-
#endif
187+
182188
if (NULL != decoded_iov) {
183189
free (decoded_iov);
184190
decoded_iov = NULL;
@@ -257,13 +263,32 @@ int mca_common_ompio_file_iread (ompio_file_t *fh,
257263
int i = 0; /* index into the decoded iovec of the buffer */
258264
int j = 0; /* index into the file vie iovec */
259265

266+
bool need_to_copy = false;
267+
260268
#if OPAL_CUDA_SUPPORT
261269
int is_gpu, is_managed;
262270
mca_common_ompio_check_gpu_buf ( fh, buf, &is_gpu, &is_managed);
263271
if ( is_gpu && !is_managed ) {
272+
need_to_copy = true;
273+
}
274+
#endif
275+
276+
if ( !( fh->f_flags & OMPIO_DATAREP_NATIVE ) &&
277+
!(datatype == &ompi_mpi_byte.dt ||
278+
datatype == &ompi_mpi_char.dt )) {
279+
/* only need to copy if any of these conditions are given:
280+
1. buffer is an unmanaged CUDA buffer (checked above).
281+
2. Datarepresentation is anything other than 'native' and
282+
3. datatype is not byte or char (i.e it does require some actual
283+
work to be done e.g. for external32.
284+
*/
285+
need_to_copy = true;
286+
}
287+
288+
if ( need_to_copy ) {
264289
char *tbuf=NULL;
265290

266-
OMPIO_CUDA_PREPARE_BUF(fh,buf,count,datatype,tbuf,&ompio_req->req_convertor,max_data,decoded_iov,iov_count);
291+
OMPIO_PREPARE_READ_BUF(fh,buf,count,datatype,tbuf,&ompio_req->req_convertor,max_data,decoded_iov,iov_count);
267292

268293
ompio_req->req_tbuf = tbuf;
269294
ompio_req->req_size = max_data;
@@ -278,16 +303,7 @@ int mca_common_ompio_file_iread (ompio_file_t *fh,
278303
&decoded_iov,
279304
&iov_count);
280305
}
281-
#else
282-
mca_common_ompio_decode_datatype (fh,
283-
datatype,
284-
count,
285-
buf,
286-
&max_data,
287-
fh->f_mem_convertor,
288-
&decoded_iov,
289-
&iov_count);
290-
#endif
306+
291307
if ( 0 < max_data && 0 == fh->f_iov_count ) {
292308
ompio_req->req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
293309
ompio_req->req_ompi.req_status._ucount = 0;

‎ompi/mca/io/ompio/io_ompio_component.c‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* University of Stuttgart. All rights reserved.
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
13-
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
13+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1414
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
1515
* reserved.
1616
* Copyright (c) 2015-2018 Research Organization for Information Science
@@ -34,14 +34,12 @@
3434
#include "ompi/mca/fs/base/base.h"
3535
#include "io_ompio.h"
3636
#include "ompi/mca/common/ompio/common_ompio_request.h"
37+
#include "ompi/mca/common/ompio/common_ompio_buffer.h"
3738

3839
#ifdef HAVE_IME_NATIVE_H
3940
#include "ompi/mca/fs/ime/fs_ime.h"
4041
#endif
4142

42-
#if OPAL_CUDA_SUPPORT
43-
#include "ompi/mca/common/ompio/common_ompio_cuda.h"
44-
#endif
4543

4644
int mca_io_ompio_cycle_buffer_size = OMPIO_DEFAULT_CYCLE_BUF_SIZE;
4745
int mca_io_ompio_bytes_per_agg = OMPIO_PREALLOC_MAX_BUF_SIZE;
@@ -280,11 +278,7 @@ static int open_component(void)
280278
static int close_component(void)
281279
{
282280
mca_common_ompio_request_fini ();
283-
284-
#if OPAL_CUDA_SUPPORT
285-
mca_common_ompio_cuda_alloc_fini();
286-
#endif
287-
281+
mca_common_ompio_buffer_alloc_fini();
288282
OBJ_DESTRUCT(&mca_io_ompio_mutex);
289283

290284
#ifdef HAVE_IME_NATIVE_H

0 commit comments

Comments
 (0)