Skip to content

Commit 52ce447

Browse files
hjelmnhjelmn
committed
Add lazy barrier code
Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
1 parent df9bd6a commit 52ce447

File tree

6 files changed

+401
-0
lines changed

6 files changed

+401
-0
lines changed

‎ompi/mca/coll/lazy/Makefile.am‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#
2+
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
# University Research and Technology
4+
# Corporation. All rights reserved.
5+
# Copyright (c) 2004-2005 The University of Tennessee and The University
6+
# of Tennessee Research Foundation. All rights
7+
# reserved.
8+
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
9+
# University of Stuttgart. All rights reserved.
10+
# Copyright (c) 2004-2005 The Regents of the University of California.
11+
# All rights reserved.
12+
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
13+
# Copyright (c) 2016 Intel, Inc. All rights reserved
14+
# Copyright (c) 2017 IBM Corporation. All rights reserved.
15+
# Copyright (c) 2018 Triad National Security, LLC. All rights
16+
# reserved.
17+
# $COPYRIGHT$
18+
#
19+
# Additional copyrights may follow
20+
#
21+
# $HEADER$
22+
#
23+
24+
sources = \
25+
coll_lazy.h \
26+
coll_lazy_component.c \
27+
coll_lazy_module.c \
28+
coll_lazy_barrier.c
29+
30+
if MCA_BUILD_ompi_coll_lazy_DSO
31+
component_noinst =
32+
component_install = mca_coll_lazy.la
33+
else
34+
component_noinst = libmca_coll_lazy.la
35+
component_install =
36+
endif
37+
38+
mcacomponentdir = $(ompilibdir)
39+
mcacomponent_LTLIBRARIES = $(component_install)
40+
mca_coll_lazy_la_SOURCES = $(sources)
41+
mca_coll_lazy_la_LDFLAGS = -module -avoid-version
42+
mca_coll_lazy_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
43+
44+
noinst_LTLIBRARIES = $(component_noinst)
45+
libmca_coll_lazy_la_SOURCES =$(sources)
46+
libmca_coll_lazy_la_LDFLAGS = -module -avoid-version

‎ompi/mca/coll/lazy/coll_lazy.h‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4+
* University Research and Technology
5+
* Corporation. All rights reserved.
6+
* Copyright (c) 2004-2006 The University of Tennessee and The University
7+
* of Tennessee Research Foundation. All rights
8+
* reserved.
9+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10+
* University of Stuttgart. All rights reserved.
11+
* Copyright (c) 2004-2005 The Regents of the University of California.
12+
* All rights reserved.
13+
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
14+
* Copyright (c) 2018 Triad National Security, LLC. All rights
15+
* reserved.
16+
* $COPYRIGHT$
17+
*
18+
* Additional copyrights may follow
19+
*
20+
* $HEADER$
21+
*/
22+
23+
#ifndef MCA_COLL_LAZY_EXPORT_H
24+
#define MCA_COLL_LAZY_EXPORT_H
25+
26+
#include "ompi_config.h"
27+
28+
#include "mpi.h"
29+
30+
#include "opal/class/opal_object.h"
31+
#include "opal/mca/mca.h"
32+
#include "opal/util/output.h"
33+
34+
#include "ompi/constants.h"
35+
#include "ompi/mca/coll/coll.h"
36+
#include "ompi/mca/coll/base/base.h"
37+
#include "ompi/communicator/communicator.h"
38+
39+
BEGIN_C_DECLS
40+
41+
/* API functions */
42+
43+
int mca_coll_lazy_init_query(bool enable_progress_threads,
44+
bool enable_mpi_threads);
45+
mca_coll_base_module_t
46+
*mca_coll_lazy_comm_query(struct ompi_communicator_t *comm,
47+
int *priority);
48+
49+
int mca_coll_lazy_module_enable(mca_coll_base_module_t *module,
50+
struct ompi_communicator_t *comm);
51+
52+
int mca_coll_lazy_barrier (struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
53+
54+
/* Types */
55+
/* Module */
56+
57+
typedef struct mca_coll_lazy_module_t {
58+
mca_coll_base_module_t super;
59+
60+
pthread_barrier_t pth_barrier;
61+
bool pth_barrier_init;
62+
} mca_coll_lazy_module_t;
63+
64+
OBJ_CLASS_DECLARATION(mca_coll_lazy_module_t);
65+
66+
/* Component */
67+
68+
typedef struct mca_coll_lazy_component_t {
69+
mca_coll_base_component_2_0_0_t super;
70+
71+
/* Priority of this component */
72+
int priority;
73+
} mca_coll_lazy_component_t;
74+
75+
/* Globally exported variables */
76+
77+
OMPI_MODULE_DECLSPEC extern mca_coll_lazy_component_t mca_coll_lazy_component;
78+
79+
END_C_DECLS
80+
81+
#endif /* MCA_COLL_LAZY_EXPORT_H */

‎ompi/mca/coll/lazy/coll_lazy_barrier.c‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4+
* University Research and Technology
5+
* Corporation. All rights reserved.
6+
* Copyright (c) 2004-2005 The University of Tennessee and The University
7+
* of Tennessee Research Foundation. All rights
8+
* reserved.
9+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10+
* University of Stuttgart. All rights reserved.
11+
* Copyright (c) 2004-2005 The Regents of the University of California.
12+
* All rights reserved.
13+
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
14+
* Copyright (c) 2018 Triad National Security, LLC. All rights
15+
* reserved.
16+
* $COPYRIGHT$
17+
*
18+
* Additional copyrights may follow
19+
*
20+
* $HEADER$
21+
*/
22+
23+
#include "ompi_config.h"
24+
25+
#include <string.h>
26+
27+
#include "opal/util/output.h"
28+
29+
#include "mpi.h"
30+
#include "ompi/constants.h"
31+
#include "coll_lazy.h"
32+
33+
int mca_coll_lazy_barrier (struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
34+
{
35+
mca_coll_lazy_module_t *lazy_module = (mca_coll_lazy_module_t *) module;
36+
ompi_request_t *request;
37+
38+
comm->c_coll->coll_ibarrier (comm, &request, comm->c_coll->coll_ibarrier_module);
39+
do {
40+
for (int i = 0 ; i < 10 ; ++i) {
41+
opal_progress ();
42+
if (REQUEST_COMPLETE(request)) {
43+
break;
44+
}
45+
}
46+
47+
if (REQUEST_COMPLETE(request)) {
48+
break;
49+
}
50+
51+
nanosleep (&(struct timespec) {.tv_sec = 0, .tv_nsec = 10000}, NULL);
52+
} while (1);
53+
54+
return OMPI_SUCCESS;
55+
}

‎ompi/mca/coll/lazy/coll_lazy_component.c‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4+
* University Research and Technology
5+
* Corporation. All rights reserved.
6+
* Copyright (c) 2004-2005 The University of Tennessee and The University
7+
* of Tennessee Research Foundation. All rights
8+
* reserved.
9+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10+
* University of Stuttgart. All rights reserved.
11+
* Copyright (c) 2004-2005 The Regents of the University of California.
12+
* All rights reserved.
13+
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
14+
* Copyright (c) 2018 Triad National Security, LLC. All rights
15+
* reserved.
16+
* $COPYRIGHT$
17+
*
18+
* Additional copyrights may follow
19+
*
20+
* $HEADER$
21+
*/
22+
23+
#include "ompi_config.h"
24+
25+
#include <string.h>
26+
27+
#include "opal/util/output.h"
28+
29+
#include "mpi.h"
30+
#include "ompi/constants.h"
31+
#include "coll_lazy.h"
32+
33+
/*
34+
* Public string showing the coll ompi_lazy component version number
35+
*/
36+
const char *mca_coll_lazy_component_version_string =
37+
"Open MPI lazy collective MCA component version " OMPI_VERSION;
38+
39+
/*
40+
* Local function
41+
*/
42+
static int lazy_register(void);
43+
44+
/*
45+
* Instantiate the public struct with all of our public information
46+
* and pointers to our public functions in it
47+
*/
48+
49+
mca_coll_lazy_component_t mca_coll_lazy_component = {
50+
{
51+
/* First, the mca_component_t struct containing meta information
52+
* about the component itself */
53+
54+
.collm_version = {
55+
MCA_COLL_BASE_VERSION_2_0_0,
56+
57+
/* Component name and version */
58+
.mca_component_name = "lazy",
59+
MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
60+
OMPI_RELEASE_VERSION),
61+
62+
/* Component open and close functions */
63+
.mca_register_component_params = lazy_register
64+
},
65+
.collm_data = {
66+
/* The component is checkpoint ready */
67+
MCA_BASE_METADATA_PARAM_CHECKPOINT
68+
},
69+
70+
/* Initialization / querying functions */
71+
72+
.collm_init_query = mca_coll_lazy_init_query,
73+
.collm_comm_query = mca_coll_lazy_comm_query
74+
},
75+
};
76+
77+
78+
static int lazy_register(void)
79+
{
80+
mca_coll_lazy_component.priority = 100;
81+
(void) mca_base_component_var_register (&mca_coll_lazy_component.super.collm_version, "priority",
82+
"Priority of the lazy coll component; only relevant for shared memory communicators with the "
83+
"lazy progress info key", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
84+
MCA_BASE_VAR_SCOPE_READONLY, &mca_coll_lazy_component.priority);
85+
86+
return OMPI_SUCCESS;
87+
}

‎ompi/mca/coll/lazy/coll_lazy_module.c‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4+
* University Research and Technology
5+
* Corporation. All rights reserved.
6+
* Copyright (c) 2004-2017 The University of Tennessee and The University
7+
* of Tennessee Research Foundation. All rights
8+
* reserved.
9+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10+
* University of Stuttgart. All rights reserved.
11+
* Copyright (c) 2004-2005 The Regents of the University of California.
12+
* All rights reserved.
13+
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
14+
* Copyright (c) 2016 Research Organization for Information Science
15+
* and Technology (RIST). All rights reserved.
16+
* Copyright (c) 2018 Intel, Inc. All rights reserved.
17+
* Copyright (c) 2018 Triad National Security, LLC. All rights
18+
* reserved.
19+
* $COPYRIGHT$
20+
*
21+
* Additional copyrights may follow
22+
*
23+
* $HEADER$
24+
*/
25+
26+
#include "ompi_config.h"
27+
28+
#ifdef HAVE_STRING_H
29+
#include <string.h>
30+
#endif
31+
#include <stdio.h>
32+
33+
#include "coll_lazy.h"
34+
35+
#include "mpi.h"
36+
37+
#include "opal/util/show_help.h"
38+
#include "ompi/mca/rte/rte.h"
39+
40+
#include "ompi/constants.h"
41+
#include "ompi/communicator/communicator.h"
42+
#include "ompi/mca/coll/coll.h"
43+
#include "ompi/mca/coll/base/base.h"
44+
#include "coll_lazy.h"
45+
46+
47+
static void mca_coll_lazy_module_construct(mca_coll_lazy_module_t *module)
48+
{
49+
module->pth_barrier_init = false;
50+
}
51+
52+
static void mca_coll_lazy_module_destruct(mca_coll_lazy_module_t *module)
53+
{
54+
if (module->pth_barrier_init) {
55+
pthread_barrier_destroy (&module->pth_barrier);
56+
module->pth_barrier_init = false;
57+
}
58+
}
59+
60+
OBJ_CLASS_INSTANCE(mca_coll_lazy_module_t, mca_coll_base_module_t,
61+
mca_coll_lazy_module_construct,
62+
mca_coll_lazy_module_destruct);
63+
64+
65+
/*
66+
* Initial query function that is invoked during MPI_INIT, allowing
67+
* this component to disqualify itself if it doesn't support the
68+
* required level of thread support.
69+
*/
70+
int mca_coll_lazy_init_query(bool enable_progress_threads,
71+
bool enable_mpi_threads)
72+
{
73+
/* Nothing to do */
74+
return OMPI_SUCCESS;
75+
}
76+
77+
78+
/*
79+
* Invoked when there's a new communicator that has been created.
80+
* Look at the communicator and decide which set of functions and
81+
* priority we want to return.
82+
*/
83+
mca_coll_base_module_t *mca_coll_lazy_comm_query (ompi_communicator_t *comm, int *priority)
84+
{
85+
mca_coll_lazy_module_t *lazy_module;
86+
87+
*priority = 0;
88+
89+
if (ompi_group_have_remote_peers (comm->c_local_group) || !OMPI_COMM_CHECK_ASSERT_LAZY_BARRIER(comm)) {
90+
return NULL;
91+
}
92+
93+
lazy_module = OBJ_NEW(mca_coll_lazy_module_t);
94+
if (NULL == lazy_module) {
95+
return NULL;
96+
}
97+
98+
*priority = mca_coll_lazy_component.priority;
99+
100+
/* Choose whether to use [intra|inter] */
101+
lazy_module->super.coll_module_enable = mca_coll_lazy_module_enable;
102+
103+
lazy_module->super.coll_barrier = mca_coll_lazy_barrier;
104+
105+
return &lazy_module->super;
106+
}
107+
108+
109+
/*
110+
* Init module on the communicator
111+
*/
112+
int mca_coll_lazy_module_enable (mca_coll_base_module_t *module, ompi_communicator_t *comm)
113+
{
114+
mca_coll_lazy_module_t *lazy_module = (mca_coll_lazy_module_t *) module;
115+
int rc;
116+
117+
rc = pthread_barrier_init (&lazy_module->pth_barrier, NULL, comm->c_local_group->grp_proc_count);
118+
if (0 != rc) {
119+
return OMPI_ERR_NOT_FOUND;
120+
}
121+
122+
lazy_module->pth_barrier_init = true;
123+
124+
return OMPI_SUCCESS;
125+
}

0 commit comments

Comments
 (0)