Skip to content

Commit d1fd1f4

Browse files
rhc54rhc54
authored
Merge pull request #6151 from nrspruit/ns_ompi_mtl_ofi_specializations
MTL_OFI: Generation of specialized functions at build time
2 parents cdc1ebd + bef5f50 commit d1fd1f4

15 files changed

+933
-29
lines changed

‎ompi/mca/mtl/ofi/Makefile.am‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,30 @@
1212
# $HEADER$
1313
#
1414

15-
EXTRA_DIST = post_configure.sh
15+
EXTRA_DIST = post_configure.sh \
16+
$(generated_source_modules)
17+
18+
MAINTAINERCLEANFILES = \
19+
$(generated_sources)
1620

1721
AM_CPPFLAGS = $(ompi_mtl_ofi_CPPFLAGS) $(opal_common_ofi_CPPFLAGS)
1822

1923
dist_ompidata_DATA = help-mtl-ofi.txt
2024

25+
generated_source_modules = \
26+
mtl_ofi_send_opt.pm \
27+
mtl_ofi_isend_opt.pm \
28+
mtl_ofi_irecv_opt.pm \
29+
mtl_ofi_iprobe_opt.pm \
30+
mtl_ofi_improbe_opt.pm
31+
32+
generated_sources = \
33+
mtl_ofi_send_opt.c \
34+
mtl_ofi_isend_opt.c \
35+
mtl_ofi_irecv_opt.c \
36+
mtl_ofi_iprobe_opt.c \
37+
mtl_ofi_improbe_opt.c
38+
2139
mtl_ofi_sources = \
2240
mtl_ofi.h \
2341
mtl_ofi.c \
@@ -26,7 +44,18 @@ mtl_ofi_sources = \
2644
mtl_ofi_endpoint.h \
2745
mtl_ofi_endpoint.c \
2846
mtl_ofi_request.h \
29-
mtl_ofi_types.h
47+
mtl_ofi_types.h \
48+
mtl_ofi_opt.h \
49+
$(generated_sources)
50+
51+
# A number of files are generated from macro expansion to minimize
52+
# branches in the critical path. These files have perl modules with the suffix
53+
# .pm that generate the corresponding .c file with all possible branches as
54+
# their own function and symbol. Additional input
55+
# files should be added to generated_source_modules, as well as adding
56+
# their .c variants to generated_sources.
57+
%.c : %.pm;
58+
$(PERL) generate-opt-funcs.pl $@
3059

3160
# Make the output library in this directory, and name it either
3261
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la

‎ompi/mca/mtl/ofi/README‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,3 +143,152 @@ Command-line syntax to set the MCA variable:
143143
variable "mtl_ofi_thread_grouping" set to 1, the MTL will use multiple
144144
contexts, but the benefits may be negligible as only one thread is driving
145145
progress.
146+
147+
SPECIALIZED FUNCTIONS:
148+
-------------------
149+
To improve performance when calling message passing APIs in the OFI mtl
150+
specialized functions are generated at compile time that eliminate all the
151+
if conditionals that can be determined at init and don't need to be
152+
queried again during the critical path. These functions are generated by
153+
perl scripts during make which generate functions and symbols for every
154+
combination of flags for each function.
155+
156+
1. ADDING NEW FLAGS FOR SPECIALIZATION OF EXISTING FUNCTION:
157+
To add a new flag to an existing specialized function for handling cases
158+
where different OFI providers may or may not support a particular feature,
159+
then you must follow these steps:
160+
1) Update the "_generic" function in mtl_ofi.h with the new flag and
161+
the if conditionals to read the new value.
162+
2) Update the *.pm file corresponding to the function with the new flag in:
163+
gen_funcs(), gen_*_function(), & gen_*_sym_init()
164+
3) Update mtl_ofi_opt.h with:
165+
The new flag as #define NEW_FLAG_TYPES #NUMBER_OF_STATES
166+
example: #define OFI_CQ_DATA 2 (only has TRUE/FALSE states)
167+
Update the function's types with:
168+
#define OMPI_MTL_OFI_FUNCTION_TYPES [NEW_FLAG_TYPES]
169+
170+
2. ADDING A NEW FUNCTION FOR SPECIALIZATION:
171+
To add a new function to be specialized you must
172+
follow these steps:
173+
1) Create a new mtl_ofi_"function_name"_opt.pm based off opt_common/mtl_ofi_opt.pm.template
174+
2) Add new .pm file to generated_source_modules in Makefile.am
175+
3) Add .c file to generated_sources in Makefile.am named the same as the corresponding .pm file
176+
4) Update existing or create function in mtl_ofi.h to _generic with new flags.
177+
5) Update mtl_ofi_opt.h with:
178+
a) New function types: #define OMPI_MTL_OFI_FUNCTION_TYPES [FLAG_TYPES]
179+
b) Add new function to the struct ompi_mtl_ofi_symtable:
180+
struct ompi_mtl_ofi_symtable {
181+
...
182+
int (*ompi_mtl_ofi_FUNCTION OMPI_MTL_OFI_FUNCTION_TYPES )
183+
}
184+
c) Add new symbol table init function definition:
185+
void ompi_mtl_ofi_FUNCTION_symtable_init(struct ompi_mtl_ofi_symtable* sym_table);
186+
6) Add calls to init the new function in the symbol table and assign the function
187+
pointer to be used based off the flags in mtl_ofi_component.c:
188+
ompi_mtl_ofi_FUNCTION_symtable_init(&ompi_mtl_ofi.sym_table);
189+
ompi_mtl_ofi.base.mtl_FUNCTION =
190+
ompi_mtl_ofi.sym_table.ompi_mtl_ofi_FUNCTION[ompi_mtl_ofi.flag];
191+
192+
3. EXAMPLE SPECIALIZED FILE:
193+
The code below is an example of what is generated by the specialization
194+
scripts for use in the OFI mtl. This code specializes the blocking
195+
send functionality based on FI_REMOTE_CQ_DATA & OFI Scalable Endpoint support
196+
provided by an OFI Provider. Only one function and symbol is used during
197+
runtime based on if FI_REMOTE_CQ_DATA is supported and/or if OFI Scalable
198+
Endpoint support is enabled.
199+
/*
200+
* Copyright (c) 2013-2018 Intel, Inc. All rights reserved
201+
*
202+
* $COPYRIGHT$
203+
*
204+
* Additional copyrights may follow
205+
*
206+
* $HEADER$
207+
*/
208+
209+
#include "mtl_ofi.h"
210+
211+
__opal_attribute_always_inline__ static inline int
212+
ompi_mtl_ofi_send_false_false(struct mca_mtl_base_module_t *mtl,
213+
struct ompi_communicator_t *comm,
214+
int dest,
215+
int tag,
216+
struct opal_convertor_t *convertor,
217+
mca_pml_base_send_mode_t mode)
218+
{
219+
const bool OFI_CQ_DATA = false;
220+
const bool OFI_SCEP_EPS = false;
221+
222+
return ompi_mtl_ofi_send_generic(mtl, comm, dest, tag,
223+
convertor, mode,
224+
OFI_CQ_DATA, OFI_SCEP_EPS);
225+
}
226+
227+
__opal_attribute_always_inline__ static inline int
228+
ompi_mtl_ofi_send_false_true(struct mca_mtl_base_module_t *mtl,
229+
struct ompi_communicator_t *comm,
230+
int dest,
231+
int tag,
232+
struct opal_convertor_t *convertor,
233+
mca_pml_base_send_mode_t mode)
234+
{
235+
const bool OFI_CQ_DATA = false;
236+
const bool OFI_SCEP_EPS = true;
237+
238+
return ompi_mtl_ofi_send_generic(mtl, comm, dest, tag,
239+
convertor, mode,
240+
OFI_CQ_DATA, OFI_SCEP_EPS);
241+
}
242+
243+
__opal_attribute_always_inline__ static inline int
244+
ompi_mtl_ofi_send_true_false(struct mca_mtl_base_module_t *mtl,
245+
struct ompi_communicator_t *comm,
246+
int dest,
247+
int tag,
248+
struct opal_convertor_t *convertor,
249+
mca_pml_base_send_mode_t mode)
250+
{
251+
const bool OFI_CQ_DATA = true;
252+
const bool OFI_SCEP_EPS = false;
253+
254+
return ompi_mtl_ofi_send_generic(mtl, comm, dest, tag,
255+
convertor, mode,
256+
OFI_CQ_DATA, OFI_SCEP_EPS);
257+
}
258+
259+
__opal_attribute_always_inline__ static inline int
260+
ompi_mtl_ofi_send_true_true(struct mca_mtl_base_module_t *mtl,
261+
struct ompi_communicator_t *comm,
262+
int dest,
263+
int tag,
264+
struct opal_convertor_t *convertor,
265+
mca_pml_base_send_mode_t mode)
266+
{
267+
const bool OFI_CQ_DATA = true;
268+
const bool OFI_SCEP_EPS = true;
269+
270+
return ompi_mtl_ofi_send_generic(mtl, comm, dest, tag,
271+
convertor, mode,
272+
OFI_CQ_DATA, OFI_SCEP_EPS);
273+
}
274+
275+
void ompi_mtl_ofi_send_symtable_init(struct ompi_mtl_ofi_symtable* sym_table)
276+
{
277+
278+
sym_table->ompi_mtl_ofi_send[false][false]
279+
= ompi_mtl_ofi_send_false_false;
280+
281+
282+
sym_table->ompi_mtl_ofi_send[false][true]
283+
= ompi_mtl_ofi_send_false_true;
284+
285+
286+
sym_table->ompi_mtl_ofi_send[true][false]
287+
= ompi_mtl_ofi_send_true_false;
288+
289+
290+
sym_table->ompi_mtl_ofi_send[true][true]
291+
= ompi_mtl_ofi_send_true_true;
292+
293+
}
294+
###

‎ompi/mca/mtl/ofi/generate-opt-funcs.pl‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env perl
2+
#
3+
# Copyright (c) 2013-2018 Intel, Inc. All rights reserved
4+
#
5+
# $COPYRIGHT$
6+
#
7+
# Additional copyrights may follow
8+
#
9+
# $HEADER$
10+
#
11+
12+
use strict;
13+
use warnings;
14+
use mtl_ofi_send_opt;
15+
use mtl_ofi_isend_opt;
16+
use mtl_ofi_irecv_opt;
17+
use mtl_ofi_iprobe_opt;
18+
use mtl_ofi_improbe_opt;
19+
use opt_common::mtl_ofi_opt_common;
20+
21+
my $MTL_OFI_HEADER =
22+
'/*
23+
* Copyright (c) 2013-2018 Intel, Inc. All rights reserved
24+
*
25+
* $COPYRIGHT$
26+
*
27+
* Additional copyrights may follow
28+
*
29+
* $HEADER$
30+
*/
31+
32+
#include "mtl_ofi.h"';
33+
34+
my $specialization_file = $ARGV[0];
35+
my $specialization_type = $specialization_file;
36+
$specialization_type =~ s{\.[^.]+$}{};
37+
my $sym_table_type = $specialization_type;
38+
$sym_table_type =~ s/_opt//g;
39+
40+
open my $gen_file, ">", $specialization_file;
41+
42+
#
43+
# Generate the Specialized functions & symbol table for the specified file.
44+
#
45+
print $gen_file "$MTL_OFI_HEADER\n\n";
46+
47+
my $GEN_FUNC = $specialization_type . "::gen_funcs\(\$gen_file, \"FUNC\"\)";
48+
my $GEN_SYM = $specialization_type . "::gen_funcs\(\$gen_file, \"SYM\"\)";
49+
my $SYM_TABLE = "ompi_" . $sym_table_type . "_symtable";
50+
51+
eval $GEN_FUNC;
52+
53+
my $SYM_FUNC_HEADER = opt_common::mtl_ofi_opt_common::gen_sym_function_header($SYM_TABLE);
54+
print $gen_file "$SYM_FUNC_HEADER\n";
55+
56+
eval $GEN_SYM;
57+
58+
my $SYM_FUNC_FOOTER = opt_common::mtl_ofi_opt_common::gen_sym_function_footer();
59+
print $gen_file "$SYM_FUNC_FOOTER\n\n";
60+
close($gen_file);
61+
exit(0);
62+
###

‎ompi/mca/mtl/ofi/mtl_ofi.c‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ mca_mtl_ofi_module_t ompi_mtl_ofi = {
2323
ompi_mtl_ofi_del_procs,
2424
ompi_mtl_ofi_finalize,
2525

26-
ompi_mtl_ofi_send,
27-
ompi_mtl_ofi_isend,
28-
ompi_mtl_ofi_irecv,
29-
ompi_mtl_ofi_iprobe,
30-
ompi_mtl_ofi_imrecv,
31-
ompi_mtl_ofi_improbe,
26+
NULL,
27+
NULL,
28+
NULL,
29+
NULL,
30+
ompi_mtl_ofi_imrecv,
31+
NULL,
3232

3333
ompi_mtl_ofi_cancel,
3434
ompi_mtl_ofi_add_comm,

0 commit comments

Comments
 (0)