OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

coll_fca.h
1 /**
2 Copyright (c) 2011 Mellanox Technologies. All rights reserved.
3 $COPYRIGHT$
4
5 Additional copyrights may follow
6
7 $HEADER$
8 */
9
10 #ifndef MCA_COLL_FCA_H
11 #define MCA_COLL_FCA_H
12
13 #include "ompi_config.h"
14
15 #include "mpi.h"
16 #include "opal/mca/mca.h"
17 #include "ompi/mca/coll/coll.h"
18 #include "ompi/request/request.h"
19 #include "ompi/mca/pml/pml.h"
20 #include "ompi/mca/coll/base/coll_tags.h"
21 #include "ompi/communicator/communicator.h"
22 #include "ompi/op/op.h"
23
24 #include "coll_fca_api.h"
25 #include "coll_fca_debug.h"
26
27
28 #ifdef OMPI_DATATYPE_MAX_PREDEFINED
29
30 #include "opal/datatype/opal_convertor.h"
31 #define FCA_CONVERTOR_T opal_convertor_t
32 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_SEND opal_convertor_copy_and_prepare_for_send
33 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_RECV opal_convertor_copy_and_prepare_for_recv
34 #define FCA_CONVERTOR_CONVERTOR_GET_PACKED_SIZE opal_convertor_get_packed_size
35 #define FCA_CONVERTOR_CONVERTOR_UNPACK opal_convertor_unpack
36 #define FCA_CONVERTOR_CONVERTOR_PACK opal_convertor_pack
37
38 #define FCA_DT_MAX_PREDEFINED OMPI_DATATYPE_MAX_PREDEFINED
39 #define FCA_DT_EXTENT ompi_datatype_type_extent
40 #define FCA_DT_GET_TRUE_EXTENT ompi_datatype_get_true_extent
41 #define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(__dtype, __count) \
42 ompi_datatype_is_contiguous_memory_layout(__dtype, __count)
43 #else
44
45 #include "ompi/datatype/convertor.h"
46 #define FCA_CONVERTOR_T ompi_convertor_t
47 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_SEND ompi_convertor_copy_and_prepare_for_send
48 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_RECV ompi_convertor_copy_and_prepare_for_recv
49 #define FCA_CONVERTOR_CONVERTOR_GET_PACKED_SIZE ompi_convertor_get_packed_size
50 #define FCA_CONVERTOR_CONVERTOR_UNPACK ompi_convertor_unpack
51 #define FCA_CONVERTOR_CONVERTOR_PACK ompi_convertor_pack
52
53 #define FCA_DT_MAX_PREDEFINED DT_MAX_PREDEFINED
54 #define FCA_DT_EXTENT ompi_ddt_type_extent
55 #define FCA_DT_GET_TRUE_EXTENT ompi_ddt_get_true_extent
56 #define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(__dtype, __count) \
57 ompi_ddt_is_contiguous_memory_layout(__dtype, __count)
58 #endif
59
60
61 #ifdef OMPI_PROC_FLAG_LOCAL
62 #define FCA_IS_LOCAL_PROCESS(n) ((n) & OMPI_PROC_FLAG_LOCAL)
63 #else
64 #define FCA_IS_LOCAL_PROCESS(n) OPAL_PROC_ON_LOCAL_NODE(n)
65 #endif
66
67
68 BEGIN_C_DECLS
69
70 /*
71 * FCA library functions.
72 * Used to load the library dynamically.
73 */
74
75
76
77 /**
78 * FCA data type information
79 */
80 struct mca_coll_fca_dtype_info_t {
81 ompi_datatype_t *mpi_dtype;
82 size_t mpi_dtype_extent;
83 int fca_dtype; /* -1 if invalid */
84 size_t fca_dtype_extent;
85
86 };
87 typedef struct mca_coll_fca_dtype_info_t mca_coll_fca_dtype_info_t;
88
89 /**
90 * FCA operator information
91 */
92 struct mca_coll_fca_op_info_t {
93 ompi_op_t *mpi_op;
94 int fca_op; /* -1 if invalid */
95 };
96 typedef struct mca_coll_fca_op_info_t mca_coll_fca_op_info_t;
97
98 #define FCA_MAX_OPS 32 /* Must be large enough to hold all FCA-supported operators */
99
100 /**
101 * Globally exported structure
102 */
103 struct mca_coll_fca_component_t {
104 /** Base coll component */
105 mca_coll_base_component_2_0_0_t super;
106
107 /** MCA parameter: Priority of this component */
108 int fca_priority;
109
110 /** MCA parameter: Verbose level of this component */
111 int fca_verbose;
112
113 /** MCA parameter: Comm_mLid */
114 char *fca_comm_mlid;
115
116 /** MCA parameter: Comm_mGid */
117 char *fca_comm_mgid;
118
119 /** MCA parameter: FCA_Mlid */
120 char *fca_fmm_mlid;
121
122 /** MCA parameter: Path to fca spec file */
123 char* fca_spec_file;
124
125 /** MCA parameter: Path to libfca.so */
126 char* fca_lib_path;
127
128 /** MCA parameter: FCA device */
129 char* fca_dev;
130
131 /** MCA parameter: Enable FCA */
132 int fca_enable;
133
134 /** MCA parameter: Enable FCA Barrier */
135 int fca_enable_barrier;
136
137 /** MCA parameter: Enable FCA Bcast */
138 int fca_enable_bcast;
139
140 /** MCA parameter: Enable FCA Reduce */
141 int fca_enable_reduce;
142
143 /** MCA parameter: Enable FCA Reduce_Scatter */
144 int fca_enable_reduce_scatter;
145
146 /** MCA parameter: Enable FCA Allreduce */
147 int fca_enable_allreduce;
148
149 /** MCA parameter: Enable FCA Allgather */
150 int fca_enable_allgather;
151
152 /** MCA parameter: Enable FCA Allgatherv */
153 int fca_enable_allgatherv;
154
155 /** MCA parameter: Enable FCA Gather */
156 int fca_enable_gather;
157
158 /** MCA parameter: Enable FCA Gatherv */
159 int fca_enable_gatherv;
160
161 /** MCA parameter: Enable FCA AlltoAll */
162 int fca_enable_alltoall;
163
164 /** MCA parameter: Enable FCA AlltoAllv */
165 int fca_enable_alltoallv;
166
167 /** MCA parameter: Enable FCA AlltoAllw */
168 int fca_enable_alltoallw;
169
170 /** MCA parameter: FCA NP */
171 int fca_np;
172
173 /* FCA global stuff */
174 void *fca_lib_handle; /* FCA dynamic library */
175 mca_coll_fca_ops_t fca_ops; /* FCA operations */
176 fca_t *fca_context; /* FCA context handle */
177 mca_coll_fca_dtype_info_t fca_dtypes[FCA_DT_MAX_PREDEFINED]; /* FCA dtype translation */
178 mca_coll_fca_op_info_t fca_reduce_ops[FCA_MAX_OPS]; /* FCA op translation */
179 };
180 typedef struct mca_coll_fca_component_t mca_coll_fca_component_t;
181
182 OMPI_MODULE_DECLSPEC extern mca_coll_fca_component_t mca_coll_fca_component;
183
184
185 /**
186 * FCA enabled communicator
187 */
188 struct mca_coll_fca_module_t {
189 mca_coll_base_module_t super;
190
191 MPI_Comm comm;
192 int rank;
193 int local_proc_idx;
194 int num_local_procs;
195 int *local_ranks;
196 fca_comm_t *fca_comm;
197 fca_comm_desc_t fca_comm_desc;
198 fca_comm_caps_t fca_comm_caps;
199
200 /* Saved handlers - for fallback */
201 mca_coll_base_module_reduce_fn_t previous_reduce;
202 mca_coll_base_module_t *previous_reduce_module;
203 mca_coll_base_module_allreduce_fn_t previous_allreduce;
204 mca_coll_base_module_t *previous_allreduce_module;
205 mca_coll_base_module_bcast_fn_t previous_bcast;
206 mca_coll_base_module_t *previous_bcast_module;
207 mca_coll_base_module_barrier_fn_t previous_barrier;
208 mca_coll_base_module_t *previous_barrier_module;
209 mca_coll_base_module_allgather_fn_t previous_allgather;
210 mca_coll_base_module_t *previous_allgather_module;
211 mca_coll_base_module_allgatherv_fn_t previous_allgatherv;
212 mca_coll_base_module_t *previous_allgatherv_module;
213 mca_coll_base_module_alltoall_fn_t previous_alltoall;
214 mca_coll_base_module_t *previous_alltoall_module;
215 mca_coll_base_module_alltoallv_fn_t previous_alltoallv;
216 mca_coll_base_module_t *previous_alltoallv_module;
217 mca_coll_base_module_alltoallw_fn_t previous_alltoallw;
218 mca_coll_base_module_t *previous_alltoallw_module;
219 mca_coll_base_module_gather_fn_t previous_gather;
220 mca_coll_base_module_t *previous_gather_module;
221 mca_coll_base_module_gatherv_fn_t previous_gatherv;
222 mca_coll_base_module_t *previous_gatherv_module;
223 mca_coll_base_module_reduce_scatter_fn_t previous_reduce_scatter;
224 mca_coll_base_module_t *previous_reduce_scatter_module;
225 };
226 typedef struct mca_coll_fca_module_t mca_coll_fca_module_t;
227
228 OBJ_CLASS_DECLARATION(mca_coll_fca_module_t);
229
230
231 /* API functions */
232 int mca_coll_fca_init_query(bool enable_progress_threads, bool enable_mpi_threads);
233 mca_coll_base_module_t *mca_coll_fca_comm_query(struct ompi_communicator_t *comm, int *priority);
234 int mca_coll_fca_get_fca_lib(struct ompi_communicator_t *comm);
235
236
237 /* Collective functions */
238 int mca_coll_fca_allreduce(void *sbuf, void *rbuf, int count,
239 struct ompi_datatype_t *dtype, struct ompi_op_t *op,
240 struct ompi_communicator_t *comm,
241 mca_coll_base_module_t *module);
242 int mca_coll_fca_bcast(void *buff, int count, struct ompi_datatype_t *datatype,
243 int root, struct ompi_communicator_t *comm,
244 mca_coll_base_module_t *module);
245
246 int mca_coll_fca_reduce(void *sbuf, void* rbuf, int count,
247 struct ompi_datatype_t *dtype, struct ompi_op_t *op,
248 int root, struct ompi_communicator_t *comm,
249 mca_coll_base_module_t *module);
250
251 int mca_coll_fca_barrier(struct ompi_communicator_t *comm,
252 mca_coll_base_module_t *module);
253
254 int mca_coll_fca_allgather(void *sbuf, int scount, struct ompi_datatype_t *sdtype,
255 void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
256 struct ompi_communicator_t *comm,
257 mca_coll_base_module_t *module);
258
259 int mca_coll_fca_allgatherv(void *sbuf, int scount,
260 struct ompi_datatype_t *sdtype,
261 void *rbuf, int *rcounts, int *disps,
262 struct ompi_datatype_t *rdtype,
263 struct ompi_communicator_t *comm,
264 mca_coll_base_module_t *module);
265
266 int mca_coll_fca_alltoall(void *sbuf, int scount,
267 struct ompi_datatype_t *sdtype,
268 void *rbuf, int rcount,
269 struct ompi_datatype_t *rdtype,
270 struct ompi_communicator_t *comm,
271 mca_coll_base_module_t *module);
272
273 int mca_coll_fca_alltoallv(void *sbuf, int *scounts, int *sdisps,
274 struct ompi_datatype_t *sdtype,
275 void *rbuf, int *rcounts, int *rdisps,
276 struct ompi_datatype_t *rdtype,
277 struct ompi_communicator_t *comm,
278 mca_coll_base_module_t *module);
279
280 int mca_coll_fca_alltoallw(void *sbuf, int *scounts, int *sdisps,
281 struct ompi_datatype_t **sdtypes,
282 void *rbuf, int *rcounts, int *rdisps,
283 struct ompi_datatype_t **rdtypes,
284 struct ompi_communicator_t *comm,
285 mca_coll_base_module_t *module);
286
287 int mca_coll_fca_gather(void *sbuf, int scount,
288 struct ompi_datatype_t *sdtype,
289 void *rbuf, int rcount,
290 struct ompi_datatype_t *rdtype,
291 int root, struct ompi_communicator_t *comm,
292 mca_coll_base_module_t *module);
293
294 int mca_coll_fca_gatherv(void *sbuf, int scount,
295 struct ompi_datatype_t *sdtype,
296 void *rbuf, int *rcounts, int *disps,
297 struct ompi_datatype_t *rdtype, int root,
298 struct ompi_communicator_t *comm,
299 mca_coll_base_module_t *module);
300
301 int mca_coll_fca_reduce_scatter(void *sbuf, void *rbuf, int *rcounts,
302 struct ompi_datatype_t *dtype,
303 struct ompi_op_t *op,
304 struct ompi_communicator_t *comm,
305 mca_coll_base_module_t *module);
306
307 END_C_DECLS
308
309 #endif
mca_coll_fca_component_t::fca_enable_gather
int fca_enable_gather
MCA parameter: Enable FCA Gather.
Definition: coll_fca.h:156
mca_coll_fca_ops_t
Definition: coll_fca_api.h:49
mca_coll_fca_component_t::fca_enable_allgather
int fca_enable_allgather
MCA parameter: Enable FCA Allgather.
Definition: coll_fca.h:150
mca_coll_fca_component_t::fca_enable_barrier
int fca_enable_barrier
MCA parameter: Enable FCA Barrier.
Definition: coll_fca.h:135
pml.h
P2P Management Layer (PML)
mca_coll_fca_component_t::fca_spec_file
char * fca_spec_file
MCA parameter: Path to fca spec file.
Definition: coll_fca.h:123
mca_coll_base_module_2_0_0_t
Collective module interface.
Definition: coll.h:316
ompi_datatype_t
Definition: ompi_datatype.h:68
mca_coll_base_component_2_0_0_t
Collective component interface.
Definition: coll.h:283
mca_coll_fca_component_t::fca_enable_bcast
int fca_enable_bcast
MCA parameter: Enable FCA Bcast.
Definition: coll_fca.h:138
mca_coll_fca_component_t::fca_enable_gatherv
int fca_enable_gatherv
MCA parameter: Enable FCA Gatherv.
Definition: coll_fca.h:159
mca_coll_fca_component_t
Globally exported structure.
Definition: coll_fca.h:103
mca_coll_fca_component_t::super
mca_coll_base_component_2_0_0_t super
Base coll component.
Definition: coll_fca.h:105
mca_coll_fca_component_t::fca_enable_alltoall
int fca_enable_alltoall
MCA parameter: Enable FCA AlltoAll.
Definition: coll_fca.h:162
mca_coll_fca_component_t::fca_comm_mgid
char * fca_comm_mgid
MCA parameter: Comm_mGid.
Definition: coll_fca.h:117
mca_coll_fca_component_t::fca_np
int fca_np
MCA parameter: FCA NP.
Definition: coll_fca.h:171
mca_coll_fca_component_t::fca_enable_reduce
int fca_enable_reduce
MCA parameter: Enable FCA Reduce.
Definition: coll_fca.h:141
mca.h
Top-level interface for all MCA components.
mca_coll_fca_component_t::fca_enable_allreduce
int fca_enable_allreduce
MCA parameter: Enable FCA Allreduce.
Definition: coll_fca.h:147
mca_coll_fca_component_t::fca_fmm_mlid
char * fca_fmm_mlid
MCA parameter: FCA_Mlid.
Definition: coll_fca.h:120
coll.h
Collective Communication Interface.
mca_coll_fca_component_t::fca_lib_path
char * fca_lib_path
MCA parameter: Path to libfca.so.
Definition: coll_fca.h:126
op.h
Public interface for the MPI_Op handle.
request.h
Top-level description of requests.
mca_coll_fca_module_t
FCA enabled communicator.
Definition: coll_fca.h:188
mca_coll_fca_op_info_t
FCA operator information.
Definition: coll_fca.h:92
mca_coll_fca_component_t::fca_comm_mlid
char * fca_comm_mlid
MCA parameter: Comm_mLid.
Definition: coll_fca.h:114
mca_coll_fca_component_t::fca_enable_alltoallv
int fca_enable_alltoallv
MCA parameter: Enable FCA AlltoAllv.
Definition: coll_fca.h:165
mca_coll_fca_component_t::fca_priority
int fca_priority
MCA parameter: Priority of this component.
Definition: coll_fca.h:108
mca_coll_fca_component_t::fca_enable_alltoallw
int fca_enable_alltoallw
MCA parameter: Enable FCA AlltoAllw.
Definition: coll_fca.h:168
mca_coll_fca_component_t::fca_enable
int fca_enable
MCA parameter: Enable FCA.
Definition: coll_fca.h:132
mca_coll_fca_dtype_info_t
FCA data type information.
Definition: coll_fca.h:80
mca_coll_fca_component_t::fca_verbose
int fca_verbose
MCA parameter: Verbose level of this component.
Definition: coll_fca.h:111
mca_coll_fca_component_t::fca_dev
char * fca_dev
MCA parameter: FCA device.
Definition: coll_fca.h:129
ompi_communicator_t
Definition: communicator.h:118
mca_coll_fca_component_t::fca_enable_reduce_scatter
int fca_enable_reduce_scatter
MCA parameter: Enable FCA Reduce_Scatter.
Definition: coll_fca.h:144
ompi_op_t
Back-end type of MPI_Op.
Definition: op.h:100
OBJ_CLASS_DECLARATION
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
mca_coll_fca_component_t::fca_enable_allgatherv
int fca_enable_allgatherv
MCA parameter: Enable FCA Allgatherv.
Definition: coll_fca.h:153