OpenMPI  0.1.1
coll_fca.h
1 /**
2  Copyright (c) 2011 Mellanox Technologies. All rights reserved.
3  $COPYRIGHT$
4 
5  Additional copyrights may follow
6 
7  $HEADER$
8  */
9 
10 #ifndef MCA_COLL_FCA_H
11 #define MCA_COLL_FCA_H
12 
13 #include "ompi_config.h"
14 
15 #include "mpi.h"
16 #include "opal/mca/mca.h"
17 #include "ompi/mca/coll/coll.h"
18 #include "ompi/request/request.h"
19 #include "ompi/mca/pml/pml.h"
20 #include "ompi/mca/coll/base/coll_tags.h"
21 #include "ompi/communicator/communicator.h"
22 #include "ompi/op/op.h"
23 
24 #include "coll_fca_api.h"
25 #include "coll_fca_debug.h"
26 
27 
28 #ifdef OMPI_DATATYPE_MAX_PREDEFINED
29 
30 #include "opal/datatype/opal_convertor.h"
31 #define FCA_CONVERTOR_T opal_convertor_t
32 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_SEND opal_convertor_copy_and_prepare_for_send
33 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_RECV opal_convertor_copy_and_prepare_for_recv
34 #define FCA_CONVERTOR_CONVERTOR_GET_PACKED_SIZE opal_convertor_get_packed_size
35 #define FCA_CONVERTOR_CONVERTOR_UNPACK opal_convertor_unpack
36 #define FCA_CONVERTOR_CONVERTOR_PACK opal_convertor_pack
37 
38 #define FCA_DT_MAX_PREDEFINED OMPI_DATATYPE_MAX_PREDEFINED
39 #define FCA_DT_EXTENT ompi_datatype_type_extent
40 #define FCA_DT_GET_TRUE_EXTENT ompi_datatype_get_true_extent
41 #define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(__dtype, __count) \
42  ompi_datatype_is_contiguous_memory_layout(__dtype, __count)
43 #else
44 
45 #include "ompi/datatype/convertor.h"
46 #define FCA_CONVERTOR_T ompi_convertor_t
47 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_SEND ompi_convertor_copy_and_prepare_for_send
48 #define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_RECV ompi_convertor_copy_and_prepare_for_recv
49 #define FCA_CONVERTOR_CONVERTOR_GET_PACKED_SIZE ompi_convertor_get_packed_size
50 #define FCA_CONVERTOR_CONVERTOR_UNPACK ompi_convertor_unpack
51 #define FCA_CONVERTOR_CONVERTOR_PACK ompi_convertor_pack
52 
53 #define FCA_DT_MAX_PREDEFINED DT_MAX_PREDEFINED
54 #define FCA_DT_EXTENT ompi_ddt_type_extent
55 #define FCA_DT_GET_TRUE_EXTENT ompi_ddt_get_true_extent
56 #define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(__dtype, __count) \
57  ompi_ddt_is_contiguous_memory_layout(__dtype, __count)
58 #endif
59 
60 
61 #ifdef OMPI_PROC_FLAG_LOCAL
62 #define FCA_IS_LOCAL_PROCESS(n) ((n) & OMPI_PROC_FLAG_LOCAL)
63 #else
64 #define FCA_IS_LOCAL_PROCESS(n) OPAL_PROC_ON_LOCAL_NODE(n)
65 #endif
66 
67 
68 BEGIN_C_DECLS
69 
70 /*
71  * FCA library functions.
72  * Used to load the library dynamically.
73  */
74 
75 
76 
77 /**
78  * FCA data type information
79  */
81  ompi_datatype_t *mpi_dtype;
82  size_t mpi_dtype_extent;
83  int fca_dtype; /* -1 if invalid */
84  size_t fca_dtype_extent;
85 
86 };
88 
89 /**
90  * FCA operator information
91  */
93  ompi_op_t *mpi_op;
94  int fca_op; /* -1 if invalid */
95 };
97 
98 #define FCA_MAX_OPS 32 /* Must be large enough to hold all FCA-supported operators */
99 
100 /**
101  * Globally exported structure
102  */
104  /** Base coll component */
106 
107  /** MCA parameter: Priority of this component */
109 
110  /** MCA parameter: Verbose level of this component */
112 
113  /** MCA parameter: Comm_mLid */
115 
116  /** MCA parameter: Comm_mGid */
118 
119  /** MCA parameter: FCA_Mlid */
121 
122  /** MCA parameter: Path to fca spec file */
124 
125  /** MCA parameter: Path to libfca.so */
127 
128  /** MCA parameter: FCA device */
129  char* fca_dev;
130 
131  /** MCA parameter: Enable FCA */
133 
134  /** MCA parameter: Enable FCA Barrier */
136 
137  /** MCA parameter: Enable FCA Bcast */
139 
140  /** MCA parameter: Enable FCA Reduce */
142 
143  /** MCA parameter: Enable FCA Reduce_Scatter */
145 
146  /** MCA parameter: Enable FCA Allreduce */
148 
149  /** MCA parameter: Enable FCA Allgather */
151 
152  /** MCA parameter: Enable FCA Allgatherv */
154 
155  /** MCA parameter: Enable FCA Gather */
157 
158  /** MCA parameter: Enable FCA Gatherv */
160 
161  /** MCA parameter: Enable FCA AlltoAll */
163 
164  /** MCA parameter: Enable FCA AlltoAllv */
166 
167  /** MCA parameter: Enable FCA AlltoAllw */
169 
170  /** MCA parameter: FCA NP */
171  int fca_np;
172 
173  /* FCA global stuff */
174  void *fca_lib_handle; /* FCA dynamic library */
175  mca_coll_fca_ops_t fca_ops; /* FCA operations */
176  fca_t *fca_context; /* FCA context handle */
177  mca_coll_fca_dtype_info_t fca_dtypes[FCA_DT_MAX_PREDEFINED]; /* FCA dtype translation */
178  mca_coll_fca_op_info_t fca_reduce_ops[FCA_MAX_OPS]; /* FCA op translation */
179 };
181 
182 OMPI_MODULE_DECLSPEC extern mca_coll_fca_component_t mca_coll_fca_component;
183 
184 
185 /**
186  * FCA enabled communicator
187  */
190 
191  MPI_Comm comm;
192  int rank;
193  int local_proc_idx;
194  int num_local_procs;
195  int *local_ranks;
196  fca_comm_t *fca_comm;
197  fca_comm_desc_t fca_comm_desc;
198  fca_comm_caps_t fca_comm_caps;
199 
200  /* Saved handlers - for fallback */
201  mca_coll_base_module_reduce_fn_t previous_reduce;
202  mca_coll_base_module_t *previous_reduce_module;
203  mca_coll_base_module_allreduce_fn_t previous_allreduce;
204  mca_coll_base_module_t *previous_allreduce_module;
205  mca_coll_base_module_bcast_fn_t previous_bcast;
206  mca_coll_base_module_t *previous_bcast_module;
207  mca_coll_base_module_barrier_fn_t previous_barrier;
208  mca_coll_base_module_t *previous_barrier_module;
209  mca_coll_base_module_allgather_fn_t previous_allgather;
210  mca_coll_base_module_t *previous_allgather_module;
211  mca_coll_base_module_allgatherv_fn_t previous_allgatherv;
212  mca_coll_base_module_t *previous_allgatherv_module;
213  mca_coll_base_module_alltoall_fn_t previous_alltoall;
214  mca_coll_base_module_t *previous_alltoall_module;
215  mca_coll_base_module_alltoallv_fn_t previous_alltoallv;
216  mca_coll_base_module_t *previous_alltoallv_module;
217  mca_coll_base_module_alltoallw_fn_t previous_alltoallw;
218  mca_coll_base_module_t *previous_alltoallw_module;
219  mca_coll_base_module_gather_fn_t previous_gather;
220  mca_coll_base_module_t *previous_gather_module;
221  mca_coll_base_module_gatherv_fn_t previous_gatherv;
222  mca_coll_base_module_t *previous_gatherv_module;
223  mca_coll_base_module_reduce_scatter_fn_t previous_reduce_scatter;
224  mca_coll_base_module_t *previous_reduce_scatter_module;
225 };
227 
229 
230 
231 /* API functions */
232 int mca_coll_fca_init_query(bool enable_progress_threads, bool enable_mpi_threads);
233 mca_coll_base_module_t *mca_coll_fca_comm_query(struct ompi_communicator_t *comm, int *priority);
234 int mca_coll_fca_get_fca_lib(struct ompi_communicator_t *comm);
235 
236 
237 /* Collective functions */
238 int mca_coll_fca_allreduce(void *sbuf, void *rbuf, int count,
239  struct ompi_datatype_t *dtype, struct ompi_op_t *op,
240  struct ompi_communicator_t *comm,
241  mca_coll_base_module_t *module);
242 int mca_coll_fca_bcast(void *buff, int count, struct ompi_datatype_t *datatype,
243  int root, struct ompi_communicator_t *comm,
244  mca_coll_base_module_t *module);
245 
246 int mca_coll_fca_reduce(void *sbuf, void* rbuf, int count,
247  struct ompi_datatype_t *dtype, struct ompi_op_t *op,
248  int root, struct ompi_communicator_t *comm,
249  mca_coll_base_module_t *module);
250 
251 int mca_coll_fca_barrier(struct ompi_communicator_t *comm,
252  mca_coll_base_module_t *module);
253 
254 int mca_coll_fca_allgather(void *sbuf, int scount, struct ompi_datatype_t *sdtype,
255  void *rbuf, int rcount, struct ompi_datatype_t *rdtype,
256  struct ompi_communicator_t *comm,
257  mca_coll_base_module_t *module);
258 
259 int mca_coll_fca_allgatherv(void *sbuf, int scount,
260  struct ompi_datatype_t *sdtype,
261  void *rbuf, int *rcounts, int *disps,
262  struct ompi_datatype_t *rdtype,
263  struct ompi_communicator_t *comm,
264  mca_coll_base_module_t *module);
265 
266 int mca_coll_fca_alltoall(void *sbuf, int scount,
267  struct ompi_datatype_t *sdtype,
268  void *rbuf, int rcount,
269  struct ompi_datatype_t *rdtype,
270  struct ompi_communicator_t *comm,
271  mca_coll_base_module_t *module);
272 
273 int mca_coll_fca_alltoallv(void *sbuf, int *scounts, int *sdisps,
274  struct ompi_datatype_t *sdtype,
275  void *rbuf, int *rcounts, int *rdisps,
276  struct ompi_datatype_t *rdtype,
277  struct ompi_communicator_t *comm,
278  mca_coll_base_module_t *module);
279 
280 int mca_coll_fca_alltoallw(void *sbuf, int *scounts, int *sdisps,
281  struct ompi_datatype_t **sdtypes,
282  void *rbuf, int *rcounts, int *rdisps,
283  struct ompi_datatype_t **rdtypes,
284  struct ompi_communicator_t *comm,
285  mca_coll_base_module_t *module);
286 
287 int mca_coll_fca_gather(void *sbuf, int scount,
288  struct ompi_datatype_t *sdtype,
289  void *rbuf, int rcount,
290  struct ompi_datatype_t *rdtype,
291  int root, struct ompi_communicator_t *comm,
292  mca_coll_base_module_t *module);
293 
294 int mca_coll_fca_gatherv(void *sbuf, int scount,
295  struct ompi_datatype_t *sdtype,
296  void *rbuf, int *rcounts, int *disps,
297  struct ompi_datatype_t *rdtype, int root,
298  struct ompi_communicator_t *comm,
299  mca_coll_base_module_t *module);
300 
301 int mca_coll_fca_reduce_scatter(void *sbuf, void *rbuf, int *rcounts,
302  struct ompi_datatype_t *dtype,
303  struct ompi_op_t *op,
304  struct ompi_communicator_t *comm,
305  mca_coll_base_module_t *module);
306 
307 END_C_DECLS
308 
309 #endif
int fca_enable_gather
MCA parameter: Enable FCA Gather.
Definition: coll_fca.h:156
Definition: coll_fca_api.h:49
int fca_enable_allgather
MCA parameter: Enable FCA Allgather.
Definition: coll_fca.h:150
int fca_enable_barrier
MCA parameter: Enable FCA Barrier.
Definition: coll_fca.h:135
P2P Management Layer (PML)
char * fca_spec_file
MCA parameter: Path to fca spec file.
Definition: coll_fca.h:123
Collective module interface.
Definition: coll.h:316
Definition: ompi_datatype.h:68
Collective component interface.
Definition: coll.h:283
int fca_enable_bcast
MCA parameter: Enable FCA Bcast.
Definition: coll_fca.h:138
int fca_enable_gatherv
MCA parameter: Enable FCA Gatherv.
Definition: coll_fca.h:159
Globally exported structure.
Definition: coll_fca.h:103
mca_coll_base_component_2_0_0_t super
Base coll component.
Definition: coll_fca.h:105
int fca_enable_alltoall
MCA parameter: Enable FCA AlltoAll.
Definition: coll_fca.h:162
char * fca_comm_mgid
MCA parameter: Comm_mGid.
Definition: coll_fca.h:117
int fca_np
MCA parameter: FCA NP.
Definition: coll_fca.h:171
int fca_enable_reduce
MCA parameter: Enable FCA Reduce.
Definition: coll_fca.h:141
Top-level interface for all MCA components.
int fca_enable_allreduce
MCA parameter: Enable FCA Allreduce.
Definition: coll_fca.h:147
char * fca_fmm_mlid
MCA parameter: FCA_Mlid.
Definition: coll_fca.h:120
Collective Communication Interface.
char * fca_lib_path
MCA parameter: Path to libfca.so.
Definition: coll_fca.h:126
Public interface for the MPI_Op handle.
Top-level description of requests.
FCA enabled communicator.
Definition: coll_fca.h:188
FCA operator information.
Definition: coll_fca.h:92
char * fca_comm_mlid
MCA parameter: Comm_mLid.
Definition: coll_fca.h:114
int fca_enable_alltoallv
MCA parameter: Enable FCA AlltoAllv.
Definition: coll_fca.h:165
int fca_priority
MCA parameter: Priority of this component.
Definition: coll_fca.h:108
int fca_enable_alltoallw
MCA parameter: Enable FCA AlltoAllw.
Definition: coll_fca.h:168
int fca_enable
MCA parameter: Enable FCA.
Definition: coll_fca.h:132
FCA data type information.
Definition: coll_fca.h:80
int fca_verbose
MCA parameter: Verbose level of this component.
Definition: coll_fca.h:111
char * fca_dev
MCA parameter: FCA device.
Definition: coll_fca.h:129
Definition: communicator.h:118
int fca_enable_reduce_scatter
MCA parameter: Enable FCA Reduce_Scatter.
Definition: coll_fca.h:144
Back-end type of MPI_Op.
Definition: op.h:100
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
int fca_enable_allgatherv
MCA parameter: Enable FCA Allgatherv.
Definition: coll_fca.h:153