OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_mx.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2009 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
13 * $COPYRIGHT$
14 *
15 * Additional copyrights may follow
16 *
17 * $HEADER$
18 */
19 /**
20 * @file
21 */
22 #ifndef MCA_PTL_MX_H
23 #define MCA_PTL_MX_H
24
25 #include "ompi_config.h"
26 #include <sys/types.h>
27 #include <string.h>
28
29 /* Open MPI includes */
30 #include "ompi/class/ompi_free_list.h"
31 #include "opal/mca/event/event.h"
32 #include "ompi/mca/btl/btl.h"
33 #include "ompi/mca/btl/base/base.h"
34 #include "ompi/mca/mpool/mpool.h"
35
36 #include <myriexpress.h>
37
38 #ifdef HAVE_MX_EXTENSIONS_H
39 #include <mx_extensions.h>
40 #endif /* HAVE_MX_EXTENSIONS_H */
41
42 BEGIN_C_DECLS
43
44 /**
45 * The mask used for receive and for the PUT protocol
46 */
47 #define BTL_MX_RECV_MASK 0x00000000000000ffULL
48 #define BTL_MX_PUT_MASK 0xffffffffffffffffULL
49
50 /**
51 * MX BTL component.
52 */
53
54 struct mca_btl_mx_component_t {
55 mca_btl_base_component_2_0_0_t super; /**< base BTL component */
56
57 int32_t mx_num_btls;
58 int32_t mx_max_btls;
59 /**< number of hcas available to the MX component */
60
61 struct mca_btl_mx_module_t** mx_btls;
62 /**< array of available BTL modules */
63
64 int32_t mx_free_list_num;
65 /**< initial size of free lists */
66
67 int32_t mx_free_list_max;
68 /**< maximum size of free lists */
69
70 int32_t mx_max_posted_recv;
71 /**< number of posted receives on each NIC */
72
73 int32_t mx_free_list_inc;
74 /**< number of elements to alloc when growing free lists */
75
76 int32_t mx_support_sharedmem;
77 /**< true if we want to activate the MX support for shared memory */
78 int32_t mx_support_self;
79 /**< true if we want to activate the MX support for self communications */
80 int32_t mx_bonding;
81 /**< true if MX is in charge of doing the device bonding */
82 int32_t mx_use_unexpected;
83 /**< true if Open MPI is allowed to register an unexpected handler with the MX library */
84
85 opal_list_t mx_procs; /**< list of mx proc structures */
86
87 int32_t mx_filter;
88 int32_t mx_timeout;
89 int32_t mx_connection_retries;
90
91 ompi_free_list_t mx_send_eager_frags; /**< free list of mx eager send fragments */
92 ompi_free_list_t mx_send_user_frags; /**< free list of mx user send fragments */
93
94 opal_mutex_t mx_lock; /**< lock for accessing module state */
95
96 #if MX_HAVE_MAPPER_STATE
97 char* mx_if_include; /**< include the following NICs */
98 char* mx_if_exclude; /**< Exclude the following NICs. These
99 * values are based on the last 6
100 * digits in hexadecimal of the MAC
101 * address of the mapper.
102 */
103 #endif /* MX_HAVE_MAPPER_STATE */
104 };
105 typedef struct mca_btl_mx_component_t mca_btl_mx_component_t;
106
107 OMPI_MODULE_DECLSPEC extern mca_btl_mx_component_t mca_btl_mx_component;
108
109 /**
110 * BTL Module Interface.
111 * Each BTL correspond to a high level vision of a network interface. The
112 * current version of the MX BTL is not able to handle stripping of the
113 * messages by itself. Therefore, it rely on the PML layer for that.
114 */
115 struct mca_btl_mx_module_t {
116 mca_btl_base_module_t super; /**< base BTL interface */
117 mx_endpoint_t mx_endpoint; /**< local MX endpoint */
118 mx_endpoint_addr_t mx_endpoint_addr; /**< local MX endpoint address */
119 uint32_t mx_unique_network_id; /**< unique identifier for this BTL,
120 * based on the MAC address of the
121 * mapper used to route messages.
122 */
123 opal_list_t mx_peers; /**< list of peers */
124
125 int32_t mx_posted_request; /**< number of posted MX request */
126 opal_mutex_t mx_lock; /**< lock for accessing module state */
127 };
128 typedef struct mca_btl_mx_module_t mca_btl_mx_module_t;
129 extern mca_btl_mx_module_t mca_btl_mx_module;
130
131 /**
132 * MX component initialization.
133 *
134 * @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
135 * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
136 * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
137 */
138 extern mca_btl_base_module_t** mca_btl_mx_component_init(
139 int *num_btl_modules,
140 bool allow_multi_user_threads,
141 bool have_hidden_threads
142 );
143
144
145 /**
146 * MX component progress.
147 */
148 extern int mca_btl_mx_component_progress(void);
149
150
151
152 /**
153 * Cleanup any resources held by the BTL.
154 *
155 * @param btl BTL instance.
156 * @return OMPI_SUCCESS or error status on failure.
157 */
158
159 extern int mca_btl_mx_finalize(
160 struct mca_btl_base_module_t* btl
161 );
162
163
164 /**
165 * PML->BTL notification of change in the process list.
166 *
167 * @param btl (IN)
168 * @param nprocs (IN) Number of processes
169 * @param procs (IN) Set of processes
170 * @param peers (OUT) Set of (optional) peer addressing info.
171 * @param peers (IN/OUT) Set of processes that are reachable via this BTL.
172 * @return OMPI_SUCCESS or error status on failure.
173 *
174 */
175
176 extern int mca_btl_mx_add_procs(
177 struct mca_btl_base_module_t* btl,
178 size_t nprocs,
179 struct ompi_proc_t **procs,
180 struct mca_btl_base_endpoint_t** peers,
181 opal_bitmap_t* reachable
182 );
183
184 /**
185 * PML->BTL notification of change in the process list.
186 *
187 * @param btl (IN) BTL instance
188 * @param nproc (IN) Number of processes.
189 * @param procs (IN) Set of processes.
190 * @param peers (IN) Set of peer data structures.
191 * @return Status indicating if cleanup was successful
192 *
193 */
194
195 extern int mca_btl_mx_del_procs(
196 struct mca_btl_base_module_t* btl,
197 size_t nprocs,
198 struct ompi_proc_t **procs,
199 struct mca_btl_base_endpoint_t** peers
200 );
201
202
203 /**
204 * Initiate an asynchronous send.
205 *
206 * @param btl (IN) BTL module
207 * @param endpoint (IN) BTL addressing information
208 * @param descriptor (IN) Description of the data to be transfered
209 * @param tag (IN) The tag value used to notify the peer.
210 */
211
212 extern int mca_btl_mx_send(
213 struct mca_btl_base_module_t* btl,
214 struct mca_btl_base_endpoint_t* btl_peer,
215 struct mca_btl_base_descriptor_t* descriptor,
216 mca_btl_base_tag_t tag
217 );
218
219
220 /**
221 * Register a callback function that is called on receipt
222 * of a fragment.
223 *
224 * @param btl (IN) BTL module
225 * @return Status indicating if registration was successful
226 *
227 */
228
229 extern int mca_btl_mx_register(
230 struct mca_btl_base_module_t* btl,
231 mca_btl_base_tag_t tag,
232 mca_btl_base_module_recv_cb_fn_t cbfunc,
233 void* cbdata);
234
235 /**
236 * Allocate a descriptor with a segment of the requested size.
237 * Note that the BTL layer may choose to return a smaller size
238 * if it cannot support the request.
239 *
240 * @param btl (IN) BTL module
241 * @param size (IN) Request segment size.
242 */
243
244 mca_btl_base_descriptor_t* mca_btl_mx_alloc( struct mca_btl_base_module_t* btl,
245 struct mca_btl_base_endpoint_t* endpoint,
246 uint8_t order,
247 size_t size,
248 uint32_t flags);
249
250
251 /**
252 * Return a segment allocated by this BTL.
253 *
254 * @param btl (IN) BTL module
255 * @param descriptor (IN) Allocated descriptor.
256 */
257
258 int mca_btl_mx_free( struct mca_btl_base_module_t* btl,
259 mca_btl_base_descriptor_t* des );
260
261
262 /**
263 * Prepare a descriptor for send/rdma using the supplied
264 * convertor. If the convertor references data that is contigous,
265 * the descriptor may simply point to the user buffer. Otherwise,
266 * this routine is responsible for allocating buffer space and
267 * packing if required.
268 *
269 * @param btl (IN) BTL module
270 * @param endpoint (IN) BTL peer addressing
271 * @param convertor (IN) Data type convertor
272 * @param reserve (IN) Additional bytes requested by upper layer to precede user data
273 * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
274 */
275 mca_btl_base_descriptor_t*
276 mca_btl_mx_prepare_src( struct mca_btl_base_module_t* btl,
277 struct mca_btl_base_endpoint_t* peer,
278 struct mca_mpool_base_registration_t*,
279 struct opal_convertor_t* convertor,
280 uint8_t order,
281 size_t reserve,
282 size_t* size,
283 uint32_t flags);
284
285 mca_btl_base_descriptor_t*
286 mca_btl_mx_prepare_dst( struct mca_btl_base_module_t* btl,
287 struct mca_btl_base_endpoint_t* peer,
288 struct mca_mpool_base_registration_t*,
289 struct opal_convertor_t* convertor,
290 uint8_t order,
291 size_t reserve,
292 size_t* size,
293 uint32_t flags);
294
295 /**
296 * Fault Tolerance Event Notification Function
297 * @param state Checkpoint Stae
298 * @return OMPI_SUCCESS or failure status
299 */
300 int mca_btl_mx_ft_event(int state);
301
302 END_C_DECLS
303
304 #endif
mca_btl_mx_component_t::mx_btls
struct mca_btl_mx_module_t ** mx_btls
array of available BTL modules
Definition: btl_mx.h:61
mca_btl_mx_send
int mca_btl_mx_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
Initiate an asynchronous send.
Definition: btl_mx.c:509
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
mca_btl_base_module_recv_cb_fn_t
void(* mca_btl_base_module_recv_cb_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t *descriptor, void *cbdata)
Callback function that is called asynchronously on receipt of data by the transport layer...
Definition: btl.h:391
opal_bitmap_t
Definition: opal_bitmap.h:53
mca_btl_mx_component_t::mx_send_user_frags
ompi_free_list_t mx_send_user_frags
free list of mx user send fragments
Definition: btl_mx.h:92
mca_btl_mx_component_t::mx_support_self
int32_t mx_support_self
true if we want to activate the MX support for self communications
Definition: btl_mx.h:78
mca_btl_mx_add_procs
int mca_btl_mx_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
PML->BTL notification of change in the process list.
Definition: btl_mx.c:36
mca_btl_mx_del_procs
int mca_btl_mx_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
PML->BTL notification of change in the process list.
Definition: btl_mx.c:97
mca_btl_mx_component_init
mca_btl_base_module_t ** mca_btl_mx_component_init(int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
MX component initialization.
Definition: btl_mx_component.c:424
opal_mutex_t
Definition: mutex_unix.h:53
mca_btl_mx_ft_event
int mca_btl_mx_ft_event(int state)
Fault Tolerance Event Notification Function.
Definition: btl_mx.c:625
mca_btl_mx_component_t
MX BTL component.
Definition: btl_mx.h:54
mca_mpool_base_registration_t
Definition: mpool.h:44
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
myriexpress.h
Macros to help interaction between hwloc and Myrinet Express.
mca_btl_mx_alloc
mca_btl_base_descriptor_t * mca_btl_mx_alloc(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
Allocate a descriptor with a segment of the requested size.
Definition: btl_mx.c:170
mca_btl_mx_module_t::mx_posted_request
int32_t mx_posted_request
number of posted MX request
Definition: btl_mx.h:125
mca_btl_mx_register
int mca_btl_mx_register(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata)
Register a callback function that is called on receipt of a fragment.
Definition: btl_mx.c:112
mca_btl_mx_component_t::mx_bonding
int32_t mx_bonding
true if MX is in charge of doing the device bonding
Definition: btl_mx.h:80
mca_btl_mx_prepare_dst
mca_btl_base_descriptor_t * mca_btl_mx_prepare_dst(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_mx.c:304
mca_btl_mx_component_t::mx_max_posted_recv
int32_t mx_max_posted_recv
number of posted receives on each NIC
Definition: btl_mx.h:70
mca_btl_mx_component_t::super
mca_btl_base_component_2_0_0_t super
base BTL component
Definition: btl_mx.h:55
mca_btl_mx_component_progress
int mca_btl_mx_component_progress(void)
MX component progress.
Definition: btl_mx_component.c:577
mca_btl_mx_component_t::mx_lock
opal_mutex_t mx_lock
lock for accessing module state
Definition: btl_mx.h:94
mca_btl_mx_finalize
int mca_btl_mx_finalize(struct mca_btl_base_module_t *btl)
Cleanup any resources held by the BTL.
Definition: btl_mx.c:610
mca_btl_mx_module_t::mx_endpoint_addr
mx_endpoint_addr_t mx_endpoint_addr
local MX endpoint address
Definition: btl_mx.h:118
mca_btl_mx_component_t::mx_max_btls
int32_t mx_max_btls
number of hcas available to the MX component
Definition: btl_mx.h:58
mca_btl_mx_component_t::mx_procs
opal_list_t mx_procs
list of mx proc structures
Definition: btl_mx.h:85
btl.h
Byte Transfer Layer (BTL)
ompi_free_list_t
Definition: ompi_free_list.h:39
mca_btl_mx_prepare_src
mca_btl_base_descriptor_t * mca_btl_mx_prepare_src(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_mx.c:219
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_mx_module_t::mx_unique_network_id
uint32_t mx_unique_network_id
unique identifier for this BTL, based on the MAC address of the mapper used to route messages...
Definition: btl_mx.h:119
mca_btl_base_component_2_0_0_t
BTL component descriptor.
Definition: btl.h:411
mpool.h
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
opal_convertor_t
Definition: opal_convertor.h:90
opal_list_t
Definition: opal_list.h:147
base.h
mca_btl_mx_free
int mca_btl_mx_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
Return a segment allocated by this BTL.
Definition: btl_mx.c:200
mca_btl_mx_component_t::mx_support_sharedmem
int32_t mx_support_sharedmem
true if we want to activate the MX support for shared memory
Definition: btl_mx.h:76
mca_btl_mx_component_t::mx_free_list_num
int32_t mx_free_list_num
initial size of free lists
Definition: btl_mx.h:64
mca_btl_mx_module_t
BTL Module Interface.
Definition: btl_mx.h:115
mca_btl_mx_module_t::mx_lock
opal_mutex_t mx_lock
lock for accessing module state
Definition: btl_mx.h:126
mca_btl_mx_component_t::mx_free_list_max
int32_t mx_free_list_max
maximum size of free lists
Definition: btl_mx.h:67
mca_btl_mx_component_t::mx_free_list_inc
int32_t mx_free_list_inc
number of elements to alloc when growing free lists
Definition: btl_mx.h:73
mca_btl_base_module_t
BTL module interface functions and attributes.
Definition: btl.h:786
mca_btl_mx_component_t::mx_use_unexpected
int32_t mx_use_unexpected
true if Open MPI is allowed to register an unexpected handler with the MX library ...
Definition: btl_mx.h:82
mca_btl_mx_module_t::mx_endpoint
mx_endpoint_t mx_endpoint
local MX endpoint
Definition: btl_mx.h:117
mca_btl_mx_module_t::mx_peers
opal_list_t mx_peers
list of peers
Definition: btl_mx.h:123
mca_btl_mx_component_t::mx_send_eager_frags
ompi_free_list_t mx_send_eager_frags
free list of mx eager send fragments
Definition: btl_mx.h:91
mca_btl_mx_module_t::super
mca_btl_base_module_t super
base BTL interface
Definition: btl_mx.h:116