OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_gm.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2009 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
13 * $COPYRIGHT$
14 *
15 * Additional copyrights may follow
16 *
17 * $HEADER$
18 */
19 /**
20 * @file
21 */
22 #ifndef MCA_BTL_GM_H
23 #define MCA_BTL_GM_H
24
25 #include "ompi_config.h"
26 #include <sys/types.h>
27 #include <string.h>
28 #include <gm.h>
29
30 /* Open MPI includes */
31 #include "opal/mca/event/event.h"
32 #include "ompi/class/ompi_free_list.h"
33 #include "ompi/mca/btl/btl.h"
34 #include "ompi/mca/btl/base/base.h"
35 #include "ompi/mca/mpool/mpool.h"
36 #include "ompi/mca/btl/btl.h"
37 #include "btl_gm_endpoint.h"
38
39 BEGIN_C_DECLS
40
41 #define GM_BUFFER_SIZE 7
42 #define GM_BUFFER_LENGTH gm_max_length_for_size(GM_BUFFER_SIZE)
43
44 /**
45 * Myrinet (GM) BTL component.
46 */
47
48 struct mca_btl_gm_component_t {
49 mca_btl_base_component_2_0_0_t super; /**< base BTL component */
50
51 size_t gm_num_btls; /**< number of hcas available to the GM component */
52 size_t gm_max_btls; /**< maximum number of supported hcas */
53 struct mca_btl_gm_module_t **gm_btls; /**< array of available BTL modules */
54 size_t gm_max_ports; /**< maximum number of ports per board */
55 size_t gm_max_boards; /**< maximum number of boards */
56 size_t gm_eager_frag_size;
57 size_t gm_max_frag_size;
58 char* gm_port_name;
59 int32_t gm_num_repost;
60 int32_t gm_num_high_priority; /**< number of receive descriptors at high priority */
61 int gm_debug; /**< turn on debug output */
62
63 int gm_free_list_num; /**< initial size of free lists */
64 int gm_free_list_max; /**< maximum size of free lists */
65 int gm_free_list_inc; /**< number of elements to alloc when growing free lists */
66 opal_list_t gm_procs; /**< list of gm proc structures */
67 opal_mutex_t gm_lock; /**< lock for accessing module state */
68 char* gm_mpool_name; /**< name of memory pool */
69 };
70 typedef struct mca_btl_gm_component_t mca_btl_gm_component_t;
71
72 OMPI_MODULE_DECLSPEC extern mca_btl_gm_component_t mca_btl_gm_component;
73
74 /**
75 * BTL Module Interface
76 */
77 struct mca_btl_gm_module_t {
78 mca_btl_base_module_t super; /**< base BTL interface */
79
80 /* local port handle/address */
81 struct gm_port *port;
82 mca_btl_gm_addr_t gm_addr;
83
84 /* free list of fragment descriptors */
85 ompi_free_list_t gm_frag_eager;
86 ompi_free_list_t gm_frag_max;
87 ompi_free_list_t gm_frag_user;
88
89 /* number of send/recv tokens */
90 int32_t gm_num_send_tokens;
91 int32_t gm_max_send_tokens;
92 int32_t gm_num_recv_tokens;
93 int32_t gm_max_recv_tokens;
94 int32_t gm_num_repost;
95
96 /* lock for accessing module state */
97 opal_list_t gm_pending; /**< list of pending send descriptors */
98 opal_list_t gm_repost; /**< list of pending fragments */
99
100 #if OMPI_ENABLE_PROGRESS_THREADS
101 opal_thread_t gm_thread;
102 bool gm_progress;
103 #endif
104 mca_btl_base_module_error_cb_fn_t error_cb;
105 };
106 typedef struct mca_btl_gm_module_t mca_btl_gm_module_t;
107 extern mca_btl_gm_module_t mca_btl_gm_module;
108
109
110 /**
111 * Register GM component parameters with the MCA framework
112 */
113 extern int mca_btl_gm_component_open(void);
114
115 /**
116 * Any final cleanup before being unloaded.
117 */
118 extern int mca_btl_gm_component_close(void);
119
120 /**
121 * GM component initialization.
122 *
123 * @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
124 * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
125 * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
126 */
127 extern mca_btl_base_module_t** mca_btl_gm_component_init(
128 int *num_btl_modules,
129 bool allow_multi_user_threads,
130 bool have_hidden_threads
131 );
132
133
134 /**
135 * GM component progress.
136 */
137 extern int mca_btl_gm_component_progress(void);
138
139
140
141 /**
142 * Cleanup any resources held by the BTL.
143 *
144 * @param btl BTL instance.
145 * @return OMPI_SUCCESS or error status on failure.
146 */
147
148 extern int mca_btl_gm_finalize(
149 struct mca_btl_base_module_t* btl
150 );
151
152
153 /**
154 * PML->BTL notification of change in the process list.
155 *
156 * @param btl (IN)
157 * @param nprocs (IN) Number of processes
158 * @param procs (IN) Set of processes
159 * @param peers (OUT) Set of (optional) peer addressing info.
160 * @param peers (IN/OUT) Set of processes that are reachable via this BTL.
161 * @return OMPI_SUCCESS or error status on failure.
162 *
163 */
164
165 extern int mca_btl_gm_add_procs(
166 struct mca_btl_base_module_t* btl,
167 size_t nprocs,
168 struct ompi_proc_t **procs,
169 struct mca_btl_base_endpoint_t** peers,
170 opal_bitmap_t* reachable
171 );
172
173 /**
174 * PML->BTL notification of change in the process list.
175 *
176 * @param btl (IN) BTL instance
177 * @param nproc (IN) Number of processes.
178 * @param procs (IN) Set of processes.
179 * @param peers (IN) Set of peer data structures.
180 * @return Status indicating if cleanup was successful
181 *
182 */
183
184 extern int mca_btl_gm_del_procs(
185 struct mca_btl_base_module_t* btl,
186 size_t nprocs,
187 struct ompi_proc_t **procs,
188 struct mca_btl_base_endpoint_t** peers
189 );
190
191
192 /**
193 * Initiate an asynchronous send.
194 *
195 * @param btl (IN) BTL module
196 * @param endpoint (IN) BTL addressing information
197 * @param descriptor (IN) Description of the data to be transfered
198 * @param tag (IN) The tag value used to notify the peer.
199 */
200
201 extern int mca_btl_gm_send(
202 struct mca_btl_base_module_t* btl,
203 struct mca_btl_base_endpoint_t* btl_peer,
204 struct mca_btl_base_descriptor_t* descriptor,
205 mca_btl_base_tag_t tag
206 );
207
208
209 /**
210 * Initiate an asynchronous put.
211 *
212 * @param btl (IN) BTL module
213 * @param endpoint (IN) BTL addressing information
214 * @param descriptor (IN) Description of the data to be transferred
215 */
216
217 extern int mca_btl_gm_put(
218 struct mca_btl_base_module_t* btl,
219 struct mca_btl_base_endpoint_t* btl_peer,
220 struct mca_btl_base_descriptor_t* decriptor
221 );
222
223
224 /**
225 * Initiate an asynchronous get.
226 *
227 * @param btl (IN) BTL module
228 * @param endpoint (IN) BTL addressing information
229 * @param descriptor (IN) Description of the data to be transferred
230 */
231
232 extern int mca_btl_gm_get(
233 struct mca_btl_base_module_t* btl,
234 struct mca_btl_base_endpoint_t* btl_peer,
235 struct mca_btl_base_descriptor_t* decriptor
236 );
237
238 /**
239 * Register a callback function that is called on error.
240 *
241 * @param btl (IN) BTL module
242 * @return Status indicating if registration was successful
243 *
244 */
245
246 extern int mca_btl_gm_register_error_cb(
247 struct mca_btl_base_module_t* btl,
248 mca_btl_base_module_error_cb_fn_t cbfunc);
249
250 /**
251 * Allocate a descriptor with a segment of the requested size.
252 * Note that the BTL layer may choose to return a smaller size
253 * if it cannot support the request.
254 *
255 * @param btl (IN) BTL module
256 * @param size (IN) Request segment size.
257 */
258
259 extern mca_btl_base_descriptor_t* mca_btl_gm_alloc(
260 struct mca_btl_base_module_t* btl,
261 struct mca_btl_base_endpoint_t* endpoint,
262 uint8_t order,
263 size_t size,
264 uint32_t flags);
265
266
267 /**
268 * Return a segment allocated by this BTL.
269 *
270 * @param btl (IN) BTL module
271 * @param descriptor (IN) Allocated descriptor.
272 */
273
274 extern int mca_btl_gm_free(
275 struct mca_btl_base_module_t* btl,
276 mca_btl_base_descriptor_t* des);
277
278
279 /**
280 * Prepare a descriptor for send/rdma using the supplied
281 * convertor. If the convertor references data that is contigous,
282 * the descriptor may simply point to the user buffer. Otherwise,
283 * this routine is responsible for allocating buffer space and
284 * packing if required.
285 *
286 * @param btl (IN) BTL module
287 * @param endpoint (IN) BTL peer addressing
288 * @param convertor (IN) Data type convertor
289 * @param reserve (IN) Additional bytes requested by upper layer to precede user data
290 * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
291 */
292
293 mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
294 struct mca_btl_base_module_t* btl,
295 struct mca_btl_base_endpoint_t* peer,
296 struct mca_mpool_base_registration_t*,
297 struct opal_convertor_t* convertor,
298 uint8_t order,
299 size_t reserve,
300 size_t* size,
301 uint32_t flags
302 );
303
304 extern mca_btl_base_descriptor_t* mca_btl_gm_prepare_dst(
305 struct mca_btl_base_module_t* btl,
306 struct mca_btl_base_endpoint_t* peer,
307 struct mca_mpool_base_registration_t*,
308 struct opal_convertor_t* convertor,
309 uint8_t order,
310 size_t reserve,
311 size_t* size,
312 uint32_t flags);
313
314 /**
315 * Fault Tolerance Event Notification Function
316 * @param state Checkpoint Stae
317 * @return OMPI_SUCCESS or failure status
318 */
319 int mca_btl_gm_ft_event(int state);
320
321 /**
322 * Acquire a send token - queue the fragment if none available
323 */
324
325 #define MCA_BTL_GM_ACQUIRE_TOKEN_NL(btl, frag) \
326 do { \
327 /* queue the descriptor if there are no send tokens */ \
328 if(OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, -1) < 0) { \
329 opal_list_append(&gm_btl->gm_pending, (opal_list_item_t*)frag); \
330 OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, 1); \
331 return OMPI_SUCCESS; \
332 } \
333 } while (0) \
334
335
336 #define MCA_BTL_GM_ACQUIRE_TOKEN(btl, frag) \
337 do { \
338 /* queue the descriptor if there are no send tokens */ \
339 if(OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, -1) < 0) { \
340 opal_list_append(&gm_btl->gm_pending, (opal_list_item_t*)frag); \
341 OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, 1); \
342 OPAL_THREAD_UNLOCK(&mca_btl_gm_component.gm_lock); \
343 return OMPI_SUCCESS; \
344 } \
345 } while (0) \
346
347 /**
348 * Return send token and dequeue and pending fragments
349 * mca_btl_gm_component.gm_lock is already held.
350 */
351
352 #define MCA_BTL_GM_RETURN_TOKEN(btl) \
353 do { \
354 OPAL_THREAD_ADD32( &btl->gm_num_send_tokens, 1 ); \
355 if(opal_list_get_size(&btl->gm_pending)) { \
356 mca_btl_gm_frag_t* frag; \
357 frag = (mca_btl_gm_frag_t*)opal_list_remove_first(&btl->gm_pending); \
358 if(NULL != frag) { \
359 switch(frag->type) { \
360 case MCA_BTL_GM_SEND: \
361 case MCA_BTL_GM_EAGER: \
362 mca_btl_gm_send_nl(&btl->super, frag->endpoint, &frag->base, frag->hdr->tag); \
363 break; \
364 case MCA_BTL_GM_PUT: \
365 mca_btl_gm_put_nl(&btl->super, frag->endpoint, &frag->base); \
366 break; \
367 case MCA_BTL_GM_GET: \
368 mca_btl_gm_get_nl(&btl->super, frag->endpoint, &frag->base); \
369 break; \
370 } \
371 } \
372 } \
373 } while (0)
374
375
376 END_C_DECLS
377 #endif
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
mca_btl_gm_module_t::super
mca_btl_base_module_t super
base BTL interface
Definition: btl_gm.h:78
mca_btl_gm_free
int mca_btl_gm_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
Return a segment allocated by this BTL.
Definition: btl_gm.c:229
mca_btl_gm_component_t::super
mca_btl_base_component_2_0_0_t super
base BTL component
Definition: btl_gm.h:49
opal_bitmap_t
Definition: opal_bitmap.h:53
mca_btl_gm_component_t::gm_free_list_max
int gm_free_list_max
maximum size of free lists
Definition: btl_gm.h:64
mca_btl_gm_put
int mca_btl_gm_put(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous put.
Definition: btl_gm.c:747
mca_btl_gm_component_t::gm_lock
opal_mutex_t gm_lock
lock for accessing module state
Definition: btl_gm.h:67
mca_btl_gm_component_t::gm_max_btls
size_t gm_max_btls
maximum number of supported hcas
Definition: btl_gm.h:52
mca_btl_gm_register_error_cb
int mca_btl_gm_register_error_cb(struct mca_btl_base_module_t *btl, mca_btl_base_module_error_cb_fn_t cbfunc)
Register a callback function that is called on error.
Definition: btl_gm.c:168
mca_btl_base_module_error_cb_fn_t
void(* mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)
Callback function that is called asynchronously on receipt of an error from the transport layer...
Definition: btl.h:538
mca_btl_gm_component_t::gm_free_list_num
int gm_free_list_num
initial size of free lists
Definition: btl_gm.h:63
opal_mutex_t
Definition: mutex_unix.h:53
mca_btl_gm_add_procs
int mca_btl_gm_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
PML->BTL notification of change in the process list.
Definition: btl_gm.c:101
mca_btl_gm_prepare_src
mca_btl_base_descriptor_t * mca_btl_gm_prepare_src(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_gm.c:250
mca_btl_gm_component_t::gm_max_ports
size_t gm_max_ports
maximum number of ports per board
Definition: btl_gm.h:54
mca_mpool_base_registration_t
Definition: mpool.h:44
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
mca_btl_gm_finalize
int mca_btl_gm_finalize(struct mca_btl_base_module_t *btl)
Cleanup any resources held by the BTL.
Definition: btl_gm.c:946
mca_btl_gm_component_t::gm_num_btls
size_t gm_num_btls
number of hcas available to the GM component
Definition: btl_gm.h:51
mca_btl_gm_component_t::gm_free_list_inc
int gm_free_list_inc
number of elements to alloc when growing free lists
Definition: btl_gm.h:65
mca_btl_gm_ft_event
int mca_btl_gm_ft_event(int state)
Fault Tolerance Event Notification Function.
Definition: btl_gm.c:967
mca_btl_gm_module_t::gm_repost
opal_list_t gm_repost
list of pending fragments
Definition: btl_gm.h:98
btl.h
Byte Transfer Layer (BTL)
mca_btl_gm_get
int mca_btl_gm_get(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous get.
Definition: btl_gm.c:903
ompi_free_list_t
Definition: ompi_free_list.h:39
mca_btl_gm_component_open
int mca_btl_gm_component_open(void)
Register GM component parameters with the MCA framework.
Definition: btl_gm_component.c:109
mca_btl_gm_del_procs
int mca_btl_gm_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
PML->BTL notification of change in the process list.
Definition: btl_gm.c:156
mca_btl_gm_addr_t
Structure used to publish GM id information to peers.
Definition: btl_gm_endpoint.h:37
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_gm_module_t::gm_pending
opal_list_t gm_pending
list of pending send descriptors
Definition: btl_gm.h:97
mca_btl_gm_component_progress
int mca_btl_gm_component_progress(void)
GM component progress.
Definition: btl_gm_component.c:568
mca_btl_base_component_2_0_0_t
BTL component descriptor.
Definition: btl.h:411
mpool.h
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
opal_convertor_t
Definition: opal_convertor.h:90
mca_btl_gm_alloc
mca_btl_base_descriptor_t * mca_btl_gm_alloc(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
Allocate a descriptor with a segment of the requested size.
Definition: btl_gm.c:185
mca_btl_gm_component_t::gm_max_boards
size_t gm_max_boards
maximum number of boards
Definition: btl_gm.h:55
mca_btl_gm_component_t::gm_debug
int gm_debug
turn on debug output
Definition: btl_gm.h:61
mca_btl_gm_component_t::gm_procs
opal_list_t gm_procs
list of gm proc structures
Definition: btl_gm.h:66
opal_list_t
Definition: opal_list.h:147
base.h
mca_btl_gm_component_t::gm_btls
struct mca_btl_gm_module_t ** gm_btls
array of available BTL modules
Definition: btl_gm.h:53
opal_thread_t
Definition: threads.h:46
mca_btl_gm_component_t::gm_num_high_priority
int32_t gm_num_high_priority
number of receive descriptors at high priority
Definition: btl_gm.h:60
mca_btl_gm_prepare_dst
mca_btl_base_descriptor_t * mca_btl_gm_prepare_dst(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_gm.c:370
mca_btl_gm_component_t
Myrinet (GM) BTL component.
Definition: btl_gm.h:48
mca_btl_base_module_t
BTL module interface functions and attributes.
Definition: btl.h:786
mca_btl_gm_component_close
int mca_btl_gm_component_close(void)
Any final cleanup before being unloaded.
Definition: btl_gm_component.c:184
mca_btl_gm_module_t
BTL Module Interface.
Definition: btl_gm.h:77
mca_btl_gm_send
int mca_btl_gm_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
Initiate an asynchronous send.
Definition: btl_gm.c:574
mca_btl_gm_component_init
mca_btl_base_module_t ** mca_btl_gm_component_init(int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
GM component initialization.
Definition: btl_gm_component.c:491
mca_btl_gm_component_t::gm_mpool_name
char * gm_mpool_name
name of memory pool
Definition: btl_gm.h:68