OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_wv.h
1 /* -*- Mode: C; c-basic-offset:4 ; -*- */
2 /*
3 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2009 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved.
14 * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
15 * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
16 * reserved.
17 * Copyright (c) 2006-2007 Voltaire All rights reserved.
18 * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved.
19 * $COPYRIGHT$
20 *
21 * Additional copyrights may follow
22 *
23 * $HEADER$
24 *
25 * @file
26 */
27
28 #ifndef MCA_BTL_IB_H
29 #define MCA_BTL_IB_H
30
31 #include "ompi_config.h"
32 #include <sys/types.h>
33 #include <string.h>
34 #include "btl_wv_def.h"
35
36 /* Open MPI includes */
37 #include "ompi/class/ompi_free_list.h"
38 #include "opal/class/opal_pointer_array.h"
39 #include "opal/class/opal_hash_table.h"
40 #include "opal/util/output.h"
41 #include "opal/mca/event/event.h"
42 #include "opal/threads/threads.h"
43 #include "ompi/mca/btl/btl.h"
44 #include "ompi/mca/mpool/mpool.h"
45 #include "ompi/mca/btl/base/btl_base_error.h"
46
47 #include "ompi/mca/btl/btl.h"
48 #include "ompi/mca/btl/base/base.h"
49
50 #include "connect/connect.h"
51
52 BEGIN_C_DECLS
53
54
55 #define MCA_BTL_IB_LEAVE_PINNED 1
56 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll
57 #define MCA_BTL_IB_PKEY_MASK 0x7fff
58
59
60 /*--------------------------------------------------------------------*/
61
62 #if OPAL_ENABLE_DEBUG
63 #define ATTACH() do { \
64 int i = 0; \
65 opal_output(0, "WAITING TO DEBUG ATTACH"); \
66 while (i == 0) sleep(5); \
67 } while(0);
68 #else
69 #define ATTACH()
70 #endif
71
72 /*--------------------------------------------------------------------*/
73
74 /**
75 * Infiniband (IB) BTL component.
76 */
77
78 typedef enum {
79 MCA_BTL_WV_TRANSPORT_IB,
80 MCA_BTL_WV_TRANSPORT_IWARP,
81 MCA_BTL_WV_TRANSPORT_RDMAOE,
82 MCA_BTL_WV_TRANSPORT_UNKNOWN,
83 MCA_BTL_WV_TRANSPORT_SIZE
84 } mca_btl_wv_transport_type_t;
85
86 typedef enum {
87 MCA_BTL_WV_PP_QP,
88 MCA_BTL_WV_SRQ_QP,
89 } mca_btl_wv_qp_type_t;
90
91 struct mca_btl_wv_pp_qp_info_t {
92 int32_t rd_win;
93 int32_t rd_rsv;
94 }; typedef struct mca_btl_wv_pp_qp_info_t mca_btl_wv_pp_qp_info_t;
95
96 struct mca_btl_wv_srq_qp_info_t {
97 int32_t sd_max;
98 /* The init value for rd_curr_num variables of all SRQs */
99 int32_t rd_init;
100 /* The watermark, threshold - if the number of WQEs in SRQ is less then this value =>
101 the SRQ limit event will be generated on corresponding SRQ.
102 As result the maximal number of pre-posted WQEs on the SRQ will be increased */
103 int32_t srq_limit;
104 }; typedef struct mca_btl_wv_srq_qp_info_t mca_btl_wv_srq_qp_info_t;
105
106 struct mca_btl_wv_qp_info_t {
107 mca_btl_wv_qp_type_t type;
108 size_t size;
109 int32_t rd_num;
110 int32_t rd_low;
111 union {
112 mca_btl_wv_pp_qp_info_t pp_qp;
113 mca_btl_wv_srq_qp_info_t srq_qp;
114 } u;
115 }; typedef struct mca_btl_wv_qp_info_t mca_btl_wv_qp_info_t;
116
117 #define BTL_WV_QP_TYPE(Q) (mca_btl_wv_component.qp_infos[(Q)].type)
118 #define BTL_WV_QP_TYPE_PP(Q) \
119 (BTL_WV_QP_TYPE(Q) == MCA_BTL_WV_PP_QP)
120 #define BTL_WV_QP_TYPE_SRQ(Q) \
121 (BTL_WV_QP_TYPE(Q) == MCA_BTL_WV_SRQ_QP)
122
123 typedef enum {
124 BTL_WV_RQ_SOURCE_DEFAULT,
125 BTL_WV_RQ_SOURCE_MCA,
126 BTL_WV_RQ_SOURCE_DEVICE_INI,
127 BTL_WV_RQ_SOURCE_MAX
128 } btl_wv_receive_queues_source_t;
129
130 typedef enum {
131 BTL_WV_DT_IB,
132 BTL_WV_DT_IWARP,
133 BTL_WV_DT_ALL
134 } btl_wv_device_type_t;
135
136 struct mca_btl_wv_component_t {
137 /**< base BTL component */
138 mca_btl_base_component_2_0_0_t super;
139
140 /**< maximum number of devices available to wv component */
141 int ib_max_btls;
142
143 /**< number of devices available to the wv component */
144 int ib_num_btls;
145
146 /**< array of available BTLs */
147 struct mca_btl_wv_module_t **wv_btls;
148
149 /**< array of available devices */
150 opal_pointer_array_t devices;
151 int devices_count;
152
153 /**< initial size of free lists */
154 int ib_free_list_num;
155
156 /**< maximum size of free lists */
157 int ib_free_list_max;
158
159 /**< number of elements to alloc when growing free lists */
160 int ib_free_list_inc;
161
162 /**< list of ib proc structures */
163 opal_list_t ib_procs;
164
165 /**< event structure for sends */
166 opal_event_t ib_send_event;
167
168 /**< event structure for recvs */
169 opal_event_t ib_recv_event;
170
171 /**< lock for accessing module state */
172 opal_mutex_t ib_lock;
173
174 /**< name of ib memory pool */
175 char* ib_mpool_name;
176
177 /**< number of pp qp's */
178 uint8_t num_pp_qps;
179 /**< number of srq qp's */
180 uint8_t num_srq_qps;
181 /**< total number of qp's */
182 uint8_t num_qps;
183
184 mca_btl_wv_qp_info_t *qp_infos;
185
186 /**< Eager send limit of first fragment, in Bytes */
187 size_t eager_limit;
188 /**< Maximum send size, in Bytes */
189 size_t max_send_size;
190 /**< Maximum message size for RDMA protocols in Bytes */
191 int32_t max_hw_msg_size;
192 /**< Length of the registration cache most recently used list */
193 uint32_t reg_mru_len;
194 /**< Use the Shared Receive Queue (SRQ mode) */
195 uint32_t use_srq;
196
197 /**< Max outstanding CQE on the CQ */
198 uint32_t ib_cq_size[2];
199
200 /**< Max size of inline data */
201 int32_t ib_max_inline_data;
202
203 uint32_t ib_pkey_val;
204 uint32_t ib_psn;
205 uint32_t ib_qp_ous_rd_atom;
206 uint32_t ib_mtu;
207 uint32_t ib_min_rnr_timer;
208 uint32_t ib_timeout;
209 uint32_t ib_retry_count;
210 uint32_t ib_rnr_retry;
211 uint32_t ib_max_rdma_dst_ops;
212 uint32_t ib_service_level;
213 uint32_t ib_path_rec_service_level;
214 int32_t use_eager_rdma;
215 /**< After this number of msg, use RDMA for short messages, always */
216 int32_t eager_rdma_threshold;
217 int32_t eager_rdma_num;
218 int32_t max_eager_rdma;
219 uint32_t btls_per_lid;
220 uint32_t max_lmc;
221 int32_t apm_lmc;
222 int32_t apm_ports;
223 /**< Preferred communication buffer alignment in Bytes (must be power of two) */
224 uint32_t buffer_alignment;
225 btl_wv_device_type_t device_type;
226 char *if_include;
227 char **if_include_list;
228 char *if_exclude;
229 char **if_exclude_list;
230 char *ipaddr_include;
231 char *ipaddr_exclude;
232
233 /* MCA param btl_wv_receive_queues */
234 char *receive_queues;
235 /* Whether we got a non-default value of btl_wv_receive_queues */
236 btl_wv_receive_queues_source_t receive_queues_source;
237
238 /** Colon-delimited list of filenames for device parameters */
239 char *device_params_file_names;
240
241 /** Whether we're in verbose mode or not */
242 bool verbose;
243
244 /** Whether we want a warning if no device-specific parameters are
245 found in INI files */
246 bool warn_no_device_params_found;
247 /** Whether we want a warning if non default GID prefix is not configured
248 on multiport setup */
249 bool warn_default_gid_prefix;
250 /** Whether we want a warning if the user specifies a non-existent
251 device and/or port via btl_wv_if_[in|ex]clude MCA params */
252 bool warn_nonexistent_if;
253 /** Dummy argv-style list; a copy of names from the
254 if_[in|ex]clude list that we use for error checking (to ensure
255 that they all exist) */
256 char **if_list;
257 bool use_message_coalescing;
258 uint32_t cq_poll_ratio;
259 uint32_t cq_poll_progress;
260 uint32_t eager_rdma_poll_ratio;
261 int rdma_qp;
262 /* qp used for software flow control */
263 int credits_qp;
264 bool cpc_explicitly_defined;
265 /**< free list of frags only; used for pining user memory */
266 ompi_free_list_t send_user_free;
267 /**< free list of frags only; used for pining user memory */
268 ompi_free_list_t recv_user_free;
269 /**< frags for coalesced massages */
270 ompi_free_list_t send_free_coalesced;
271 /** Default receive queues */
272 char *default_recv_qps;
273 /** Whether we want a dynamically resizing srq, enabled by default */
274 bool enable_srq_resize;
275 #if BTL_WV_FAILOVER_ENABLED
276 int verbose_failover;
277 #endif
278 }; typedef struct mca_btl_wv_component_t mca_btl_wv_component_t;
279
280 OMPI_MODULE_DECLSPEC extern mca_btl_wv_component_t mca_btl_wv_component;
281
282 typedef mca_btl_base_recv_reg_t mca_btl_wv_recv_reg_t;
283
284 /**
285 * Common information for all ports that is sent in the modex message
286 */
287 typedef struct mca_btl_wv_modex_message_t {
288 /** The subnet ID of this port */
289 uint64_t subnet_id;
290 /** LID of this port */
291 uint16_t lid;
292 /** APM LID for this port */
293 uint16_t apm_lid;
294 /** The MTU used by this port */
295 uint8_t mtu;
296 /** vendor id define device type and tuning */
297 uint32_t vendor_id;
298 /** vendor part id define device type and tuning */
299 uint32_t vendor_part_id;
300 /** Transport type of remote port */
301 uint8_t transport_type;
302 /** Dummy field used to calculate the real length */
303 uint8_t end;
304 } mca_btl_wv_modex_message_t;
305
306 #define MCA_BTL_WV_MODEX_MSG_NTOH(hdr) \
307 do { \
308 (hdr).subnet_id = ntoh64((hdr).subnet_id); \
309 (hdr).lid = ntohs((hdr).lid); \
310 } while (0)
311 #define MCA_BTL_WV_MODEX_MSG_HTON(hdr) \
312 do { \
313 (hdr).subnet_id = hton64((hdr).subnet_id); \
314 (hdr).lid = htons((hdr).lid); \
315 } while (0)
316
317 typedef struct mca_btl_wv_device_qp_t {
318 ompi_free_list_t send_free; /**< free lists of send buffer descriptors */
319 ompi_free_list_t recv_free; /**< free lists of receive buffer descriptors */
320 } mca_btl_wv_device_qp_t;
321
322 struct mca_btl_base_endpoint_t;
323
324 typedef struct mca_btl_wv_device_t {
325 opal_object_t super;
326 struct wv_device *ib_dev; /* the ib device */
327 opal_mutex_t device_lock; /* device level lock */
328 struct wv_context *ib_dev_context;
329 WV_DEVICE_ATTRIBUTES ib_dev_attr;
330 struct wv_pd *ib_pd;
331 struct wv_cq *ib_cq[2];
332 uint32_t cq_size[2];
333 mca_mpool_base_module_t *mpool;
334 /* MTU for this device */
335 uint32_t mtu;
336 /* Whether this device supports eager RDMA */
337 uint8_t use_eager_rdma;
338 uint8_t btls; /** < number of btls using this device */
339 opal_pointer_array_t *endpoints;
340 opal_pointer_array_t *device_btls;
341 uint16_t hp_cq_polls;
342 uint16_t eager_rdma_polls;
343 bool pollme;
344 int32_t non_eager_rdma_endpoints;
345 int32_t eager_rdma_buffers_count;
346 struct mca_btl_base_endpoint_t **eager_rdma_buffers;
347 /**< frags for control massages */
348 ompi_free_list_t send_free_control;
349 /* QP types and attributes that will be used on this device */
350 mca_btl_wv_device_qp_t *qps;
351 /* Maximum value supported by this device for max_inline_data */
352 uint32_t max_inline_data;
353 } mca_btl_wv_device_t;
354 OBJ_CLASS_DECLARATION(mca_btl_wv_device_t);
355
356 struct mca_btl_wv_module_pp_qp_t {
357 int32_t dummy;
358 }; typedef struct mca_btl_wv_module_pp_qp_t mca_btl_wv_module_pp_qp_t;
359
360 struct mca_btl_wv_module_srq_qp_t {
361 struct wv_srq *srq;
362 int32_t rd_posted;
363 int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */
364 /* i.e. the number of frags that can be outstanding (down counter) */
365 opal_list_t pending_frags[2]; /**< list of high/low prio frags */
366 /** The number of receive buffers that can be post in the current time.
367 The value starts from (rd_num / 4) and increased up to rd_num */
368 int32_t rd_curr_num;
369 /** We post additional WQEs only if a number of WQEs (in specific SRQ) is less of this value.
370 The value increased together with rd_curr_num. The value is unique for every SRQ. */
371 int32_t rd_low_local;
372
373 bool srq_limit_event_flag;
374 /**< In difference of the "--mca enable_srq_resize" parameter that says, if we want(or no)
375 to start with small num of pre-posted receive buffers (rd_curr_num) and to increase this number by needs
376 (the max of this value is rd_num – the whole size of SRQ), the "srq_limit_event_flag" says if we want to get limit event
377 from device if the defined srq limit was reached (signal to the main thread) and we put off this flag if the rd_curr_num
378 was increased up to rd_num.*/
379 }; typedef struct mca_btl_wv_module_srq_qp_t mca_btl_wv_module_srq_qp_t;
380
381 struct mca_btl_wv_module_qp_t {
382 union {
383 mca_btl_wv_module_pp_qp_t pp_qp;
384 mca_btl_wv_module_srq_qp_t srq_qp;
385 } u;
386 }; typedef struct mca_btl_wv_module_qp_t mca_btl_wv_module_qp_t;
387
388 /**
389 * IB BTL Interface
390 */
391 struct mca_btl_wv_module_t {
392 /* Base BTL module */
393 mca_btl_base_module_t super;
394
395 bool btl_inited;
396
397 /** Common information about all ports */
398 mca_btl_wv_modex_message_t port_info;
399
400 /** Array of CPCs on this port */
401 ompi_btl_wv_connect_base_module_t **cpcs;
402
403 /** Number of elements in the cpcs array */
404 uint8_t num_cpcs;
405
406 mca_btl_wv_device_t *device;
407 uint8_t port_num; /**< ID of the PORT */
408 uint16_t pkey_index;
409 WV_PORT_ATTRIBUTES ib_port_attr;
410 uint16_t lid; /**< lid that is actually used (for LMC) */
411 int apm_port; /**< Alternative port that may be used for APM */
412 uint8_t src_path_bits; /**< offset from base lid (for LMC) */
413
414 int32_t num_peers;
415
416 opal_mutex_t ib_lock; /**< module level lock */
417
418 size_t eager_rdma_frag_size; /**< length of eager frag */
419 volatile int32_t eager_rdma_channels; /**< number of open RDMA channels */
420
421 mca_btl_base_module_error_cb_fn_t error_cb; /**< error handler */
422
423 mca_btl_wv_module_qp_t * qps;
424 };
425 typedef struct mca_btl_wv_module_t mca_btl_wv_module_t;
426
427 extern mca_btl_wv_module_t mca_btl_wv_module;
428
429 struct mca_btl_wv_reg_t {
430 mca_mpool_base_registration_t base;
431 struct wv_mr *mr;
432 };
433 typedef struct mca_btl_wv_reg_t mca_btl_wv_reg_t;
434 /**
435 * Register a callback function that is called on error..
436 *
437 * @param btl (IN) BTL module
438 * @return Status indicating if cleanup was successful
439 */
440
441 int mca_btl_wv_register_error_cb(
442 struct mca_btl_base_module_t* btl,
443 mca_btl_base_module_error_cb_fn_t cbfunc
444 );
445
446
447 /**
448 * Cleanup any resources held by the BTL.
449 *
450 * @param btl BTL instance.
451 * @return OMPI_SUCCESS or error status on failure.
452 */
453
454 extern int mca_btl_wv_finalize(
455 struct mca_btl_base_module_t* btl
456 );
457
458
459 /**
460 * PML->BTL notification of change in the process list.
461 *
462 * @param btl (IN) BTL module
463 * @param nprocs (IN) Number of processes
464 * @param procs (IN) Set of processes
465 * @param peers (OUT) Set of (optional) peer addressing info.
466 * @param reachable (IN/OUT) Set of processes that are reachable via this BTL.
467 * @return OMPI_SUCCESS or error status on failure.
468 *
469 */
470
471 extern int mca_btl_wv_add_procs(
472 struct mca_btl_base_module_t* btl,
473 size_t nprocs,
474 struct ompi_proc_t **procs,
475 struct mca_btl_base_endpoint_t** peers,
476 opal_bitmap_t* reachable
477 );
478
479 /**
480 * PML->BTL notification of change in the process list.
481 *
482 * @param btl (IN) BTL instance
483 * @param nproc (IN) Number of processes.
484 * @param procs (IN) Set of processes.
485 * @param peers (IN) Set of peer data structures.
486 * @return Status indicating if cleanup was successful
487 *
488 */
489 extern int mca_btl_wv_del_procs(
490 struct mca_btl_base_module_t* btl,
491 size_t nprocs,
492 struct ompi_proc_t **procs,
493 struct mca_btl_base_endpoint_t** peers
494 );
495
496
497 /**
498 * PML->BTL Initiate a send of the specified size.
499 *
500 * @param btl (IN) BTL instance
501 * @param btl_peer (IN) BTL peer addressing
502 * @param descriptor (IN) Descriptor of data to be transmitted.
503 * @param tag (IN) Tag.
504 */
505 extern int mca_btl_wv_send(
506 struct mca_btl_base_module_t* btl,
507 struct mca_btl_base_endpoint_t* btl_peer,
508 struct mca_btl_base_descriptor_t* descriptor,
509 mca_btl_base_tag_t tag
510 );
511
512 /**
513 * PML->BTL Initiate a immediate send of the specified size.
514 *
515 * @param btl (IN) BTL instance
516 * @param ep (IN) Endpoint
517 * @param convertor (IN) Datatypes converter
518 * @param header (IN) PML header
519 * @param header_size (IN) PML header size
520 * @param payload_size (IN) Payload size
521 * @param order (IN) Order
522 * @param flags (IN) Flags
523 * @param tag (IN) Tag
524 * @param descriptor (OUT) Messages descriptor
525 */
526 extern int mca_btl_wv_sendi( struct mca_btl_base_module_t* btl,
527 struct mca_btl_base_endpoint_t* ep,
528 struct opal_convertor_t* convertor,
529 void* header,
530 size_t header_size,
531 size_t payload_size,
532 uint8_t order,
533 uint32_t flags,
534 mca_btl_base_tag_t tag,
535 mca_btl_base_descriptor_t** descriptor
536 );
537
538 /**
539 * PML->BTL Initiate a put of the specified size.
540 *
541 * @param btl (IN) BTL instance
542 * @param btl_peer (IN) BTL peer addressing
543 * @param descriptor (IN) Descriptor of data to be transmitted.
544 */
545 extern int mca_btl_wv_put(
546 struct mca_btl_base_module_t* btl,
547 struct mca_btl_base_endpoint_t* btl_peer,
548 struct mca_btl_base_descriptor_t* descriptor
549 );
550
551 /**
552 * PML->BTL Initiate a get of the specified size.
553 *
554 * @param btl (IN) BTL instance
555 * @param btl_base_peer (IN) BTL peer addressing
556 * @param descriptor (IN) Descriptor of data to be transmitted.
557 */
558 extern int mca_btl_wv_get(
559 struct mca_btl_base_module_t* btl,
560 struct mca_btl_base_endpoint_t* btl_peer,
561 struct mca_btl_base_descriptor_t* descriptor
562 );
563
564
565 /**
566 * Allocate a descriptor.
567 *
568 * @param btl (IN) BTL module
569 * @param size (IN) Requested descriptor size.
570 */
571 extern mca_btl_base_descriptor_t* mca_btl_wv_alloc(
572 struct mca_btl_base_module_t* btl,
573 struct mca_btl_base_endpoint_t* endpoint,
574 uint8_t order,
575 size_t size,
576 uint32_t flags);
577
578
579 /**
580 * Return a segment allocated by this BTL.
581 *
582 * @param btl (IN) BTL module
583 * @param descriptor (IN) Allocated descriptor.
584 */
585 extern int mca_btl_wv_free(
586 struct mca_btl_base_module_t* btl,
587 mca_btl_base_descriptor_t* des);
588
589
590 /**
591 * Pack data and return a descriptor that can be
592 * used for send/put.
593 *
594 * @param btl (IN) BTL module
595 * @param peer (IN) BTL peer addressing
596 */
597 mca_btl_base_descriptor_t* mca_btl_wv_prepare_src(
598 struct mca_btl_base_module_t* btl,
599 struct mca_btl_base_endpoint_t* peer,
600 mca_mpool_base_registration_t* registration,
601 struct opal_convertor_t* convertor,
602 uint8_t order,
603 size_t reserve,
604 size_t* size,
605 uint32_t flags
606 );
607
608 /**
609 * Allocate a descriptor initialized for RDMA write.
610 *
611 * @param btl (IN) BTL module
612 * @param peer (IN) BTL peer addressing
613 */
614 extern mca_btl_base_descriptor_t* mca_btl_wv_prepare_dst(
615 struct mca_btl_base_module_t* btl,
616 struct mca_btl_base_endpoint_t* peer,
617 mca_mpool_base_registration_t* registration,
618 struct opal_convertor_t* convertor,
619 uint8_t order,
620 size_t reserve,
621 size_t* size,
622 uint32_t flags);
623
624 extern void mca_btl_wv_frag_progress_pending_put_get(
625 struct mca_btl_base_endpoint_t*, const int);
626
627 /**
628 * Fault Tolerance Event Notification Function
629 *
630 * @param state (IN) Checkpoint State
631 * @return OMPI_SUCCESS or failure status
632 */
633 extern int mca_btl_wv_ft_event(int state);
634
635
636 /**
637 * Show an error during init, particularly when running out of
638 * registered memory.
639 */
640 void mca_btl_wv_show_init_error(const char *file, int line,
641 const char *func, const char *dev);
642
643 #define BTL_WV_HP_CQ 0
644 #define BTL_WV_LP_CQ 1
645
646
647 /**
648 * Post to Shared Receive Queue with certain priority
649 *
650 * @param wv_btl (IN) BTL module
651 * @param additional (IN) Additional Bytes to reserve
652 * @param prio (IN) Priority (either BTL_WV_HP_QP or BTL_WV_LP_QP)
653 * @return OMPI_SUCCESS or failure status
654 */
655
656 int mca_btl_wv_post_srr(mca_btl_wv_module_t* wv_btl, const int qp);
657
658 /**
659 * Get a transport name of btl by its transport type.
660 */
661
662 const char* btl_wv_get_transport_name(mca_btl_wv_transport_type_t transport_type);
663
664 /**
665 * Get a transport type of btl.
666 */
667
668 mca_btl_wv_transport_type_t mca_btl_wv_get_transport_type(mca_btl_wv_module_t* wv_btl);
669
670 static inline int qp_cq_prio(const int qp)
671 {
672 if(0 == qp)
673 return BTL_WV_HP_CQ; /* smallest qp is always HP */
674
675 /* If the size for this qp is <= the eager limit, make it a
676 high priority QP. Otherwise, make it a low priority QP. */
677 return (mca_btl_wv_component.qp_infos[qp].size <=
678 mca_btl_wv_component.eager_limit) ?
679 BTL_WV_HP_CQ : BTL_WV_LP_CQ;
680 }
681
682 #define BTL_WV_RDMA_QP(QP) \
683 ((QP) == mca_btl_wv_component.rdma_qp)
684
685 END_C_DECLS
686
687 #endif /* MCA_BTL_IB_H */
mca_btl_wv_component_t::recv_user_free
ompi_free_list_t recv_user_free
frags for coalesced massages
Definition: btl_wv.h:268
mca_btl_wv_module_t::ib_lock
opal_mutex_t ib_lock
module level lock
Definition: btl_wv.h:416
mca_btl_wv_module_t
IB BTL Interface.
Definition: btl_wv.h:391
mca_btl_wv_module_t::apm_port
int apm_port
Alternative port that may be used for APM.
Definition: btl_wv.h:411
mca_btl_wv_reg_t
Definition: btl_wv.h:429
mca_btl_wv_device_t::eager_rdma_buffers
struct mca_btl_base_endpoint_t ** eager_rdma_buffers
frags for control massages
Definition: btl_wv.h:346
mca_btl_wv_module_srq_qp_t::rd_curr_num
int32_t rd_curr_num
The number of receive buffers that can be post in the current time.
Definition: btl_wv.h:368
wv_cq
Definition: btl_wv_def.h:126
mca_btl_wv_device_qp_t::send_free
ompi_free_list_t send_free
free lists of send buffer descriptors
Definition: btl_wv.h:318
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
output.h
OPAL output stream facility.
mca_btl_wv_module_t::port_info
mca_btl_wv_modex_message_t port_info
Common information about all ports.
Definition: btl_wv.h:398
mca_btl_wv_module_srq_qp_t::pending_frags
opal_list_t pending_frags[2]
list of high/low prio frags
Definition: btl_wv.h:365
mca_btl_wv_component_t::ib_recv_event
opal_event_t ib_recv_event
lock for accessing module state
Definition: btl_wv.h:169
opal_pointer_array_t
dynamic pointer array
Definition: opal_pointer_array.h:45
mca_btl_wv_component_t::apm_ports
int32_t apm_ports
Preferred communication buffer alignment in Bytes (must be power of two)
Definition: btl_wv.h:222
mca_btl_wv_modex_message_t::apm_lid
uint16_t apm_lid
APM LID for this port.
Definition: btl_wv.h:293
opal_bitmap_t
Definition: opal_bitmap.h:53
mca_btl_base_recv_reg_t
Definition: base.h:44
mca_btl_wv_component_t
Definition: btl_wv.h:136
mca_btl_wv_component_t::enable_srq_resize
bool enable_srq_resize
Whether we want a dynamically resizing srq, enabled by default.
Definition: btl_wv.h:274
mca_btl_wv_module_t::num_cpcs
uint8_t num_cpcs
Number of elements in the cpcs array.
Definition: btl_wv.h:404
mca_btl_wv_component_t::verbose
bool verbose
Whether we're in verbose mode or not.
Definition: btl_wv.h:242
mca_btl_wv_module_t::error_cb
mca_btl_base_module_error_cb_fn_t error_cb
error handler
Definition: btl_wv.h:421
mca_btl_wv_module_t::lid
uint16_t lid
lid that is actually used (for LMC)
Definition: btl_wv.h:410
event
Structure to represent a single event.
Definition: event_struct.h:87
mca_btl_wv_component_t::max_send_size
size_t max_send_size
Maximum message size for RDMA protocols in Bytes.
Definition: btl_wv.h:189
mca_btl_wv_device_qp_t::recv_free
ompi_free_list_t recv_free
free lists of receive buffer descriptors
Definition: btl_wv.h:319
mca_btl_wv_component_t::eager_limit
size_t eager_limit
Maximum send size, in Bytes.
Definition: btl_wv.h:187
mca_btl_wv_component_t::use_srq
uint32_t use_srq
Max outstanding CQE on the CQ.
Definition: btl_wv.h:195
mca_btl_base_module_error_cb_fn_t
void(* mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)
Callback function that is called asynchronously on receipt of an error from the transport layer...
Definition: btl.h:538
opal_mutex_t
Definition: mutex_unix.h:53
mca_btl_wv_modex_message_t::subnet_id
uint64_t subnet_id
The subnet ID of this port.
Definition: btl_wv.h:289
mca_btl_wv_component_t::if_list
char ** if_list
Dummy argv-style list; a copy of names from the if_[in|ex]clude list that we use for error checking (...
Definition: btl_wv.h:256
mca_btl_wv_component_t::qp_infos
mca_btl_wv_qp_info_t * qp_infos
Eager send limit of first fragment, in Bytes.
Definition: btl_wv.h:184
mca_btl_wv_module_t::eager_rdma_channels
volatile int32_t eager_rdma_channels
number of open RDMA channels
Definition: btl_wv.h:419
mca_mpool_base_registration_t
Definition: mpool.h:44
opal_pointer_array.h
See opal_bitmap.h for an explanation of why there is a split between OPAL and ORTE for this generic c...
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
mca_btl_wv_module_t::src_path_bits
uint8_t src_path_bits
offset from base lid (for LMC)
Definition: btl_wv.h:412
mca_btl_wv_component_t::cpc_explicitly_defined
bool cpc_explicitly_defined
free list of frags only; used for pining user memory
Definition: btl_wv.h:264
mca_btl_wv_srq_qp_info_t
Definition: btl_wv.h:96
mca_btl_wv_component_t::max_hw_msg_size
int32_t max_hw_msg_size
Length of the registration cache most recently used list.
Definition: btl_wv.h:191
mca_btl_wv_module_pp_qp_t
Definition: btl_wv.h:356
mca_btl_wv_device_t
Definition: btl_wv.h:324
mca_btl_wv_component_t::num_pp_qps
uint8_t num_pp_qps
number of srq qp's
Definition: btl_wv.h:178
mca_btl_wv_component_t::ib_send_event
opal_event_t ib_send_event
event structure for recvs
Definition: btl_wv.h:166
mca_btl_wv_component_t::ib_num_btls
int ib_num_btls
array of available BTLs
Definition: btl_wv.h:144
mca_btl_wv_device_t::endpoints
opal_pointer_array_t * endpoints
< number of btls using this device
Definition: btl_wv.h:339
mca_btl_wv_module_t::cpcs
ompi_btl_wv_connect_base_module_t ** cpcs
Array of CPCs on this port.
Definition: btl_wv.h:401
mca_btl_wv_component_t::ib_max_btls
int ib_max_btls
number of devices available to the wv component
Definition: btl_wv.h:141
mca_btl_wv_pp_qp_info_t
Definition: btl_wv.h:91
wv_mr
Definition: btl_wv_def.h:62
mca_btl_wv_module_t::eager_rdma_frag_size
size_t eager_rdma_frag_size
length of eager frag
Definition: btl_wv.h:418
mca_btl_wv_modex_message_t::end
uint8_t end
Dummy field used to calculate the real length.
Definition: btl_wv.h:303
mca_btl_wv_component_t::ib_mpool_name
char * ib_mpool_name
number of pp qp's
Definition: btl_wv.h:175
mca_btl_wv_component_t::ib_cq_size
uint32_t ib_cq_size[2]
Max size of inline data.
Definition: btl_wv.h:198
mca_btl_wv_module_srq_qp_t
Definition: btl_wv.h:360
mca_btl_wv_component_t::ib_procs
opal_list_t ib_procs
event structure for sends
Definition: btl_wv.h:163
btl.h
Byte Transfer Layer (BTL)
wv_context
Definition: btl_wv_def.h:40
mca_btl_wv_module_srq_qp_t::srq_limit_event_flag
bool srq_limit_event_flag
In difference of the "--mca enable_srq_resize" parameter that says, if we want(or no) to start with s...
Definition: btl_wv.h:373
mca_btl_wv_component_t::devices_count
int devices_count
initial size of free lists
Definition: btl_wv.h:151
mca_btl_wv_component_t::wv_btls
struct mca_btl_wv_module_t ** wv_btls
array of available devices
Definition: btl_wv.h:147
mca_btl_wv_qp_info_t
Definition: btl_wv.h:106
mca_btl_wv_component_t::ib_free_list_num
int ib_free_list_num
maximum size of free lists
Definition: btl_wv.h:154
mca_btl_wv_module_qp_t
Definition: btl_wv.h:381
ompi_free_list_t
Definition: ompi_free_list.h:39
opal_hash_table.h
A hash table that may be indexed with either fixed length (e.g.
mca_btl_wv_component_t::send_user_free
ompi_free_list_t send_user_free
free list of frags only; used for pining user memory
Definition: btl_wv.h:266
mca_btl_wv_component_t::device_params_file_names
char * device_params_file_names
Colon-delimited list of filenames for device parameters.
Definition: btl_wv.h:239
mca_btl_wv_component_t::warn_nonexistent_if
bool warn_nonexistent_if
Whether we want a warning if the user specifies a non-existent device and/or port via btl_wv_if_[in|e...
Definition: btl_wv.h:252
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_wv_modex_message_t::vendor_id
uint32_t vendor_id
vendor id define device type and tuning
Definition: btl_wv.h:297
mca_btl_base_component_2_0_0_t
BTL component descriptor.
Definition: btl.h:411
mpool.h
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
opal_object_t
Base object.
Definition: opal_object.h:182
opal_convertor_t
Definition: opal_convertor.h:90
mca_btl_wv_component_t::ib_free_list_inc
int ib_free_list_inc
list of ib proc structures
Definition: btl_wv.h:160
wv_pd
Definition: btl_wv_def.h:56
opal_list_t
Definition: opal_list.h:147
wv_device
Definition: btl_wv_def.h:33
mca_btl_wv_component_t::super
mca_btl_base_component_2_0_0_t super
< base BTL component
Definition: btl_wv.h:138
mca_btl_wv_component_t::default_recv_qps
char * default_recv_qps
Default receive queues.
Definition: btl_wv.h:272
base.h
mca_btl_wv_component_t::warn_default_gid_prefix
bool warn_default_gid_prefix
Whether we want a warning if non default GID prefix is not configured on multiport setup...
Definition: btl_wv.h:249
mca_btl_wv_module_srq_qp_t::rd_low_local
int32_t rd_low_local
We post additional WQEs only if a number of WQEs (in specific SRQ) is less of this value...
Definition: btl_wv.h:371
mca_btl_wv_module_t::port_num
uint8_t port_num
ID of the PORT.
Definition: btl_wv.h:407
mca_btl_wv_modex_message_t::lid
uint16_t lid
LID of this port.
Definition: btl_wv.h:291
mca_btl_wv_component_t::ib_lock
opal_mutex_t ib_lock
name of ib memory pool
Definition: btl_wv.h:172
mca_btl_wv_component_t::reg_mru_len
uint32_t reg_mru_len
Use the Shared Receive Queue (SRQ mode)
Definition: btl_wv.h:193
ompi_btl_wv_connect_base_module_t
Struct for holding CPC module and associated meta data.
Definition: connect.h:328
mca_btl_wv_component_t::ib_free_list_max
int ib_free_list_max
number of elements to alloc when growing free lists
Definition: btl_wv.h:157
mca_btl_wv_modex_message_t::transport_type
uint8_t transport_type
Transport type of remote port.
Definition: btl_wv.h:301
mca_btl_wv_component_t::warn_no_device_params_found
bool warn_no_device_params_found
Whether we want a warning if no device-specific parameters are found in INI files.
Definition: btl_wv.h:246
mca_btl_base_module_t
BTL module interface functions and attributes.
Definition: btl.h:786
mca_btl_wv_modex_message_t::vendor_part_id
uint32_t vendor_part_id
vendor part id define device type and tuning
Definition: btl_wv.h:299
mca_btl_wv_component_t::use_eager_rdma
int32_t use_eager_rdma
After this number of msg, use RDMA for short messages, always.
Definition: btl_wv.h:214
wv_srq
Definition: btl_wv_def.h:93
mca_btl_wv_modex_message_t
Common information for all ports that is sent in the modex message.
Definition: btl_wv.h:287
mca_btl_wv_component_t::num_srq_qps
uint8_t num_srq_qps
total number of qp's
Definition: btl_wv.h:180
mca_btl_wv_device_qp_t
Definition: btl_wv.h:317
OBJ_CLASS_DECLARATION
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
mca_mpool_base_module_t
mpool module descriptor.
Definition: mpool.h:174
mca_btl_wv_modex_message_t::mtu
uint8_t mtu
The MTU used by this port.
Definition: btl_wv.h:295