OpenMPI  0.1.1
btl_wv.h
1 /* -*- Mode: C; c-basic-offset:4 ; -*- */
2 /*
3  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4  * University Research and Technology
5  * Corporation. All rights reserved.
6  * Copyright (c) 2004-2009 The University of Tennessee and The University
7  * of Tennessee Research Foundation. All rights
8  * reserved.
9  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
10  * University of Stuttgart. All rights reserved.
11  * Copyright (c) 2004-2005 The Regents of the University of California.
12  * All rights reserved.
13  * Copyright (c) 2006-2008 Cisco Systems, Inc. All rights reserved.
14  * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
15  * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
16  * reserved.
17  * Copyright (c) 2006-2007 Voltaire All rights reserved.
18  * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved.
19  * $COPYRIGHT$
20  *
21  * Additional copyrights may follow
22  *
23  * $HEADER$
24  *
25  * @file
26  */
27 
28 #ifndef MCA_BTL_IB_H
29 #define MCA_BTL_IB_H
30 
31 #include "ompi_config.h"
32 #include <sys/types.h>
33 #include <string.h>
34 #include "btl_wv_def.h"
35 
36 /* Open MPI includes */
37 #include "ompi/class/ompi_free_list.h"
40 #include "opal/util/output.h"
41 #include "opal/mca/event/event.h"
42 #include "opal/threads/threads.h"
43 #include "ompi/mca/btl/btl.h"
44 #include "ompi/mca/mpool/mpool.h"
45 #include "ompi/mca/btl/base/btl_base_error.h"
46 
47 #include "ompi/mca/btl/btl.h"
48 #include "ompi/mca/btl/base/base.h"
49 
50 #include "connect/connect.h"
51 
52 BEGIN_C_DECLS
53 
54 
55 #define MCA_BTL_IB_LEAVE_PINNED 1
56 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll
57 #define MCA_BTL_IB_PKEY_MASK 0x7fff
58 
59 
60 /*--------------------------------------------------------------------*/
61 
62 #if OPAL_ENABLE_DEBUG
63 #define ATTACH() do { \
64  int i = 0; \
65  opal_output(0, "WAITING TO DEBUG ATTACH"); \
66  while (i == 0) sleep(5); \
67  } while(0);
68 #else
69 #define ATTACH()
70 #endif
71 
72 /*--------------------------------------------------------------------*/
73 
74 /**
75  * Infiniband (IB) BTL component.
76  */
77 
78 typedef enum {
79  MCA_BTL_WV_TRANSPORT_IB,
80  MCA_BTL_WV_TRANSPORT_IWARP,
81  MCA_BTL_WV_TRANSPORT_RDMAOE,
82  MCA_BTL_WV_TRANSPORT_UNKNOWN,
83  MCA_BTL_WV_TRANSPORT_SIZE
84 } mca_btl_wv_transport_type_t;
85 
86 typedef enum {
87  MCA_BTL_WV_PP_QP,
88  MCA_BTL_WV_SRQ_QP,
89 } mca_btl_wv_qp_type_t;
90 
92  int32_t rd_win;
93  int32_t rd_rsv;
95 
97  int32_t sd_max;
98  /* The init value for rd_curr_num variables of all SRQs */
99  int32_t rd_init;
100  /* The watermark, threshold - if the number of WQEs in SRQ is less then this value =>
101  the SRQ limit event will be generated on corresponding SRQ.
102  As result the maximal number of pre-posted WQEs on the SRQ will be increased */
103  int32_t srq_limit;
105 
107  mca_btl_wv_qp_type_t type;
108  size_t size;
109  int32_t rd_num;
110  int32_t rd_low;
111  union {
114  } u;
116 
117 #define BTL_WV_QP_TYPE(Q) (mca_btl_wv_component.qp_infos[(Q)].type)
118 #define BTL_WV_QP_TYPE_PP(Q) \
119  (BTL_WV_QP_TYPE(Q) == MCA_BTL_WV_PP_QP)
120 #define BTL_WV_QP_TYPE_SRQ(Q) \
121  (BTL_WV_QP_TYPE(Q) == MCA_BTL_WV_SRQ_QP)
122 
123 typedef enum {
124  BTL_WV_RQ_SOURCE_DEFAULT,
125  BTL_WV_RQ_SOURCE_MCA,
126  BTL_WV_RQ_SOURCE_DEVICE_INI,
127  BTL_WV_RQ_SOURCE_MAX
128 } btl_wv_receive_queues_source_t;
129 
130 typedef enum {
131  BTL_WV_DT_IB,
132  BTL_WV_DT_IWARP,
133  BTL_WV_DT_ALL
134 } btl_wv_device_type_t;
135 
137  /**< base BTL component */
139 
140  /**< maximum number of devices available to wv component */
142 
143  /**< number of devices available to the wv component */
145 
146  /**< array of available BTLs */
148 
149  /**< array of available devices */
150  opal_pointer_array_t devices;
152 
153  /**< initial size of free lists */
155 
156  /**< maximum size of free lists */
158 
159  /**< number of elements to alloc when growing free lists */
161 
162  /**< list of ib proc structures */
164 
165  /**< event structure for sends */
167 
168  /**< event structure for recvs */
170 
171  /**< lock for accessing module state */
173 
174  /**< name of ib memory pool */
176 
177  /**< number of pp qp's */
178  uint8_t num_pp_qps;
179  /**< number of srq qp's */
180  uint8_t num_srq_qps;
181  /**< total number of qp's */
182  uint8_t num_qps;
183 
185 
186  /**< Eager send limit of first fragment, in Bytes */
187  size_t eager_limit;
188  /**< Maximum send size, in Bytes */
190  /**< Maximum message size for RDMA protocols in Bytes */
192  /**< Length of the registration cache most recently used list */
193  uint32_t reg_mru_len;
194  /**< Use the Shared Receive Queue (SRQ mode) */
195  uint32_t use_srq;
196 
197  /**< Max outstanding CQE on the CQ */
198  uint32_t ib_cq_size[2];
199 
200  /**< Max size of inline data */
201  int32_t ib_max_inline_data;
202 
203  uint32_t ib_pkey_val;
204  uint32_t ib_psn;
205  uint32_t ib_qp_ous_rd_atom;
206  uint32_t ib_mtu;
207  uint32_t ib_min_rnr_timer;
208  uint32_t ib_timeout;
209  uint32_t ib_retry_count;
210  uint32_t ib_rnr_retry;
211  uint32_t ib_max_rdma_dst_ops;
212  uint32_t ib_service_level;
213  uint32_t ib_path_rec_service_level;
214  int32_t use_eager_rdma;
215  /**< After this number of msg, use RDMA for short messages, always */
216  int32_t eager_rdma_threshold;
217  int32_t eager_rdma_num;
218  int32_t max_eager_rdma;
219  uint32_t btls_per_lid;
220  uint32_t max_lmc;
221  int32_t apm_lmc;
222  int32_t apm_ports;
223  /**< Preferred communication buffer alignment in Bytes (must be power of two) */
224  uint32_t buffer_alignment;
225  btl_wv_device_type_t device_type;
226  char *if_include;
227  char **if_include_list;
228  char *if_exclude;
229  char **if_exclude_list;
230  char *ipaddr_include;
231  char *ipaddr_exclude;
232 
233  /* MCA param btl_wv_receive_queues */
234  char *receive_queues;
235  /* Whether we got a non-default value of btl_wv_receive_queues */
236  btl_wv_receive_queues_source_t receive_queues_source;
237 
238  /** Colon-delimited list of filenames for device parameters */
240 
241  /** Whether we're in verbose mode or not */
242  bool verbose;
243 
244  /** Whether we want a warning if no device-specific parameters are
245  found in INI files */
247  /** Whether we want a warning if non default GID prefix is not configured
248  on multiport setup */
250  /** Whether we want a warning if the user specifies a non-existent
251  device and/or port via btl_wv_if_[in|ex]clude MCA params */
253  /** Dummy argv-style list; a copy of names from the
254  if_[in|ex]clude list that we use for error checking (to ensure
255  that they all exist) */
256  char **if_list;
257  bool use_message_coalescing;
258  uint32_t cq_poll_ratio;
259  uint32_t cq_poll_progress;
260  uint32_t eager_rdma_poll_ratio;
261  int rdma_qp;
262  /* qp used for software flow control */
263  int credits_qp;
265  /**< free list of frags only; used for pining user memory */
267  /**< free list of frags only; used for pining user memory */
269  /**< frags for coalesced massages */
270  ompi_free_list_t send_free_coalesced;
271  /** Default receive queues */
273  /** Whether we want a dynamically resizing srq, enabled by default */
275 #if BTL_WV_FAILOVER_ENABLED
276  int verbose_failover;
277 #endif
279 
280 OMPI_MODULE_DECLSPEC extern mca_btl_wv_component_t mca_btl_wv_component;
281 
283 
284 /**
285  * Common information for all ports that is sent in the modex message
286  */
288  /** The subnet ID of this port */
289  uint64_t subnet_id;
290  /** LID of this port */
291  uint16_t lid;
292  /** APM LID for this port */
293  uint16_t apm_lid;
294  /** The MTU used by this port */
295  uint8_t mtu;
296  /** vendor id define device type and tuning */
297  uint32_t vendor_id;
298  /** vendor part id define device type and tuning */
299  uint32_t vendor_part_id;
300  /** Transport type of remote port */
301  uint8_t transport_type;
302  /** Dummy field used to calculate the real length */
303  uint8_t end;
305 
306 #define MCA_BTL_WV_MODEX_MSG_NTOH(hdr) \
307  do { \
308  (hdr).subnet_id = ntoh64((hdr).subnet_id); \
309  (hdr).lid = ntohs((hdr).lid); \
310  } while (0)
311 #define MCA_BTL_WV_MODEX_MSG_HTON(hdr) \
312  do { \
313  (hdr).subnet_id = hton64((hdr).subnet_id); \
314  (hdr).lid = htons((hdr).lid); \
315  } while (0)
316 
317 typedef struct mca_btl_wv_device_qp_t {
318  ompi_free_list_t send_free; /**< free lists of send buffer descriptors */
319  ompi_free_list_t recv_free; /**< free lists of receive buffer descriptors */
321 
323 
324 typedef struct mca_btl_wv_device_t {
325  opal_object_t super;
326  struct wv_device *ib_dev; /* the ib device */
327  opal_mutex_t device_lock; /* device level lock */
328  struct wv_context *ib_dev_context;
329  WV_DEVICE_ATTRIBUTES ib_dev_attr;
330  struct wv_pd *ib_pd;
331  struct wv_cq *ib_cq[2];
332  uint32_t cq_size[2];
334  /* MTU for this device */
335  uint32_t mtu;
336  /* Whether this device supports eager RDMA */
337  uint8_t use_eager_rdma;
338  uint8_t btls; /** < number of btls using this device */
340  opal_pointer_array_t *device_btls;
341  uint16_t hp_cq_polls;
342  uint16_t eager_rdma_polls;
343  bool pollme;
344  int32_t non_eager_rdma_endpoints;
345  int32_t eager_rdma_buffers_count;
347  /**< frags for control massages */
348  ompi_free_list_t send_free_control;
349  /* QP types and attributes that will be used on this device */
351  /* Maximum value supported by this device for max_inline_data */
352  uint32_t max_inline_data;
355 
357  int32_t dummy;
359 
361  struct wv_srq *srq;
362  int32_t rd_posted;
363  int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */
364  /* i.e. the number of frags that can be outstanding (down counter) */
365  opal_list_t pending_frags[2]; /**< list of high/low prio frags */
366  /** The number of receive buffers that can be post in the current time.
367  The value starts from (rd_num / 4) and increased up to rd_num */
368  int32_t rd_curr_num;
369  /** We post additional WQEs only if a number of WQEs (in specific SRQ) is less of this value.
370  The value increased together with rd_curr_num. The value is unique for every SRQ. */
371  int32_t rd_low_local;
372 
374  /**< In difference of the "--mca enable_srq_resize" parameter that says, if we want(or no)
375  to start with small num of pre-posted receive buffers (rd_curr_num) and to increase this number by needs
376  (the max of this value is rd_num – the whole size of SRQ), the "srq_limit_event_flag" says if we want to get limit event
377  from device if the defined srq limit was reached (signal to the main thread) and we put off this flag if the rd_curr_num
378  was increased up to rd_num.*/
380 
382  union {
385  } u;
387 
388 /**
389  * IB BTL Interface
390  */
392  /* Base BTL module */
393  mca_btl_base_module_t super;
394 
395  bool btl_inited;
396 
397  /** Common information about all ports */
399 
400  /** Array of CPCs on this port */
402 
403  /** Number of elements in the cpcs array */
404  uint8_t num_cpcs;
405 
406  mca_btl_wv_device_t *device;
407  uint8_t port_num; /**< ID of the PORT */
408  uint16_t pkey_index;
409  WV_PORT_ATTRIBUTES ib_port_attr;
410  uint16_t lid; /**< lid that is actually used (for LMC) */
411  int apm_port; /**< Alternative port that may be used for APM */
412  uint8_t src_path_bits; /**< offset from base lid (for LMC) */
413 
414  int32_t num_peers;
415 
416  opal_mutex_t ib_lock; /**< module level lock */
417 
418  size_t eager_rdma_frag_size; /**< length of eager frag */
419  volatile int32_t eager_rdma_channels; /**< number of open RDMA channels */
420 
422 
424 };
426 
427 extern mca_btl_wv_module_t mca_btl_wv_module;
428 
431  struct wv_mr *mr;
432 };
433 typedef struct mca_btl_wv_reg_t mca_btl_wv_reg_t;
434 /**
435  * Register a callback function that is called on error..
436  *
437  * @param btl (IN) BTL module
438  * @return Status indicating if cleanup was successful
439  */
440 
441 int mca_btl_wv_register_error_cb(
442  struct mca_btl_base_module_t* btl,
444 );
445 
446 
447 /**
448  * Cleanup any resources held by the BTL.
449  *
450  * @param btl BTL instance.
451  * @return OMPI_SUCCESS or error status on failure.
452  */
453 
454 extern int mca_btl_wv_finalize(
455  struct mca_btl_base_module_t* btl
456 );
457 
458 
459 /**
460  * PML->BTL notification of change in the process list.
461  *
462  * @param btl (IN) BTL module
463  * @param nprocs (IN) Number of processes
464  * @param procs (IN) Set of processes
465  * @param peers (OUT) Set of (optional) peer addressing info.
466  * @param reachable (IN/OUT) Set of processes that are reachable via this BTL.
467  * @return OMPI_SUCCESS or error status on failure.
468  *
469  */
470 
471 extern int mca_btl_wv_add_procs(
472  struct mca_btl_base_module_t* btl,
473  size_t nprocs,
474  struct ompi_proc_t **procs,
475  struct mca_btl_base_endpoint_t** peers,
476  opal_bitmap_t* reachable
477 );
478 
479 /**
480  * PML->BTL notification of change in the process list.
481  *
482  * @param btl (IN) BTL instance
483  * @param nproc (IN) Number of processes.
484  * @param procs (IN) Set of processes.
485  * @param peers (IN) Set of peer data structures.
486  * @return Status indicating if cleanup was successful
487  *
488  */
489 extern int mca_btl_wv_del_procs(
490  struct mca_btl_base_module_t* btl,
491  size_t nprocs,
492  struct ompi_proc_t **procs,
493  struct mca_btl_base_endpoint_t** peers
494 );
495 
496 
497 /**
498  * PML->BTL Initiate a send of the specified size.
499  *
500  * @param btl (IN) BTL instance
501  * @param btl_peer (IN) BTL peer addressing
502  * @param descriptor (IN) Descriptor of data to be transmitted.
503  * @param tag (IN) Tag.
504  */
505 extern int mca_btl_wv_send(
506  struct mca_btl_base_module_t* btl,
507  struct mca_btl_base_endpoint_t* btl_peer,
508  struct mca_btl_base_descriptor_t* descriptor,
509  mca_btl_base_tag_t tag
510 );
511 
512 /**
513  * PML->BTL Initiate a immediate send of the specified size.
514  *
515  * @param btl (IN) BTL instance
516  * @param ep (IN) Endpoint
517  * @param convertor (IN) Datatypes converter
518  * @param header (IN) PML header
519  * @param header_size (IN) PML header size
520  * @param payload_size (IN) Payload size
521  * @param order (IN) Order
522  * @param flags (IN) Flags
523  * @param tag (IN) Tag
524  * @param descriptor (OUT) Messages descriptor
525  */
526 extern int mca_btl_wv_sendi( struct mca_btl_base_module_t* btl,
527  struct mca_btl_base_endpoint_t* ep,
528  struct opal_convertor_t* convertor,
529  void* header,
530  size_t header_size,
531  size_t payload_size,
532  uint8_t order,
533  uint32_t flags,
534  mca_btl_base_tag_t tag,
535  mca_btl_base_descriptor_t** descriptor
536 );
537 
538 /**
539  * PML->BTL Initiate a put of the specified size.
540  *
541  * @param btl (IN) BTL instance
542  * @param btl_peer (IN) BTL peer addressing
543  * @param descriptor (IN) Descriptor of data to be transmitted.
544  */
545 extern int mca_btl_wv_put(
546  struct mca_btl_base_module_t* btl,
547  struct mca_btl_base_endpoint_t* btl_peer,
548  struct mca_btl_base_descriptor_t* descriptor
549  );
550 
551 /**
552  * PML->BTL Initiate a get of the specified size.
553  *
554  * @param btl (IN) BTL instance
555  * @param btl_base_peer (IN) BTL peer addressing
556  * @param descriptor (IN) Descriptor of data to be transmitted.
557  */
558 extern int mca_btl_wv_get(
559  struct mca_btl_base_module_t* btl,
560  struct mca_btl_base_endpoint_t* btl_peer,
561  struct mca_btl_base_descriptor_t* descriptor
562  );
563 
564 
565 /**
566  * Allocate a descriptor.
567  *
568  * @param btl (IN) BTL module
569  * @param size (IN) Requested descriptor size.
570  */
571 extern mca_btl_base_descriptor_t* mca_btl_wv_alloc(
572  struct mca_btl_base_module_t* btl,
573  struct mca_btl_base_endpoint_t* endpoint,
574  uint8_t order,
575  size_t size,
576  uint32_t flags);
577 
578 
579 /**
580  * Return a segment allocated by this BTL.
581  *
582  * @param btl (IN) BTL module
583  * @param descriptor (IN) Allocated descriptor.
584  */
585 extern int mca_btl_wv_free(
586  struct mca_btl_base_module_t* btl,
588 
589 
590 /**
591  * Pack data and return a descriptor that can be
592  * used for send/put.
593  *
594  * @param btl (IN) BTL module
595  * @param peer (IN) BTL peer addressing
596  */
597 mca_btl_base_descriptor_t* mca_btl_wv_prepare_src(
598  struct mca_btl_base_module_t* btl,
599  struct mca_btl_base_endpoint_t* peer,
600  mca_mpool_base_registration_t* registration,
601  struct opal_convertor_t* convertor,
602  uint8_t order,
603  size_t reserve,
604  size_t* size,
605  uint32_t flags
606  );
607 
608 /**
609  * Allocate a descriptor initialized for RDMA write.
610  *
611  * @param btl (IN) BTL module
612  * @param peer (IN) BTL peer addressing
613  */
614 extern mca_btl_base_descriptor_t* mca_btl_wv_prepare_dst(
615  struct mca_btl_base_module_t* btl,
616  struct mca_btl_base_endpoint_t* peer,
617  mca_mpool_base_registration_t* registration,
618  struct opal_convertor_t* convertor,
619  uint8_t order,
620  size_t reserve,
621  size_t* size,
622  uint32_t flags);
623 
624 extern void mca_btl_wv_frag_progress_pending_put_get(
625  struct mca_btl_base_endpoint_t*, const int);
626 
627 /**
628  * Fault Tolerance Event Notification Function
629  *
630  * @param state (IN) Checkpoint State
631  * @return OMPI_SUCCESS or failure status
632  */
633 extern int mca_btl_wv_ft_event(int state);
634 
635 
636 /**
637  * Show an error during init, particularly when running out of
638  * registered memory.
639  */
640 void mca_btl_wv_show_init_error(const char *file, int line,
641  const char *func, const char *dev);
642 
643 #define BTL_WV_HP_CQ 0
644 #define BTL_WV_LP_CQ 1
645 
646 
647 /**
648  * Post to Shared Receive Queue with certain priority
649  *
650  * @param wv_btl (IN) BTL module
651  * @param additional (IN) Additional Bytes to reserve
652  * @param prio (IN) Priority (either BTL_WV_HP_QP or BTL_WV_LP_QP)
653  * @return OMPI_SUCCESS or failure status
654  */
655 
656 int mca_btl_wv_post_srr(mca_btl_wv_module_t* wv_btl, const int qp);
657 
658 /**
659  * Get a transport name of btl by its transport type.
660  */
661 
662 const char* btl_wv_get_transport_name(mca_btl_wv_transport_type_t transport_type);
663 
664 /**
665  * Get a transport type of btl.
666  */
667 
668 mca_btl_wv_transport_type_t mca_btl_wv_get_transport_type(mca_btl_wv_module_t* wv_btl);
669 
670 static inline int qp_cq_prio(const int qp)
671 {
672  if(0 == qp)
673  return BTL_WV_HP_CQ; /* smallest qp is always HP */
674 
675  /* If the size for this qp is <= the eager limit, make it a
676  high priority QP. Otherwise, make it a low priority QP. */
677  return (mca_btl_wv_component.qp_infos[qp].size <=
678  mca_btl_wv_component.eager_limit) ?
679  BTL_WV_HP_CQ : BTL_WV_LP_CQ;
680 }
681 
682 #define BTL_WV_RDMA_QP(QP) \
683  ((QP) == mca_btl_wv_component.rdma_qp)
684 
685 END_C_DECLS
686 
687 #endif /* MCA_BTL_IB_H */
ompi_free_list_t recv_user_free
frags for coalesced massages
Definition: btl_wv.h:268
opal_mutex_t ib_lock
module level lock
Definition: btl_wv.h:416
IB BTL Interface.
Definition: btl_wv.h:391
int apm_port
Alternative port that may be used for APM.
Definition: btl_wv.h:411
Definition: btl_wv.h:429
struct mca_btl_base_endpoint_t ** eager_rdma_buffers
frags for control massages
Definition: btl_wv.h:346
int32_t rd_curr_num
The number of receive buffers that can be post in the current time.
Definition: btl_wv.h:368
Definition: btl_wv_def.h:126
ompi_free_list_t send_free
free lists of send buffer descriptors
Definition: btl_wv.h:318
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
OPAL output stream facility.
mca_btl_wv_modex_message_t port_info
Common information about all ports.
Definition: btl_wv.h:398
opal_list_t pending_frags[2]
list of high/low prio frags
Definition: btl_wv.h:365
opal_event_t ib_recv_event
lock for accessing module state
Definition: btl_wv.h:169
dynamic pointer array
Definition: opal_pointer_array.h:45
int32_t apm_ports
Preferred communication buffer alignment in Bytes (must be power of two)
Definition: btl_wv.h:222
uint16_t apm_lid
APM LID for this port.
Definition: btl_wv.h:293
Definition: opal_bitmap.h:53
Definition: base.h:44
Definition: btl_wv.h:136
bool enable_srq_resize
Whether we want a dynamically resizing srq, enabled by default.
Definition: btl_wv.h:274
uint8_t num_cpcs
Number of elements in the cpcs array.
Definition: btl_wv.h:404
bool verbose
Whether we're in verbose mode or not.
Definition: btl_wv.h:242
mca_btl_base_module_error_cb_fn_t error_cb
error handler
Definition: btl_wv.h:421
uint16_t lid
lid that is actually used (for LMC)
Definition: btl_wv.h:410
Structure to represent a single event.
Definition: event_struct.h:87
size_t max_send_size
Maximum message size for RDMA protocols in Bytes.
Definition: btl_wv.h:189
ompi_free_list_t recv_free
free lists of receive buffer descriptors
Definition: btl_wv.h:319
size_t eager_limit
Maximum send size, in Bytes.
Definition: btl_wv.h:187
uint32_t use_srq
Max outstanding CQE on the CQ.
Definition: btl_wv.h:195
void(* mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)
Callback function that is called asynchronously on receipt of an error from the transport layer...
Definition: btl.h:538
Definition: mutex_unix.h:53
uint64_t subnet_id
The subnet ID of this port.
Definition: btl_wv.h:289
char ** if_list
Dummy argv-style list; a copy of names from the if_[in|ex]clude list that we use for error checking (...
Definition: btl_wv.h:256
mca_btl_wv_qp_info_t * qp_infos
Eager send limit of first fragment, in Bytes.
Definition: btl_wv.h:184
volatile int32_t eager_rdma_channels
number of open RDMA channels
Definition: btl_wv.h:419
Definition: mpool.h:44
See opal_bitmap.h for an explanation of why there is a split between OPAL and ORTE for this generic c...
Remote Open MPI process structure.
Definition: proc.h:56
uint8_t src_path_bits
offset from base lid (for LMC)
Definition: btl_wv.h:412
bool cpc_explicitly_defined
free list of frags only; used for pining user memory
Definition: btl_wv.h:264
Definition: btl_wv.h:96
int32_t max_hw_msg_size
Length of the registration cache most recently used list.
Definition: btl_wv.h:191
Definition: btl_wv.h:356
Definition: btl_wv.h:324
uint8_t num_pp_qps
number of srq qp's
Definition: btl_wv.h:178
opal_event_t ib_send_event
event structure for recvs
Definition: btl_wv.h:166
int ib_num_btls
array of available BTLs
Definition: btl_wv.h:144
opal_pointer_array_t * endpoints
< number of btls using this device
Definition: btl_wv.h:339
ompi_btl_wv_connect_base_module_t ** cpcs
Array of CPCs on this port.
Definition: btl_wv.h:401
int ib_max_btls
number of devices available to the wv component
Definition: btl_wv.h:141
Definition: btl_wv.h:91
Definition: btl_wv_def.h:62
size_t eager_rdma_frag_size
length of eager frag
Definition: btl_wv.h:418
uint8_t end
Dummy field used to calculate the real length.
Definition: btl_wv.h:303
char * ib_mpool_name
number of pp qp's
Definition: btl_wv.h:175
uint32_t ib_cq_size[2]
Max size of inline data.
Definition: btl_wv.h:198
Definition: btl_wv.h:360
opal_list_t ib_procs
event structure for sends
Definition: btl_wv.h:163
Byte Transfer Layer (BTL)
Definition: btl_wv_def.h:40
bool srq_limit_event_flag
In difference of the "--mca enable_srq_resize" parameter that says, if we want(or no) to start with s...
Definition: btl_wv.h:373
int devices_count
initial size of free lists
Definition: btl_wv.h:151
struct mca_btl_wv_module_t ** wv_btls
array of available devices
Definition: btl_wv.h:147
Definition: btl_wv.h:106
int ib_free_list_num
maximum size of free lists
Definition: btl_wv.h:154
Definition: btl_wv.h:381
Definition: ompi_free_list.h:39
A hash table that may be indexed with either fixed length (e.g.
ompi_free_list_t send_user_free
free list of frags only; used for pining user memory
Definition: btl_wv.h:266
char * device_params_file_names
Colon-delimited list of filenames for device parameters.
Definition: btl_wv.h:239
bool warn_nonexistent_if
Whether we want a warning if the user specifies a non-existent device and/or port via btl_wv_if_[in|e...
Definition: btl_wv.h:252
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
uint32_t vendor_id
vendor id define device type and tuning
Definition: btl_wv.h:297
BTL component descriptor.
Definition: btl.h:411
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
Base object.
Definition: opal_object.h:182
Definition: opal_convertor.h:90
int ib_free_list_inc
list of ib proc structures
Definition: btl_wv.h:160
Definition: btl_wv_def.h:56
Definition: opal_list.h:147
Definition: btl_wv_def.h:33
mca_btl_base_component_2_0_0_t super
< base BTL component
Definition: btl_wv.h:138
char * default_recv_qps
Default receive queues.
Definition: btl_wv.h:272
bool warn_default_gid_prefix
Whether we want a warning if non default GID prefix is not configured on multiport setup...
Definition: btl_wv.h:249
int32_t rd_low_local
We post additional WQEs only if a number of WQEs (in specific SRQ) is less of this value...
Definition: btl_wv.h:371
uint8_t port_num
ID of the PORT.
Definition: btl_wv.h:407
uint16_t lid
LID of this port.
Definition: btl_wv.h:291
opal_mutex_t ib_lock
name of ib memory pool
Definition: btl_wv.h:172
uint32_t reg_mru_len
Use the Shared Receive Queue (SRQ mode)
Definition: btl_wv.h:193
Struct for holding CPC module and associated meta data.
Definition: connect.h:328
int ib_free_list_max
number of elements to alloc when growing free lists
Definition: btl_wv.h:157
uint8_t transport_type
Transport type of remote port.
Definition: btl_wv.h:301
bool warn_no_device_params_found
Whether we want a warning if no device-specific parameters are found in INI files.
Definition: btl_wv.h:246
BTL module interface functions and attributes.
Definition: btl.h:786
uint32_t vendor_part_id
vendor part id define device type and tuning
Definition: btl_wv.h:299
int32_t use_eager_rdma
After this number of msg, use RDMA for short messages, always.
Definition: btl_wv.h:214
Definition: btl_wv_def.h:93
Common information for all ports that is sent in the modex message.
Definition: btl_wv.h:287
uint8_t num_srq_qps
total number of qp's
Definition: btl_wv.h:180
Definition: btl_wv.h:317
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
mpool module descriptor.
Definition: mpool.h:174
uint8_t mtu
The MTU used by this port.
Definition: btl_wv.h:295