30 #define MCA_BTL_UD_NUM_QP 4
32 #include "ompi_config.h"
33 #include <sys/types.h>
34 #include <infiniband/verbs.h>
38 #include "ompi/class/ompi_free_list.h"
40 #include "ompi/mca/btl/base/btl_base_error.h"
45 #include "btl_ofud_endpoint.h"
61 char** if_include_list;
63 char** if_exclude_list;
84 uint32_t ib_service_level;
85 uint32_t ib_src_path_bits;
99 #define MCA_BTL_UD_ENABLE_PROFILE 0
101 #define MCA_BTL_UD_ENABLE_PROFILE 0
104 #if MCA_BTL_UD_ENABLE_PROFILE
106 #define MCA_BTL_UD_PROFILE_VAR(var) \
107 opal_timer_t avg_ ## var; \
108 opal_timer_t cnt_ ## var; \
109 opal_timer_t tmp_ ## var
111 struct mca_btl_ud_profile_t {
112 MCA_BTL_UD_PROFILE_VAR(post_send);
113 MCA_BTL_UD_PROFILE_VAR(ibv_post_send);
116 typedef struct mca_btl_ud_profile_t mca_btl_ud_profile_t;
117 extern mca_btl_ud_profile_t mca_btl_ud_profile;
130 struct ibv_device* ib_dev;
131 struct ibv_context* ib_dev_context;
132 struct ibv_pd* ib_pd;
133 struct ibv_cq* ib_cq;
156 struct ibv_qp*
ib_qp[MCA_BTL_UD_NUM_QP];
182 int *num_btl_modules,
183 bool allow_multi_user_threads,
184 bool have_hidden_threads);
258 mca_btl_base_tag_t tag);
323 #if MCA_BTL_UD_ENABLE_PROFILE
325 #define MCA_BTL_UD_START_TIME(var) \
326 ((mca_btl_ud_profile.tmp_ ## var) = opal_timer_base_get_cycles())
328 #define MCA_BTL_UD_END_TIME(var) \
330 mca_btl_ud_profile.avg_ ## var += \
331 opal_timer_base_get_cycles() - mca_btl_ud_profile.tmp_ ## var; \
332 mca_btl_ud_profile.cnt_ ## var++; \
335 #define MCA_BTL_UD_SHOW_TIME(var) \
336 OPAL_OUTPUT((0, " " #var " avg %lu cnt %lu", \
337 (mca_btl_ud_profile.avg_ ## var) / (mca_btl_ud_profile.cnt_ ## var), \
338 mca_btl_ud_profile.cnt_ ## var));
341 #define MCA_BTL_UD_START_TIME(var)
342 #define MCA_BTL_UD_END_TIME(var)
343 #define MCA_BTL_UD_SHOW_TIME(var)
opal_list_t ud_procs
list of ib proc structures
Definition: btl_ofud.h:68
int32_t sd_wqe
< number of receives currently posted
Definition: btl_ofud.h:149
int mca_btl_ud_ft_event(int state)
Fault Tolerance Event Notification Function.
Definition: btl_ofud.c:689
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
mca_btl_base_component_2_0_0_t super
base BTL component
Definition: btl_ofud.h:55
Definition: btl_ofud_endpoint.h:34
ompi_free_list_t send_frags
send fragments & buffers
Definition: btl_ofud.h:137
opal_list_t pending_frags
list of pending send frags
Definition: btl_ofud.h:141
Definition: opal_bitmap.h:53
uint32_t ib_qp_next
Local QPs and stripe counters.
Definition: btl_ofud.h:157
size_t ib_inline_max
max size of IB inline send
Definition: btl_ofud.h:145
int mca_btl_ud_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
PML->BTL notification of change in the process list.
Definition: btl_ofud.c:80
opal_mutex_t ud_lock
lock for accessing component state
Definition: btl_ofud.h:69
mca_btl_base_descriptor_t * mca_btl_ud_alloc(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
Allocate a descriptor.
Definition: btl_ofud.c:202
UD/IB BTL Interface.
Definition: btl_ofud.h:126
Definition: mutex_unix.h:53
struct mca_btl_ud_addr_t addr
local address information
Definition: btl_ofud.h:135
int mca_btl_ud_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
PML->BTL Initiate a send of the specified size.
Definition: btl_ofud.c:394
int32_t sd_num
max send descriptors to post per BTL
Definition: btl_ofud.h:73
struct mca_btl_ud_module_t * ud_btls
array of available BTLs
Definition: btl_ofud.h:66
Remote Open MPI process structure.
Definition: proc.h:56
ompi_free_list_t recv_frags
receive fragments & buffers
Definition: btl_ofud.h:139
int mca_btl_ud_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
PML->BTL notification of change in the process list.
Definition: btl_ofud.c:166
struct ibv_qp * ib_qp[MCA_BTL_UD_NUM_QP]
< hash table for fast lookup of endpoint structures in recv path
Definition: btl_ofud.h:156
uint32_t num_btls
Number of available/initialized BTL modules.
Definition: btl_ofud.h:58
mca_btl_base_module_t ** mca_btl_ud_component_init(int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
IB component initialization.
Definition: btl_ofud_component.c:390
Byte Transfer Layer (BTL)
Definition: ompi_free_list.h:39
A hash table that may be indexed with either fixed length (e.g.
opal_mutex_t ud_lock
lock for pending_frags
Definition: btl_ofud.h:143
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
BTL component descriptor.
Definition: btl.h:411
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
Definition: opal_convertor.h:90
IB send fragment derived type.
Definition: btl_ofud_frag.h:73
ompi_free_list_t user_frags
user data fragments
Definition: btl_ofud.h:138
char * ud_mpool_name
name of memory pool
Definition: btl_ofud.h:71
Definition: opal_list.h:147
uint32_t max_btls
Maximum number of BTL modules.
Definition: btl_ofud.h:57
int32_t rd_num
number of receive descriptors per BTL
Definition: btl_ofud.h:75
int mca_btl_ud_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
Return a segment allocated by this BTL.
Definition: btl_ofud.c:233
int mca_btl_ud_component_progress(void)
UD/IB component progress.
Definition: btl_ofud_component.c:626
BTL module interface functions and attributes.
Definition: btl.h:786
int mca_btl_ud_finalize(struct mca_btl_base_module_t *btl)
Cleanup any resources held by the BTL.
Definition: btl_ofud.c:369
mca_btl_base_descriptor_t * mca_btl_ud_prepare_src(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Pack data and return a descriptor that can be used for send/put.
Definition: btl_ofud.c:277
UD Infiniband (IB) BTL component.
Definition: btl_ofud.h:54