OpenMPI
0.1.1
|
#include "ompi_config.h"
#include <sys/types.h>
#include <infiniband/verbs.h>
#include "opal/class/opal_hash_table.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include "btl_ofud_endpoint.h"
Go to the source code of this file.
Data Structures | |
struct | mca_btl_ud_component_t |
UD Infiniband (IB) BTL component. More... | |
struct | mca_btl_ud_module_t |
UD/IB BTL Interface. More... | |
Macros | |
#define | MCA_BTL_UD_NUM_QP 4 |
#define | MCA_BTL_UD_ENABLE_PROFILE 0 |
Profiling variables. | |
#define | MCA_BTL_UD_START_TIME(var) |
#define | MCA_BTL_UD_END_TIME(var) |
#define | MCA_BTL_UD_SHOW_TIME(var) |
Typedefs | |
typedef struct mca_btl_ud_component_t | mca_btl_ud_component_t |
typedef mca_btl_base_recv_reg_t | mca_btl_ud_recv_reg_t |
typedef struct mca_btl_ud_module_t | mca_btl_ud_module_t |
Functions | |
mca_btl_base_module_t ** | mca_btl_ud_component_init (int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads) |
IB component initialization. More... | |
int | mca_btl_ud_component_progress (void) |
UD/IB component progress. | |
int | mca_btl_ud_finalize (struct mca_btl_base_module_t *btl) |
Cleanup any resources held by the BTL. More... | |
int | mca_btl_ud_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable) |
PML->BTL notification of change in the process list. More... | |
int | mca_btl_ud_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers) |
PML->BTL notification of change in the process list. More... | |
int | mca_btl_ud_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag) |
PML->BTL Initiate a send of the specified size. More... | |
mca_btl_base_descriptor_t * | mca_btl_ud_alloc (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags) |
Allocate a descriptor. More... | |
int | mca_btl_ud_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des) |
Return a segment allocated by this BTL. More... | |
mca_btl_base_descriptor_t * | mca_btl_ud_prepare_src (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags) |
Pack data and return a descriptor that can be used for send/put. More... | |
int | mca_btl_ud_module_init (mca_btl_ud_module_t *ud_btl) |
int | mca_btl_ud_ft_event (int state) |
Fault Tolerance Event Notification Function. More... | |
Variables | |
OMPI_MODULE_DECLSPEC mca_btl_ud_component_t | mca_btl_ofud_component |
mca_btl_ud_module_t | mca_btl_ofud_module |
int mca_btl_ud_add_procs | ( | struct mca_btl_base_module_t * | btl, |
size_t | nprocs, | ||
struct ompi_proc_t ** | procs, | ||
struct mca_btl_base_endpoint_t ** | peers, | ||
opal_bitmap_t * | reachable | ||
) |
PML->BTL notification of change in the process list.
btl | (IN) |
nprocs | (IN) Number of processes |
procs | (IN) Set of processes |
peers | (OUT) Set of (optional) peer addressing info. |
peers | (IN/OUT) Set of processes that are reachable via this BTL. |
References OBJ_CONSTRUCT, OBJ_RELEASE, opal_bitmap_set_bit(), opal_hash_table_init(), opal_hash_table_set_value_uint64(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, mca_btl_ud_proc_t::proc_lock, mca_btl_base_endpoint_t::rem_addr, and mca_btl_base_endpoint_t::rmt_ah.
mca_btl_base_descriptor_t* mca_btl_ud_alloc | ( | struct mca_btl_base_module_t * | btl, |
struct mca_btl_base_endpoint_t * | endpoint, | ||
uint8_t | order, | ||
size_t | size, | ||
uint32_t | flags | ||
) |
Allocate a descriptor.
btl | (IN) BTL module |
size | (IN) Requested descriptor size. |
Allocate a descriptor.
btl | (IN) BTL module |
size | (IN) Request segment size. |
When allocating a segment we pull a pre-alllocated segment from one of two free lists, an eager list and a max list
References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::order, and mca_btl_base_segment_t::seg_len.
mca_btl_base_module_t** mca_btl_ud_component_init | ( | int * | num_btl_modules, |
bool | allow_multi_user_threads, | ||
bool | have_hidden_threads | ||
) |
IB component initialization.
num_btl_modules | (OUT) Number of BTLs returned in BTL array. |
allow_multi_user_threads | (OUT) Flag indicating wether BTL supports user threads (TRUE) |
have_hidden_threads | (OUT) Flag indicating whether BTL uses threads (TRUE) |
(1) read interface list from verbs and compare against component parameters then create a BTL instance for selected interfaces (2) publish BTL addressing info
We must loop through all the hca id's, get their handles and for each hca we query the number of ports on the hca and set up a distinct btl module for each hca port
References mca_btl_ud_module_t::addr, mca_btl_ud_component_t::max_btls, mca_base_param_find(), mca_base_param_lookup_string(), mca_btl_ud_component_t::num_btls, OBJ_CONSTRUCT, OBJ_DESTRUCT, ompi_mpi_thread_multiple, opal_argv_copy(), opal_argv_free(), opal_argv_split(), opal_list_append, opal_list_remove_first(), ORTE_PROC_MY_NAME, orte_show_help(), and mca_btl_ud_component_t::ud_btls.
int mca_btl_ud_del_procs | ( | struct mca_btl_base_module_t * | btl, |
size_t | nprocs, | ||
struct ompi_proc_t ** | procs, | ||
struct mca_btl_base_endpoint_t ** | peers | ||
) |
PML->BTL notification of change in the process list.
btl | (IN) BTL instance |
nproc | (IN) Number of processes. |
procs | (IN) Set of processes. |
peers | (IN) Set of peer data structures. |
References OBJ_RELEASE, opal_hash_table_remove_value_uint64(), and mca_btl_base_endpoint_t::rem_addr.
int mca_btl_ud_finalize | ( | struct mca_btl_base_module_t * | btl | ) |
Cleanup any resources held by the BTL.
btl | BTL instance. |
References mca_btl_base_module_t::btl_mpool, mca_btl_ud_module_t::ib_qp, OBJ_DESTRUCT, mca_btl_ud_module_t::pending_frags, mca_btl_ud_module_t::recv_frags, mca_btl_ud_module_t::send_frags, mca_btl_ud_module_t::ud_lock, and mca_btl_ud_module_t::user_frags.
int mca_btl_ud_free | ( | struct mca_btl_base_module_t * | btl, |
mca_btl_base_descriptor_t * | des | ||
) |
Return a segment allocated by this BTL.
btl | (IN) BTL module |
descriptor | (IN) Allocated descriptor. |
Return a segment allocated by this BTL.
Return the segment to the appropriate preallocated segment list
References mca_btl_base_module_t::btl_mpool, and mca_mpool_base_module_t::mpool_deregister.
Referenced by mca_btl_ud_component_progress().
int mca_btl_ud_ft_event | ( | int | state | ) |
mca_btl_base_descriptor_t* mca_btl_ud_prepare_src | ( | struct mca_btl_base_module_t * | btl, |
struct mca_btl_base_endpoint_t * | endpoint, | ||
mca_mpool_base_registration_t * | registration, | ||
struct opal_convertor_t * | convertor, | ||
uint8_t | order, | ||
size_t | reserve, | ||
size_t * | size, | ||
uint32_t | flags | ||
) |
Pack data and return a descriptor that can be used for send/put.
btl | (IN) BTL module |
peer | (IN) BTL peer addressing |
Pack data and return a descriptor that can be used for send/put.
btl | (IN) BTL module |
peer | (IN) BTL peer addressing |
prepare source's behavior depends on the following: Has a valid memory registration been passed to prepare_src? if so we attempt to use the pre-registred user-buffer, if the memory registration is to small (only a portion of the user buffer) then we must reregister the user buffer Has the user requested the memory to be left pinned? if so we insert the memory registration into a memory tree for later lookup, we may also remove a previous registration if a MRU (most recently used) list of registions is full, this prevents resources from being exhausted. Is the requested size larger than the btl's max send size? if so and we aren't asked to leave the registration pinned then we register the memory if the user's buffer is contiguous. Otherwise we choose from two free lists of pre-registered memory in which to pack the data into.
References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_module_t::btl_max_send_size, mca_btl_base_module_t::btl_mpool, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_mpool_base_module_t::mpool_register, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.
int mca_btl_ud_send | ( | struct mca_btl_base_module_t * | btl, |
struct mca_btl_base_endpoint_t * | btl_peer, | ||
struct mca_btl_base_descriptor_t * | descriptor, | ||
mca_btl_base_tag_t | tag | ||
) |
PML->BTL Initiate a send of the specified size.
btl | (IN) BTL instance |
btl_base_peer | (IN) BTL peer addressing |
send_request | (IN/OUT) Send request (allocated by PML via mca_btl_base_request_alloc_fn_t) |
size | (IN) Number of bytes PML is requesting BTL to deliver |
flags | (IN) Flags that should be passed to the peer via the message header. |
request | (OUT) OMPI_SUCCESS if the BTL was able to queue one or more fragments |