OpenMPI  0.1.1
btl_ofud.h File Reference
#include "ompi_config.h"
#include <sys/types.h>
#include <infiniband/verbs.h>
#include "opal/class/opal_hash_table.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include "btl_ofud_endpoint.h"

Go to the source code of this file.

Data Structures

struct  mca_btl_ud_component_t
 UD Infiniband (IB) BTL component. More...
 
struct  mca_btl_ud_module_t
 UD/IB BTL Interface. More...
 

Macros

#define MCA_BTL_UD_NUM_QP   4
 
#define MCA_BTL_UD_ENABLE_PROFILE   0
 Profiling variables.
 
#define MCA_BTL_UD_START_TIME(var)
 
#define MCA_BTL_UD_END_TIME(var)
 
#define MCA_BTL_UD_SHOW_TIME(var)
 

Typedefs

typedef struct
mca_btl_ud_component_t 
mca_btl_ud_component_t
 
typedef mca_btl_base_recv_reg_t mca_btl_ud_recv_reg_t
 
typedef struct mca_btl_ud_module_t mca_btl_ud_module_t
 

Functions

mca_btl_base_module_t ** mca_btl_ud_component_init (int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
 IB component initialization. More...
 
int mca_btl_ud_component_progress (void)
 UD/IB component progress.
 
int mca_btl_ud_finalize (struct mca_btl_base_module_t *btl)
 Cleanup any resources held by the BTL. More...
 
int mca_btl_ud_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
 PML->BTL notification of change in the process list. More...
 
int mca_btl_ud_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
 PML->BTL notification of change in the process list. More...
 
int mca_btl_ud_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 PML->BTL Initiate a send of the specified size. More...
 
mca_btl_base_descriptor_tmca_btl_ud_alloc (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
 Allocate a descriptor. More...
 
int mca_btl_ud_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 Return a segment allocated by this BTL. More...
 
mca_btl_base_descriptor_tmca_btl_ud_prepare_src (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
 Pack data and return a descriptor that can be used for send/put. More...
 
int mca_btl_ud_module_init (mca_btl_ud_module_t *ud_btl)
 
int mca_btl_ud_ft_event (int state)
 Fault Tolerance Event Notification Function. More...
 

Variables

OMPI_MODULE_DECLSPEC
mca_btl_ud_component_t 
mca_btl_ofud_component
 
mca_btl_ud_module_t mca_btl_ofud_module
 

Function Documentation

int mca_btl_ud_add_procs ( struct mca_btl_base_module_t btl,
size_t  nprocs,
struct ompi_proc_t **  procs,
struct mca_btl_base_endpoint_t **  peers,
opal_bitmap_t reachable 
)

PML->BTL notification of change in the process list.

Parameters
btl(IN)
nprocs(IN) Number of processes
procs(IN) Set of processes
peers(OUT) Set of (optional) peer addressing info.
peers(IN/OUT) Set of processes that are reachable via this BTL.
Returns
OMPI_SUCCESS or error status on failure.

References OBJ_CONSTRUCT, OBJ_RELEASE, opal_bitmap_set_bit(), opal_hash_table_init(), opal_hash_table_set_value_uint64(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, mca_btl_ud_proc_t::proc_lock, mca_btl_base_endpoint_t::rem_addr, and mca_btl_base_endpoint_t::rmt_ah.

mca_btl_base_descriptor_t* mca_btl_ud_alloc ( struct mca_btl_base_module_t btl,
struct mca_btl_base_endpoint_t endpoint,
uint8_t  order,
size_t  size,
uint32_t  flags 
)

Allocate a descriptor.

Parameters
btl(IN) BTL module
size(IN) Requested descriptor size.

Allocate a descriptor.

Parameters
btl(IN) BTL module
size(IN) Request segment size.

When allocating a segment we pull a pre-alllocated segment from one of two free lists, an eager list and a max list

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::order, and mca_btl_base_segment_t::seg_len.

mca_btl_base_module_t** mca_btl_ud_component_init ( int *  num_btl_modules,
bool  allow_multi_user_threads,
bool  have_hidden_threads 
)

IB component initialization.

Parameters
num_btl_modules(OUT) Number of BTLs returned in BTL array.
allow_multi_user_threads(OUT) Flag indicating wether BTL supports user threads (TRUE)
have_hidden_threads(OUT) Flag indicating whether BTL uses threads (TRUE)

(1) read interface list from verbs and compare against component parameters then create a BTL instance for selected interfaces (2) publish BTL addressing info

We must loop through all the hca id's, get their handles and for each hca we query the number of ports on the hca and set up a distinct btl module for each hca port

References mca_btl_ud_module_t::addr, mca_btl_ud_component_t::max_btls, mca_base_param_find(), mca_base_param_lookup_string(), mca_btl_ud_component_t::num_btls, OBJ_CONSTRUCT, OBJ_DESTRUCT, ompi_mpi_thread_multiple, opal_argv_copy(), opal_argv_free(), opal_argv_split(), opal_list_append, opal_list_remove_first(), ORTE_PROC_MY_NAME, orte_show_help(), and mca_btl_ud_component_t::ud_btls.

int mca_btl_ud_del_procs ( struct mca_btl_base_module_t btl,
size_t  nprocs,
struct ompi_proc_t **  procs,
struct mca_btl_base_endpoint_t **  peers 
)

PML->BTL notification of change in the process list.

Parameters
btl(IN) BTL instance
nproc(IN) Number of processes.
procs(IN) Set of processes.
peers(IN) Set of peer data structures.
Returns
Status indicating if cleanup was successful

References OBJ_RELEASE, opal_hash_table_remove_value_uint64(), and mca_btl_base_endpoint_t::rem_addr.

int mca_btl_ud_finalize ( struct mca_btl_base_module_t btl)
int mca_btl_ud_free ( struct mca_btl_base_module_t btl,
mca_btl_base_descriptor_t des 
)

Return a segment allocated by this BTL.

Parameters
btl(IN) BTL module
descriptor(IN) Allocated descriptor.

Return a segment allocated by this BTL.

Return the segment to the appropriate preallocated segment list

References mca_btl_base_module_t::btl_mpool, and mca_mpool_base_module_t::mpool_deregister.

Referenced by mca_btl_ud_component_progress().

int mca_btl_ud_ft_event ( int  state)

Fault Tolerance Event Notification Function.

Parameters
stateCheckpoint State
Returns
OMPI_SUCCESS or failure status
mca_btl_base_descriptor_t* mca_btl_ud_prepare_src ( struct mca_btl_base_module_t btl,
struct mca_btl_base_endpoint_t endpoint,
mca_mpool_base_registration_t registration,
struct opal_convertor_t convertor,
uint8_t  order,
size_t  reserve,
size_t *  size,
uint32_t  flags 
)

Pack data and return a descriptor that can be used for send/put.

Parameters
btl(IN) BTL module
peer(IN) BTL peer addressing

Pack data and return a descriptor that can be used for send/put.

Parameters
btl(IN) BTL module
peer(IN) BTL peer addressing

prepare source's behavior depends on the following: Has a valid memory registration been passed to prepare_src? if so we attempt to use the pre-registred user-buffer, if the memory registration is to small (only a portion of the user buffer) then we must reregister the user buffer Has the user requested the memory to be left pinned? if so we insert the memory registration into a memory tree for later lookup, we may also remove a previous registration if a MRU (most recently used) list of registions is full, this prevents resources from being exhausted. Is the requested size larger than the btl's max send size? if so and we aren't asked to leave the registration pinned then we register the memory if the user's buffer is contiguous. Otherwise we choose from two free lists of pre-registered memory in which to pack the data into.

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_module_t::btl_max_send_size, mca_btl_base_module_t::btl_mpool, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_mpool_base_module_t::mpool_register, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

int mca_btl_ud_send ( struct mca_btl_base_module_t btl,
struct mca_btl_base_endpoint_t btl_peer,
struct mca_btl_base_descriptor_t descriptor,
mca_btl_base_tag_t  tag 
)

PML->BTL Initiate a send of the specified size.

Parameters
btl(IN) BTL instance
btl_base_peer(IN) BTL peer addressing
send_request(IN/OUT) Send request (allocated by PML via mca_btl_base_request_alloc_fn_t)
size(IN) Number of bytes PML is requesting BTL to deliver
flags(IN) Flags that should be passed to the peer via the message header.
request(OUT) OMPI_SUCCESS if the BTL was able to queue one or more fragments