OpenMPI  0.1.1
btl_mx.h File Reference
#include "ompi_config.h"
#include <sys/types.h>
#include <string.h>
#include "ompi/class/ompi_free_list.h"
#include "opal/mca/event/event.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include <myriexpress.h>

Go to the source code of this file.

Data Structures

struct  mca_btl_mx_component_t
 MX BTL component. More...
 
struct  mca_btl_mx_module_t
 BTL Module Interface. More...
 

Macros

#define BTL_MX_RECV_MASK   0x00000000000000ffULL
 The mask used for receive and for the PUT protocol.
 
#define BTL_MX_PUT_MASK   0xffffffffffffffffULL
 

Typedefs

typedef struct
mca_btl_mx_component_t 
mca_btl_mx_component_t
 
typedef struct mca_btl_mx_module_t mca_btl_mx_module_t
 

Functions

mca_btl_base_module_t ** mca_btl_mx_component_init (int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
 MX component initialization. More...
 
int mca_btl_mx_component_progress (void)
 MX component progress. More...
 
int mca_btl_mx_finalize (struct mca_btl_base_module_t *btl)
 Cleanup any resources held by the BTL. More...
 
int mca_btl_mx_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
 PML->BTL notification of change in the process list. More...
 
int mca_btl_mx_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
 PML->BTL notification of change in the process list. More...
 
int mca_btl_mx_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 Initiate an asynchronous send. More...
 
int mca_btl_mx_register (struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata)
 Register a callback function that is called on receipt of a fragment. More...
 
mca_btl_base_descriptor_tmca_btl_mx_alloc (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
 Allocate a descriptor with a segment of the requested size. More...
 
int mca_btl_mx_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
 Return a segment allocated by this BTL. More...
 
mca_btl_base_descriptor_tmca_btl_mx_prepare_src (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
 Prepare a descriptor for send/rdma using the supplied convertor. More...
 
mca_btl_base_descriptor_tmca_btl_mx_prepare_dst (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
 Prepare a descriptor for send/rdma using the supplied convertor. More...
 
int mca_btl_mx_ft_event (int state)
 Fault Tolerance Event Notification Function. More...
 

Variables

OMPI_MODULE_DECLSPEC
mca_btl_mx_component_t 
mca_btl_mx_component
 
mca_btl_mx_module_t mca_btl_mx_module
 

Function Documentation

int mca_btl_mx_add_procs ( struct mca_btl_base_module_t btl,
size_t  nprocs,
struct ompi_proc_t **  procs,
struct mca_btl_base_endpoint_t **  peers,
opal_bitmap_t reachable 
)

PML->BTL notification of change in the process list.

Parameters
btl(IN)
nprocs(IN) Number of processes
procs(IN) Set of processes
peers(OUT) Set of (optional) peer addressing info.
peers(IN/OUT) Set of processes that are reachable via this BTL.
Returns
OMPI_SUCCESS or error status on failure.

By default don't allow communications with self nor with any other processes on the same node. The BTL self and sm are supposed to take care of such communications.

References mca_btl_base_endpoint_t::endpoint_btl, mca_btl_mx_component_t::mx_support_self, mca_btl_mx_component_t::mx_support_sharedmem, OBJ_RELEASE, ompi_proc_local_proc, opal_bitmap_set_bit(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, ompi_proc_t::proc_flags, and mca_btl_mx_proc_t::proc_lock.

mca_btl_base_descriptor_t* mca_btl_mx_alloc ( struct mca_btl_base_module_t btl,
struct mca_btl_base_endpoint_t endpoint,
uint8_t  order,
size_t  size,
uint32_t  flags 
)

Allocate a descriptor with a segment of the requested size.

Note that the BTL layer may choose to return a smaller size if it cannot support the request.

Parameters
btl(IN) BTL module
size(IN) Request segment size.

Allocate a descriptor with a segment of the requested size.

Parameters
btl(IN) BTL module
size(IN) Request segment size.

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_len, and mca_btl_mx_module_t::super.

mca_btl_base_module_t** mca_btl_mx_component_init ( int *  num_btl_modules,
bool  allow_multi_user_threads,
bool  have_hidden_threads 
)

MX component initialization.

Parameters
num_btl_modules(OUT) Number of BTLs returned in BTL array.
allow_multi_user_threads(OUT) Flag indicating wether BTL supports user threads (TRUE)
have_hidden_threads(OUT) Flag indicating wether BTL uses threads (TRUE)

As the MX MTL get initialized before the MX BTL it will call the mx_init and the environment variables set by the BTL will be useless. Closing the MX will force the next call to mx_init to take these environment variables into account.

References mca_btl_base_module_t::btl_eager_limit, mca_btl_mx_finalize(), mca_btl_mx_component_t::mx_bonding, mca_btl_mx_component_t::mx_btls, mca_btl_mx_module_t::mx_endpoint_addr, mca_btl_mx_component_t::mx_free_list_inc, mca_btl_mx_component_t::mx_free_list_max, mca_btl_mx_component_t::mx_free_list_num, mca_btl_mx_component_t::mx_lock, mca_btl_mx_component_t::mx_max_btls, mca_btl_mx_component_t::mx_procs, mca_btl_mx_component_t::mx_send_eager_frags, mca_btl_mx_component_t::mx_send_user_frags, mca_btl_mx_module_t::mx_unique_network_id, OBJ_CLASS, OBJ_CONSTRUCT, ompi_modex_send(), opal_output(), mca_btl_mx_component_t::super, and mca_btl_mx_module_t::super.

int mca_btl_mx_del_procs ( struct mca_btl_base_module_t btl,
size_t  nprocs,
struct ompi_proc_t **  procs,
struct mca_btl_base_endpoint_t **  peers 
)

PML->BTL notification of change in the process list.

Parameters
btl(IN) BTL instance
nproc(IN) Number of processes.
procs(IN) Set of processes.
peers(IN) Set of peer data structures.
Returns
Status indicating if cleanup was successful

References opal_output().

int mca_btl_mx_finalize ( struct mca_btl_base_module_t btl)

Cleanup any resources held by the BTL.

Parameters
btlBTL instance.
Returns
OMPI_SUCCESS or error status on failure.

References mca_btl_mx_module_t::mx_endpoint, mca_btl_mx_module_t::mx_lock, mca_btl_mx_module_t::mx_peers, and OBJ_DESTRUCT.

Referenced by mca_btl_mx_component_init().

int mca_btl_mx_free ( struct mca_btl_base_module_t btl,
mca_btl_base_descriptor_t des 
)

Return a segment allocated by this BTL.

Parameters
btl(IN) BTL module
descriptor(IN) Allocated descriptor.

Return a segment allocated by this BTL.

int mca_btl_mx_ft_event ( int  state)

Fault Tolerance Event Notification Function.

Parameters
stateCheckpoint Stae
Returns
OMPI_SUCCESS or failure status
mca_btl_base_descriptor_t* mca_btl_mx_prepare_dst ( struct mca_btl_base_module_t btl,
struct mca_btl_base_endpoint_t endpoint,
struct mca_mpool_base_registration_t registration,
struct opal_convertor_t convertor,
uint8_t  order,
size_t  reserve,
size_t *  size,
uint32_t  flags 
)

Prepare a descriptor for send/rdma using the supplied convertor.

If the convertor references data that is contigous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

Parameters
btl(IN) BTL module
endpoint(IN) BTL peer addressing
convertor(IN) Data type convertor
reserve(IN) Additional bytes requested by upper layer to precede user data
size(IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)

References mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_mx_module_t::mx_endpoint, opal_output(), mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_key, and mca_btl_base_segment_t::seg_len.

mca_btl_base_descriptor_t* mca_btl_mx_prepare_src ( struct mca_btl_base_module_t btl,
struct mca_btl_base_endpoint_t endpoint,
struct mca_mpool_base_registration_t registration,
struct opal_convertor_t convertor,
uint8_t  order,
size_t  reserve,
size_t *  size,
uint32_t  flags 
)

Prepare a descriptor for send/rdma using the supplied convertor.

If the convertor references data that is contigous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

Parameters
btl(IN) BTL module
endpoint(IN) BTL peer addressing
convertor(IN) Data type convertor
reserve(IN) Additional bytes requested by upper layer to precede user data
size(IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)

Prepare a descriptor for send/rdma using the supplied convertor.

Parameters
btl(IN) BTL module
peer(IN) BTL peer addressing

let the convertor figure out the correct pointer depending on the data layout

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

int mca_btl_mx_register ( struct mca_btl_base_module_t btl,
mca_btl_base_tag_t  tag,
mca_btl_base_module_recv_cb_fn_t  cbfunc,
void *  cbdata 
)

Register a callback function that is called on receipt of a fragment.

Parameters
btl(IN) BTL module
Returns
Status indicating if registration was successful

Register a callback function that is called on receipt of a fragment.

References mca_btl_base_module_t::btl_eager_limit, BTL_MX_RECV_MASK, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_mx_module_t::mx_endpoint, mca_btl_mx_component_t::mx_max_posted_recv, mca_btl_mx_component_t::mx_use_unexpected, opal_output(), and mca_btl_mx_module_t::super.

int mca_btl_mx_send ( struct mca_btl_base_module_t btl,
struct mca_btl_base_endpoint_t endpoint,
struct mca_btl_base_descriptor_t descriptor,
mca_btl_base_tag_t  tag 
)

Initiate an asynchronous send.

Parameters
btl(IN) BTL module
endpoint(IN) BTL addressing information
descriptor(IN) Description of the data to be transfered
tag(IN) The tag value used to notify the peer.

References mca_btl_base_descriptor_t::des_cbfunc, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_mx_module_t::mx_endpoint, mca_btl_base_endpoint_t::mx_peer_addr, opal_output(), mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_len, and mca_btl_mx_module_t::super.