OpenMPI
0.1.1
|
#include "ompi_config.h"
#include <sys/types.h>
#include <string.h>
#include "ompi/class/ompi_free_list.h"
#include "opal/mca/event/event.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/mpool/mpool.h"
#include <myriexpress.h>
Go to the source code of this file.
Data Structures | |
struct | mca_btl_mx_component_t |
MX BTL component. More... | |
struct | mca_btl_mx_module_t |
BTL Module Interface. More... | |
Macros | |
#define | BTL_MX_RECV_MASK 0x00000000000000ffULL |
The mask used for receive and for the PUT protocol. | |
#define | BTL_MX_PUT_MASK 0xffffffffffffffffULL |
Typedefs | |
typedef struct mca_btl_mx_component_t | mca_btl_mx_component_t |
typedef struct mca_btl_mx_module_t | mca_btl_mx_module_t |
Functions | |
mca_btl_base_module_t ** | mca_btl_mx_component_init (int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads) |
MX component initialization. More... | |
int | mca_btl_mx_component_progress (void) |
MX component progress. More... | |
int | mca_btl_mx_finalize (struct mca_btl_base_module_t *btl) |
Cleanup any resources held by the BTL. More... | |
int | mca_btl_mx_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable) |
PML->BTL notification of change in the process list. More... | |
int | mca_btl_mx_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers) |
PML->BTL notification of change in the process list. More... | |
int | mca_btl_mx_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag) |
Initiate an asynchronous send. More... | |
int | mca_btl_mx_register (struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata) |
Register a callback function that is called on receipt of a fragment. More... | |
mca_btl_base_descriptor_t * | mca_btl_mx_alloc (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags) |
Allocate a descriptor with a segment of the requested size. More... | |
int | mca_btl_mx_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des) |
Return a segment allocated by this BTL. More... | |
mca_btl_base_descriptor_t * | mca_btl_mx_prepare_src (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags) |
Prepare a descriptor for send/rdma using the supplied convertor. More... | |
mca_btl_base_descriptor_t * | mca_btl_mx_prepare_dst (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags) |
Prepare a descriptor for send/rdma using the supplied convertor. More... | |
int | mca_btl_mx_ft_event (int state) |
Fault Tolerance Event Notification Function. More... | |
Variables | |
OMPI_MODULE_DECLSPEC mca_btl_mx_component_t | mca_btl_mx_component |
mca_btl_mx_module_t | mca_btl_mx_module |
int mca_btl_mx_add_procs | ( | struct mca_btl_base_module_t * | btl, |
size_t | nprocs, | ||
struct ompi_proc_t ** | procs, | ||
struct mca_btl_base_endpoint_t ** | peers, | ||
opal_bitmap_t * | reachable | ||
) |
PML->BTL notification of change in the process list.
btl | (IN) |
nprocs | (IN) Number of processes |
procs | (IN) Set of processes |
peers | (OUT) Set of (optional) peer addressing info. |
peers | (IN/OUT) Set of processes that are reachable via this BTL. |
By default don't allow communications with self nor with any other processes on the same node. The BTL self and sm are supposed to take care of such communications.
References mca_btl_base_endpoint_t::endpoint_btl, mca_btl_mx_component_t::mx_support_self, mca_btl_mx_component_t::mx_support_sharedmem, OBJ_RELEASE, ompi_proc_local_proc, opal_bitmap_set_bit(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, ompi_proc_t::proc_flags, and mca_btl_mx_proc_t::proc_lock.
mca_btl_base_descriptor_t* mca_btl_mx_alloc | ( | struct mca_btl_base_module_t * | btl, |
struct mca_btl_base_endpoint_t * | endpoint, | ||
uint8_t | order, | ||
size_t | size, | ||
uint32_t | flags | ||
) |
Allocate a descriptor with a segment of the requested size.
Note that the BTL layer may choose to return a smaller size if it cannot support the request.
btl | (IN) BTL module |
size | (IN) Request segment size. |
Allocate a descriptor with a segment of the requested size.
btl | (IN) BTL module |
size | (IN) Request segment size. |
References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_len, and mca_btl_mx_module_t::super.
mca_btl_base_module_t** mca_btl_mx_component_init | ( | int * | num_btl_modules, |
bool | allow_multi_user_threads, | ||
bool | have_hidden_threads | ||
) |
MX component initialization.
num_btl_modules | (OUT) Number of BTLs returned in BTL array. |
allow_multi_user_threads | (OUT) Flag indicating wether BTL supports user threads (TRUE) |
have_hidden_threads | (OUT) Flag indicating wether BTL uses threads (TRUE) |
As the MX MTL get initialized before the MX BTL it will call the mx_init and the environment variables set by the BTL will be useless. Closing the MX will force the next call to mx_init to take these environment variables into account.
References mca_btl_base_module_t::btl_eager_limit, mca_btl_mx_finalize(), mca_btl_mx_component_t::mx_bonding, mca_btl_mx_component_t::mx_btls, mca_btl_mx_module_t::mx_endpoint_addr, mca_btl_mx_component_t::mx_free_list_inc, mca_btl_mx_component_t::mx_free_list_max, mca_btl_mx_component_t::mx_free_list_num, mca_btl_mx_component_t::mx_lock, mca_btl_mx_component_t::mx_max_btls, mca_btl_mx_component_t::mx_procs, mca_btl_mx_component_t::mx_send_eager_frags, mca_btl_mx_component_t::mx_send_user_frags, mca_btl_mx_module_t::mx_unique_network_id, OBJ_CLASS, OBJ_CONSTRUCT, ompi_modex_send(), opal_output(), mca_btl_mx_component_t::super, and mca_btl_mx_module_t::super.
int mca_btl_mx_component_progress | ( | void | ) |
MX component progress.
The upper level extract the data from the fragment. Now we can register the fragment again with the MX BTL.
References mca_btl_base_module_t::btl_eager_limit, BTL_MX_RECV_MASK, mca_btl_base_descriptor_t::des_cbfunc, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_flags, mca_btl_mx_component_t::mx_btls, mca_btl_mx_module_t::mx_endpoint, opal_output(), mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_len, and mca_btl_mx_module_t::super.
int mca_btl_mx_del_procs | ( | struct mca_btl_base_module_t * | btl, |
size_t | nprocs, | ||
struct ompi_proc_t ** | procs, | ||
struct mca_btl_base_endpoint_t ** | peers | ||
) |
PML->BTL notification of change in the process list.
btl | (IN) BTL instance |
nproc | (IN) Number of processes. |
procs | (IN) Set of processes. |
peers | (IN) Set of peer data structures. |
References opal_output().
int mca_btl_mx_finalize | ( | struct mca_btl_base_module_t * | btl | ) |
Cleanup any resources held by the BTL.
btl | BTL instance. |
References mca_btl_mx_module_t::mx_endpoint, mca_btl_mx_module_t::mx_lock, mca_btl_mx_module_t::mx_peers, and OBJ_DESTRUCT.
Referenced by mca_btl_mx_component_init().
int mca_btl_mx_free | ( | struct mca_btl_base_module_t * | btl, |
mca_btl_base_descriptor_t * | des | ||
) |
Return a segment allocated by this BTL.
btl | (IN) BTL module |
descriptor | (IN) Allocated descriptor. |
Return a segment allocated by this BTL.
int mca_btl_mx_ft_event | ( | int | state | ) |
Fault Tolerance Event Notification Function.
state | Checkpoint Stae |
mca_btl_base_descriptor_t* mca_btl_mx_prepare_dst | ( | struct mca_btl_base_module_t * | btl, |
struct mca_btl_base_endpoint_t * | endpoint, | ||
struct mca_mpool_base_registration_t * | registration, | ||
struct opal_convertor_t * | convertor, | ||
uint8_t | order, | ||
size_t | reserve, | ||
size_t * | size, | ||
uint32_t | flags | ||
) |
Prepare a descriptor for send/rdma using the supplied convertor.
If the convertor references data that is contigous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.
btl | (IN) BTL module |
endpoint | (IN) BTL peer addressing |
convertor | (IN) Data type convertor |
reserve | (IN) Additional bytes requested by upper layer to precede user data |
size | (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) |
References mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_mx_module_t::mx_endpoint, opal_output(), mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_key, and mca_btl_base_segment_t::seg_len.
mca_btl_base_descriptor_t* mca_btl_mx_prepare_src | ( | struct mca_btl_base_module_t * | btl, |
struct mca_btl_base_endpoint_t * | endpoint, | ||
struct mca_mpool_base_registration_t * | registration, | ||
struct opal_convertor_t * | convertor, | ||
uint8_t | order, | ||
size_t | reserve, | ||
size_t * | size, | ||
uint32_t | flags | ||
) |
Prepare a descriptor for send/rdma using the supplied convertor.
If the convertor references data that is contigous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.
btl | (IN) BTL module |
endpoint | (IN) BTL peer addressing |
convertor | (IN) Data type convertor |
reserve | (IN) Additional bytes requested by upper layer to precede user data |
size | (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT) |
Prepare a descriptor for send/rdma using the supplied convertor.
btl | (IN) BTL module |
peer | (IN) BTL peer addressing |
let the convertor figure out the correct pointer depending on the data layout
References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.
int mca_btl_mx_register | ( | struct mca_btl_base_module_t * | btl, |
mca_btl_base_tag_t | tag, | ||
mca_btl_base_module_recv_cb_fn_t | cbfunc, | ||
void * | cbdata | ||
) |
Register a callback function that is called on receipt of a fragment.
btl | (IN) BTL module |
Register a callback function that is called on receipt of a fragment.
References mca_btl_base_module_t::btl_eager_limit, BTL_MX_RECV_MASK, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_mx_module_t::mx_endpoint, mca_btl_mx_component_t::mx_max_posted_recv, mca_btl_mx_component_t::mx_use_unexpected, opal_output(), and mca_btl_mx_module_t::super.
int mca_btl_mx_send | ( | struct mca_btl_base_module_t * | btl, |
struct mca_btl_base_endpoint_t * | endpoint, | ||
struct mca_btl_base_descriptor_t * | descriptor, | ||
mca_btl_base_tag_t | tag | ||
) |
Initiate an asynchronous send.
btl | (IN) BTL module |
endpoint | (IN) BTL addressing information |
descriptor | (IN) Description of the data to be transfered |
tag | (IN) The tag value used to notify the peer. |
References mca_btl_base_descriptor_t::des_cbfunc, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_mx_module_t::mx_endpoint, mca_btl_base_endpoint_t::mx_peer_addr, opal_output(), mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_len, and mca_btl_mx_module_t::super.