OpenMPI  0.1.1
btl.h File Reference

Byte Transfer Layer (BTL) More...

#include "ompi_config.h"
#include "opal/mca/mca.h"
#include "opal/class/opal_bitmap.h"
#include "opal/datatype/opal_convertor.h"
#include "opal/prefetch.h"
#include "ompi/mca/mpool/mpool.h"
#include "ompi/types.h"
#include "opal/types.h"
#include "opal/mca/crs/crs.h"
#include "opal/mca/crs/base/base.h"

Go to the source code of this file.

Data Structures

struct  mca_btl_base_segment_t
 Describes a region/segment of memory that is addressable by an BTL. More...
 
struct  mca_btl_base_descriptor_t
 A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that is called on completion of the request. More...
 
struct  mca_btl_base_header_t
 
struct  mca_btl_active_message_callback_t
 
struct  mca_btl_base_component_2_0_0_t
 BTL component descriptor. More...
 
struct  mca_btl_base_module_t
 BTL module interface functions and attributes. More...
 

Macros

#define MCA_BTL_NO_ORDER   255
 
#define MCA_BTL_AM_FRAMEWORK_MASK   0xD0
 
#define MCA_BTL_TAG_BTL   0x20
 
#define MCA_BTL_TAG_FT   0x30
 
#define MCA_BTL_TAG_PML   0x40
 
#define MCA_BTL_TAG_OSC_RDMA   0x60
 
#define MCA_BTL_TAG_USR   0x80
 
#define MCA_BTL_TAG_MAX   255 /* 1 + highest allowed tag num */
 
#define MCA_BTL_TAG_IB   (MCA_BTL_TAG_BTL + 0)
 
#define MCA_BTL_TAG_UDAPL   (MCA_BTL_TAG_BTL + 1)
 
#define MCA_BTL_FLAGS_SEND   0x0001
 
#define MCA_BTL_FLAGS_PUT   0x0002
 
#define MCA_BTL_FLAGS_GET   0x0004
 
#define MCA_BTL_FLAGS_RDMA   (MCA_BTL_FLAGS_GET|MCA_BTL_FLAGS_PUT)
 
#define MCA_BTL_FLAGS_SEND_INPLACE   0x0008
 
#define MCA_BTL_FLAGS_NEED_ACK   0x0010
 
#define MCA_BTL_FLAGS_NEED_CSUM   0x0020
 
#define MCA_BTL_FLAGS_RDMA_MATCHED   0x0040
 RDMA put/get calls must have a matching prepare_{src,dst} call on the target with the same base (and possibly bound). More...
 
#define MCA_BTL_FLAGS_RDMA_COMPLETION   0x0080
 
#define MCA_BTL_FLAGS_HETEROGENEOUS_RDMA   0x0100
 
#define MCA_BTL_FLAGS_FAILOVER_SUPPORT   0x0200
 
#define MCA_BTL_FLAGS_CUDA_PUT   0x0400
 
#define MCA_BTL_FLAGS_CUDA_GET   0x0800
 
#define MCA_BTL_FLAGS_CUDA_RDMA   (MCA_BTL_FLAGS_CUDA_GET|MCA_BTL_FLAGS_CUDA_PUT)
 
#define MCA_BTL_EXCLUSIVITY_HIGH   (64*1024) /* internal loopback */
 
#define MCA_BTL_EXCLUSIVITY_DEFAULT   1024 /* GM/IB/etc. */
 
#define MCA_BTL_EXCLUSIVITY_LOW   0 /* TCP used as a last resort */
 
#define MCA_BTL_ERROR_FLAGS_FATAL   0x1
 
#define MCA_BTL_ERROR_FLAGS_NONFATAL   0x2
 
#define MCA_BTL_DES_FLAGS_PRIORITY   0x0001
 
#define MCA_BTL_DES_FLAGS_BTL_OWNERSHIP   0x0002
 
#define MCA_BTL_DES_SEND_ALWAYS_CALLBACK   0x0004
 
#define MCA_BTL_DES_FLAGS_PUT   0x0010
 
#define MCA_BTL_DES_FLAGS_GET   0x0020
 
#define MCA_BTL_DES_MAX_SEGMENTS   16
 Maximum number of allowed segments in src/dst fields of a descriptor.
 
#define MCA_BTL_BASE_HEADER_HTON(hdr)
 
#define MCA_BTL_BASE_HEADER_NTOH(hdr)
 
#define MCA_BTL_BASE_VERSION_2_0_0
 

Typedefs

typedef uint8_t mca_btl_base_tag_t
 
typedef void(* mca_btl_base_completion_fn_t )(struct mca_btl_base_module_t *module, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, int status)
 Asynchronous callback function on completion of an operation. More...
 
typedef struct
mca_btl_base_segment_t 
mca_btl_base_segment_t
 
typedef struct
mca_btl_base_descriptor_t 
mca_btl_base_descriptor_t
 
typedef struct
mca_btl_base_header_t 
mca_btl_base_header_t
 
typedef struct
mca_btl_base_module_t **(* 
mca_btl_base_component_init_fn_t )(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads)
 MCA->BTL Initializes the BTL component and creates specific BTL module(s). More...
 
typedef int(* mca_btl_base_component_progress_fn_t )(void)
 MCA->BTL Called to progress outstanding requests for non-threaded polling environments. More...
 
typedef void(* mca_btl_base_module_recv_cb_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t *descriptor, void *cbdata)
 Callback function that is called asynchronously on receipt of data by the transport layer. More...
 
typedef struct
mca_btl_active_message_callback_t 
mca_btl_active_message_callback_t
 
typedef struct
mca_btl_base_component_2_0_0_t 
mca_btl_base_component_2_0_0_t
 
typedef struct
mca_btl_base_component_2_0_0_t 
mca_btl_base_component_t
 
typedef struct
mca_btl_base_component_2_0_0_t 
mca_btl_base_component_1_0_1_t
 
typedef struct
mca_btl_base_component_2_0_0_t 
mca_btl_base_component_1_0_0_t
 
typedef int(* mca_btl_base_module_finalize_fn_t )(struct mca_btl_base_module_t *btl)
 MCA->BTL Clean up any resources held by BTL module before the module is unloaded. More...
 
typedef int(* mca_btl_base_module_add_procs_fn_t )(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **endpoints, struct opal_bitmap_t *reachable)
 BML->BTL notification of change in the process list. More...
 
typedef int(* mca_btl_base_module_del_procs_fn_t )(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peer)
 Notification of change to the process list. More...
 
typedef int(* mca_btl_base_module_register_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata)
 Register a callback function that is called on receipt of a fragment. More...
 
typedef void(* mca_btl_base_module_error_cb_fn_t )(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)
 Callback function that is called asynchronously on receipt of an error from the transport layer. More...
 
typedef int(* mca_btl_base_module_register_error_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_module_error_cb_fn_t cbfunc)
 Register a callback function that is called on receipt of an error. More...
 
typedef
mca_btl_base_descriptor_t *(* 
mca_btl_base_module_alloc_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
 Allocate a descriptor with a segment of the requested size. More...
 
typedef int(* mca_btl_base_module_free_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *descriptor)
 Return a descriptor allocated from this BTL via alloc/prepare. More...
 
typedef struct
mca_btl_base_descriptor_t *(* 
mca_btl_base_module_prepare_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
 Prepare a descriptor for send/put/get using the supplied convertor. More...
 
typedef int(* mca_btl_base_module_send_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
 Initiate an asynchronous send. More...
 
typedef int(* mca_btl_base_module_sendi_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct opal_convertor_t *convertor, void *header, size_t header_size, size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor)
 Initiate an immediate blocking send. More...
 
typedef int(* mca_btl_base_module_put_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)
 Initiate an asynchronous put. More...
 
typedef int(* mca_btl_base_module_get_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)
 Initiate an asynchronous get. More...
 
typedef void(* mca_btl_base_module_dump_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, int verbose)
 Diagnostic dump of btl state. More...
 
typedef int(* mca_btl_base_module_ft_event_fn_t )(int state)
 Fault Tolerance Event Notification Function. More...
 
typedef struct
mca_btl_base_module_t 
mca_btl_base_module_t
 

Functions

OMPI_DECLSPEC OBJ_CLASS_DECLARATION (mca_btl_base_descriptor_t)
 

Variables

OMPI_DECLSPEC
mca_btl_active_message_callback_t 
mca_btl_base_active_message_trigger [MCA_BTL_TAG_MAX]
 

Detailed Description

Byte Transfer Layer (BTL)

BTL Initialization:

During library initialization, all available BTL components are loaded and opened via their mca_base_open_component_fn_t function. The BTL open function should register any mca parameters used to tune/adjust the behaviour of the BTL (mca_base_param_register_int(), mca_base_param_register_string()). Note that the open function may fail if the resources (e.g. shared libraries, etc) required by the network transport are not available.

The mca_btl_base_component_init_fn_t() is then called for each of the components that are succesfully opened. The component init function may return either:

(1) a NULL list of BTL modules if the transport is not available, (2) a list containing a one or more single BTL modules, where the BTL provides a layer of abstraction over one or more physical devices (e.g. NICs),

During module initialization, the module should post any addressing information required by its peers. An example would be the TCP listen port opened by the TCP module for incoming connection requests. This information is published to peers via the ompi_modex_send() interface. Note that peer information is not guaranteed to be available via ompi_modex_recv() during the module's init function. However, it will be available during BTL selection (mca_btl_base_add_proc_fn_t()).

BTL Selection:

The upper layer builds an ordered list of the available BTL modules sorted by their exclusivity ranking. This is a relative ranking that is used to determine the set of BTLs that may be used to reach a given destination. During startup the BTL modules are queried via their mca_btl_base_add_proc_fn_t() to determine if they are able to reach a given destination. The BTL module with the highest ranking that returns success is selected. Subsequent BTL modules are selected only if they have the same exclusivity ranking.

An example of how this might be used:

BTL Exclusivity Comments


LO 100 Selected exclusively for local process SM 50 Selected exclusively for other processes on host IB 0 Selected based on network reachability IB 0 Selected based on network reachability TCP 0 Selected based on network reachability TCP 0 Selected based on network reachability

When mca_btl_base_add_proc_fn_t() is called on a BTL module, the BTL will populate an OUT variable with mca_btl_base_endpoint_t pointers. Each pointer is treated as an opaque handle by the upper layer and is returned to the BTL on subsequent data transfer calls to the corresponding destination process. The actual contents of the data structure are defined on a per BTL basis, and may be used to cache addressing or connection information, such as a TCP socket or IB queue pair.

Progress:

By default, the library provides for polling based progress of outstanding requests. The BTL component exports an interface function (btl_progress) that is called in a polling mode by the PML during calls into the MPI library. Note that the btl_progress() function is called on the BTL component rather than each BTL module. This implies that the BTL author is responsible for iterating over the pending operations in each of the BTL modules associated with the component.

On platforms where threading support is provided, the library provides the option of building with asynchronous threaded progress. In this case, the BTL author is responsible for providing a thread to progress pending operations. A thread is associated with the BTL component/module such that transport specific functionality/APIs may be used to block the thread until a pending operation completes. This thread MUST NOT poll for completion as this would oversubscribe the CPU.

Note that in the threaded case the PML may choose to use a hybrid approach, such that polling is implemented from the user thread for a fixed number of cycles before relying on the background thread(s) to complete requests. If possible the BTL should support the use of both modes concurrently.

Macro Definition Documentation

#define MCA_BTL_BASE_VERSION_2_0_0
Value:
MCA_BASE_VERSION_2_0_0, \
"btl", 2, 0, 0
#define MCA_BTL_FLAGS_RDMA_MATCHED   0x0040

RDMA put/get calls must have a matching prepare_{src,dst} call on the target with the same base (and possibly bound).

Typedef Documentation

typedef void(* mca_btl_base_completion_fn_t)(struct mca_btl_base_module_t *module, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, int status)

Asynchronous callback function on completion of an operation.

Completion Semantics: The descriptor can be reused or returned to the BTL via mca_btl_base_module_free_fn_t. The operation has been queued to the network device or will otherwise make asynchronous progress without subsequent calls to btl_progress.

Parameters
IN]module the BTL module
IN]endpoint the BTL endpoint
IN]descriptor the BTL descriptor
typedef struct mca_btl_base_module_t**(* mca_btl_base_component_init_fn_t)(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads)

MCA->BTL Initializes the BTL component and creates specific BTL module(s).

Parameters
num_btls(OUT) Returns the number of btl modules created, or 0 if the transport is not available.
enable_progress_threads(IN) Whether this component is allowed to run a hidden/progress thread or not.
enable_mpi_threads(IN) Whether support for multiple MPI threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which indicates whether multiple threads may invoke this component simultaneously or not.
Returns
Array of pointers to BTL modules, or NULL if the transport is not available.

During component initialization, the BTL component should discover the physical devices that are available for the given transport, and create a BTL module to represent each device. Any addressing information required by peers to reach the device should be published during this function via the ompi_modex_send() interface.

typedef int(* mca_btl_base_component_progress_fn_t)(void)

MCA->BTL Called to progress outstanding requests for non-threaded polling environments.

Returns
Count of "completions", a metric of how many items where completed in the call to progress.
typedef int(* mca_btl_base_module_add_procs_fn_t)(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **endpoints, struct opal_bitmap_t *reachable)

BML->BTL notification of change in the process list.

Parameters
btl(IN) BTL module
nprocs(IN) Number of processes
procs(IN) Array of processes
endpoint(OUT) Array of mca_btl_base_endpoint_t structures by BTL.
reachable(OUT) Bitmask indicating set of peer processes that are reachable by this BTL.
Returns
OMPI_SUCCESS or error status on failure.

The mca_btl_base_module_add_procs_fn_t() is called by the BML to determine the set of BTLs that should be used to reach each process. Any addressing information exported by the peer via the ompi_modex_send() function should be available during this call via the corresponding ompi_modex_recv() function. The BTL may utilize this information to determine reachability of each peer process.

For each process that is reachable by the BTL, the bit corresponding to the index into the proc array (nprocs) should be set in the reachable bitmask. The BTL will return an array of pointers to a data structure defined by the BTL that is then returned to the BTL on subsequent calls to the BTL data transfer functions (e.g btl_send). This may be used by the BTL to cache any addressing or connection information (e.g. TCP socket, IB queue pair).

typedef mca_btl_base_descriptor_t*(* mca_btl_base_module_alloc_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)

Allocate a descriptor with a segment of the requested size.

Note that the BTL layer may choose to return a smaller size if it cannot support the request. The order tag value ensures that operations on the descriptor that is allocated will be ordered w.r.t. a previous operation on a particular descriptor. Ordering is only guaranteed if the previous descriptor had its local completion callback function called and the order tag of that descriptor is only valid upon the local completion callback function.

Parameters
btl(IN) BTL module
size(IN) Request segment size.
order(IN) The ordering tag (may be MCA_BTL_NO_ORDER)
typedef int(* mca_btl_base_module_del_procs_fn_t)(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peer)

Notification of change to the process list.

Parameters
btl(IN) BTL module
nprocs(IN) Number of processes
proc(IN) Set of processes
peer(IN) Set of peer addressing information.
Returns
Status indicating if cleanup was successful

When the process list changes, the BML notifies the BTL of the change, to provide the opportunity to cleanup or release any resources associated with the peer.

typedef void(* mca_btl_base_module_dump_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, int verbose)

Diagnostic dump of btl state.

Parameters
btl(IN) BTL module
endpoint(IN) BTL endpoint
verbose(IN) Verbosity level
typedef void(* mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)

Callback function that is called asynchronously on receipt of an error from the transport layer.

Parameters
IN]btl BTL module
IN]flags type of error
IN]errproc process that had an error
IN]btlinfo descriptive string from the BTL
typedef int(* mca_btl_base_module_finalize_fn_t)(struct mca_btl_base_module_t *btl)

MCA->BTL Clean up any resources held by BTL module before the module is unloaded.

Parameters
btl(IN) BTL module.
Returns
OMPI_SUCCESS or error status on failure.

Prior to unloading a BTL module, the MCA framework will call the BTL finalize method of the module. Any resources held by the BTL should be released and if required the memory corresponding to the BTL module freed.

typedef int(* mca_btl_base_module_free_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *descriptor)

Return a descriptor allocated from this BTL via alloc/prepare.

A descriptor can only be deallocated after its local completion callback function has called for all send/put/get operations.

Parameters
btl(IN) BTL module
segment(IN) Descriptor allocated from the BTL
typedef int(* mca_btl_base_module_ft_event_fn_t)(int state)

Fault Tolerance Event Notification Function.

Parameters
stateCheckpoint Status
Returns
OMPI_SUCCESS or failure status
typedef int(* mca_btl_base_module_get_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)

Initiate an asynchronous get.

Completion Semantics: the descriptor has been queued for a get operation the BTL now controls the descriptor until local completion callback is made on the descriptor

BTLs that do not have the MCA_BTL_FLAGS_RDMA_MATCHED flag set allow multiple concurrent get operations on the same descriptor. BTLs that do have the MCA_BTL_FLAGS_RDMA_MATCHED flag set require a corresponding prepare_src/dst call for each get operation and therefore prohibit multiple concurrent get operations.

Parameters
btl(IN) BTL module
endpoint(IN) BTL addressing information
descriptor(IN) Description of the data to be transferred
Return values
OMPI_SUCCESSThe descriptor was successfully queued for a get
OMPI_ERRORThe descriptor was NOT successfully queued for a get
typedef struct mca_btl_base_descriptor_t*(* mca_btl_base_module_prepare_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)

Prepare a descriptor for send/put/get using the supplied convertor.

If the convertor references data that is contiguous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

The descriptor returned can be used in multiple concurrent operations (send/put/get) unless the BTL has the MCA_BTL_FLAGS_RDMA_MATCHED flag set in which case a corresponding prepare call must accompany the put/get call in addition, the address and length that is put/get must match the address and length which is prepared.

The order tag value ensures that operations on the descriptor that is prepared will be ordered w.r.t. a previous operation on a particular descriptor. Ordering is only guaranteed if the previous descriptor had its local completion callback function called and the order tag of that descriptor is only valid upon the local completion callback function.

Parameters
btl(IN) BTL module
endpoint(IN) BTL peer addressing
registration(IN) Memory registration
convertor(IN) Data type convertor
order(IN) The ordering tag (may be MCA_BTL_NO_ORDER)
reserve(IN) Additional bytes requested by upper layer to precede user data
size(IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
typedef int(* mca_btl_base_module_put_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)

Initiate an asynchronous put.

Completion Semantics: the descriptor has been queued for a put operation the BTL now controls the descriptor until local completion callback is made on the descriptor

BTLs that do not have the MCA_BTL_FLAGS_RDMA_MATCHED flag set allow multiple concurrent put operations on the same descriptor. BTLs that do have the MCA_BTL_FLAGS_RDMA_MATCHED flag set require a corresponding prepare_src/dst call for each put operation and therefore prohibit multiple concurrent put operations.

Parameters
btl(IN) BTL module
endpoint(IN) BTL addressing information
descriptor(IN) Description of the data to be transferred
Return values
OMPI_SUCCESSThe descriptor was successfully queued for a put
OMPI_ERRORThe descriptor was NOT successfully queued for a put
typedef void(* mca_btl_base_module_recv_cb_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t *descriptor, void *cbdata)

Callback function that is called asynchronously on receipt of data by the transport layer.

Note that the the mca_btl_base_descriptor_t is only valid within the completion function, this implies that all data payload in the mca_btl_base_descriptor_t must be copied out within this callback or forfeited back to the BTL.

Parameters
IN]btl BTL module
IN]tag The active message receive callback tag value
IN]descriptor The BTL descriptor (contains the receive payload)
IN]cbdata Opaque callback data
typedef int(* mca_btl_base_module_register_error_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_module_error_cb_fn_t cbfunc)

Register a callback function that is called on receipt of an error.

Parameters
IN]btl BTL module
IN]cbfunc The callback function
Returns
OMPI_SUCCESS The callback was registered successfully
OMPI_ERROR The callback was NOT registered successfully
typedef int(* mca_btl_base_module_register_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata)

Register a callback function that is called on receipt of a fragment.

Parameters
IN]btl BTL module
IN]tag tag value of this callback (specified on subsequent send operations)
IN]cbfunc The callback function
IN]cbdata Opaque callback data
Returns
OMPI_SUCCESS The callback was registered successfully
OMPI_ERROR The callback was NOT registered successfully
typedef int(* mca_btl_base_module_send_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)

Initiate an asynchronous send.

Completion Semantics: the descriptor has been queued for a send operation the BTL now controls the descriptor until local completion callback is made on the descriptor

All BTLs allow multiple concurrent asynchronous send operations on a descriptor

Parameters
btl(IN) BTL module
endpoint(IN) BTL addressing information
descriptor(IN) Description of the data to be transfered
tag(IN) The tag value used to notify the peer.
Return values
OMPI_SUCCESSThe descriptor was successfully queued for a send
OMPI_ERRORThe descriptor was NOT successfully queued for a send
OMPI_ERR_UNREACHThe endpoint is not reachable
typedef int(* mca_btl_base_module_sendi_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct opal_convertor_t *convertor, void *header, size_t header_size, size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor)

Initiate an immediate blocking send.

Completion Semantics: the BTL will make a best effort to send the header and "size" bytes from the datatype using the convertor. The header is guaranteed to be delivered entirely in the first segment. Should the BTL be unable to deliver the data due to resource constraints the BTL will return a descriptor (via the OUT param) of size "payload_size + header_size".

Parameters
btl(IN) BTL module
endpoint(IN) BTL addressing information
convertor(IN) Data type convertor
header(IN) Pointer to header.
header_size(IN) Size of header.
payload_size(IN) Size of payload (from convertor).
order(IN) The ordering tag (may be MCA_BTL_NO_ORDER)
flags(IN) Flags.
tag(IN) The tag value used to notify the peer.
descriptor(OUT) The descriptor to be returned unable to be sent immediately
Return values
OMPI_SUCCESSThe send was successfully queued
OMPI_ERRORThe send failed
OMPI_ERR_UNREACHThe endpoint is not reachable
OMPI_ERR_RESOURCE_BUSYThe BTL is busy a descriptor will be returned (via the OUT param) if descriptors are available