OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

Data Structures | Macros | Typedefs | Functions | Variables
btl.h File Reference

Byte Transfer Layer (BTL) More...

`#include "ompi_config.h"`
`#include "opal/mca/mca.h"`
`#include "opal/class/opal_bitmap.h"`
`#include "opal/datatype/opal_convertor.h"`
`#include "opal/prefetch.h"`
`#include "ompi/mca/mpool/mpool.h"`
`#include "ompi/types.h"`
`#include "opal/types.h"`
`#include "opal/mca/crs/crs.h"`
`#include "opal/mca/crs/base/base.h"`

Go to the source code of this file.

## Data Structures
struct | mca_btl_base_segment_t
| Describes a region/segment of memory that is addressable by an BTL. More...
struct | mca_btl_base_descriptor_t
| A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that is called on completion of the request. More...
struct | mca_btl_base_header_t
struct | mca_btl_active_message_callback_t
struct | mca_btl_base_component_2_0_0_t
| BTL component descriptor. More...
struct | mca_btl_base_module_t
| BTL module interface functions and attributes. More...

## Macros
#define | MCA_BTL_NO_ORDER 255
#define | MCA_BTL_AM_FRAMEWORK_MASK 0xD0
#define | MCA_BTL_TAG_BTL 0x20
#define | MCA_BTL_TAG_FT 0x30
#define | MCA_BTL_TAG_PML 0x40
#define | MCA_BTL_TAG_OSC_RDMA 0x60
#define | MCA_BTL_TAG_USR 0x80
#define | MCA_BTL_TAG_MAX 255 /* 1 + highest allowed tag num */
#define | MCA_BTL_TAG_IB (MCA_BTL_TAG_BTL + 0)
#define | MCA_BTL_TAG_UDAPL (MCA_BTL_TAG_BTL + 1)
#define | MCA_BTL_FLAGS_SEND 0x0001
#define | MCA_BTL_FLAGS_PUT 0x0002
#define | MCA_BTL_FLAGS_GET 0x0004
#define | MCA_BTL_FLAGS_RDMA (MCA_BTL_FLAGS_GET|MCA_BTL_FLAGS_PUT)
#define | MCA_BTL_FLAGS_SEND_INPLACE 0x0008
#define | MCA_BTL_FLAGS_NEED_ACK 0x0010
#define | MCA_BTL_FLAGS_NEED_CSUM 0x0020
#define | MCA_BTL_FLAGS_RDMA_MATCHED 0x0040
| RDMA put/get calls must have a matching prepare_{src,dst} call on the target with the same base (and possibly bound). More...
#define | MCA_BTL_FLAGS_RDMA_COMPLETION 0x0080
#define | MCA_BTL_FLAGS_HETEROGENEOUS_RDMA 0x0100
#define | MCA_BTL_FLAGS_FAILOVER_SUPPORT 0x0200
#define | MCA_BTL_FLAGS_CUDA_PUT 0x0400
#define | MCA_BTL_FLAGS_CUDA_GET 0x0800
#define | MCA_BTL_FLAGS_CUDA_RDMA (MCA_BTL_FLAGS_CUDA_GET|MCA_BTL_FLAGS_CUDA_PUT)
#define | MCA_BTL_EXCLUSIVITY_HIGH (64*1024) /* internal loopback */
#define | MCA_BTL_EXCLUSIVITY_DEFAULT 1024 /* GM/IB/etc. */
#define | MCA_BTL_EXCLUSIVITY_LOW 0 /* TCP used as a last resort */
#define | MCA_BTL_ERROR_FLAGS_FATAL 0x1
#define | MCA_BTL_ERROR_FLAGS_NONFATAL 0x2
#define | MCA_BTL_DES_FLAGS_PRIORITY 0x0001
#define | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP 0x0002
#define | MCA_BTL_DES_SEND_ALWAYS_CALLBACK 0x0004
#define | MCA_BTL_DES_FLAGS_PUT 0x0010
#define | MCA_BTL_DES_FLAGS_GET 0x0020
#define | MCA_BTL_DES_MAX_SEGMENTS 16
| Maximum number of allowed segments in src/dst fields of a descriptor.
#define | MCA_BTL_BASE_HEADER_HTON(hdr)
#define | MCA_BTL_BASE_HEADER_NTOH(hdr)
#define | MCA_BTL_BASE_VERSION_2_0_0

## Typedefs
typedef uint8_t | mca_btl_base_tag_t
typedef void(* | mca_btl_base_completion_fn_t )(struct mca_btl_base_module_t *module, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, int status)
| Asynchronous callback function on completion of an operation. More...
typedef struct mca_btl_base_segment_t | mca_btl_base_segment_t
typedef struct mca_btl_base_descriptor_t | mca_btl_base_descriptor_t
typedef struct mca_btl_base_header_t | mca_btl_base_header_t
typedef struct mca_btl_base_module_t **(* | mca_btl_base_component_init_fn_t )(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads)
| MCA->BTL Initializes the BTL component and creates specific BTL module(s). More...
typedef int(* | mca_btl_base_component_progress_fn_t )(void)
| MCA->BTL Called to progress outstanding requests for non-threaded polling environments. More...
typedef void(* | mca_btl_base_module_recv_cb_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t *descriptor, void *cbdata)
| Callback function that is called asynchronously on receipt of data by the transport layer. More...
typedef struct mca_btl_active_message_callback_t | mca_btl_active_message_callback_t
typedef struct mca_btl_base_component_2_0_0_t | mca_btl_base_component_2_0_0_t
typedef struct mca_btl_base_component_2_0_0_t | mca_btl_base_component_t
typedef struct mca_btl_base_component_2_0_0_t | mca_btl_base_component_1_0_1_t
typedef struct mca_btl_base_component_2_0_0_t | mca_btl_base_component_1_0_0_t
typedef int(* | mca_btl_base_module_finalize_fn_t )(struct mca_btl_base_module_t *btl)
| MCA->BTL Clean up any resources held by BTL module before the module is unloaded. More...
typedef int(* | mca_btl_base_module_add_procs_fn_t )(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **endpoints, struct opal_bitmap_t *reachable)
| BML->BTL notification of change in the process list. More...
typedef int(* | mca_btl_base_module_del_procs_fn_t )(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peer)
| Notification of change to the process list. More...
typedef int(* | mca_btl_base_module_register_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata)
| Register a callback function that is called on receipt of a fragment. More...
typedef void(* | mca_btl_base_module_error_cb_fn_t )(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)
| Callback function that is called asynchronously on receipt of an error from the transport layer. More...
typedef int(* | mca_btl_base_module_register_error_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_module_error_cb_fn_t cbfunc)
| Register a callback function that is called on receipt of an error. More...
typedef mca_btl_base_descriptor_t *(* | mca_btl_base_module_alloc_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
| Allocate a descriptor with a segment of the requested size. More...
typedef int(* | mca_btl_base_module_free_fn_t )(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *descriptor)
| Return a descriptor allocated from this BTL via alloc/prepare. More...
typedef struct mca_btl_base_descriptor_t *(* | mca_btl_base_module_prepare_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
| Prepare a descriptor for send/put/get using the supplied convertor. More...
typedef int(* | mca_btl_base_module_send_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
| Initiate an asynchronous send. More...
typedef int(* | mca_btl_base_module_sendi_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct opal_convertor_t *convertor, void *header, size_t header_size, size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor)
| Initiate an immediate blocking send. More...
typedef int(* | mca_btl_base_module_put_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)
| Initiate an asynchronous put. More...
typedef int(* | mca_btl_base_module_get_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)
| Initiate an asynchronous get. More...
typedef void(* | mca_btl_base_module_dump_fn_t )(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, int verbose)
| Diagnostic dump of btl state. More...
typedef int(* | mca_btl_base_module_ft_event_fn_t )(int state)
| Fault Tolerance Event Notification Function. More...
typedef struct mca_btl_base_module_t | mca_btl_base_module_t

## Functions
OMPI_DECLSPEC | OBJ_CLASS_DECLARATION (mca_btl_base_descriptor_t)

## Variables
OMPI_DECLSPEC mca_btl_active_message_callback_t | mca_btl_base_active_message_trigger [MCA_BTL_TAG_MAX]

## Detailed Description

Byte Transfer Layer (BTL)

BTL Initialization:

During library initialization, all available BTL components are loaded and opened via their mca_base_open_component_fn_t function. The BTL open function should register any mca parameters used to tune/adjust the behaviour of the BTL (mca_base_param_register_int(), mca_base_param_register_string()). Note that the open function may fail if the resources (e.g. shared libraries, etc) required by the network transport are not available.

The mca_btl_base_component_init_fn_t() is then called for each of the components that are succesfully opened. The component init function may return either:

(1) a NULL list of BTL modules if the transport is not available, (2) a list containing a one or more single BTL modules, where the BTL provides a layer of abstraction over one or more physical devices (e.g. NICs),

During module initialization, the module should post any addressing information required by its peers. An example would be the TCP listen port opened by the TCP module for incoming connection requests. This information is published to peers via the ompi_modex_send() interface. Note that peer information is not guaranteed to be available via ompi_modex_recv() during the module's init function. However, it will be available during BTL selection (mca_btl_base_add_proc_fn_t()).

BTL Selection:

The upper layer builds an ordered list of the available BTL modules sorted by their exclusivity ranking. This is a relative ranking that is used to determine the set of BTLs that may be used to reach a given destination. During startup the BTL modules are queried via their mca_btl_base_add_proc_fn_t() to determine if they are able to reach a given destination. The BTL module with the highest ranking that returns success is selected. Subsequent BTL modules are selected only if they have the same exclusivity ranking.

An example of how this might be used:

BTL Exclusivity Comments

LO 100 Selected exclusively for local process SM 50 Selected exclusively for other processes on host IB 0 Selected based on network reachability IB 0 Selected based on network reachability TCP 0 Selected based on network reachability TCP 0 Selected based on network reachability

When mca_btl_base_add_proc_fn_t() is called on a BTL module, the BTL will populate an OUT variable with mca_btl_base_endpoint_t pointers. Each pointer is treated as an opaque handle by the upper layer and is returned to the BTL on subsequent data transfer calls to the corresponding destination process. The actual contents of the data structure are defined on a per BTL basis, and may be used to cache addressing or connection information, such as a TCP socket or IB queue pair.

Progress:

By default, the library provides for polling based progress of outstanding requests. The BTL component exports an interface function (btl_progress) that is called in a polling mode by the PML during calls into the MPI library. Note that the btl_progress() function is called on the BTL component rather than each BTL module. This implies that the BTL author is responsible for iterating over the pending operations in each of the BTL modules associated with the component.

On platforms where threading support is provided, the library provides the option of building with asynchronous threaded progress. In this case, the BTL author is responsible for providing a thread to progress pending operations. A thread is associated with the BTL component/module such that transport specific functionality/APIs may be used to block the thread until a pending operation completes. This thread MUST NOT poll for completion as this would oversubscribe the CPU.

Note that in the threaded case the PML may choose to use a hybrid approach, such that polling is implemented from the user thread for a fixed number of cycles before relying on the background thread(s) to complete requests. If possible the BTL should support the use of both modes concurrently.

## Macro Definition Documentation

#define MCA_BTL_BASE_VERSION_2_0_0

Value:
MCA_BASE_VERSION_2_0_0, \
"btl", 2, 0, 0

#define MCA_BTL_FLAGS_RDMA_MATCHED 0x0040

RDMA put/get calls must have a matching prepare_{src,dst} call on the target with the same base (and possibly bound).

## Typedef Documentation

typedef void(* mca_btl_base_completion_fn_t)(struct mca_btl_base_module_t *module, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, int status)

Asynchronous callback function on completion of an operation.

Completion Semantics: The descriptor can be reused or returned to the BTL via mca_btl_base_module_free_fn_t. The operation has been queued to the network device or will otherwise make asynchronous progress without subsequent calls to btl_progress.

Parameters

  IN] | module the BTL module
  IN] | endpoint the BTL endpoint
  IN] | descriptor the BTL descriptor

typedef struct mca_btl_base_module_t**(* mca_btl_base_component_init_fn_t)(int *num_btls, bool enable_progress_threads, bool enable_mpi_threads)

MCA->BTL Initializes the BTL component and creates specific BTL module(s).

Parameters

  num_btls | (OUT) Returns the number of btl modules created, or 0 if the transport is not available.
  enable_progress_threads | (IN) Whether this component is allowed to run a hidden/progress thread or not.
  enable_mpi_threads | (IN) Whether support for multiple MPI threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which indicates whether multiple threads may invoke this component simultaneously or not.

Returns
  Array of pointers to BTL modules, or NULL if the transport is not available.

During component initialization, the BTL component should discover the physical devices that are available for the given transport, and create a BTL module to represent each device. Any addressing information required by peers to reach the device should be published during this function via the ompi_modex_send() interface.

typedef int(* mca_btl_base_component_progress_fn_t)(void)

MCA->BTL Called to progress outstanding requests for non-threaded polling environments.

Returns
  Count of "completions", a metric of how many items where completed in the call to progress.

typedef int(* mca_btl_base_module_add_procs_fn_t)(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **endpoints, struct opal_bitmap_t *reachable)

BML->BTL notification of change in the process list.

Parameters

  btl | (IN) BTL module
  nprocs | (IN) Number of processes
  procs | (IN) Array of processes
  endpoint | (OUT) Array of mca_btl_base_endpoint_t structures by BTL.
  reachable | (OUT) Bitmask indicating set of peer processes that are reachable by this BTL.

Returns
  OMPI_SUCCESS or error status on failure.

The mca_btl_base_module_add_procs_fn_t() is called by the BML to determine the set of BTLs that should be used to reach each process. Any addressing information exported by the peer via the ompi_modex_send() function should be available during this call via the corresponding ompi_modex_recv() function. The BTL may utilize this information to determine reachability of each peer process.

For each process that is reachable by the BTL, the bit corresponding to the index into the proc array (nprocs) should be set in the reachable bitmask. The BTL will return an array of pointers to a data structure defined by the BTL that is then returned to the BTL on subsequent calls to the BTL data transfer functions (e.g btl_send). This may be used by the BTL to cache any addressing or connection information (e.g. TCP socket, IB queue pair).

typedef mca_btl_base_descriptor_t*(* mca_btl_base_module_alloc_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)

Allocate a descriptor with a segment of the requested size.

Note that the BTL layer may choose to return a smaller size if it cannot support the request. The order tag value ensures that operations on the descriptor that is allocated will be ordered w.r.t. a previous operation on a particular descriptor. Ordering is only guaranteed if the previous descriptor had its local completion callback function called and the order tag of that descriptor is only valid upon the local completion callback function.

Parameters

  btl | (IN) BTL module
  size | (IN) Request segment size.
  order | (IN) The ordering tag (may be MCA_BTL_NO_ORDER)

typedef int(* mca_btl_base_module_del_procs_fn_t)(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peer)

Notification of change to the process list.

Parameters

  btl | (IN) BTL module
  nprocs | (IN) Number of processes
  proc | (IN) Set of processes
  peer | (IN) Set of peer addressing information.

Returns
  Status indicating if cleanup was successful

When the process list changes, the BML notifies the BTL of the change, to provide the opportunity to cleanup or release any resources associated with the peer.

typedef void(* mca_btl_base_module_dump_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, int verbose)

Diagnostic dump of btl state.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL endpoint
  verbose | (IN) Verbosity level

typedef void(* mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)

Callback function that is called asynchronously on receipt of an error from the transport layer.

Parameters

  IN] | btl BTL module
  IN] | flags type of error
  IN] | errproc process that had an error
  IN] | btlinfo descriptive string from the BTL

typedef int(* mca_btl_base_module_finalize_fn_t)(struct mca_btl_base_module_t *btl)

MCA->BTL Clean up any resources held by BTL module before the module is unloaded.

Parameters

  btl | (IN) BTL module.

Returns
  OMPI_SUCCESS or error status on failure.

Prior to unloading a BTL module, the MCA framework will call the BTL finalize method of the module. Any resources held by the BTL should be released and if required the memory corresponding to the BTL module freed.

typedef int(* mca_btl_base_module_free_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *descriptor)

Return a descriptor allocated from this BTL via alloc/prepare.

A descriptor can only be deallocated after its local completion callback function has called for all send/put/get operations.

Parameters

  btl | (IN) BTL module
  segment | (IN) Descriptor allocated from the BTL

typedef int(* mca_btl_base_module_ft_event_fn_t)(int state)

Fault Tolerance Event Notification Function.

Parameters

  state | Checkpoint Status

Returns
  OMPI_SUCCESS or failure status

typedef int(* mca_btl_base_module_get_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)

Initiate an asynchronous get.

Completion Semantics: the descriptor has been queued for a get operation the BTL now controls the descriptor until local completion callback is made on the descriptor

BTLs that do not have the MCA_BTL_FLAGS_RDMA_MATCHED flag set allow multiple concurrent get operations on the same descriptor. BTLs that do have the MCA_BTL_FLAGS_RDMA_MATCHED flag set require a corresponding prepare_src/dst call for each get operation and therefore prohibit multiple concurrent get operations.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transferred

Return values

  OMPI_SUCCESS | The descriptor was successfully queued for a get
  OMPI_ERROR | The descriptor was NOT successfully queued for a get

typedef struct mca_btl_base_descriptor_t*(* mca_btl_base_module_prepare_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)

Prepare a descriptor for send/put/get using the supplied convertor.

If the convertor references data that is contiguous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

The descriptor returned can be used in multiple concurrent operations (send/put/get) unless the BTL has the MCA_BTL_FLAGS_RDMA_MATCHED flag set in which case a corresponding prepare call must accompany the put/get call in addition, the address and length that is put/get must match the address and length which is prepared.

The order tag value ensures that operations on the descriptor that is prepared will be ordered w.r.t. a previous operation on a particular descriptor. Ordering is only guaranteed if the previous descriptor had its local completion callback function called and the order tag of that descriptor is only valid upon the local completion callback function.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL peer addressing
  registration | (IN) Memory registration
  convertor | (IN) Data type convertor
  order | (IN) The ordering tag (may be MCA_BTL_NO_ORDER)
  reserve | (IN) Additional bytes requested by upper layer to precede user data
  size | (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)

typedef int(* mca_btl_base_module_put_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor)

Initiate an asynchronous put.

Completion Semantics: the descriptor has been queued for a put operation the BTL now controls the descriptor until local completion callback is made on the descriptor

BTLs that do not have the MCA_BTL_FLAGS_RDMA_MATCHED flag set allow multiple concurrent put operations on the same descriptor. BTLs that do have the MCA_BTL_FLAGS_RDMA_MATCHED flag set require a corresponding prepare_src/dst call for each put operation and therefore prohibit multiple concurrent put operations.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transferred

Return values

  OMPI_SUCCESS | The descriptor was successfully queued for a put
  OMPI_ERROR | The descriptor was NOT successfully queued for a put

typedef void(* mca_btl_base_module_recv_cb_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t *descriptor, void *cbdata)

Callback function that is called asynchronously on receipt of data by the transport layer.

Note that the the mca_btl_base_descriptor_t is only valid within the completion function, this implies that all data payload in the mca_btl_base_descriptor_t must be copied out within this callback or forfeited back to the BTL.

Parameters

  IN] | btl BTL module
  IN] | tag The active message receive callback tag value
  IN] | descriptor The BTL descriptor (contains the receive payload)
  IN] | cbdata Opaque callback data

typedef int(* mca_btl_base_module_register_error_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_module_error_cb_fn_t cbfunc)

Register a callback function that is called on receipt of an error.

Parameters

  IN] | btl BTL module
  IN] | cbfunc The callback function

Returns
  OMPI_SUCCESS The callback was registered successfully
  OMPI_ERROR The callback was NOT registered successfully

typedef int(* mca_btl_base_module_register_fn_t)(struct mca_btl_base_module_t *btl, mca_btl_base_tag_t tag, mca_btl_base_module_recv_cb_fn_t cbfunc, void *cbdata)

Register a callback function that is called on receipt of a fragment.

Parameters

  IN] | btl BTL module
  IN] | tag tag value of this callback (specified on subsequent send operations)
  IN] | cbfunc The callback function
  IN] | cbdata Opaque callback data

Returns
  OMPI_SUCCESS The callback was registered successfully
  OMPI_ERROR The callback was NOT registered successfully

typedef int(* mca_btl_base_module_send_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)

Initiate an asynchronous send.

Completion Semantics: the descriptor has been queued for a send operation the BTL now controls the descriptor until local completion callback is made on the descriptor

All BTLs allow multiple concurrent asynchronous send operations on a descriptor

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transfered
  tag | (IN) The tag value used to notify the peer.

Return values

  OMPI_SUCCESS | The descriptor was successfully queued for a send
  OMPI_ERROR | The descriptor was NOT successfully queued for a send
  OMPI_ERR_UNREACH | The endpoint is not reachable

typedef int(* mca_btl_base_module_sendi_fn_t)(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct opal_convertor_t *convertor, void *header, size_t header_size, size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor)

Initiate an immediate blocking send.

Completion Semantics: the BTL will make a best effort to send the header and "size" bytes from the datatype using the convertor. The header is guaranteed to be delivered entirely in the first segment. Should the BTL be unable to deliver the data due to resource constraints the BTL will return a descriptor (via the OUT param) of size "payload_size + header_size".

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  convertor | (IN) Data type convertor
  header | (IN) Pointer to header.
  header_size | (IN) Size of header.
  payload_size | (IN) Size of payload (from convertor).
  order | (IN) The ordering tag (may be MCA_BTL_NO_ORDER)
  flags | (IN) Flags.
  tag | (IN) The tag value used to notify the peer.
  descriptor | (OUT) The descriptor to be returned unable to be sent immediately

Return values

  OMPI_SUCCESS | The send was successfully queued
  OMPI_ERROR | The send failed
  OMPI_ERR_UNREACH | The endpoint is not reachable
  OMPI_ERR_RESOURCE_BUSY | The BTL is busy a descriptor will be returned (via the OUT param) if descriptors are available