20 #ifndef OMPI_PML_BFO_SEND_REQUEST_H
21 #define OMPI_PML_BFO_SEND_REQUEST_H
30 #include "opal/datatype/opal_convertor.h"
36 MCA_PML_BFO_SEND_PENDING_NONE,
37 MCA_PML_BFO_SEND_PENDING_SCHEDULE,
38 MCA_PML_BFO_SEND_PENDING_START
39 } mca_pml_bfo_send_pending_t;
47 int32_t req_restartseq;
53 bool req_throttle_sends;
54 size_t req_pipeline_depth;
55 size_t req_bytes_delivered;
56 uint32_t req_rdma_cnt;
57 mca_pml_bfo_send_pending_t req_pending;
68 uint64_t range_send_offset;
69 uint64_t range_send_length;
89 const mca_pml_bfo_send_pending_t type,
95 sendreq->req_pending = type;
105 get_request_from_send_pending(mca_pml_bfo_send_pending_t *type)
113 *type = sendreq->req_pending;
114 sendreq->req_pending = MCA_PML_BFO_SEND_PENDING_NONE;
121 #define MCA_PML_BFO_SEND_REQUEST_ALLOC( comm, \
126 ompi_proc_t *proc = ompi_comm_peer_lookup( comm, dst ); \
127 ompi_free_list_item_t* item; \
129 rc = OMPI_ERR_OUT_OF_RESOURCE; \
130 if( OPAL_LIKELY(NULL != proc) ) { \
132 OMPI_FREE_LIST_WAIT(&mca_pml_base_send_requests, item, rc); \
133 sendreq = (mca_pml_bfo_send_request_t*)item; \
134 sendreq->req_send.req_base.req_proc = proc; \
139 #define MCA_PML_BFO_SEND_REQUEST_INIT( sendreq, \
149 MCA_PML_BASE_SEND_REQUEST_INIT(&sendreq->req_send, \
159 (sendreq)->req_recv.pval = NULL; \
168 for(r = 0; r < sendreq->req_rdma_cnt; r++) {
170 if( NULL != reg && reg->mpool != NULL ) {
174 sendreq->req_rdma_cnt = 0;
182 #define MCA_PML_BFO_SEND_REQUEST_START(sendreq, rc) \
184 rc = mca_pml_bfo_send_request_start(sendreq); \
192 #define MCA_PML_BFO_SEND_REQUEST_MPI_COMPLETE(sendreq, with_signal) \
194 (sendreq)->req_send.req_base.req_ompi.req_status.MPI_SOURCE = \
195 (sendreq)->req_send.req_base.req_comm->c_my_rank; \
196 (sendreq)->req_send.req_base.req_ompi.req_status.MPI_TAG = \
197 (sendreq)->req_send.req_base.req_tag; \
198 (sendreq)->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS; \
199 (sendreq)->req_send.req_base.req_ompi.req_status._ucount = \
200 (sendreq)->req_send.req_bytes_packed; \
201 ompi_request_complete( &((sendreq)->req_send.req_base.req_ompi), (with_signal) ); \
203 PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_COMPLETE, \
204 &(sendreq->req_send.req_base), PERUSE_SEND); \
211 #define MCA_PML_BFO_SEND_REQUEST_RETURN(sendreq) \
214 MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send)); \
215 OMPI_FREE_LIST_RETURN( &mca_pml_base_send_requests, \
216 (ompi_free_list_item_t*)sendreq); \
234 PERUSE_TRACE_COMM_EVENT( PERUSE_COMM_REQ_XFER_END,
235 &(sendreq->req_send.
req_base), PERUSE_SEND);
239 mca_pml_bfo_free_rdma_resources(sendreq);
241 if (sendreq->req_send.
req_send_mode == MCA_PML_BASE_SEND_BUFFERED &&
249 MCA_PML_BFO_SEND_REQUEST_MPI_COMPLETE(sendreq,
true);
257 MCA_PML_BFO_SEND_REQUEST_RETURN(sendreq);
266 #if OPAL_ENABLE_MULTI_THREADS
274 if(sendreq->req_state == 0 &&
276 && lock_send_request(sendreq)) {
277 send_request_pml_complete(sendreq);
295 rc = mca_pml_bfo_send_request_schedule_once(sendreq);
296 if(OPAL_SOS_GET_ERROR_CODE(rc) == OMPI_ERR_OUT_OF_RESOURCE)
298 }
while(!unlock_send_request(sendreq));
300 if(OMPI_SUCCESS == rc)
301 send_request_pml_complete_check(sendreq);
316 if(!lock_send_request(sendreq))
319 mca_pml_bfo_send_request_schedule_exclusive(sendreq);
322 #if OMPI_CUDA_SUPPORT
323 int mca_pml_bfo_send_request_start_cuda(
333 int mca_pml_bfo_send_request_start_buffered(
338 int mca_pml_bfo_send_request_start_copy(
343 int mca_pml_bfo_send_request_start_prepare(
348 int mca_pml_bfo_send_request_start_rdma(
353 int mca_pml_bfo_send_request_start_rndv(
368 if( OPAL_LIKELY(size <= eager_limit) ) {
370 case MCA_PML_BASE_SEND_SYNCHRONOUS:
371 rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, 0);
373 case MCA_PML_BASE_SEND_BUFFERED:
374 rc = mca_pml_bfo_send_request_start_copy(sendreq, bml_btl, size);
376 case MCA_PML_BASE_SEND_COMPLETE:
377 rc = mca_pml_bfo_send_request_start_prepare(sendreq, bml_btl, size);
380 if (size != 0 && bml_btl->
btl_flags & MCA_BTL_FLAGS_SEND_INPLACE) {
381 rc = mca_pml_bfo_send_request_start_prepare(sendreq, bml_btl, size);
383 rc = mca_pml_bfo_send_request_start_copy(sendreq, bml_btl, size);
391 if(sendreq->req_send.
req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
392 rc = mca_pml_bfo_send_request_start_buffered(sendreq, bml_btl, size);
398 if( 0 != (sendreq->req_rdma_cnt = (uint32_t)mca_pml_bfo_rdma_btls(
399 sendreq->req_endpoint,
402 sendreq->req_rdma))) {
403 rc = mca_pml_bfo_send_request_start_rdma(sendreq, bml_btl,
405 if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
406 mca_pml_bfo_free_rdma_resources(sendreq);
409 rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size,
410 MCA_PML_BFO_HDR_FLAGS_CONTIG);
413 #if OMPI_CUDA_SUPPORT
415 return mca_pml_bfo_send_request_start_cuda(sendreq, bml_btl, size);
418 rc = mca_pml_bfo_send_request_start_rndv(sendreq, bml_btl, size, 0);
433 if( OPAL_UNLIKELY(endpoint == NULL) ) {
434 return OMPI_ERR_UNREACH;
437 sendreq->req_endpoint = endpoint;
438 sendreq->req_state = 0;
439 sendreq->req_lock = 0;
440 sendreq->req_pipeline_depth = 0;
441 sendreq->req_bytes_delivered = 0;
442 sendreq->req_pending = MCA_PML_BFO_SEND_PENDING_NONE;
446 sendreq->req_restartseq = 0;
447 sendreq->req_restart = 0;
448 sendreq->req_error = 0;
449 sendreq->req_events = 0;
460 rc = mca_pml_bfo_send_request_start_btl(sendreq, bml_btl);
461 if( OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != OPAL_SOS_GET_ERROR_CODE(rc)) )
464 add_request_to_send_pending(sendreq, MCA_PML_BFO_SEND_PENDING_START,
true);
490 uint64_t send_offset, uint64_t send_length);
#define OPAL_THREAD_ADD32(x, y)
Use an atomic operation for increment/decrement if opal_using_threads() indicates that threads are in...
Definition: mutex.h:367
struct mca_bml_base_endpoint_t * proc_bml
BML specific proc data.
Definition: proc.h:64
Definition: pml_bfo.h:299
uint32_t btl_flags
support for put/get?
Definition: bml.h:59
void opal_atomic_rmb(void)
Read memory barrier.
struct ompi_communicator_t * req_comm
communicator pointer
Definition: pml_base_request.h:63
Cached on ompi_communicator_t to hold queues/state used by the PML<->PTL interface for matching logic...
Definition: pml_bfo_comm.h:51
Definition: mutex_unix.h:53
Union of defined hdr types.
Definition: pml_bfo_hdr.h:441
size_t btl_rndv_eager_limit
the size of a data sent in a first fragment of rendezvous protocol
Definition: btl.h:791
int32_t send_sequence
send side sequence number
Definition: pml_bfo_comm.h:38
int32_t req_peer
peer process - rank w/in this communicator
Definition: pml_base_request.h:71
BML Management Layer (BML)
Definition: opal_list.h:98
#define OPAL_THREAD_LOCK(mutex)
Lock a mutex if opal_using_threads() says that multiple threads may be active in the process...
Definition: mutex.h:223
Header used to initiate an RDMA operation.
Definition: pml_bfo_hdr.h:294
void * req_addr
pointer to send buffer - may not be application buffer
Definition: pml_base_sendreq.h:39
#define OPAL_THREAD_UNLOCK(mutex)
Unlock a mutex if opal_using_threads() says that multiple threads may be active in the process...
Definition: mutex.h:309
opal_convertor_t req_convertor
always need the convertor
Definition: pml_base_request.h:66
mca_pml_base_request_t req_base
base request type - common data structure for use by wait/test
Definition: pml_base_sendreq.h:38
#define opal_list_append(l, i)
Append an item to the end of the list.
Definition: opal_list.h:410
volatile bool req_pml_complete
flag indicating if the pt-2-pt layer is done with this request
Definition: pml_base_request.h:61
Byte Transfer Layer (BTL)
struct ompi_proc_t * req_proc
peer process
Definition: pml_base_request.h:73
mca_bml_base_btl_array_t btl_eager
array of btls to use for first fragments
Definition: bml.h:228
volatile bool req_complete
Flag indicating wether request has completed.
Definition: request.h:104
mca_mpool_base_module_deregister_fn_t mpool_deregister
deregister memory
Definition: mpool.h:181
Base type for send requests.
Definition: pml_base_sendreq.h:37
Definition: ompi_free_list.h:62
Definition: pml_bfo_sendreq.h:66
Structure associated w/ ompi_proc_t that contains the set of BTLs used to reach a destination...
Definition: bml.h:222
void * req_addr
pointer to application buffer
Definition: pml_base_request.h:69
Definition: pml_bfo_rdmafrag.h:36
#define MCA_PML_BASE_SEND_START(request)
Mark the request as started from the PML base point of view.
Definition: pml_base_sendreq.h:120
Definition: opal_list.h:147
static opal_list_item_t * opal_list_remove_first(opal_list_t *list)
Remove the first item from the list and return it.
Definition: opal_list.h:522
static void opal_list_prepend(opal_list_t *list, opal_list_item_t *item)
Prepend an item to the beginning of the list.
Definition: opal_list.h:469
struct mca_btl_base_module_t * btl
BTL module.
Definition: bml.h:61
Definition: pml_bfo_sendreq.h:41
ompi_request_t req_ompi
base request
Definition: pml_base_request.h:60
static size_t mca_bml_base_btl_array_get_size(mca_bml_base_btl_array_t *array)
If required, reallocate (grow) the array to the indicate size.
Definition: bml.h:91
BTL module interface functions and attributes.
Definition: btl.h:786
size_t req_bytes_packed
packed size of a message given the datatype and count
Definition: pml_base_sendreq.h:40
uint32_t flags
the properties of this convertor
Definition: opal_convertor.h:93
mca_pml_base_send_mode_t req_send_mode
type of send
Definition: pml_base_sendreq.h:41
Main top-level request struct definition.
Definition: request.h:100
size_t btl_eager_limit
maximum size of first fragment – eager send
Definition: btl.h:790
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
uint64_t req_sequence
sequence number for MPI pt-2-pt ordering
Definition: pml_base_request.h:74
volatile bool req_free_called
flag indicating if the user has freed this request
Definition: pml_base_request.h:65
static mca_bml_base_btl_t * mca_bml_base_btl_array_get_next(mca_bml_base_btl_array_t *array)
Return the next LRU index in the array.
Definition: bml.h:179