OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

pml_cm_sendreq.h
1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2010 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2006 The Regents of the University of California.
11 * All rights reserved.
12 * $COPYRIGHT$
13 *
14 * Additional copyrights may follow
15 *
16 * $HEADER$
17 */
18
19 #ifndef PML_CM_SENDREQ_H
20 #define PML_CM_SENDREQ_H
21
22 #include "pml_cm_request.h"
23 #include "ompi/mca/pml/base/pml_base_sendreq.h"
24 #include "ompi/mca/pml/base/pml_base_bsend.h"
25 #include "ompi/mca/pml/pml.h"
26 #include "ompi/mca/mtl/mtl.h"
27 #include "opal/prefetch.h"
28
29 struct mca_pml_cm_send_request_t {
30 mca_pml_cm_request_t req_base;
31 mca_pml_base_send_mode_t req_send_mode;
32 };
33 typedef struct mca_pml_cm_send_request_t mca_pml_cm_send_request_t;
34 OBJ_CLASS_DECLARATION(mca_pml_cm_send_request_t);
35
36
37 struct mca_pml_cm_thin_send_request_t {
38 mca_pml_cm_send_request_t req_send;
39 mca_mtl_request_t req_mtl; /**< the mtl specific memory. This field should be the last in the struct */
40 };
41 typedef struct mca_pml_cm_thin_send_request_t mca_pml_cm_thin_send_request_t;
42 OBJ_CLASS_DECLARATION(mca_pml_cm_thin_send_request_t);
43
44
45 struct mca_pml_cm_hvy_send_request_t {
46 mca_pml_cm_send_request_t req_send;
47 void *req_addr; /**< pointer to application buffer */
48 size_t req_count; /**< count of user datatype elements */
49 int32_t req_peer; /**< peer process - rank w/in this communicator */
50 int32_t req_tag; /**< user defined tag */
51 void *req_buff; /**< pointer to send buffer - may not be application buffer */
52 bool req_blocking;
53 mca_mtl_request_t req_mtl; /**< the mtl specific memory. This field should be the last in the struct */
54 };
55 typedef struct mca_pml_cm_hvy_send_request_t mca_pml_cm_hvy_send_request_t;
56 OBJ_CLASS_DECLARATION(mca_pml_cm_hvy_send_request_t);
57
58
59 #define MCA_PML_CM_THIN_SEND_REQUEST_ALLOC(sendreq, comm, dst, \
60 ompi_proc, rc) \
61 do { \
62 ompi_free_list_item_t* item; \
63 ompi_proc = ompi_comm_peer_lookup( comm, dst ); \
64 \
65 if(OPAL_UNLIKELY(NULL == ompi_proc)) { \
66 rc = OMPI_ERR_OUT_OF_RESOURCE; \
67 sendreq = NULL; \
68 } else { \
69 rc = OMPI_SUCCESS; \
70 OMPI_FREE_LIST_WAIT(&mca_pml_base_send_requests, \
71 item, rc); \
72 sendreq = (mca_pml_cm_thin_send_request_t*)item; \
73 sendreq->req_send.req_base.req_pml_type = MCA_PML_CM_REQUEST_SEND_THIN; \
74 sendreq->req_mtl.ompi_req = (ompi_request_t*) sendreq; \
75 sendreq->req_mtl.completion_callback = mca_pml_cm_send_request_completion; \
76 } \
77 } while(0)
78
79
80 #define MCA_PML_CM_HVY_SEND_REQUEST_ALLOC(sendreq, comm, dst, \
81 ompi_proc, rc) \
82 { \
83 ompi_free_list_item_t* item; \
84 ompi_proc = ompi_comm_peer_lookup( comm, dst ); \
85 if(OPAL_UNLIKELY(NULL == ompi_proc)) { \
86 rc = OMPI_ERR_OUT_OF_RESOURCE; \
87 sendreq = NULL; \
88 } else { \
89 rc = OMPI_SUCCESS; \
90 OMPI_FREE_LIST_WAIT(&mca_pml_base_send_requests, \
91 item, rc); \
92 sendreq = (mca_pml_cm_hvy_send_request_t*)item; \
93 sendreq->req_send.req_base.req_pml_type = MCA_PML_CM_REQUEST_SEND_HEAVY; \
94 sendreq->req_mtl.ompi_req = (ompi_request_t*) sendreq; \
95 sendreq->req_mtl.completion_callback = mca_pml_cm_send_request_completion; \
96 } \
97 }
98
99
100 #define MCA_PML_CM_SEND_REQUEST_INIT_COMMON(req_send, \
101 ompi_proc, \
102 comm, \
103 tag, \
104 datatype, \
105 sendmode, \
106 buf, \
107 count) \
108 { \
109 OBJ_RETAIN(comm); \
110 OBJ_RETAIN(datatype); \
111 (req_send)->req_base.req_comm = comm; \
112 (req_send)->req_base.req_datatype = datatype; \
113 opal_convertor_copy_and_prepare_for_send( \
114 ompi_proc->proc_convertor, \
115 &(datatype->super), \
116 count, \
117 buf, \
118 0, \
119 &(req_send)->req_base.req_convertor ); \
120 (req_send)->req_base.req_ompi.req_mpi_object.comm = comm; \
121 (req_send)->req_base.req_ompi.req_status.MPI_SOURCE = \
122 comm->c_my_rank; \
123 (req_send)->req_base.req_ompi.req_status.MPI_TAG = tag; \
124 (req_send)->req_base.req_ompi.req_status._ucount = count; \
125 (req_send)->req_send_mode = sendmode; \
126 (req_send)->req_base.req_free_called = false; \
127 }
128
129 #define MCA_PML_CM_HVY_SEND_REQUEST_INIT( sendreq, \
130 ompi_proc, \
131 comm, \
132 tag, \
133 dst, \
134 datatype, \
135 sendmode, \
136 persistent, \
137 blocking, \
138 buf, \
139 count) \
140 do { \
141 OMPI_REQUEST_INIT(&(sendreq->req_send.req_base.req_ompi), \
142 persistent); \
143 sendreq->req_tag = tag; \
144 sendreq->req_peer = dst; \
145 sendreq->req_addr = buf; \
146 sendreq->req_count = count; \
147 MCA_PML_CM_SEND_REQUEST_INIT_COMMON( (&sendreq->req_send), \
148 ompi_proc, \
149 comm, \
150 tag, \
151 datatype, \
152 sendmode, \
153 buf, \
154 count); \
155 opal_convertor_get_packed_size( \
156 &sendreq->req_send.req_base.req_convertor, \
157 &sendreq->req_count ); \
158 \
159 sendreq->req_blocking = blocking; \
160 sendreq->req_send.req_base.req_pml_complete = \
161 (persistent ? true:false); \
162 } while(0)
163
164
165 #define MCA_PML_CM_THIN_SEND_REQUEST_INIT( sendreq, \
166 ompi_proc, \
167 comm, \
168 tag, \
169 dst, \
170 datatype, \
171 sendmode, \
172 buf, \
173 count) \
174 do { \
175 OMPI_REQUEST_INIT(&(sendreq->req_send.req_base.req_ompi), \
176 false); \
177 MCA_PML_CM_SEND_REQUEST_INIT_COMMON( (&sendreq->req_send), \
178 ompi_proc, \
179 comm, \
180 tag, \
181 datatype, \
182 sendmode, \
183 buf, \
184 count); \
185 sendreq->req_send.req_base.req_pml_complete = false; \
186 } while(0)
187
188
189 #define MCA_PML_CM_SEND_REQUEST_START_SETUP(req_send) \
190 do { \
191 (req_send)->req_base.req_pml_complete = false; \
192 (req_send)->req_base.req_ompi.req_complete = false; \
193 (req_send)->req_base.req_ompi.req_state = \
194 OMPI_REQUEST_ACTIVE; \
195 (req_send)->req_base.req_ompi.req_status._cancelled = 0; \
196 } while (0)
197
198
199 #define MCA_PML_CM_THIN_SEND_REQUEST_START(sendreq, \
200 comm, \
201 tag, \
202 dst, \
203 sendmode, \
204 blocking, \
205 ret) \
206 do { \
207 MCA_PML_CM_SEND_REQUEST_START_SETUP(&(sendreq)->req_send); \
208 ret = OMPI_MTL_CALL(isend(ompi_mtl, \
209 comm, \
210 dst, \
211 tag, \
212 &sendreq->req_send.req_base.req_convertor, \
213 sendmode, \
214 blocking, \
215 &sendreq->req_mtl)); \
216 } while (0)
217
218 #define MCA_PML_CM_HVY_SEND_REQUEST_BSEND_ALLOC(sendreq, ret) \
219 do { \
220 struct iovec iov; \
221 unsigned int iov_count; \
222 size_t max_data; \
223 \
224 if(sendreq->req_count > 0) { \
225 sendreq->req_buff = \
226 mca_pml_base_bsend_request_alloc_buf(sendreq->req_count); \
227 if (NULL == sendreq->req_buff) { \
228 ret = MPI_ERR_BUFFER; \
229 } else { \
230 iov.iov_base = (IOVBASE_TYPE*)sendreq->req_buff; \
231 max_data = iov.iov_len = sendreq->req_count; \
232 iov_count = 1; \
233 opal_convertor_pack( &sendreq->req_send.req_base.req_convertor, \
234 &iov, \
235 &iov_count, \
236 &max_data ); \
237 opal_convertor_prepare_for_send( &sendreq->req_send.req_base.req_convertor, \
238 &(ompi_mpi_packed.dt.super), \
239 max_data, sendreq->req_buff ); \
240 } \
241 } \
242 } while(0);
243
244
245 #define MCA_PML_CM_HVY_SEND_REQUEST_START(sendreq, ret) \
246 do { \
247 ret = OMPI_SUCCESS; \
248 MCA_PML_CM_SEND_REQUEST_START_SETUP(&(sendreq)->req_send); \
249 if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
250 MCA_PML_CM_HVY_SEND_REQUEST_BSEND_ALLOC(sendreq, ret); \
251 } \
252 if (OMPI_SUCCESS == ret) { \
253 ret = OMPI_MTL_CALL(isend(ompi_mtl, \
254 sendreq->req_send.req_base.req_comm, \
255 sendreq->req_peer, \
256 sendreq->req_tag, \
257 &sendreq->req_send.req_base.req_convertor, \
258 sendreq->req_send.req_send_mode, \
259 sendreq->req_blocking, \
260 &sendreq->req_mtl)); \
261 if(OMPI_SUCCESS == ret && \
262 sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
263 sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR = 0; \
264 ompi_request_complete(&(sendreq)->req_send.req_base.req_ompi, true); \
265 } \
266 } \
267 } while (0)
268
269 /*
270 * The PML has completed a send request. Note that this request
271 * may have been orphaned by the user or have already completed
272 * at the MPI level.
273 * This macro will never be called directly from the upper level, as it should
274 * only be an internal call to the PML.
275 */
276 #define MCA_PML_CM_HVY_SEND_REQUEST_PML_COMPLETE(sendreq) \
277 do { \
278 assert( false == sendreq->req_send.req_base.req_pml_complete ); \
279 \
280 if (sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED && \
281 sendreq->req_count > 0 ) { \
282 mca_pml_base_bsend_request_free(sendreq->req_buff); \
283 } \
284 \
285 OPAL_THREAD_LOCK(&ompi_request_lock); \
286 if( false == sendreq->req_send.req_base.req_ompi.req_complete ) { \
287 /* Should only be called for long messages (maybe synchronous) */ \
288 ompi_request_complete(&(sendreq->req_send.req_base.req_ompi), true); \
289 } \
290 sendreq->req_send.req_base.req_pml_complete = true; \
291 \
292 if( sendreq->req_send.req_base.req_free_called ) { \
293 MCA_PML_CM_HVY_SEND_REQUEST_RETURN( sendreq ); \
294 } else { \
295 if(sendreq->req_send.req_base.req_ompi.req_persistent) { \
296 /* rewind convertor */ \
297 size_t offset = 0; \
298 opal_convertor_set_position(&sendreq->req_send.req_base.req_convertor, \
299 &offset); \
300 } \
301 } \
302 OPAL_THREAD_UNLOCK(&ompi_request_lock); \
303 } while (0)
304
305
306 /*
307 * Release resources associated with a request
308 */
309 #define MCA_PML_CM_HVY_SEND_REQUEST_RETURN(sendreq) \
310 { \
311 /* Let the base handle the reference counts */ \
312 OBJ_RELEASE(sendreq->req_send.req_base.req_datatype); \
313 OBJ_RELEASE(sendreq->req_send.req_base.req_comm); \
314 OMPI_REQUEST_FINI(&sendreq->req_send.req_base.req_ompi); \
315 opal_convertor_cleanup( &(sendreq->req_send.req_base.req_convertor) ); \
316 OMPI_FREE_LIST_RETURN( &mca_pml_base_send_requests, \
317 (ompi_free_list_item_t*)sendreq); \
318 }
319
320 /*
321 * The PML has completed a send request. Note that this request
322 * may have been orphaned by the user or have already completed
323 * at the MPI level.
324 * This macro will never be called directly from the upper level, as it should
325 * only be an internal call to the PML.
326 */
327 #define MCA_PML_CM_THIN_SEND_REQUEST_PML_COMPLETE(sendreq) \
328 do { \
329 assert( false == sendreq->req_send.req_base.req_pml_complete ); \
330 \
331 OPAL_THREAD_LOCK(&ompi_request_lock); \
332 if( false == sendreq->req_send.req_base.req_ompi.req_complete ) { \
333 /* Should only be called for long messages (maybe synchronous) */ \
334 ompi_request_complete(&(sendreq->req_send.req_base.req_ompi), true); \
335 } \
336 sendreq->req_send.req_base.req_pml_complete = true; \
337 \
338 if( sendreq->req_send.req_base.req_free_called ) { \
339 MCA_PML_CM_THIN_SEND_REQUEST_RETURN( sendreq ); \
340 } \
341 OPAL_THREAD_UNLOCK(&ompi_request_lock); \
342 } while (0)
343
344
345 /*
346 * Release resources associated with a request
347 */
348 #define MCA_PML_CM_THIN_SEND_REQUEST_RETURN(sendreq) \
349 { \
350 /* Let the base handle the reference counts */ \
351 OBJ_RELEASE(sendreq->req_send.req_base.req_datatype); \
352 OBJ_RELEASE(sendreq->req_send.req_base.req_comm); \
353 OMPI_REQUEST_FINI(&sendreq->req_send.req_base.req_ompi); \
354 opal_convertor_cleanup( &(sendreq->req_send.req_base.req_convertor) ); \
355 OMPI_FREE_LIST_RETURN( &mca_pml_base_send_requests, \
356 (ompi_free_list_item_t*)sendreq); \
357 }
358
359 extern void
360 mca_pml_cm_send_request_completion(struct mca_mtl_request_t *mtl_request);
361
362 #endif
mca_pml_cm_hvy_send_request_t::req_tag
int32_t req_tag
user defined tag
Definition: pml_cm_sendreq.h:50
mca_pml_cm_send_request_t
Definition: pml_cm_sendreq.h:29
pml.h
P2P Management Layer (PML)
mca_pml_cm_thin_send_request_t
Definition: pml_cm_sendreq.h:37
mca_pml_cm_thin_send_request_t::req_mtl
mca_mtl_request_t req_mtl
the mtl specific memory.
Definition: pml_cm_sendreq.h:39
mca_mtl_request_t
Definition: mtl.h:51
mca_pml_cm_hvy_send_request_t::req_buff
void * req_buff
pointer to send buffer - may not be application buffer
Definition: pml_cm_sendreq.h:51
mca_pml_cm_hvy_send_request_t::req_peer
int32_t req_peer
peer process - rank w/in this communicator
Definition: pml_cm_sendreq.h:49
mca_pml_cm_hvy_send_request_t::req_addr
void * req_addr
pointer to application buffer
Definition: pml_cm_sendreq.h:47
mtl.h
Matching Transport Layer.
mca_pml_cm_hvy_send_request_t
Definition: pml_cm_sendreq.h:45
prefetch.h
Compiler-specific prefetch functions.
pml_base_sendreq.h
mca_pml_cm_hvy_send_request_t::req_mtl
mca_mtl_request_t req_mtl
the mtl specific memory.
Definition: pml_cm_sendreq.h:53
mca_pml_cm_hvy_send_request_t::req_count
size_t req_count
count of user datatype elements
Definition: pml_cm_sendreq.h:48
mca_pml_cm_request_t
Base type for PML CM P2P requests.
Definition: pml_cm_request.h:41
OBJ_CLASS_DECLARATION
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236