OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

oob_tcp_msg.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2005 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * $COPYRIGHT$
13 *
14 * Additional copyrights may follow
15 *
16 * $HEADER$
17 */
18 /** @file:
19 *
20 * contains the data structure we will use to describe a message
21 */
22
23 #ifndef _MCA_OOB_TCP_MESSAGE_H_
24 #define _MCA_OOB_TCP_MESSAGE_H_
25
26 #include "orte_config.h"
27 #include "orte/types.h"
28
29 #include <errno.h>
30
31
32 #include "orte/mca/oob/oob.h"
33 #include "oob_tcp_hdr.h"
34
35 BEGIN_C_DECLS
36
37 struct mca_oob_tcp_peer_t;
38
39 #define MCA_OOB_TCP_IOV_MAX 16
40
41 typedef enum { MCA_OOB_TCP_POSTED, MCA_OOB_TCP_UNEXPECTED } mca_oob_tcp_type_t;
42
43
44 /**
45 * describes each message being progressed.
46 */
47 struct mca_oob_tcp_msg_t {
48 opal_free_list_item_t super; /**< allow this item to be put on a list */
49 mca_oob_tcp_type_t msg_type; /**< posted receive or unexpected */
50 int msg_flags; /**< flags to send/recv */
51 int msg_rc; /**< the return code for the send/recv (amount sent/recvd or errno) */
52 mca_oob_tcp_hdr_t msg_hdr; /**< header used to convey message properties to peer */
53 struct iovec* msg_uiov; /**< the user supplied iovec array */
54 int msg_ucnt; /**< the number of items in the user iovec array */
55 struct iovec * msg_rwiov; /**< copy of iovec array - not data */
56 struct iovec * msg_rwptr; /**< current read/write pointer into msg_iov */
57 int msg_rwnum; /**< number of iovecs left for read/write */
58 int msg_rwcnt; /**< total number of iovecs for read/write */
59 void* msg_rwbuf; /**< optional buffer for send/recv */
60 orte_rml_callback_fn_t msg_cbfunc; /**< the callback function for the send/receive */
61 void * msg_cbdata; /**< the data for the callback fnuction */
62 bool msg_complete; /**< whether the message is done sending or not */
63 orte_process_name_t msg_peer; /**< the name of the peer */
64 opal_mutex_t msg_lock; /**< lock for the condition variable */
65 opal_condition_t msg_condition; /**< condition variable for completion */
66 struct iovec msg_iov[MCA_OOB_TCP_IOV_MAX]; /** preallocate space for iovec array */
67 };
68
69 /**
70 * Convenience typedef
71 */
72 typedef struct mca_oob_tcp_msg_t mca_oob_tcp_msg_t;
73
74 OBJ_CLASS_DECLARATION(mca_oob_tcp_msg_t);
75
76 /**
77 * Get a new structure for use with a message
78 */
79 #define MCA_OOB_TCP_MSG_ALLOC(msg, rc) \
80 { \
81 opal_free_list_item_t* item; \
82 OPAL_FREE_LIST_GET(&mca_oob_tcp_component.tcp_msgs, item, rc); \
83 msg = (mca_oob_tcp_msg_t*)item; \
84 }
85
86 /**
87 * return a message structure that is no longer needed
88 */
89 #define MCA_OOB_TCP_MSG_RETURN(msg) \
90 { \
91 /* frees the iovec allocated during the send/receive */ \
92 if(NULL != msg->msg_rwiov) \
93 mca_oob_tcp_msg_iov_return(msg,msg->msg_rwiov); \
94 if(NULL != msg->msg_rwbuf) \
95 free(msg->msg_rwbuf); \
96 OPAL_FREE_LIST_RETURN(&mca_oob_tcp_component.tcp_msgs, \
97 &msg->super); \
98 }
99
100 /**
101 * Wait for a msg to complete.
102 * @param msg (IN) Message to wait on.
103 * @param size (OUT) Number of bytes delivered.
104 * @retval ORTE_SUCCESS or error code on failure.
105 */
106 int mca_oob_tcp_msg_wait(mca_oob_tcp_msg_t* msg, int* size);
107
108 /**
109 * Wait - up to a timeout - for a msg to complete.
110 * @param msg (IN) Message to wait on.
111 * @param size (OUT) Number of bytes delivered.
112 * @retval ORTE_SUCCESS or error code on failure.
113 */
114 int mca_oob_tcp_msg_timedwait(mca_oob_tcp_msg_t* msg, int* size, struct timespec* ts);
115
116 /**
117 * Signal that a message has completed. Wakes up any pending threads (for blocking send)
118 * or invokes callbacks for non-blocking case.
119 * @param msg (IN) Message send/recv that has completed.
120 * @param peer (IN) The peer the send/receive was from
121 * @retval ORTE_SUCCESS or error code on failure.
122 */
123 int mca_oob_tcp_msg_complete(mca_oob_tcp_msg_t* msg, orte_process_name_t * peer);
124
125 /**
126 * Called to copy the results of a message into user supplied iovec array.
127 * @param msg (IN) Message send that is in progress.
128 * @param iov (IN) Iovec array of user supplied buffers.
129 * @retval count Number of elements in iovec array.
130 */
131
132 int mca_oob_tcp_msg_copy(mca_oob_tcp_msg_t* msg, struct iovec* iov, int count);
133
134 /**
135 * Called asynchronously to progress sending a message from the event library thread.
136 * @param msg (IN) Message send that is in progress.
137 * @param peer (IN) Peer we are sending to.
138 * @retval Number of bytes copied.
139 */
140 bool mca_oob_tcp_msg_send_handler(mca_oob_tcp_msg_t* msg, struct mca_oob_tcp_peer_t * peer);
141
142 /**
143 * Called asynchronously to progress sending a message from the event library thread.
144 * @param msg (IN) Message send that is in progress.
145 * @param peer (IN) Peer theat we are recieving from.
146 * @retval bool Bool flag indicating wether operation has completed.
147 */
148
149 bool mca_oob_tcp_msg_recv_handler(mca_oob_tcp_msg_t* msg, struct mca_oob_tcp_peer_t * peer);
150
151 /**
152 * The message has been completely received - so attempt to match
153 * against posted recvs.
154 */
155
156 void mca_oob_tcp_msg_recv_complete(mca_oob_tcp_msg_t* msg, struct mca_oob_tcp_peer_t* peer);
157
158 /**
159 * Match name to a message that has been received asynchronously (unexpected).
160 *
161 * @param name (IN) Name associated with peer or wildcard to match first posted recv.
162 * @param tag (IN) Message tag.
163 * @return msg Matched message or NULL.
164 *
165 * Note - this routine requires the caller to be holding the module lock.
166 */
167
168 mca_oob_tcp_msg_t* mca_oob_tcp_msg_match_recv(orte_process_name_t* name, int tag);
169
170 /**
171 * Match name to a posted recv request.
172 *
173 * @param name (IN) Name associated with peer or wildcard to match first posted recv.
174 * @param tag (IN) Message tag.
175 * @return msg Matched message or NULL.
176 *
177 * Note - this routine requires the caller to be holding the module lock.
178 */
179
180 mca_oob_tcp_msg_t* mca_oob_tcp_msg_match_post(orte_process_name_t* name, int tag);
181
182 /**
183 * Allocate space for iovec array - if the request number of elements is less than
184 * MCA_OOB_TCP_IOV_MAX then use the array allocated along w/ the message - otherwise
185 * allocate count elements.
186 *
187 * @param msg (IN) Message to allocate array.
188 * @param count (IN) the number of iovecs
189 * @return Array of iovec elements.
190 *
191 */
192 static inline struct iovec* mca_oob_tcp_msg_iov_alloc(mca_oob_tcp_msg_t* msg, int count)
193 {
194 if(count <= MCA_OOB_TCP_IOV_MAX)
195 return msg->msg_iov;
196 return (struct iovec *)malloc(sizeof(struct iovec) * count);
197 }
198
199
200 /**
201 * Release resource held by iovec array if this is not part of the message.
202 *
203 * @param msg (IN) Message to allocate array.
204 * @param iov (IN) Iovec array to return.
205 *
206 */
207
208 static inline void mca_oob_tcp_msg_iov_return(mca_oob_tcp_msg_t* msg, struct iovec* iov)
209 {
210 if(iov != msg->msg_iov)
211 free(iov);
212 }
213
214 END_C_DECLS
215
216 #endif /* _MCA_OOB_TCP_MESSAGE_H_ */
217
mca_oob_tcp_msg_t::msg_cbdata
void * msg_cbdata
the data for the callback fnuction
Definition: oob_tcp_msg.h:61
mca_oob_tcp_msg_t::msg_peer
orte_process_name_t msg_peer
the name of the peer
Definition: oob_tcp_msg.h:63
mca_oob_tcp_msg_t::msg_hdr
mca_oob_tcp_hdr_t msg_hdr
header used to convey message properties to peer
Definition: oob_tcp_msg.h:52
mca_oob_tcp_msg_send_handler
bool mca_oob_tcp_msg_send_handler(mca_oob_tcp_msg_t *msg, struct mca_oob_tcp_peer_t *peer)
Called asynchronously to progress sending a message from the event library thread.
Definition: oob_tcp_msg.c:138
oob.h
Contains the internal functions and typedefs for the use of the oob.
opal_condition_t
Definition: condition.h:49
orte_process_name_t
Definition: types.h:146
mca_oob_tcp_msg_t::msg_rwptr
struct iovec * msg_rwptr
current read/write pointer into msg_iov
Definition: oob_tcp_msg.h:56
mca_oob_tcp_msg_t::msg_rwbuf
void * msg_rwbuf
optional buffer for send/recv
Definition: oob_tcp_msg.h:59
opal_mutex_t
Definition: mutex_unix.h:53
mca_oob_tcp_msg_t::msg_rwcnt
int msg_rwcnt
total number of iovecs for read/write
Definition: oob_tcp_msg.h:58
mca_oob_tcp_msg_match_recv
mca_oob_tcp_msg_t * mca_oob_tcp_msg_match_recv(orte_process_name_t *name, int tag)
Match name to a message that has been received asynchronously (unexpected).
Definition: oob_tcp_msg.c:515
mca_oob_tcp_msg_t::msg_lock
opal_mutex_t msg_lock
lock for the condition variable
Definition: oob_tcp_msg.h:64
mca_oob_tcp_hdr_t
Header used by tcp oob protocol.
Definition: oob_tcp_hdr.h:40
mca_oob_tcp_msg_iov_alloc
static struct iovec * mca_oob_tcp_msg_iov_alloc(mca_oob_tcp_msg_t *msg, int count)
Allocate space for iovec array - if the request number of elements is less than MCA_OOB_TCP_IOV_MAX t...
Definition: oob_tcp_msg.h:192
mca_oob_tcp_msg_t::msg_type
mca_oob_tcp_type_t msg_type
posted receive or unexpected
Definition: oob_tcp_msg.h:49
orte_rml_callback_fn_t
void(* orte_rml_callback_fn_t)(int status, struct orte_process_name_t *peer, struct iovec *msg, int count, orte_rml_tag_t tag, void *cbdata)
Funtion prototype for callback from non-blocking iovec send and receive.
Definition: rml.h:123
mca_oob_tcp_msg_match_post
mca_oob_tcp_msg_t * mca_oob_tcp_msg_match_post(orte_process_name_t *name, int tag)
Match name to a posted recv request.
Definition: oob_tcp_msg.c:540
types.h
mca_oob_tcp_msg_t
describes each message being progressed.
Definition: oob_tcp_msg.h:47
opal_free_list_item_t
Definition: opal_free_list.h:47
mca_oob_tcp_msg_wait
int mca_oob_tcp_msg_wait(mca_oob_tcp_msg_t *msg, int *size)
Wait for a msg to complete.
mca_oob_tcp_msg_complete
int mca_oob_tcp_msg_complete(mca_oob_tcp_msg_t *msg, orte_process_name_t *peer)
Signal that a message has completed.
Definition: oob_tcp_msg.c:77
mca_oob_tcp_msg_t::msg_rc
int msg_rc
the return code for the send/recv (amount sent/recvd or errno)
Definition: oob_tcp_msg.h:51
mca_oob_tcp_msg_iov_return
static void mca_oob_tcp_msg_iov_return(mca_oob_tcp_msg_t *msg, struct iovec *iov)
Release resource held by iovec array if this is not part of the message.
Definition: oob_tcp_msg.h:208
mca_oob_tcp_msg_recv_handler
bool mca_oob_tcp_msg_recv_handler(mca_oob_tcp_msg_t *msg, struct mca_oob_tcp_peer_t *peer)
Called asynchronously to progress sending a message from the event library thread.
Definition: oob_tcp_msg.c:195
mca_oob_tcp_msg_t::msg_rwiov
struct iovec * msg_rwiov
copy of iovec array - not data
Definition: oob_tcp_msg.h:55
mca_oob_tcp_msg_timedwait
int mca_oob_tcp_msg_timedwait(mca_oob_tcp_msg_t *msg, int *size, struct timespec *ts)
Wait - up to a timeout - for a msg to complete.
oob_tcp_hdr.h
Contains header used by tcp oob.
iovec
Definition: ompi_uio.h:29
mca_oob_tcp_msg_t::msg_uiov
struct iovec * msg_uiov
the user supplied iovec array
Definition: oob_tcp_msg.h:53
mca_oob_tcp_msg_t::msg_flags
int msg_flags
flags to send/recv
Definition: oob_tcp_msg.h:50
mca_oob_tcp_peer_t
This structure describes a peer.
Definition: oob_tcp_peer.h:59
mca_oob_tcp_msg_t::msg_ucnt
int msg_ucnt
the number of items in the user iovec array
Definition: oob_tcp_msg.h:54
mca_oob_tcp_msg_t::msg_complete
bool msg_complete
whether the message is done sending or not
Definition: oob_tcp_msg.h:62
mca_oob_tcp_msg_t::super
opal_free_list_item_t super
allow this item to be put on a list
Definition: oob_tcp_msg.h:48
timespec
Definition: ompi_time.h:160
mca_oob_tcp_msg_t::msg_rwnum
int msg_rwnum
number of iovecs left for read/write
Definition: oob_tcp_msg.h:57
mca_oob_tcp_msg_t::msg_condition
opal_condition_t msg_condition
condition variable for completion
Definition: oob_tcp_msg.h:65
mca_oob_tcp_msg_t::msg_cbfunc
orte_rml_callback_fn_t msg_cbfunc
the callback function for the send/receive
Definition: oob_tcp_msg.h:60
mca_oob_tcp_msg_copy
int mca_oob_tcp_msg_copy(mca_oob_tcp_msg_t *msg, struct iovec *iov, int count)
Called to copy the results of a message into user supplied iovec array.
Definition: oob_tcp_msg.c:470
OBJ_CLASS_DECLARATION
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
mca_oob_tcp_msg_recv_complete
void mca_oob_tcp_msg_recv_complete(mca_oob_tcp_msg_t *msg, struct mca_oob_tcp_peer_t *peer)
The message has been completely received - so attempt to match against posted recvs.
Definition: oob_tcp_msg.c:322