OpenMPI  0.1.1
oob_tcp_msg.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2005 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * $COPYRIGHT$
13  *
14  * Additional copyrights may follow
15  *
16  * $HEADER$
17  */
18 /** @file:
19  *
20  * contains the data structure we will use to describe a message
21  */
22 
23 #ifndef _MCA_OOB_TCP_MESSAGE_H_
24 #define _MCA_OOB_TCP_MESSAGE_H_
25 
26 #include "orte_config.h"
27 #include "orte/types.h"
28 
29 #include <errno.h>
30 
31 
32 #include "orte/mca/oob/oob.h"
33 #include "oob_tcp_hdr.h"
34 
35 BEGIN_C_DECLS
36 
37 struct mca_oob_tcp_peer_t;
38 
39 #define MCA_OOB_TCP_IOV_MAX 16
40 
41 typedef enum { MCA_OOB_TCP_POSTED, MCA_OOB_TCP_UNEXPECTED } mca_oob_tcp_type_t;
42 
43 
44 /**
45  * describes each message being progressed.
46  */
48  opal_free_list_item_t super; /**< allow this item to be put on a list */
49  mca_oob_tcp_type_t msg_type; /**< posted receive or unexpected */
50  int msg_flags; /**< flags to send/recv */
51  int msg_rc; /**< the return code for the send/recv (amount sent/recvd or errno) */
52  mca_oob_tcp_hdr_t msg_hdr; /**< header used to convey message properties to peer */
53  struct iovec* msg_uiov; /**< the user supplied iovec array */
54  int msg_ucnt; /**< the number of items in the user iovec array */
55  struct iovec * msg_rwiov; /**< copy of iovec array - not data */
56  struct iovec * msg_rwptr; /**< current read/write pointer into msg_iov */
57  int msg_rwnum; /**< number of iovecs left for read/write */
58  int msg_rwcnt; /**< total number of iovecs for read/write */
59  void* msg_rwbuf; /**< optional buffer for send/recv */
60  orte_rml_callback_fn_t msg_cbfunc; /**< the callback function for the send/receive */
61  void * msg_cbdata; /**< the data for the callback fnuction */
62  bool msg_complete; /**< whether the message is done sending or not */
63  orte_process_name_t msg_peer; /**< the name of the peer */
64  opal_mutex_t msg_lock; /**< lock for the condition variable */
65  opal_condition_t msg_condition; /**< condition variable for completion */
66  struct iovec msg_iov[MCA_OOB_TCP_IOV_MAX]; /** preallocate space for iovec array */
67 };
68 
69 /**
70  * Convenience typedef
71  */
73 
75 
76 /**
77  * Get a new structure for use with a message
78  */
79 #define MCA_OOB_TCP_MSG_ALLOC(msg, rc) \
80 { \
81  opal_free_list_item_t* item; \
82  OPAL_FREE_LIST_GET(&mca_oob_tcp_component.tcp_msgs, item, rc); \
83  msg = (mca_oob_tcp_msg_t*)item; \
84 }
85 
86 /**
87  * return a message structure that is no longer needed
88  */
89 #define MCA_OOB_TCP_MSG_RETURN(msg) \
90 { \
91  /* frees the iovec allocated during the send/receive */ \
92  if(NULL != msg->msg_rwiov) \
93  mca_oob_tcp_msg_iov_return(msg,msg->msg_rwiov); \
94  if(NULL != msg->msg_rwbuf) \
95  free(msg->msg_rwbuf); \
96  OPAL_FREE_LIST_RETURN(&mca_oob_tcp_component.tcp_msgs, \
97  &msg->super); \
98 }
99 
100 /**
101  * Wait for a msg to complete.
102  * @param msg (IN) Message to wait on.
103  * @param size (OUT) Number of bytes delivered.
104  * @retval ORTE_SUCCESS or error code on failure.
105  */
106 int mca_oob_tcp_msg_wait(mca_oob_tcp_msg_t* msg, int* size);
107 
108 /**
109  * Wait - up to a timeout - for a msg to complete.
110  * @param msg (IN) Message to wait on.
111  * @param size (OUT) Number of bytes delivered.
112  * @retval ORTE_SUCCESS or error code on failure.
113  */
114 int mca_oob_tcp_msg_timedwait(mca_oob_tcp_msg_t* msg, int* size, struct timespec* ts);
115 
116 /**
117  * Signal that a message has completed. Wakes up any pending threads (for blocking send)
118  * or invokes callbacks for non-blocking case.
119  * @param msg (IN) Message send/recv that has completed.
120  * @param peer (IN) The peer the send/receive was from
121  * @retval ORTE_SUCCESS or error code on failure.
122  */
124 
125 /**
126  * Called to copy the results of a message into user supplied iovec array.
127  * @param msg (IN) Message send that is in progress.
128  * @param iov (IN) Iovec array of user supplied buffers.
129  * @retval count Number of elements in iovec array.
130  */
131 
132 int mca_oob_tcp_msg_copy(mca_oob_tcp_msg_t* msg, struct iovec* iov, int count);
133 
134 /**
135  * Called asynchronously to progress sending a message from the event library thread.
136  * @param msg (IN) Message send that is in progress.
137  * @param peer (IN) Peer we are sending to.
138  * @retval Number of bytes copied.
139  */
141 
142 /**
143  * Called asynchronously to progress sending a message from the event library thread.
144  * @param msg (IN) Message send that is in progress.
145  * @param peer (IN) Peer theat we are recieving from.
146  * @retval bool Bool flag indicating wether operation has completed.
147  */
148 
150 
151 /**
152  * The message has been completely received - so attempt to match
153  * against posted recvs.
154  */
155 
157 
158 /**
159  * Match name to a message that has been received asynchronously (unexpected).
160  *
161  * @param name (IN) Name associated with peer or wildcard to match first posted recv.
162  * @param tag (IN) Message tag.
163  * @return msg Matched message or NULL.
164  *
165  * Note - this routine requires the caller to be holding the module lock.
166  */
167 
169 
170 /**
171  * Match name to a posted recv request.
172  *
173  * @param name (IN) Name associated with peer or wildcard to match first posted recv.
174  * @param tag (IN) Message tag.
175  * @return msg Matched message or NULL.
176  *
177  * Note - this routine requires the caller to be holding the module lock.
178  */
179 
181 
182 /**
183  * Allocate space for iovec array - if the request number of elements is less than
184  * MCA_OOB_TCP_IOV_MAX then use the array allocated along w/ the message - otherwise
185  * allocate count elements.
186  *
187  * @param msg (IN) Message to allocate array.
188  * @param count (IN) the number of iovecs
189  * @return Array of iovec elements.
190  *
191  */
192 static inline struct iovec* mca_oob_tcp_msg_iov_alloc(mca_oob_tcp_msg_t* msg, int count)
193 {
194  if(count <= MCA_OOB_TCP_IOV_MAX)
195  return msg->msg_iov;
196  return (struct iovec *)malloc(sizeof(struct iovec) * count);
197 }
198 
199 
200 /**
201  * Release resource held by iovec array if this is not part of the message.
202  *
203  * @param msg (IN) Message to allocate array.
204  * @param iov (IN) Iovec array to return.
205  *
206  */
207 
208 static inline void mca_oob_tcp_msg_iov_return(mca_oob_tcp_msg_t* msg, struct iovec* iov)
209 {
210  if(iov != msg->msg_iov)
211  free(iov);
212 }
213 
214 END_C_DECLS
215 
216 #endif /* _MCA_OOB_TCP_MESSAGE_H_ */
217 
void * msg_cbdata
the data for the callback fnuction
Definition: oob_tcp_msg.h:61
orte_process_name_t msg_peer
the name of the peer
Definition: oob_tcp_msg.h:63
mca_oob_tcp_hdr_t msg_hdr
header used to convey message properties to peer
Definition: oob_tcp_msg.h:52
bool mca_oob_tcp_msg_send_handler(mca_oob_tcp_msg_t *msg, struct mca_oob_tcp_peer_t *peer)
Called asynchronously to progress sending a message from the event library thread.
Definition: oob_tcp_msg.c:138
Contains the internal functions and typedefs for the use of the oob.
Definition: condition.h:49
Definition: types.h:146
struct iovec * msg_rwptr
current read/write pointer into msg_iov
Definition: oob_tcp_msg.h:56
void * msg_rwbuf
optional buffer for send/recv
Definition: oob_tcp_msg.h:59
Definition: mutex_unix.h:53
int msg_rwcnt
total number of iovecs for read/write
Definition: oob_tcp_msg.h:58
mca_oob_tcp_msg_t * mca_oob_tcp_msg_match_recv(orte_process_name_t *name, int tag)
Match name to a message that has been received asynchronously (unexpected).
Definition: oob_tcp_msg.c:515
opal_mutex_t msg_lock
lock for the condition variable
Definition: oob_tcp_msg.h:64
Header used by tcp oob protocol.
Definition: oob_tcp_hdr.h:40
static struct iovec * mca_oob_tcp_msg_iov_alloc(mca_oob_tcp_msg_t *msg, int count)
Allocate space for iovec array - if the request number of elements is less than MCA_OOB_TCP_IOV_MAX t...
Definition: oob_tcp_msg.h:192
mca_oob_tcp_type_t msg_type
posted receive or unexpected
Definition: oob_tcp_msg.h:49
void(* orte_rml_callback_fn_t)(int status, struct orte_process_name_t *peer, struct iovec *msg, int count, orte_rml_tag_t tag, void *cbdata)
Funtion prototype for callback from non-blocking iovec send and receive.
Definition: rml.h:123
mca_oob_tcp_msg_t * mca_oob_tcp_msg_match_post(orte_process_name_t *name, int tag)
Match name to a posted recv request.
Definition: oob_tcp_msg.c:540
describes each message being progressed.
Definition: oob_tcp_msg.h:47
Definition: opal_free_list.h:47
int mca_oob_tcp_msg_wait(mca_oob_tcp_msg_t *msg, int *size)
Wait for a msg to complete.
int mca_oob_tcp_msg_complete(mca_oob_tcp_msg_t *msg, orte_process_name_t *peer)
Signal that a message has completed.
Definition: oob_tcp_msg.c:77
int msg_rc
the return code for the send/recv (amount sent/recvd or errno)
Definition: oob_tcp_msg.h:51
static void mca_oob_tcp_msg_iov_return(mca_oob_tcp_msg_t *msg, struct iovec *iov)
Release resource held by iovec array if this is not part of the message.
Definition: oob_tcp_msg.h:208
bool mca_oob_tcp_msg_recv_handler(mca_oob_tcp_msg_t *msg, struct mca_oob_tcp_peer_t *peer)
Called asynchronously to progress sending a message from the event library thread.
Definition: oob_tcp_msg.c:195
struct iovec * msg_rwiov
copy of iovec array - not data
Definition: oob_tcp_msg.h:55
int mca_oob_tcp_msg_timedwait(mca_oob_tcp_msg_t *msg, int *size, struct timespec *ts)
Wait - up to a timeout - for a msg to complete.
Contains header used by tcp oob.
Definition: ompi_uio.h:29
struct iovec * msg_uiov
the user supplied iovec array
Definition: oob_tcp_msg.h:53
int msg_flags
flags to send/recv
Definition: oob_tcp_msg.h:50
This structure describes a peer.
Definition: oob_tcp_peer.h:59
int msg_ucnt
the number of items in the user iovec array
Definition: oob_tcp_msg.h:54
bool msg_complete
whether the message is done sending or not
Definition: oob_tcp_msg.h:62
opal_free_list_item_t super
allow this item to be put on a list
Definition: oob_tcp_msg.h:48
Definition: ompi_time.h:160
int msg_rwnum
number of iovecs left for read/write
Definition: oob_tcp_msg.h:57
opal_condition_t msg_condition
condition variable for completion
Definition: oob_tcp_msg.h:65
orte_rml_callback_fn_t msg_cbfunc
the callback function for the send/receive
Definition: oob_tcp_msg.h:60
int mca_oob_tcp_msg_copy(mca_oob_tcp_msg_t *msg, struct iovec *iov, int count)
Called to copy the results of a message into user supplied iovec array.
Definition: oob_tcp_msg.c:470
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
void mca_oob_tcp_msg_recv_complete(mca_oob_tcp_msg_t *msg, struct mca_oob_tcp_peer_t *peer)
The message has been completely received - so attempt to match against posted recvs.
Definition: oob_tcp_msg.c:322