OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

oob_tcp.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2006 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2006-2007 Los Alamos National Security, LLC.
13 * All rights reserved.
14 * Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
15 * $COPYRIGHT$
16 *
17 * Additional copyrights may follow
18 *
19 * $HEADER$
20 */
21 /** @file:
22 *
23 * Defines the functions for the tcp module.
24 */
25
26 #ifndef _MCA_OOB_TCP_H_
27 #define _MCA_OOB_TCP_H_
28
29 #include "orte_config.h"
30
31 #ifdef HAVE_SYS_TYPES_H
32 #include <sys/types.h>
33 #endif
34 #ifdef HAVE_SYS_SOCKET_H
35 #include <sys/socket.h>
36 #endif
37
38 #include "orte/types.h"
39
40 #include "opal/mca/base/base.h"
41 #include "opal/class/opal_free_list.h"
42 #include "opal/class/opal_hash_table.h"
43 #include "opal/threads/mutex.h"
44 #include "opal/threads/condition.h"
45 #include "opal/threads/threads.h"
46 #include "opal/mca/timer/base/base.h"
47
48 #include "orte/mca/oob/oob.h"
49 #include "orte/mca/oob/base/base.h"
50 #include "orte/mca/oob/tcp/oob_tcp_peer.h"
51 #include "orte/mca/oob/tcp/oob_tcp_msg.h"
52
53
54 BEGIN_C_DECLS
55
56
57 #define OOB_TCP_DEBUG_CONNECT_FAIL 1 /* debug connection establishment failures */
58 #define OOB_TCP_DEBUG_CONNECT 2 /* other connection information */
59 #define OOB_TCP_DEBUG_INFO 3 /* information about startup, connection establish, etc. */
60 #define OOB_TCP_DEBUG_ALL 4 /* everything else */
61
62 extern mca_oob_t mca_oob_tcp;
63
64 /*
65 * standard component functions
66 */
67 mca_oob_t* mca_oob_tcp_component_init(int* priority);
68
69 /**
70 * Hook function to allow the selected oob components
71 * to register their contact info with the registry
72 */
73
74 int mca_oob_tcp_init(void);
75
76 /**
77 * Cleanup resources during shutdown.
78 */
79 int mca_oob_tcp_fini(void);
80
81 /**
82 * Compare two process names for equality.
83 *
84 * @param n1 Process name 1.
85 * @param n2 Process name 2.
86 * @return (-1 for n1<n2 0 for equality, 1 for n1>n2)
87 *
88 * Note that the definition of < or > is somewhat arbitrary -
89 * just needs to be consistently applied to maintain an ordering
90 * when process names are used as indices.
91 */
92 int mca_oob_tcp_process_name_compare(const orte_process_name_t* n1, const orte_process_name_t* n2);
93
94 /**
95 * Obtain contact information for this host (e.g. <ipaddress>:<port>)
96 */
97
98 char* mca_oob_tcp_get_addr(void);
99
100 /**
101 * Setup cached addresses for the peers.
102 */
103
104 int mca_oob_tcp_set_addr(const orte_process_name_t*, const char*);
105
106 int mca_oob_tcp_get_new_name(orte_process_name_t* name);
107
108 /**
109 * A routine to ping a given process name to determine if it is reachable.
110 *
111 * @param name The peer name.
112 * @param tv The length of time to wait on a connection/response.
113 *
114 * Note that this routine blocks up to the specified timeout waiting for a
115 * connection / response from the specified peer. If the peer is unavailable
116 * an error status is returned.
117 */
118
119 int mca_oob_tcp_ping(const orte_process_name_t*, const char* uri, const struct timeval* tv);
120
121
122 /*
123 * Non-blocking versions of send/recv.
124 */
125
126 int mca_oob_tcp_send_nb(orte_process_name_t* target,
127 orte_process_name_t* origin,
128 struct iovec* msg,
129 int count,
130 int tag,
131 int flags,
132 orte_rml_callback_fn_t cbfunc,
133 void* cbdata);
134
135 /**
136 * Non-blocking version of mca_oob_recv().
137 *
138 * @param peer (IN) Opaque name of peer process or ORTE_NAME_WILDCARD for wildcard receive.
139 * @param msg (IN) Array of iovecs describing user buffers and lengths.
140 * @param count (IN) Number of elements in iovec array.
141 * @param tag (IN) User defined tag for matching send/recv.
142 * @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue,
143 * @param cbfunc (IN) Callback function on recv completion.
144 * @param cbdata (IN) User data that is passed to callback function.
145 * @return OMPI error code (<0) on error or number of bytes actually received.
146 */
147
148 int mca_oob_tcp_recv_nb(
149 orte_process_name_t* peer,
150 struct iovec* msg,
151 int count,
152 int tag,
153 int flags,
154 orte_rml_callback_fn_t cbfunc,
155 void* cbdata);
156
157 /**
158 * Cancel non-blocking receive.
159 *
160 * @param peer (IN) Opaque name of peer process or ORTE_NAME_WILDCARD for wildcard receive.
161 * @param tag (IN) User defined tag for matching send/recv.
162 * @return OMPI error code (<0) on error or number of bytes actually received.
163 */
164
165 int mca_oob_tcp_recv_cancel(
166 orte_process_name_t* peer,
167 int tag);
168
169 /**
170 * Attempt to map a peer name to its corresponding address.
171 */
172
173 int mca_oob_tcp_resolve(mca_oob_tcp_peer_t*);
174
175 /**
176 * Parse a URI string into an IP address and port number.
177 */
178 int mca_oob_tcp_parse_uri(
179 const char* uri,
180 struct sockaddr* inaddr
181 );
182
183 /**
184 * Setup socket options
185 */
186
187 void mca_oob_tcp_set_socket_options(int sd);
188
189 int mca_oob_tcp_ft_event(int state);
190
191 typedef enum { OOB_TCP_EVENT, OOB_TCP_LISTEN_THREAD } mca_oob_tcp_listen_type_t;
192
193 /**
194 * OOB TCP Component
195 */
196 struct mca_oob_tcp_component_t {
197 mca_oob_base_component_2_0_0_t super; /**< base OOB component */
198 char* tcp_include; /**< list of ip interfaces to include */
199 char* tcp_exclude; /**< list of ip interfaces to exclude */
200 opal_list_t tcp_peer_list; /**< list of peers sorted in mru order */
201 opal_hash_table_t tcp_peers; /**< peers sorted by name */
202 opal_hash_table_t tcp_peer_names; /**< cache of peer contact info sorted by name */
203 opal_free_list_t tcp_peer_free; /**< free list of peers */
204 int tcp_peer_limit; /**< max size of tcp peer cache */
205 int tcp_peer_retries; /**< max number of retries before declaring peer gone */
206 int tcp_sndbuf; /**< socket send buffer size */
207 int tcp_rcvbuf; /**< socket recv buffer size */
208 opal_free_list_t tcp_msgs; /**< free list of messages */
209 opal_event_t tcp_recv_event; /**< event structure for IPv4 recvs */
210 int tcp_listen_sd; /**< listen socket for incoming IPv4 connection requests */
211 unsigned short tcp_listen_port; /**< IPv4 listen port */
212 char** tcp4_static_ports; /**< Static ports - IPV4 */
213 char** tcp4_dyn_ports; /**< Dynamic ports - IPV4 */
214 int disable_family; /**< disable AF: 0-nothing, 4-IPv4, 6-IPv6 */
215 #if OPAL_WANT_IPV6
216 opal_event_t tcp6_recv_event; /**< event structure for IPv6 recvs */
217 int tcp6_listen_sd; /**< listen socket for incoming IPv6 connection requests */
218 unsigned short tcp6_listen_port; /**< IPv6 listen port */
219 char** tcp6_static_ports; /**< Static ports - IPV6 */
220 char** tcp6_dyn_ports; /**< Dynamic ports - IPV6 */
221 #endif /* OPAL_WANT_IPV6 */
222 opal_mutex_t tcp_lock; /**< lock for accessing module state */
223 opal_list_t tcp_events; /**< list of pending events (accepts) */
224 opal_list_t tcp_msg_post; /**< list of recieves user has posted */
225 opal_list_t tcp_msg_recv; /**< list of recieved messages */
226 opal_list_t tcp_msg_completed; /**< list of completed messages */
227 opal_mutex_t tcp_match_lock; /**< lock held while searching/posting messages */
228 opal_condition_t tcp_match_cond; /**< condition variable used in finalize */
229 int tcp_match_count; /**< number of matched recvs in progress */
230 int tcp_debug; /**< debug level */
231
232 bool tcp_shutdown;
233 mca_oob_tcp_listen_type_t tcp_listen_type;
234
235 opal_list_t tcp_available_devices;
236
237 opal_thread_t tcp_listen_thread; /** handle to the listening thread */
238 opal_list_t tcp_pending_connections; /**< List of accepted connections waiting for processing */
239 opal_list_t tcp_connections_return; /**< List of connection fragments being returned to accept thread */
240 opal_mutex_t tcp_connections_lock; /**< Lock protecting pending_connections and connections_return */
241 int tcp_connections_pipe[2];
242 opal_event_t tcp_listen_thread_event;
243
244 int tcp_copy_max_size; /**< Max size of the copy list before copying must commence */
245 int tcp_listen_thread_num_sockets; /**< Number of sockets in tcp_listen_thread_sds */
246 int tcp_listen_thread_sds[2]; /**< Room for IPv4 and IPv6. Might need to make more dynamic. */
247 struct timeval tcp_listen_thread_tv; /**< Timeout when using listen thread */
248
249 int connect_sleep;
250 };
251
252 /**
253 * Convenience Typedef
254 */
255 typedef struct mca_oob_tcp_component_t mca_oob_tcp_component_t;
256
257 ORTE_MODULE_DECLSPEC extern mca_oob_tcp_component_t mca_oob_tcp_component;
258
259 extern int mca_oob_tcp_output_handle;
260
261 #if defined(__WINDOWS__)
262 #define CLOSE_THE_SOCKET(socket) closesocket(socket)
263 #else
264
265 #define CLOSE_THE_SOCKET(socket) \
266 do { \
267 shutdown(socket, 2); \
268 close(socket); \
269 } while(0)
270
271 #endif /* defined(__WINDOWS__) */
272
273
274 struct mca_oob_tcp_pending_connection_t {
275 opal_free_list_item_t super;
276 int fd;
277 struct sockaddr_storage addr;
278 };
279 typedef struct mca_oob_tcp_pending_connection_t mca_oob_tcp_pending_connection_t;
280 OBJ_CLASS_DECLARATION(mca_oob_tcp_pending_connection_t);
281
282 struct mca_oob_tcp_device_t {
283 opal_list_item_t super;
284 int if_index;
285 bool if_local;
286 struct sockaddr_storage if_addr;
287 };
288 typedef struct mca_oob_tcp_device_t mca_oob_tcp_device_t;
289 OBJ_CLASS_DECLARATION(mca_oob_tcp_device_t);
290
291
292 END_C_DECLS
293
294 #endif /* MCA_OOB_TCP_H_ */
295
mca_oob_tcp_component_t::tcp_peer_names
opal_hash_table_t tcp_peer_names
cache of peer contact info sorted by name
Definition: oob_tcp.h:202
mca_oob_tcp_component_t::tcp_msg_completed
opal_list_t tcp_msg_completed
list of completed messages
Definition: oob_tcp.h:226
opal_hash_table_t
Definition: opal_hash_table.h:42
mca_oob_tcp_set_socket_options
void mca_oob_tcp_set_socket_options(int sd)
Setup socket options.
Definition: oob_tcp_peer.c:1189
mca_oob_tcp_get_addr
char * mca_oob_tcp_get_addr(void)
Obtain contact information for this host (e.g.
Definition: oob_tcp.c:1934
mca_oob_tcp_component_t::tcp_match_lock
opal_mutex_t tcp_match_lock
lock held while searching/posting messages
Definition: oob_tcp.h:227
mca_oob_tcp_component_t::tcp_rcvbuf
int tcp_rcvbuf
socket recv buffer size
Definition: oob_tcp.h:207
mca_oob_tcp_component_t::tcp_connections_return
opal_list_t tcp_connections_return
List of connection fragments being returned to accept thread.
Definition: oob_tcp.h:239
mca_oob_tcp_component_t::tcp_match_cond
opal_condition_t tcp_match_cond
condition variable used in finalize
Definition: oob_tcp.h:228
oob.h
Contains the internal functions and typedefs for the use of the oob.
mca_oob_tcp_fini
int mca_oob_tcp_fini(void)
Cleanup resources during shutdown.
Definition: oob_tcp.c:1847
opal_condition_t
Definition: condition.h:49
mca_oob_tcp_component_t::tcp_peers
opal_hash_table_t tcp_peers
peers sorted by name
Definition: oob_tcp.h:201
mca_oob_tcp_component_t::tcp_recv_event
opal_event_t tcp_recv_event
event structure for IPv4 recvs
Definition: oob_tcp.h:209
orte_process_name_t
Definition: types.h:146
mca_oob_tcp_component_t::tcp_msg_post
opal_list_t tcp_msg_post
list of recieves user has posted
Definition: oob_tcp.h:224
mca_oob_tcp_component_t::tcp_msg_recv
opal_list_t tcp_msg_recv
list of recieved messages
Definition: oob_tcp.h:225
mca_oob_tcp_component_t::tcp_listen_port
unsigned short tcp_listen_port
IPv4 listen port.
Definition: oob_tcp.h:211
base.h
the oob framework
event
Structure to represent a single event.
Definition: event_struct.h:87
mca_oob_tcp_component_t::tcp_listen_thread_tv
struct timeval tcp_listen_thread_tv
Timeout when using listen thread.
Definition: oob_tcp.h:247
oob_tcp_peer.h
Contains the data structure which describes each connection.
opal_mutex_t
Definition: mutex_unix.h:53
mca_oob_tcp_component_t::disable_family
int disable_family
disable AF: 0-nothing, 4-IPv4, 6-IPv6
Definition: oob_tcp.h:214
mca_oob_tcp_component_t::tcp_msgs
opal_free_list_t tcp_msgs
free list of messages
Definition: oob_tcp.h:208
mca_oob_tcp_component_t::tcp_peer_limit
int tcp_peer_limit
max size of tcp peer cache
Definition: oob_tcp.h:204
mca_oob_tcp_recv_cancel
int mca_oob_tcp_recv_cancel(orte_process_name_t *peer, int tag)
Cancel non-blocking receive.
Definition: oob_tcp_recv.c:173
mca_oob_tcp_component_t::tcp_match_count
int tcp_match_count
number of matched recvs in progress
Definition: oob_tcp.h:229
mca_oob_tcp_component_t::tcp4_dyn_ports
char ** tcp4_dyn_ports
Dynamic ports - IPV4.
Definition: oob_tcp.h:213
mca_oob_tcp_component_t::tcp_listen_thread_num_sockets
int tcp_listen_thread_num_sockets
Number of sockets in tcp_listen_thread_sds.
Definition: oob_tcp.h:245
mca_oob_tcp_component_t::tcp_peer_list
opal_list_t tcp_peer_list
list of peers sorted in mru order
Definition: oob_tcp.h:200
opal_list_item_t
Definition: opal_list.h:98
mca_oob_tcp_set_addr
int mca_oob_tcp_set_addr(const orte_process_name_t *, const char *)
Setup cached addresses for the peers.
Definition: oob_tcp.c:2084
mca_oob_tcp_process_name_compare
int mca_oob_tcp_process_name_compare(const orte_process_name_t *n1, const orte_process_name_t *n2)
Compare two process names for equality.
Definition: oob_tcp.c:1924
orte_rml_callback_fn_t
void(* orte_rml_callback_fn_t)(int status, struct orte_process_name_t *peer, struct iovec *msg, int count, orte_rml_tag_t tag, void *cbdata)
Funtion prototype for callback from non-blocking iovec send and receive.
Definition: rml.h:123
types.h
mca_oob_tcp_parse_uri
int mca_oob_tcp_parse_uri(const char *uri, struct sockaddr *inaddr)
Parse a URI string into an IP address and port number.
Definition: oob_tcp.c:1971
opal_free_list_item_t
Definition: opal_free_list.h:47
mca_oob_tcp_component_t::tcp_peer_free
opal_free_list_t tcp_peer_free
free list of peers
Definition: oob_tcp.h:203
oob_tcp_msg.h
contains the data structure we will use to describe a message
mca_oob_tcp_recv_nb
int mca_oob_tcp_recv_nb(orte_process_name_t *peer, struct iovec *msg, int count, int tag, int flags, orte_rml_callback_fn_t cbfunc, void *cbdata)
Non-blocking version of mca_oob_recv().
Definition: oob_tcp_recv.c:92
opal_free_list_t
Definition: opal_free_list.h:31
mca_oob_tcp_component_t::tcp_sndbuf
int tcp_sndbuf
socket send buffer size
Definition: oob_tcp.h:206
mca_oob_tcp_ping
int mca_oob_tcp_ping(const orte_process_name_t *, const char *uri, const struct timeval *tv)
A routine to ping a given process name to determine if it is reachable.
Definition: oob_tcp_ping.c:79
mca_oob_tcp_component_t::tcp_exclude
char * tcp_exclude
list of ip interfaces to exclude
Definition: oob_tcp.h:199
mca_oob_tcp_pending_connection_t
Definition: oob_tcp.h:274
mca_oob_tcp_component_t::tcp4_static_ports
char ** tcp4_static_ports
Static ports - IPV4.
Definition: oob_tcp.h:212
sockaddr_storage
Definition: util-internal.h:214
opal_hash_table.h
A hash table that may be indexed with either fixed length (e.g.
mca_oob_tcp_component_t::tcp_listen_sd
int tcp_listen_sd
listen socket for incoming IPv4 connection requests
Definition: oob_tcp.h:210
mca_oob_base_component_2_0_0_t
Definition: oob.h:51
mca_oob_tcp_component_t::tcp_copy_max_size
int tcp_copy_max_size
Max size of the copy list before copying must commence.
Definition: oob_tcp.h:244
iovec
Definition: ompi_uio.h:29
mca_oob_tcp_resolve
int mca_oob_tcp_resolve(mca_oob_tcp_peer_t *)
Attempt to map a peer name to its corresponding address.
Definition: oob_tcp.c:1586
mca_oob_tcp_component_t::tcp_peer_retries
int tcp_peer_retries
max number of retries before declaring peer gone
Definition: oob_tcp.h:205
mca_oob_tcp_component_t::tcp_events
opal_list_t tcp_events
list of pending events (accepts)
Definition: oob_tcp.h:223
mca_oob_tcp_peer_t
This structure describes a peer.
Definition: oob_tcp_peer.h:59
opal_list_t
Definition: opal_list.h:147
mca_oob_tcp_component_t
OOB TCP Component.
Definition: oob_tcp.h:196
mca_oob_1_0_0_t
OOB Module.
Definition: oob.h:161
mca_oob_tcp_component_t::tcp_lock
opal_mutex_t tcp_lock
lock for accessing module state
Definition: oob_tcp.h:222
mca_oob_tcp_component_t::tcp_connections_lock
opal_mutex_t tcp_connections_lock
Lock protecting pending_connections and connections_return.
Definition: oob_tcp.h:240
opal_thread_t
Definition: threads.h:46
mca_oob_tcp_component_t::super
mca_oob_base_component_2_0_0_t super
base OOB component
Definition: oob_tcp.h:197
mca_oob_tcp_device_t
Definition: oob_tcp.h:282
mca_oob_tcp_component_t::tcp_pending_connections
opal_list_t tcp_pending_connections
handle to the listening thread
Definition: oob_tcp.h:238
mca_oob_tcp_init
int mca_oob_tcp_init(void)
Hook function to allow the selected oob components to register their contact info with the registry...
Definition: oob_tcp.c:1719
mca_oob_tcp_component_t::tcp_listen_thread_sds
int tcp_listen_thread_sds[2]
Room for IPv4 and IPv6.
Definition: oob_tcp.h:246
mutex.h
Mutual exclusion functions.
mca_oob_tcp_component_t::tcp_debug
int tcp_debug
debug level
Definition: oob_tcp.h:230
mca_oob_tcp_component_t::tcp_include
char * tcp_include
list of ip interfaces to include
Definition: oob_tcp.h:198
OBJ_CLASS_DECLARATION
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236