OpenMPI  0.1.1
oob_tcp.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2006 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2006-2007 Los Alamos National Security, LLC.
13  * All rights reserved.
14  * Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
15  * $COPYRIGHT$
16  *
17  * Additional copyrights may follow
18  *
19  * $HEADER$
20  */
21 /** @file:
22  *
23  * Defines the functions for the tcp module.
24  */
25 
26 #ifndef _MCA_OOB_TCP_H_
27 #define _MCA_OOB_TCP_H_
28 
29 #include "orte_config.h"
30 
31 #ifdef HAVE_SYS_TYPES_H
32 #include <sys/types.h>
33 #endif
34 #ifdef HAVE_SYS_SOCKET_H
35 #include <sys/socket.h>
36 #endif
37 
38 #include "orte/types.h"
39 
40 #include "opal/mca/base/base.h"
41 #include "opal/class/opal_free_list.h"
43 #include "opal/threads/mutex.h"
44 #include "opal/threads/condition.h"
45 #include "opal/threads/threads.h"
46 #include "opal/mca/timer/base/base.h"
47 
48 #include "orte/mca/oob/oob.h"
49 #include "orte/mca/oob/base/base.h"
52 
53 
54 BEGIN_C_DECLS
55 
56 
57 #define OOB_TCP_DEBUG_CONNECT_FAIL 1 /* debug connection establishment failures */
58 #define OOB_TCP_DEBUG_CONNECT 2 /* other connection information */
59 #define OOB_TCP_DEBUG_INFO 3 /* information about startup, connection establish, etc. */
60 #define OOB_TCP_DEBUG_ALL 4 /* everything else */
61 
62 extern mca_oob_t mca_oob_tcp;
63 
64 /*
65  * standard component functions
66  */
67 mca_oob_t* mca_oob_tcp_component_init(int* priority);
68 
69 /**
70  * Hook function to allow the selected oob components
71  * to register their contact info with the registry
72 */
73 
74 int mca_oob_tcp_init(void);
75 
76 /**
77  * Cleanup resources during shutdown.
78  */
79 int mca_oob_tcp_fini(void);
80 
81 /**
82 * Compare two process names for equality.
83 *
84 * @param n1 Process name 1.
85 * @param n2 Process name 2.
86 * @return (-1 for n1<n2 0 for equality, 1 for n1>n2)
87 *
88 * Note that the definition of < or > is somewhat arbitrary -
89 * just needs to be consistently applied to maintain an ordering
90 * when process names are used as indices.
91 */
93 
94 /**
95  * Obtain contact information for this host (e.g. <ipaddress>:<port>)
96  */
97 
98 char* mca_oob_tcp_get_addr(void);
99 
100 /**
101  * Setup cached addresses for the peers.
102  */
103 
104 int mca_oob_tcp_set_addr(const orte_process_name_t*, const char*);
105 
106 int mca_oob_tcp_get_new_name(orte_process_name_t* name);
107 
108 /**
109  * A routine to ping a given process name to determine if it is reachable.
110  *
111  * @param name The peer name.
112  * @param tv The length of time to wait on a connection/response.
113  *
114  * Note that this routine blocks up to the specified timeout waiting for a
115  * connection / response from the specified peer. If the peer is unavailable
116  * an error status is returned.
117  */
118 
119 int mca_oob_tcp_ping(const orte_process_name_t*, const char* uri, const struct timeval* tv);
120 
121 
122 /*
123  * Non-blocking versions of send/recv.
124  */
125 
126 int mca_oob_tcp_send_nb(orte_process_name_t* target,
127  orte_process_name_t* origin,
128  struct iovec* msg,
129  int count,
130  int tag,
131  int flags,
132  orte_rml_callback_fn_t cbfunc,
133  void* cbdata);
134 
135 /**
136  * Non-blocking version of mca_oob_recv().
137  *
138  * @param peer (IN) Opaque name of peer process or ORTE_NAME_WILDCARD for wildcard receive.
139  * @param msg (IN) Array of iovecs describing user buffers and lengths.
140  * @param count (IN) Number of elements in iovec array.
141  * @param tag (IN) User defined tag for matching send/recv.
142  * @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue,
143  * @param cbfunc (IN) Callback function on recv completion.
144  * @param cbdata (IN) User data that is passed to callback function.
145  * @return OMPI error code (<0) on error or number of bytes actually received.
146  */
147 
149  orte_process_name_t* peer,
150  struct iovec* msg,
151  int count,
152  int tag,
153  int flags,
154  orte_rml_callback_fn_t cbfunc,
155  void* cbdata);
156 
157 /**
158  * Cancel non-blocking receive.
159  *
160  * @param peer (IN) Opaque name of peer process or ORTE_NAME_WILDCARD for wildcard receive.
161  * @param tag (IN) User defined tag for matching send/recv.
162  * @return OMPI error code (<0) on error or number of bytes actually received.
163  */
164 
166  orte_process_name_t* peer,
167  int tag);
168 
169 /**
170  * Attempt to map a peer name to its corresponding address.
171  */
172 
174 
175 /**
176  * Parse a URI string into an IP address and port number.
177  */
179  const char* uri,
180  struct sockaddr* inaddr
181 );
182 
183 /**
184  * Setup socket options
185  */
186 
187 void mca_oob_tcp_set_socket_options(int sd);
188 
189 int mca_oob_tcp_ft_event(int state);
190 
191 typedef enum { OOB_TCP_EVENT, OOB_TCP_LISTEN_THREAD } mca_oob_tcp_listen_type_t;
192 
193 /**
194  * OOB TCP Component
195 */
197  mca_oob_base_component_2_0_0_t super; /**< base OOB component */
198  char* tcp_include; /**< list of ip interfaces to include */
199  char* tcp_exclude; /**< list of ip interfaces to exclude */
200  opal_list_t tcp_peer_list; /**< list of peers sorted in mru order */
201  opal_hash_table_t tcp_peers; /**< peers sorted by name */
202  opal_hash_table_t tcp_peer_names; /**< cache of peer contact info sorted by name */
203  opal_free_list_t tcp_peer_free; /**< free list of peers */
204  int tcp_peer_limit; /**< max size of tcp peer cache */
205  int tcp_peer_retries; /**< max number of retries before declaring peer gone */
206  int tcp_sndbuf; /**< socket send buffer size */
207  int tcp_rcvbuf; /**< socket recv buffer size */
208  opal_free_list_t tcp_msgs; /**< free list of messages */
209  opal_event_t tcp_recv_event; /**< event structure for IPv4 recvs */
210  int tcp_listen_sd; /**< listen socket for incoming IPv4 connection requests */
211  unsigned short tcp_listen_port; /**< IPv4 listen port */
212  char** tcp4_static_ports; /**< Static ports - IPV4 */
213  char** tcp4_dyn_ports; /**< Dynamic ports - IPV4 */
214  int disable_family; /**< disable AF: 0-nothing, 4-IPv4, 6-IPv6 */
215 #if OPAL_WANT_IPV6
216  opal_event_t tcp6_recv_event; /**< event structure for IPv6 recvs */
217  int tcp6_listen_sd; /**< listen socket for incoming IPv6 connection requests */
218  unsigned short tcp6_listen_port; /**< IPv6 listen port */
219  char** tcp6_static_ports; /**< Static ports - IPV6 */
220  char** tcp6_dyn_ports; /**< Dynamic ports - IPV6 */
221 #endif /* OPAL_WANT_IPV6 */
222  opal_mutex_t tcp_lock; /**< lock for accessing module state */
223  opal_list_t tcp_events; /**< list of pending events (accepts) */
224  opal_list_t tcp_msg_post; /**< list of recieves user has posted */
225  opal_list_t tcp_msg_recv; /**< list of recieved messages */
226  opal_list_t tcp_msg_completed; /**< list of completed messages */
227  opal_mutex_t tcp_match_lock; /**< lock held while searching/posting messages */
228  opal_condition_t tcp_match_cond; /**< condition variable used in finalize */
229  int tcp_match_count; /**< number of matched recvs in progress */
230  int tcp_debug; /**< debug level */
231 
232  bool tcp_shutdown;
233  mca_oob_tcp_listen_type_t tcp_listen_type;
234 
235  opal_list_t tcp_available_devices;
236 
237  opal_thread_t tcp_listen_thread; /** handle to the listening thread */
238  opal_list_t tcp_pending_connections; /**< List of accepted connections waiting for processing */
239  opal_list_t tcp_connections_return; /**< List of connection fragments being returned to accept thread */
240  opal_mutex_t tcp_connections_lock; /**< Lock protecting pending_connections and connections_return */
241  int tcp_connections_pipe[2];
242  opal_event_t tcp_listen_thread_event;
243 
244  int tcp_copy_max_size; /**< Max size of the copy list before copying must commence */
245  int tcp_listen_thread_num_sockets; /**< Number of sockets in tcp_listen_thread_sds */
246  int tcp_listen_thread_sds[2]; /**< Room for IPv4 and IPv6. Might need to make more dynamic. */
247  struct timeval tcp_listen_thread_tv; /**< Timeout when using listen thread */
248 
249  int connect_sleep;
250 };
251 
252 /**
253  * Convenience Typedef
254  */
256 
257 ORTE_MODULE_DECLSPEC extern mca_oob_tcp_component_t mca_oob_tcp_component;
258 
259 extern int mca_oob_tcp_output_handle;
260 
261 #if defined(__WINDOWS__)
262 #define CLOSE_THE_SOCKET(socket) closesocket(socket)
263 #else
264 
265 #define CLOSE_THE_SOCKET(socket) \
266  do { \
267  shutdown(socket, 2); \
268  close(socket); \
269  } while(0)
270 
271 #endif /* defined(__WINDOWS__) */
272 
273 
275  opal_free_list_item_t super;
276  int fd;
277  struct sockaddr_storage addr;
278 };
281 
283  opal_list_item_t super;
284  int if_index;
285  bool if_local;
286  struct sockaddr_storage if_addr;
287 };
290 
291 
292 END_C_DECLS
293 
294 #endif /* MCA_OOB_TCP_H_ */
295 
opal_hash_table_t tcp_peer_names
cache of peer contact info sorted by name
Definition: oob_tcp.h:202
opal_list_t tcp_msg_completed
list of completed messages
Definition: oob_tcp.h:226
Definition: opal_hash_table.h:42
void mca_oob_tcp_set_socket_options(int sd)
Setup socket options.
Definition: oob_tcp_peer.c:1189
char * mca_oob_tcp_get_addr(void)
Obtain contact information for this host (e.g.
Definition: oob_tcp.c:1934
opal_mutex_t tcp_match_lock
lock held while searching/posting messages
Definition: oob_tcp.h:227
int tcp_rcvbuf
socket recv buffer size
Definition: oob_tcp.h:207
opal_list_t tcp_connections_return
List of connection fragments being returned to accept thread.
Definition: oob_tcp.h:239
opal_condition_t tcp_match_cond
condition variable used in finalize
Definition: oob_tcp.h:228
Contains the internal functions and typedefs for the use of the oob.
int mca_oob_tcp_fini(void)
Cleanup resources during shutdown.
Definition: oob_tcp.c:1847
Definition: condition.h:49
opal_hash_table_t tcp_peers
peers sorted by name
Definition: oob_tcp.h:201
opal_event_t tcp_recv_event
event structure for IPv4 recvs
Definition: oob_tcp.h:209
Definition: types.h:146
opal_list_t tcp_msg_post
list of recieves user has posted
Definition: oob_tcp.h:224
opal_list_t tcp_msg_recv
list of recieved messages
Definition: oob_tcp.h:225
unsigned short tcp_listen_port
IPv4 listen port.
Definition: oob_tcp.h:211
the oob framework
Structure to represent a single event.
Definition: event_struct.h:87
struct timeval tcp_listen_thread_tv
Timeout when using listen thread.
Definition: oob_tcp.h:247
Contains the data structure which describes each connection.
Definition: mutex_unix.h:53
int disable_family
disable AF: 0-nothing, 4-IPv4, 6-IPv6
Definition: oob_tcp.h:214
opal_free_list_t tcp_msgs
free list of messages
Definition: oob_tcp.h:208
int tcp_peer_limit
max size of tcp peer cache
Definition: oob_tcp.h:204
int mca_oob_tcp_recv_cancel(orte_process_name_t *peer, int tag)
Cancel non-blocking receive.
Definition: oob_tcp_recv.c:173
int tcp_match_count
number of matched recvs in progress
Definition: oob_tcp.h:229
char ** tcp4_dyn_ports
Dynamic ports - IPV4.
Definition: oob_tcp.h:213
int tcp_listen_thread_num_sockets
Number of sockets in tcp_listen_thread_sds.
Definition: oob_tcp.h:245
opal_list_t tcp_peer_list
list of peers sorted in mru order
Definition: oob_tcp.h:200
Definition: opal_list.h:98
int mca_oob_tcp_set_addr(const orte_process_name_t *, const char *)
Setup cached addresses for the peers.
Definition: oob_tcp.c:2084
int mca_oob_tcp_process_name_compare(const orte_process_name_t *n1, const orte_process_name_t *n2)
Compare two process names for equality.
Definition: oob_tcp.c:1924
void(* orte_rml_callback_fn_t)(int status, struct orte_process_name_t *peer, struct iovec *msg, int count, orte_rml_tag_t tag, void *cbdata)
Funtion prototype for callback from non-blocking iovec send and receive.
Definition: rml.h:123
int mca_oob_tcp_parse_uri(const char *uri, struct sockaddr *inaddr)
Parse a URI string into an IP address and port number.
Definition: oob_tcp.c:1971
Definition: opal_free_list.h:47
opal_free_list_t tcp_peer_free
free list of peers
Definition: oob_tcp.h:203
contains the data structure we will use to describe a message
int mca_oob_tcp_recv_nb(orte_process_name_t *peer, struct iovec *msg, int count, int tag, int flags, orte_rml_callback_fn_t cbfunc, void *cbdata)
Non-blocking version of mca_oob_recv().
Definition: oob_tcp_recv.c:92
Definition: opal_free_list.h:31
int tcp_sndbuf
socket send buffer size
Definition: oob_tcp.h:206
int mca_oob_tcp_ping(const orte_process_name_t *, const char *uri, const struct timeval *tv)
A routine to ping a given process name to determine if it is reachable.
Definition: oob_tcp_ping.c:79
char * tcp_exclude
list of ip interfaces to exclude
Definition: oob_tcp.h:199
Definition: oob_tcp.h:274
char ** tcp4_static_ports
Static ports - IPV4.
Definition: oob_tcp.h:212
Definition: util-internal.h:214
A hash table that may be indexed with either fixed length (e.g.
int tcp_listen_sd
listen socket for incoming IPv4 connection requests
Definition: oob_tcp.h:210
Definition: oob.h:51
int tcp_copy_max_size
Max size of the copy list before copying must commence.
Definition: oob_tcp.h:244
Definition: ompi_uio.h:29
int mca_oob_tcp_resolve(mca_oob_tcp_peer_t *)
Attempt to map a peer name to its corresponding address.
Definition: oob_tcp.c:1586
int tcp_peer_retries
max number of retries before declaring peer gone
Definition: oob_tcp.h:205
opal_list_t tcp_events
list of pending events (accepts)
Definition: oob_tcp.h:223
This structure describes a peer.
Definition: oob_tcp_peer.h:59
Definition: opal_list.h:147
OOB TCP Component.
Definition: oob_tcp.h:196
OOB Module.
Definition: oob.h:161
opal_mutex_t tcp_lock
lock for accessing module state
Definition: oob_tcp.h:222
opal_mutex_t tcp_connections_lock
Lock protecting pending_connections and connections_return.
Definition: oob_tcp.h:240
Definition: threads.h:46
mca_oob_base_component_2_0_0_t super
base OOB component
Definition: oob_tcp.h:197
Definition: oob_tcp.h:282
opal_list_t tcp_pending_connections
handle to the listening thread
Definition: oob_tcp.h:238
int mca_oob_tcp_init(void)
Hook function to allow the selected oob components to register their contact info with the registry...
Definition: oob_tcp.c:1719
int tcp_listen_thread_sds[2]
Room for IPv4 and IPv6.
Definition: oob_tcp.h:246
Mutual exclusion functions.
int tcp_debug
debug level
Definition: oob_tcp.h:230
char * tcp_include
list of ip interfaces to include
Definition: oob_tcp.h:198
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236