OpenMPI  0.1.1
btl_tcp.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2012 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
13  * $COPYRIGHT$
14  *
15  * Additional copyrights may follow
16  *
17  * $HEADER$
18  */
19 /**
20  * @file
21  */
22 #ifndef MCA_BTL_TCP_H
23 #define MCA_BTL_TCP_H
24 
25 #include "ompi_config.h"
26 #ifdef HAVE_SYS_TYPES_H
27 #include <sys/types.h>
28 #endif
29 #ifdef HAVE_SYS_SOCKET_H
30 #include <sys/socket.h>
31 #endif
32 #ifdef HAVE_NETINET_IN_H
33 #include <netinet/in.h>
34 #endif
35 
36 /* Open MPI includes */
37 #include "opal/mca/event/event.h"
38 #include "ompi/class/ompi_free_list.h"
39 #include "ompi/mca/btl/btl.h"
40 #include "ompi/mca/btl/base/base.h"
41 #include "ompi/mca/mpool/mpool.h"
42 #include "ompi/mca/btl/btl.h"
44 
45 #define MCA_BTL_TCP_STATISTICS 0
46 BEGIN_C_DECLS
47 
48 
49 /**
50  * TCP BTL component.
51  */
52 
54  mca_btl_base_component_2_0_0_t super; /**< base BTL component */
55  uint32_t tcp_addr_count; /**< total number of addresses */
56  uint32_t tcp_num_btls; /**< number of hcas available to the TCP component */
57  uint32_t tcp_num_links; /**< number of logical links per physical device */
58  struct mca_btl_tcp_module_t **tcp_btls; /**< array of available BTL modules */
59  struct mca_btl_tcp_proc_t* tcp_local; /**< local proc struct */
60  int tcp_free_list_num; /**< initial size of free lists */
61  int tcp_free_list_max; /**< maximum size of free lists */
62  int tcp_free_list_inc; /**< number of elements to alloc when growing free lists */
63  int tcp_endpoint_cache; /**< amount of cache on each endpoint */
64  opal_hash_table_t tcp_procs; /**< hash table of tcp proc structures */
65  opal_list_t tcp_events; /**< list of pending tcp events */
66  opal_mutex_t tcp_lock; /**< lock for accessing module state */
67 
68  opal_event_t tcp_recv_event; /**< recv event for IPv4 listen socket */
69  int tcp_listen_sd; /**< IPv4 listen socket for incoming connection requests */
70  unsigned short tcp_listen_port; /**< IPv4 listen port */
71  int32_t tcp_port_min; /**< IPv4 minimum port */
72  int32_t tcp_port_range; /**< IPv4 port range */
73 #if OPAL_WANT_IPV6
74  opal_event_t tcp6_recv_event; /**< recv event for IPv6 listen socket */
75  int tcp6_listen_sd; /**< IPv6 listen socket for incoming connection requests */
76  unsigned short tcp6_listen_port; /**< IPv6 listen port */
77  int32_t tcp6_port_min; /**< IPv4 minimum port */
78  int32_t tcp6_port_range; /**< IPv4 port range */
79 #endif
80  /* Port range restriction */
81 
82  char* tcp_if_include; /**< comma seperated list of interface to include */
83  char* tcp_if_exclude; /**< comma seperated list of interface to exclude */
84  int tcp_sndbuf; /**< socket sndbuf size */
85  int tcp_rcvbuf; /**< socket rcvbuf size */
86  int tcp_disable_family; /**< disabled AF_family */
87 
88  /* free list of fragment descriptors */
89  ompi_free_list_t tcp_frag_eager;
90  ompi_free_list_t tcp_frag_max;
91  ompi_free_list_t tcp_frag_user;
92 
93  /* Do we want to use TCP_NODELAY? */
94  int tcp_use_nodelay;
95 
96  /* If btl_tcp_if_seq was specified, this is the one interface
97  (name) that we're supposed to use. */
98  char *tcp_if_seq;
99 };
101 
102 OMPI_MODULE_DECLSPEC extern mca_btl_tcp_component_t mca_btl_tcp_component;
103 
104 /**
105  * BTL Module Interface
106  */
108  mca_btl_base_module_t super; /**< base BTL interface */
110  uint16_t tcp_ifkindex; /** <BTL kernel interface index */
111 #if 0
112  int tcp_ifindex; /**< BTL interface index */
113 #endif
114  struct sockaddr_storage tcp_ifaddr; /**< BTL interface address */
115  uint32_t tcp_ifmask; /**< BTL interface netmask */
116  opal_list_t tcp_endpoints;
117 #if MCA_BTL_TCP_STATISTICS
118  size_t tcp_bytes_sent;
119  size_t tcp_bytes_recv;
120  size_t tcp_send_handler;
121 #endif
122 };
124 extern mca_btl_tcp_module_t mca_btl_tcp_module;
125 
126 #if defined(__WINDOWS__)
127 #define CLOSE_THE_SOCKET(socket) closesocket(socket)
128 #else
129 #define CLOSE_THE_SOCKET(socket) close(socket)
130 #endif /* defined(__WINDOWS__) */
131 
132 /**
133  * TCP component initialization.
134  *
135  * @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
136  * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
137  * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
138  */
140  int *num_btl_modules,
141  bool allow_multi_user_threads,
142  bool have_hidden_threads
143 );
144 
145 
146 /**
147  * TCP component progress.
148  */
149 extern int mca_btl_tcp_component_progress(void);
150 
151 
152 
153 /**
154  * Cleanup any resources held by the BTL.
155  *
156  * @param btl BTL instance.
157  * @return OMPI_SUCCESS or error status on failure.
158  */
159 
160 extern int mca_btl_tcp_finalize(
161  struct mca_btl_base_module_t* btl
162 );
163 
164 
165 /**
166  * PML->BTL notification of change in the process list.
167  *
168  * @param btl (IN)
169  * @param nprocs (IN) Number of processes
170  * @param procs (IN) Set of processes
171  * @param peers (OUT) Set of (optional) peer addressing info.
172  * @param peers (IN/OUT) Set of processes that are reachable via this BTL.
173  * @return OMPI_SUCCESS or error status on failure.
174  *
175  */
176 
177 extern int mca_btl_tcp_add_procs(
178  struct mca_btl_base_module_t* btl,
179  size_t nprocs,
180  struct ompi_proc_t **procs,
181  struct mca_btl_base_endpoint_t** peers,
182  opal_bitmap_t* reachable
183 );
184 
185 /**
186  * PML->BTL notification of change in the process list.
187  *
188  * @param btl (IN) BTL instance
189  * @param nproc (IN) Number of processes.
190  * @param procs (IN) Set of processes.
191  * @param peers (IN) Set of peer data structures.
192  * @return Status indicating if cleanup was successful
193  *
194  */
195 
196 extern int mca_btl_tcp_del_procs(
197  struct mca_btl_base_module_t* btl,
198  size_t nprocs,
199  struct ompi_proc_t **procs,
200  struct mca_btl_base_endpoint_t** peers
201 );
202 
203 
204 /**
205  * Initiate an asynchronous send.
206  *
207  * @param btl (IN) BTL module
208  * @param endpoint (IN) BTL addressing information
209  * @param descriptor (IN) Description of the data to be transfered
210  * @param tag (IN) The tag value used to notify the peer.
211  */
212 
213 extern int mca_btl_tcp_send(
214  struct mca_btl_base_module_t* btl,
215  struct mca_btl_base_endpoint_t* btl_peer,
216  struct mca_btl_base_descriptor_t* descriptor,
217  mca_btl_base_tag_t tag
218 );
219 
220 
221 /**
222  * Initiate an asynchronous put.
223  *
224  * @param btl (IN) BTL module
225  * @param endpoint (IN) BTL addressing information
226  * @param descriptor (IN) Description of the data to be transferred
227  */
228 
229 extern int mca_btl_tcp_put(
230  struct mca_btl_base_module_t* btl,
231  struct mca_btl_base_endpoint_t* btl_peer,
232  struct mca_btl_base_descriptor_t* decriptor
233 );
234 
235 
236 /**
237  * Initiate an asynchronous get.
238  *
239  * @param btl (IN) BTL module
240  * @param endpoint (IN) BTL addressing information
241  * @param descriptor (IN) Description of the data to be transferred
242  */
243 
244 extern int mca_btl_tcp_get(
245  struct mca_btl_base_module_t* btl,
246  struct mca_btl_base_endpoint_t* btl_peer,
247  struct mca_btl_base_descriptor_t* decriptor
248 );
249 
250 /**
251  * Allocate a descriptor with a segment of the requested size.
252  * Note that the BTL layer may choose to return a smaller size
253  * if it cannot support the request.
254  *
255  * @param btl (IN) BTL module
256  * @param size (IN) Request segment size.
257  */
258 
260  struct mca_btl_base_module_t* btl,
261  struct mca_btl_base_endpoint_t* endpoint,
262  uint8_t order,
263  size_t size,
264  uint32_t flags);
265 
266 
267 /**
268  * Return a segment allocated by this BTL.
269  *
270  * @param btl (IN) BTL module
271  * @param descriptor (IN) Allocated descriptor.
272  */
273 
274 extern int mca_btl_tcp_free(
275  struct mca_btl_base_module_t* btl,
277 
278 
279 /**
280  * Prepare a descriptor for send/rdma using the supplied
281  * convertor. If the convertor references data that is contigous,
282  * the descriptor may simply point to the user buffer. Otherwise,
283  * this routine is responsible for allocating buffer space and
284  * packing if required.
285  *
286  * @param btl (IN) BTL module
287  * @param endpoint (IN) BTL peer addressing
288  * @param convertor (IN) Data type convertor
289  * @param reserve (IN) Additional bytes requested by upper layer to precede user data
290  * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
291 */
292 
294  struct mca_btl_base_module_t* btl,
295  struct mca_btl_base_endpoint_t* peer,
297  struct opal_convertor_t* convertor,
298  uint8_t order,
299  size_t reserve,
300  size_t* size,
301  uint32_t flags
302 );
303 
305  struct mca_btl_base_module_t* btl,
306  struct mca_btl_base_endpoint_t* peer,
308  struct opal_convertor_t* convertor,
309  uint8_t order,
310  size_t reserve,
311  size_t* size,
312  uint32_t flags);
313 
314 
315 /**
316  * Fault Tolerance Event Notification Function
317  * @param state Checkpoint Stae
318  * @return OMPI_SUCCESS or failure status
319  */
320 int mca_btl_tcp_ft_event(int state);
321 
322 END_C_DECLS
323 #endif
struct mca_btl_tcp_module_t ** tcp_btls
array of available BTL modules
Definition: btl_tcp.h:58
Definition: opal_hash_table.h:42
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
int mca_btl_tcp_component_progress(void)
TCP component progress.
mca_btl_base_component_2_0_0_t super
base BTL component
Definition: btl_tcp.h:54
Definition: opal_bitmap.h:53
mca_btl_base_module_t super
base BTL interface
Definition: btl_tcp.h:108
int32_t tcp_port_range
IPv4 port range.
Definition: btl_tcp.h:72
opal_event_t tcp_recv_event
recv event for IPv4 listen socket
Definition: btl_tcp.h:68
int mca_btl_tcp_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
PML->BTL notification of change in the process list.
Definition: btl_tcp.c:82
int tcp_free_list_num
initial size of free lists
Definition: btl_tcp.h:60
int mca_btl_tcp_get(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous get.
Definition: btl_tcp.c:452
int tcp_free_list_inc
number of elements to alloc when growing free lists
Definition: btl_tcp.h:62
Structure to represent a single event.
Definition: event_struct.h:87
int mca_btl_tcp_put(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous put.
Definition: btl_tcp.c:410
void(* mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)
Callback function that is called asynchronously on receipt of an error from the transport layer...
Definition: btl.h:538
BTL Module Interface.
Definition: btl_tcp.h:107
Definition: mutex_unix.h:53
int mca_btl_tcp_ft_event(int state)
Fault Tolerance Event Notification Function.
Definition: btl_tcp_ft.c:33
Represents the state of a remote process and the set of addresses that it exports.
Definition: btl_tcp_proc.h:38
int tcp_disable_family
disabled AF_family
Definition: btl_tcp.h:86
Definition: mpool.h:44
Remote Open MPI process structure.
Definition: proc.h:56
int mca_btl_tcp_finalize(struct mca_btl_base_module_t *btl)
Cleanup any resources held by the BTL.
Definition: btl_tcp.c:484
int tcp_sndbuf
socket sndbuf size
Definition: btl_tcp.h:84
int tcp_endpoint_cache
amount of cache on each endpoint
Definition: btl_tcp.h:63
int mca_btl_tcp_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
Initiate an asynchronous send.
Definition: btl_tcp.c:370
int32_t tcp_port_min
IPv4 minimum port.
Definition: btl_tcp.h:71
char * tcp_if_exclude
comma seperated list of interface to exclude
Definition: btl_tcp.h:83
int tcp_free_list_max
maximum size of free lists
Definition: btl_tcp.h:61
uint32_t tcp_addr_count
total number of addresses
Definition: btl_tcp.h:55
Byte Transfer Layer (BTL)
uint32_t tcp_num_links
number of logical links per physical device
Definition: btl_tcp.h:57
mca_btl_base_descriptor_t * mca_btl_tcp_alloc(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
Allocate a descriptor with a segment of the requested size.
Definition: btl_tcp.c:178
opal_list_t tcp_events
list of pending tcp events
Definition: btl_tcp.h:65
Definition: util-internal.h:214
Definition: ompi_free_list.h:39
mca_btl_base_descriptor_t * mca_btl_tcp_prepare_dst(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_tcp.c:327
mca_btl_base_descriptor_t * mca_btl_tcp_prepare_src(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_tcp.c:231
A hash table that may be indexed with either fixed length (e.g.
uint32_t tcp_ifmask
BTL interface netmask.
Definition: btl_tcp.h:115
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
BTL component descriptor.
Definition: btl.h:411
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
Definition: opal_convertor.h:90
mca_btl_base_module_t ** mca_btl_tcp_component_init(int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
TCP component initialization.
Definition: btl_tcp_component.c:1006
Definition: opal_list.h:147
opal_mutex_t tcp_lock
lock for accessing module state
Definition: btl_tcp.h:66
uint32_t tcp_num_btls
number of hcas available to the TCP component
Definition: btl_tcp.h:56
char * tcp_if_include
comma seperated list of interface to include
Definition: btl_tcp.h:82
int mca_btl_tcp_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
Return a segment allocated by this BTL.
Definition: btl_tcp.c:215
BTL module interface functions and attributes.
Definition: btl.h:786
unsigned short tcp_listen_port
IPv4 listen port.
Definition: btl_tcp.h:70
opal_hash_table_t tcp_procs
hash table of tcp proc structures
Definition: btl_tcp.h:64
int mca_btl_tcp_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
PML->BTL notification of change in the process list.
Definition: btl_tcp.c:152
int tcp_rcvbuf
socket rcvbuf size
Definition: btl_tcp.h:85
struct sockaddr_storage tcp_ifaddr
Definition: btl_tcp.h:114
int tcp_listen_sd
IPv4 listen socket for incoming connection requests.
Definition: btl_tcp.h:69
struct mca_btl_tcp_proc_t * tcp_local
local proc struct
Definition: btl_tcp.h:59
TCP BTL component.
Definition: btl_tcp.h:53