OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_tcp.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2012 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
13 * $COPYRIGHT$
14 *
15 * Additional copyrights may follow
16 *
17 * $HEADER$
18 */
19 /**
20 * @file
21 */
22 #ifndef MCA_BTL_TCP_H
23 #define MCA_BTL_TCP_H
24
25 #include "ompi_config.h"
26 #ifdef HAVE_SYS_TYPES_H
27 #include <sys/types.h>
28 #endif
29 #ifdef HAVE_SYS_SOCKET_H
30 #include <sys/socket.h>
31 #endif
32 #ifdef HAVE_NETINET_IN_H
33 #include <netinet/in.h>
34 #endif
35
36 /* Open MPI includes */
37 #include "opal/mca/event/event.h"
38 #include "ompi/class/ompi_free_list.h"
39 #include "ompi/mca/btl/btl.h"
40 #include "ompi/mca/btl/base/base.h"
41 #include "ompi/mca/mpool/mpool.h"
42 #include "ompi/mca/btl/btl.h"
43 #include "opal/class/opal_hash_table.h"
44
45 #define MCA_BTL_TCP_STATISTICS 0
46 BEGIN_C_DECLS
47
48
49 /**
50 * TCP BTL component.
51 */
52
53 struct mca_btl_tcp_component_t {
54 mca_btl_base_component_2_0_0_t super; /**< base BTL component */
55 uint32_t tcp_addr_count; /**< total number of addresses */
56 uint32_t tcp_num_btls; /**< number of hcas available to the TCP component */
57 uint32_t tcp_num_links; /**< number of logical links per physical device */
58 struct mca_btl_tcp_module_t **tcp_btls; /**< array of available BTL modules */
59 struct mca_btl_tcp_proc_t* tcp_local; /**< local proc struct */
60 int tcp_free_list_num; /**< initial size of free lists */
61 int tcp_free_list_max; /**< maximum size of free lists */
62 int tcp_free_list_inc; /**< number of elements to alloc when growing free lists */
63 int tcp_endpoint_cache; /**< amount of cache on each endpoint */
64 opal_hash_table_t tcp_procs; /**< hash table of tcp proc structures */
65 opal_list_t tcp_events; /**< list of pending tcp events */
66 opal_mutex_t tcp_lock; /**< lock for accessing module state */
67
68 opal_event_t tcp_recv_event; /**< recv event for IPv4 listen socket */
69 int tcp_listen_sd; /**< IPv4 listen socket for incoming connection requests */
70 unsigned short tcp_listen_port; /**< IPv4 listen port */
71 int32_t tcp_port_min; /**< IPv4 minimum port */
72 int32_t tcp_port_range; /**< IPv4 port range */
73 #if OPAL_WANT_IPV6
74 opal_event_t tcp6_recv_event; /**< recv event for IPv6 listen socket */
75 int tcp6_listen_sd; /**< IPv6 listen socket for incoming connection requests */
76 unsigned short tcp6_listen_port; /**< IPv6 listen port */
77 int32_t tcp6_port_min; /**< IPv4 minimum port */
78 int32_t tcp6_port_range; /**< IPv4 port range */
79 #endif
80 /* Port range restriction */
81
82 char* tcp_if_include; /**< comma seperated list of interface to include */
83 char* tcp_if_exclude; /**< comma seperated list of interface to exclude */
84 int tcp_sndbuf; /**< socket sndbuf size */
85 int tcp_rcvbuf; /**< socket rcvbuf size */
86 int tcp_disable_family; /**< disabled AF_family */
87
88 /* free list of fragment descriptors */
89 ompi_free_list_t tcp_frag_eager;
90 ompi_free_list_t tcp_frag_max;
91 ompi_free_list_t tcp_frag_user;
92
93 /* Do we want to use TCP_NODELAY? */
94 int tcp_use_nodelay;
95
96 /* If btl_tcp_if_seq was specified, this is the one interface
97 (name) that we're supposed to use. */
98 char *tcp_if_seq;
99 };
100 typedef struct mca_btl_tcp_component_t mca_btl_tcp_component_t;
101
102 OMPI_MODULE_DECLSPEC extern mca_btl_tcp_component_t mca_btl_tcp_component;
103
104 /**
105 * BTL Module Interface
106 */
107 struct mca_btl_tcp_module_t {
108 mca_btl_base_module_t super; /**< base BTL interface */
109 mca_btl_base_module_error_cb_fn_t error_cb;
110 uint16_t tcp_ifkindex; /** <BTL kernel interface index */
111 #if 0
112 int tcp_ifindex; /**< BTL interface index */
113 #endif
114 struct sockaddr_storage tcp_ifaddr; /**< BTL interface address */
115 uint32_t tcp_ifmask; /**< BTL interface netmask */
116 opal_list_t tcp_endpoints;
117 #if MCA_BTL_TCP_STATISTICS
118 size_t tcp_bytes_sent;
119 size_t tcp_bytes_recv;
120 size_t tcp_send_handler;
121 #endif
122 };
123 typedef struct mca_btl_tcp_module_t mca_btl_tcp_module_t;
124 extern mca_btl_tcp_module_t mca_btl_tcp_module;
125
126 #if defined(__WINDOWS__)
127 #define CLOSE_THE_SOCKET(socket) closesocket(socket)
128 #else
129 #define CLOSE_THE_SOCKET(socket) close(socket)
130 #endif /* defined(__WINDOWS__) */
131
132 /**
133 * TCP component initialization.
134 *
135 * @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
136 * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
137 * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
138 */
139 extern mca_btl_base_module_t** mca_btl_tcp_component_init(
140 int *num_btl_modules,
141 bool allow_multi_user_threads,
142 bool have_hidden_threads
143 );
144
145
146 /**
147 * TCP component progress.
148 */
149 extern int mca_btl_tcp_component_progress(void);
150
151
152
153 /**
154 * Cleanup any resources held by the BTL.
155 *
156 * @param btl BTL instance.
157 * @return OMPI_SUCCESS or error status on failure.
158 */
159
160 extern int mca_btl_tcp_finalize(
161 struct mca_btl_base_module_t* btl
162 );
163
164
165 /**
166 * PML->BTL notification of change in the process list.
167 *
168 * @param btl (IN)
169 * @param nprocs (IN) Number of processes
170 * @param procs (IN) Set of processes
171 * @param peers (OUT) Set of (optional) peer addressing info.
172 * @param peers (IN/OUT) Set of processes that are reachable via this BTL.
173 * @return OMPI_SUCCESS or error status on failure.
174 *
175 */
176
177 extern int mca_btl_tcp_add_procs(
178 struct mca_btl_base_module_t* btl,
179 size_t nprocs,
180 struct ompi_proc_t **procs,
181 struct mca_btl_base_endpoint_t** peers,
182 opal_bitmap_t* reachable
183 );
184
185 /**
186 * PML->BTL notification of change in the process list.
187 *
188 * @param btl (IN) BTL instance
189 * @param nproc (IN) Number of processes.
190 * @param procs (IN) Set of processes.
191 * @param peers (IN) Set of peer data structures.
192 * @return Status indicating if cleanup was successful
193 *
194 */
195
196 extern int mca_btl_tcp_del_procs(
197 struct mca_btl_base_module_t* btl,
198 size_t nprocs,
199 struct ompi_proc_t **procs,
200 struct mca_btl_base_endpoint_t** peers
201 );
202
203
204 /**
205 * Initiate an asynchronous send.
206 *
207 * @param btl (IN) BTL module
208 * @param endpoint (IN) BTL addressing information
209 * @param descriptor (IN) Description of the data to be transfered
210 * @param tag (IN) The tag value used to notify the peer.
211 */
212
213 extern int mca_btl_tcp_send(
214 struct mca_btl_base_module_t* btl,
215 struct mca_btl_base_endpoint_t* btl_peer,
216 struct mca_btl_base_descriptor_t* descriptor,
217 mca_btl_base_tag_t tag
218 );
219
220
221 /**
222 * Initiate an asynchronous put.
223 *
224 * @param btl (IN) BTL module
225 * @param endpoint (IN) BTL addressing information
226 * @param descriptor (IN) Description of the data to be transferred
227 */
228
229 extern int mca_btl_tcp_put(
230 struct mca_btl_base_module_t* btl,
231 struct mca_btl_base_endpoint_t* btl_peer,
232 struct mca_btl_base_descriptor_t* decriptor
233 );
234
235
236 /**
237 * Initiate an asynchronous get.
238 *
239 * @param btl (IN) BTL module
240 * @param endpoint (IN) BTL addressing information
241 * @param descriptor (IN) Description of the data to be transferred
242 */
243
244 extern int mca_btl_tcp_get(
245 struct mca_btl_base_module_t* btl,
246 struct mca_btl_base_endpoint_t* btl_peer,
247 struct mca_btl_base_descriptor_t* decriptor
248 );
249
250 /**
251 * Allocate a descriptor with a segment of the requested size.
252 * Note that the BTL layer may choose to return a smaller size
253 * if it cannot support the request.
254 *
255 * @param btl (IN) BTL module
256 * @param size (IN) Request segment size.
257 */
258
259 extern mca_btl_base_descriptor_t* mca_btl_tcp_alloc(
260 struct mca_btl_base_module_t* btl,
261 struct mca_btl_base_endpoint_t* endpoint,
262 uint8_t order,
263 size_t size,
264 uint32_t flags);
265
266
267 /**
268 * Return a segment allocated by this BTL.
269 *
270 * @param btl (IN) BTL module
271 * @param descriptor (IN) Allocated descriptor.
272 */
273
274 extern int mca_btl_tcp_free(
275 struct mca_btl_base_module_t* btl,
276 mca_btl_base_descriptor_t* des);
277
278
279 /**
280 * Prepare a descriptor for send/rdma using the supplied
281 * convertor. If the convertor references data that is contigous,
282 * the descriptor may simply point to the user buffer. Otherwise,
283 * this routine is responsible for allocating buffer space and
284 * packing if required.
285 *
286 * @param btl (IN) BTL module
287 * @param endpoint (IN) BTL peer addressing
288 * @param convertor (IN) Data type convertor
289 * @param reserve (IN) Additional bytes requested by upper layer to precede user data
290 * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
291 */
292
293 mca_btl_base_descriptor_t* mca_btl_tcp_prepare_src(
294 struct mca_btl_base_module_t* btl,
295 struct mca_btl_base_endpoint_t* peer,
296 struct mca_mpool_base_registration_t*,
297 struct opal_convertor_t* convertor,
298 uint8_t order,
299 size_t reserve,
300 size_t* size,
301 uint32_t flags
302 );
303
304 extern mca_btl_base_descriptor_t* mca_btl_tcp_prepare_dst(
305 struct mca_btl_base_module_t* btl,
306 struct mca_btl_base_endpoint_t* peer,
307 struct mca_mpool_base_registration_t*,
308 struct opal_convertor_t* convertor,
309 uint8_t order,
310 size_t reserve,
311 size_t* size,
312 uint32_t flags);
313
314
315 /**
316 * Fault Tolerance Event Notification Function
317 * @param state Checkpoint Stae
318 * @return OMPI_SUCCESS or failure status
319 */
320 int mca_btl_tcp_ft_event(int state);
321
322 END_C_DECLS
323 #endif
mca_btl_tcp_component_t::tcp_btls
struct mca_btl_tcp_module_t ** tcp_btls
array of available BTL modules
Definition: btl_tcp.h:58
opal_hash_table_t
Definition: opal_hash_table.h:42
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
mca_btl_tcp_component_progress
int mca_btl_tcp_component_progress(void)
TCP component progress.
mca_btl_tcp_component_t::super
mca_btl_base_component_2_0_0_t super
base BTL component
Definition: btl_tcp.h:54
opal_bitmap_t
Definition: opal_bitmap.h:53
mca_btl_tcp_module_t::super
mca_btl_base_module_t super
base BTL interface
Definition: btl_tcp.h:108
mca_btl_tcp_component_t::tcp_port_range
int32_t tcp_port_range
IPv4 port range.
Definition: btl_tcp.h:72
mca_btl_tcp_component_t::tcp_recv_event
opal_event_t tcp_recv_event
recv event for IPv4 listen socket
Definition: btl_tcp.h:68
mca_btl_tcp_add_procs
int mca_btl_tcp_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
PML->BTL notification of change in the process list.
Definition: btl_tcp.c:82
mca_btl_tcp_component_t::tcp_free_list_num
int tcp_free_list_num
initial size of free lists
Definition: btl_tcp.h:60
mca_btl_tcp_get
int mca_btl_tcp_get(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous get.
Definition: btl_tcp.c:452
mca_btl_tcp_component_t::tcp_free_list_inc
int tcp_free_list_inc
number of elements to alloc when growing free lists
Definition: btl_tcp.h:62
event
Structure to represent a single event.
Definition: event_struct.h:87
mca_btl_tcp_put
int mca_btl_tcp_put(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous put.
Definition: btl_tcp.c:410
mca_btl_base_module_error_cb_fn_t
void(* mca_btl_base_module_error_cb_fn_t)(struct mca_btl_base_module_t *btl, int32_t flags, struct ompi_proc_t *errproc, char *btlinfo)
Callback function that is called asynchronously on receipt of an error from the transport layer...
Definition: btl.h:538
mca_btl_tcp_module_t
BTL Module Interface.
Definition: btl_tcp.h:107
opal_mutex_t
Definition: mutex_unix.h:53
mca_btl_tcp_ft_event
int mca_btl_tcp_ft_event(int state)
Fault Tolerance Event Notification Function.
Definition: btl_tcp_ft.c:33
mca_btl_tcp_proc_t
Represents the state of a remote process and the set of addresses that it exports.
Definition: btl_tcp_proc.h:38
mca_btl_tcp_component_t::tcp_disable_family
int tcp_disable_family
disabled AF_family
Definition: btl_tcp.h:86
mca_mpool_base_registration_t
Definition: mpool.h:44
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
mca_btl_tcp_finalize
int mca_btl_tcp_finalize(struct mca_btl_base_module_t *btl)
Cleanup any resources held by the BTL.
Definition: btl_tcp.c:484
mca_btl_tcp_component_t::tcp_sndbuf
int tcp_sndbuf
socket sndbuf size
Definition: btl_tcp.h:84
mca_btl_tcp_component_t::tcp_endpoint_cache
int tcp_endpoint_cache
amount of cache on each endpoint
Definition: btl_tcp.h:63
mca_btl_tcp_send
int mca_btl_tcp_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
Initiate an asynchronous send.
Definition: btl_tcp.c:370
mca_btl_tcp_component_t::tcp_port_min
int32_t tcp_port_min
IPv4 minimum port.
Definition: btl_tcp.h:71
mca_btl_tcp_component_t::tcp_if_exclude
char * tcp_if_exclude
comma seperated list of interface to exclude
Definition: btl_tcp.h:83
mca_btl_tcp_component_t::tcp_free_list_max
int tcp_free_list_max
maximum size of free lists
Definition: btl_tcp.h:61
mca_btl_tcp_component_t::tcp_addr_count
uint32_t tcp_addr_count
total number of addresses
Definition: btl_tcp.h:55
btl.h
Byte Transfer Layer (BTL)
mca_btl_tcp_component_t::tcp_num_links
uint32_t tcp_num_links
number of logical links per physical device
Definition: btl_tcp.h:57
mca_btl_tcp_alloc
mca_btl_base_descriptor_t * mca_btl_tcp_alloc(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
Allocate a descriptor with a segment of the requested size.
Definition: btl_tcp.c:178
mca_btl_tcp_component_t::tcp_events
opal_list_t tcp_events
list of pending tcp events
Definition: btl_tcp.h:65
sockaddr_storage
Definition: util-internal.h:214
ompi_free_list_t
Definition: ompi_free_list.h:39
mca_btl_tcp_prepare_dst
mca_btl_base_descriptor_t * mca_btl_tcp_prepare_dst(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_tcp.c:327
mca_btl_tcp_prepare_src
mca_btl_base_descriptor_t * mca_btl_tcp_prepare_src(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_tcp.c:231
opal_hash_table.h
A hash table that may be indexed with either fixed length (e.g.
mca_btl_tcp_module_t::tcp_ifmask
uint32_t tcp_ifmask
BTL interface netmask.
Definition: btl_tcp.h:115
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_base_component_2_0_0_t
BTL component descriptor.
Definition: btl.h:411
mpool.h
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
opal_convertor_t
Definition: opal_convertor.h:90
mca_btl_tcp_component_init
mca_btl_base_module_t ** mca_btl_tcp_component_init(int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
TCP component initialization.
Definition: btl_tcp_component.c:1006
opal_list_t
Definition: opal_list.h:147
mca_btl_tcp_component_t::tcp_lock
opal_mutex_t tcp_lock
lock for accessing module state
Definition: btl_tcp.h:66
base.h
mca_btl_tcp_component_t::tcp_num_btls
uint32_t tcp_num_btls
number of hcas available to the TCP component
Definition: btl_tcp.h:56
mca_btl_tcp_component_t::tcp_if_include
char * tcp_if_include
comma seperated list of interface to include
Definition: btl_tcp.h:82
mca_btl_tcp_free
int mca_btl_tcp_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
Return a segment allocated by this BTL.
Definition: btl_tcp.c:215
mca_btl_base_module_t
BTL module interface functions and attributes.
Definition: btl.h:786
mca_btl_tcp_component_t::tcp_listen_port
unsigned short tcp_listen_port
IPv4 listen port.
Definition: btl_tcp.h:70
mca_btl_tcp_component_t::tcp_procs
opal_hash_table_t tcp_procs
hash table of tcp proc structures
Definition: btl_tcp.h:64
mca_btl_tcp_del_procs
int mca_btl_tcp_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
PML->BTL notification of change in the process list.
Definition: btl_tcp.c:152
mca_btl_tcp_component_t::tcp_rcvbuf
int tcp_rcvbuf
socket rcvbuf size
Definition: btl_tcp.h:85
mca_btl_tcp_module_t::tcp_ifaddr
struct sockaddr_storage tcp_ifaddr
Definition: btl_tcp.h:114
mca_btl_tcp_component_t::tcp_listen_sd
int tcp_listen_sd
IPv4 listen socket for incoming connection requests.
Definition: btl_tcp.h:69
mca_btl_tcp_component_t::tcp_local
struct mca_btl_tcp_proc_t * tcp_local
local proc struct
Definition: btl_tcp.h:59
mca_btl_tcp_component_t
TCP BTL component.
Definition: btl_tcp.h:53