OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_sctp.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2012 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
13 * $COPYRIGHT$
14 *
15 * Additional copyrights may follow
16 *
17 * $HEADER$
18 */
19 /**
20 * @file
21 */
22 #ifndef MCA_BTL_SCTP_H
23 #define MCA_BTL_SCTP_H
24
25 #include "ompi_config.h"
26 #ifdef HAVE_SYS_TYPES_H
27 #include <sys/types.h>
28 #endif
29 #ifdef HAVE_SYS_SOCKET_H
30 #include <sys/socket.h>
31 #endif
32 #ifdef HAVE_NETINET_IN_H
33 #include <netinet/in.h>
34 #endif
35
36 /* Open MPI includes */
37 #include "opal/mca/event/event.h"
38 #include "ompi/class/ompi_free_list.h"
39 #include "ompi/mca/btl/btl.h"
40 #include "ompi/mca/btl/base/base.h"
41 #include "ompi/mca/mpool/mpool.h"
42 #include "ompi/mca/btl/btl.h"
43 #include "opal/class/opal_hash_table.h"
44
45 /* For the assocID -> proc mapping, do we use opal_hash_table (0)
46 * or our homemade array (1)?
47 */
48 #define MCA_BTL_SCTP_DONT_USE_HASH 1
49
50 #if 1
51 /* if you do not want to see these debug messages */
52 #define SCTP_BTL_ERROR(args)
53 #else
54 #define SCTP_BTL_ERROR(args) BTL_ERROR(args)
55 #endif
56
57 #define MCA_BTL_SCTP_STATISTICS 0
58 BEGIN_C_DECLS
59
60
61 /**
62 * SCTP BTL component.
63 */
64
65 struct mca_btl_sctp_component_t {
66 mca_btl_base_component_2_0_0_t super; /**< base BTL component */
67 uint32_t sctp_num_btls; /**< number of hcas available to the SCTP component */
68 struct mca_btl_sctp_module_t **sctp_btls; /**< array of available BTL modules */
69 struct mca_btl_sctp_proc_t* sctp_local; /**< local proc struct */
70 int sctp_free_list_num; /**< initial size of free lists */
71 int sctp_free_list_max; /**< maximum size of free lists */
72 int sctp_free_list_inc; /**< number of elements to alloc when growing free lists */
73 int sctp_endpoint_cache; /**< amount of cache on each endpoint */
74 opal_hash_table_t sctp_procs; /**< hash table of sctp proc structures */
75 #if MCA_BTL_SCTP_DONT_USE_HASH
76 #else
77 opal_hash_table_t sctp_assocID_hash; /**< hash table of procs keyed on assocIDs */
78 #endif
79 opal_list_t sctp_events; /**< list of pending sctp events */
80 opal_mutex_t sctp_lock; /**< lock for accessing module state */
81 opal_event_t sctp_recv_event; /**< recv event for listen socket */
82 int sctp_listen_sd; /**< listen socket for incoming connection requests */
83 unsigned short sctp_listen_port; /**< listen port */
84 char* sctp_if_include; /**< comma seperated list of interface to include */
85 char* sctp_if_exclude; /**< comma seperated list of interface to exclude */
86 int sctp_sndbuf; /**< socket sndbuf size */
87 int sctp_rcvbuf; /**< socket rcvbuf size */
88 int sctp_use_nodelay; /**< SCTP_NODELAY value */
89 int sctp_if_11; /**< Are we going 1 to 1? 1 to many is default. */
90
91 /* free list of fragment descriptors */
92 ompi_free_list_t sctp_frag_eager;
93 ompi_free_list_t sctp_frag_max;
94 ompi_free_list_t sctp_frag_user;
95 };
96 typedef struct mca_btl_sctp_component_t mca_btl_sctp_component_t;
97
98 OMPI_MODULE_DECLSPEC extern mca_btl_sctp_component_t mca_btl_sctp_component;
99
100 /**
101 * BTL Module Interface
102 */
103 struct mca_btl_sctp_module_t {
104 mca_btl_base_module_t super; /**< base BTL interface */
105 int sctp_ifindex; /**< PTL interface index */
106 struct sockaddr_in sctp_ifaddr; /**< PTL interface address */
107 struct sockaddr_in sctp_ifmask; /**< PTL interface netmask */
108 opal_list_t sctp_endpoints;
109 #if MCA_BTL_SCTP_STATISTICS
110 size_t sctp_bytes_sent;
111 size_t sctp_bytes_recv;
112 size_t sctp_send_handler;
113 #endif
114 };
115 typedef struct mca_btl_sctp_module_t mca_btl_sctp_module_t;
116 extern mca_btl_sctp_module_t mca_btl_sctp_module;
117
118 #if defined(__WINDOWS__)
119 #define CLOSE_THE_SOCKET(socket) closesocket(socket)
120 #else
121 #define CLOSE_THE_SOCKET(socket) close(socket)
122 #endif /* defined(__WINDOWS__) */
123
124 /**
125 * SCTP component initialization.
126 *
127 * @param num_btl_modules (OUT) Number of BTLs returned in BTL array.
128 * @param allow_multi_user_threads (OUT) Flag indicating wether BTL supports user threads (TRUE)
129 * @param have_hidden_threads (OUT) Flag indicating wether BTL uses threads (TRUE)
130 */
131 extern mca_btl_base_module_t** mca_btl_sctp_component_init(
132 int *num_btl_modules,
133 bool allow_multi_user_threads,
134 bool have_hidden_threads
135 );
136
137
138
139 /**
140 * SCTP component progress.
141 */
142 extern int mca_btl_sctp_component_progress(void);
143
144
145
146 /**
147 * Cleanup any resources held by the BTL.
148 *
149 * @param btl BTL instance.
150 * @return OMPI_SUCCESS or error status on failure.
151 */
152
153 extern int mca_btl_sctp_finalize(
154 struct mca_btl_base_module_t* btl
155 );
156
157
158 /**
159 * PML->BTL notification of change in the process list.
160 *
161 * @param btl (IN)
162 * @param nprocs (IN) Number of processes
163 * @param procs (IN) Set of processes
164 * @param peers (OUT) Set of (optional) peer addressing info.
165 * @param peers (IN/OUT) Set of processes that are reachable via this BTL.
166 * @return OMPI_SUCCESS or error status on failure.
167 *
168 */
169
170 extern int mca_btl_sctp_add_procs(
171 struct mca_btl_base_module_t* btl,
172 size_t nprocs,
173 struct ompi_proc_t **procs,
174 struct mca_btl_base_endpoint_t** peers,
175 opal_bitmap_t* reachable
176 );
177
178 /**
179 * PML->BTL notification of change in the process list.
180 *
181 * @param btl (IN) BTL instance
182 * @param nproc (IN) Number of processes.
183 * @param procs (IN) Set of processes.
184 * @param peers (IN) Set of peer data structures.
185 * @return Status indicating if cleanup was successful
186 *
187 */
188
189 extern int mca_btl_sctp_del_procs(
190 struct mca_btl_base_module_t* btl,
191 size_t nprocs,
192 struct ompi_proc_t **procs,
193 struct mca_btl_base_endpoint_t** peers
194 );
195
196
197 /**
198 * Initiate an asynchronous send.
199 *
200 * @param btl (IN) BTL module
201 * @param endpoint (IN) BTL addressing information
202 * @param descriptor (IN) Description of the data to be transfered
203 * @param tag (IN) The tag value used to notify the peer.
204 */
205
206 extern int mca_btl_sctp_send(
207 struct mca_btl_base_module_t* btl,
208 struct mca_btl_base_endpoint_t* btl_peer,
209 struct mca_btl_base_descriptor_t* descriptor,
210 mca_btl_base_tag_t tag
211 );
212
213
214 /**
215 * Initiate an asynchronous put.
216 *
217 * @param btl (IN) BTL module
218 * @param endpoint (IN) BTL addressing information
219 * @param descriptor (IN) Description of the data to be transferred
220 */
221
222 extern int mca_btl_sctp_put(
223 struct mca_btl_base_module_t* btl,
224 struct mca_btl_base_endpoint_t* btl_peer,
225 struct mca_btl_base_descriptor_t* decriptor
226 );
227
228
229 /**
230 * Initiate an asynchronous get.
231 *
232 * @param btl (IN) BTL module
233 * @param endpoint (IN) BTL addressing information
234 * @param descriptor (IN) Description of the data to be transferred
235 */
236
237 extern int mca_btl_sctp_get(
238 struct mca_btl_base_module_t* btl,
239 struct mca_btl_base_endpoint_t* btl_peer,
240 struct mca_btl_base_descriptor_t* decriptor
241 );
242
243 /**
244 * Allocate a descriptor with a segment of the requested size.
245 * Note that the BTL layer may choose to return a smaller size
246 * if it cannot support the request.
247 *
248 * @param btl (IN) BTL module
249 * @param size (IN) Request segment size.
250 */
251
252 extern mca_btl_base_descriptor_t* mca_btl_sctp_alloc(
253 struct mca_btl_base_module_t* btl,
254 struct mca_btl_base_endpoint_t* endpoint,
255 uint8_t order,
256 size_t size,
257 uint32_t flags);
258
259
260 /**
261 * Return a segment allocated by this BTL.
262 *
263 * @param btl (IN) BTL module
264 * @param descriptor (IN) Allocated descriptor.
265 */
266
267 extern int mca_btl_sctp_free(
268 struct mca_btl_base_module_t* btl,
269 mca_btl_base_descriptor_t* des);
270
271
272 /**
273 * Prepare a descriptor for send/rdma using the supplied
274 * convertor. If the convertor references data that is contigous,
275 * the descriptor may simply point to the user buffer. Otherwise,
276 * this routine is responsible for allocating buffer space and
277 * packing if required.
278 *
279 * @param btl (IN) BTL module
280 * @param endpoint (IN) BTL peer addressing
281 * @param convertor (IN) Data type convertor
282 * @param reserve (IN) Additional bytes requested by upper layer to precede user data
283 * @param size (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)
284 */
285
286 mca_btl_base_descriptor_t* mca_btl_sctp_prepare_src(
287 struct mca_btl_base_module_t* btl,
288 struct mca_btl_base_endpoint_t* peer,
289 struct mca_mpool_base_registration_t*,
290 struct opal_convertor_t* convertor,
291 uint8_t order,
292 size_t reserve,
293 size_t* size,
294 uint32_t flags
295 );
296
297 extern mca_btl_base_descriptor_t* mca_btl_sctp_prepare_dst(
298 struct mca_btl_base_module_t* btl,
299 struct mca_btl_base_endpoint_t* peer,
300 struct mca_mpool_base_registration_t*,
301 struct opal_convertor_t* convertor,
302 uint8_t order,
303 size_t reserve,
304 size_t* size,
305 uint32_t flags);
306
307
308 END_C_DECLS
309 #endif
mca_btl_sctp_get
int mca_btl_sctp_get(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous get.
Definition: btl_sctp.c:450
mca_btl_sctp_component_t::sctp_listen_port
unsigned short sctp_listen_port
listen port
Definition: btl_sctp.h:83
mca_btl_sctp_component_t::sctp_local
struct mca_btl_sctp_proc_t * sctp_local
local proc struct
Definition: btl_sctp.h:69
opal_hash_table_t
Definition: opal_hash_table.h:42
mca_btl_sctp_finalize
int mca_btl_sctp_finalize(struct mca_btl_base_module_t *btl)
Cleanup any resources held by the BTL.
Definition: btl_sctp.c:481
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
mca_btl_sctp_module_t::sctp_ifindex
int sctp_ifindex
PTL interface index.
Definition: btl_sctp.h:105
mca_btl_sctp_component_t::sctp_free_list_inc
int sctp_free_list_inc
number of elements to alloc when growing free lists
Definition: btl_sctp.h:72
mca_btl_sctp_prepare_dst
mca_btl_base_descriptor_t * mca_btl_sctp_prepare_dst(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_sctp.c:321
mca_btl_sctp_component_t::sctp_endpoint_cache
int sctp_endpoint_cache
amount of cache on each endpoint
Definition: btl_sctp.h:73
opal_bitmap_t
Definition: opal_bitmap.h:53
mca_btl_sctp_component_t::sctp_use_nodelay
int sctp_use_nodelay
SCTP_NODELAY value.
Definition: btl_sctp.h:88
mca_btl_sctp_component_t::sctp_sndbuf
int sctp_sndbuf
socket sndbuf size
Definition: btl_sctp.h:86
event
Structure to represent a single event.
Definition: event_struct.h:87
opal_mutex_t
Definition: mutex_unix.h:53
mca_btl_sctp_component_t::sctp_num_btls
uint32_t sctp_num_btls
number of hcas available to the SCTP component
Definition: btl_sctp.h:67
mca_btl_sctp_put
int mca_btl_sctp_put(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
Initiate an asynchronous put.
Definition: btl_sctp.c:405
mca_btl_sctp_component_t::sctp_if_include
char * sctp_if_include
comma seperated list of interface to include
Definition: btl_sctp.h:84
mca_btl_sctp_component_t::sctp_events
opal_list_t sctp_events
list of pending sctp events
Definition: btl_sctp.h:79
mca_btl_sctp_component_t::super
mca_btl_base_component_2_0_0_t super
base BTL component
Definition: btl_sctp.h:66
mca_mpool_base_registration_t
Definition: mpool.h:44
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
mca_btl_sctp_component_t::sctp_if_11
int sctp_if_11
Are we going 1 to 1? 1 to many is default.
Definition: btl_sctp.h:89
mca_btl_sctp_module_t::super
mca_btl_base_module_t super
base BTL interface
Definition: btl_sctp.h:104
mca_btl_sctp_component_init
mca_btl_base_module_t ** mca_btl_sctp_component_init(int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
SCTP component initialization.
Definition: btl_sctp_component.c:718
mca_btl_sctp_component_progress
int mca_btl_sctp_component_progress(void)
SCTP component progress.
mca_btl_sctp_component_t::sctp_btls
struct mca_btl_sctp_module_t ** sctp_btls
array of available BTL modules
Definition: btl_sctp.h:68
mca_btl_sctp_free
int mca_btl_sctp_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
Return a segment allocated by this BTL.
Definition: btl_sctp.c:204
mca_btl_sctp_component_t::sctp_recv_event
opal_event_t sctp_recv_event
recv event for listen socket
Definition: btl_sctp.h:81
mca_btl_sctp_component_t::sctp_free_list_max
int sctp_free_list_max
maximum size of free lists
Definition: btl_sctp.h:71
mca_btl_sctp_send
int mca_btl_sctp_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
Initiate an asynchronous send.
Definition: btl_sctp.c:362
btl.h
Byte Transfer Layer (BTL)
mca_btl_sctp_component_t::sctp_lock
opal_mutex_t sctp_lock
lock for accessing module state
Definition: btl_sctp.h:80
mca_btl_sctp_prepare_src
mca_btl_base_descriptor_t * mca_btl_sctp_prepare_src(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
Prepare a descriptor for send/rdma using the supplied convertor.
Definition: btl_sctp.c:220
ompi_free_list_t
Definition: ompi_free_list.h:39
opal_hash_table.h
A hash table that may be indexed with either fixed length (e.g.
mca_btl_sctp_add_procs
int mca_btl_sctp_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
PML->BTL notification of change in the process list.
Definition: btl_sctp.c:69
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_base_component_2_0_0_t
BTL component descriptor.
Definition: btl.h:411
mpool.h
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
mca_btl_sctp_component_t::sctp_listen_sd
int sctp_listen_sd
listen socket for incoming connection requests
Definition: btl_sctp.h:82
opal_convertor_t
Definition: opal_convertor.h:90
mca_btl_sctp_component_t::sctp_free_list_num
int sctp_free_list_num
initial size of free lists
Definition: btl_sctp.h:70
opal_list_t
Definition: opal_list.h:147
mca_btl_sctp_module_t::sctp_ifaddr
struct sockaddr_in sctp_ifaddr
PTL interface address.
Definition: btl_sctp.h:106
mca_btl_sctp_component_t::sctp_if_exclude
char * sctp_if_exclude
comma seperated list of interface to exclude
Definition: btl_sctp.h:85
base.h
mca_btl_sctp_alloc
mca_btl_base_descriptor_t * mca_btl_sctp_alloc(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
Allocate a descriptor with a segment of the requested size.
Definition: btl_sctp.c:167
mca_btl_sctp_del_procs
int mca_btl_sctp_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
PML->BTL notification of change in the process list.
Definition: btl_sctp.c:141
mca_btl_sctp_component_t
SCTP BTL component.
Definition: btl_sctp.h:65
mca_btl_sctp_module_t::sctp_ifmask
struct sockaddr_in sctp_ifmask
PTL interface netmask.
Definition: btl_sctp.h:107
mca_btl_sctp_component_t::sctp_procs
opal_hash_table_t sctp_procs
hash table of sctp proc structures
Definition: btl_sctp.h:74
mca_btl_base_module_t
BTL module interface functions and attributes.
Definition: btl.h:786
mca_btl_sctp_component_t::sctp_rcvbuf
int sctp_rcvbuf
socket rcvbuf size
Definition: btl_sctp.h:87
mca_btl_sctp_module_t
BTL Module Interface.
Definition: btl_sctp.h:103
mca_btl_sctp_proc_t
Represents the state of a remote process and the set of addresses that it exports.
Definition: btl_sctp_proc.h:37