OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_udapl_endpoint.h
1 /*
2 * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2005 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2006 Sandia National Laboratories. All rights
13 * reserved.
14 * Copyright (c) 2006-2009 Sun Microsystems, Inc. All rights reserved.
15 *
16 * $COPYRIGHT$
17 *
18 * Additional copyrights may follow
19 *
20 * $HEADER$
21 */
22
23 #ifndef MCA_BTL_UDAPL_ENDPOINT_H
24 #define MCA_BTL_UDAPL_ENDPOINT_H
25
26 #include "opal/class/opal_list.h"
27 #include "opal/mca/event/event.h"
28 #include "ompi/mca/btl/base/btl_base_error.h"
29 #include "btl_udapl_frag.h"
30 #include "btl_udapl.h"
31 #include "btl_udapl_eager_rdma.h"
32
33 BEGIN_C_DECLS
34
35
36 #define BTL_UDAPL_TOKEN_AVAIL(E, C, T) \
37 do { \
38 (T) = 0; \
39 if ( (E)->endpoint_lwqe_tokens[(C)] > 0 && \
40 ((E)->endpoint_sr_tokens[(C)] + \
41 (((C) == BTL_UDAPL_EAGER_CONNECTION)?(E)->endpoint_eager_rdma_remote.tokens:0)) \
42 ) { \
43 (T) = 1; \
44 } \
45 } while (0)
46
47 /**
48 * Structure used to publish uDAPL id information to peers.
49 */
50 struct mca_btl_udapl_addr_t {
51 DAT_CONN_QUAL port;
52 DAT_SOCK_ADDR addr;
53 bool inuse;
54 };
55 typedef struct mca_btl_udapl_addr_t mca_btl_udapl_addr_t;
56
57
58 /**
59 * State of uDAPL endpoint connection.
60 */
61
62 typedef enum {
63 MCA_BTL_UDAPL_CONN_EAGER,
64 MCA_BTL_UDAPL_CONN_MAX,
65 MCA_BTL_UDAPL_CONNECTED,
66 MCA_BTL_UDAPL_CLOSED,
67 MCA_BTL_UDAPL_FAILED
68 } mca_btl_udapl_endpoint_state_t;
69
70 /*
71 * Establish a name for the 2 connections opened per peer
72 */
73 typedef enum {
74 BTL_UDAPL_EAGER_CONNECTION,
75 BTL_UDAPL_MAX_CONNECTION,
76 BTL_UDAPL_NUM_CONNECTION
77 } mca_btl_udapl_endpoint_conn_t;
78
79 /*
80 * Encapsulate data that describes sendrecv credit information.
81 */
82 struct mca_btl_udapl_sr_credit_t {
83 mca_btl_udapl_control_header_t control;
84 uint32_t credits;
85 int connection; /* 0 == BTL_UDAPL_EAGER_CONNECTION;
86 1 == BTL_UDAPL_MAX_CONNECTION */
87 };
88 typedef struct mca_btl_udapl_sr_credit_t mca_btl_udapl_sr_credit_t;
89
90 /**
91 * An abstraction that represents a connection to a endpoint process.
92 * An instance of mca_btl_base_endpoint_t is associated w/ each process
93 * and BTL pair at startup. However, connections to the endpoint
94 * are established dynamically on an as-needed basis:
95 */
96
97 struct mca_btl_base_endpoint_t {
98 opal_list_item_t super;
99
100 struct mca_btl_udapl_module_t* endpoint_btl;
101 /**< BTL instance that created this connection */
102
103 struct mca_btl_udapl_proc_t* endpoint_proc;
104 /**< proc structure corresponding to endpoint */
105
106 mca_btl_udapl_endpoint_state_t endpoint_state;
107 /**< current state of the endpoint connection */
108
109 opal_list_t endpoint_eager_frags;
110 opal_list_t endpoint_max_frags;
111 /**< pending send frags on this endpoint */
112
113 int32_t endpoint_eager_sends;
114 int32_t endpoint_max_sends;
115 /**< number of sends that may be posted */
116
117 int32_t endpoint_sr_tokens[BTL_UDAPL_NUM_CONNECTION];
118 /**< number of sends that may be posted */
119
120 int32_t endpoint_sr_credits[BTL_UDAPL_NUM_CONNECTION];
121 /**< number of recvs that are now available */
122
123 int32_t endpoint_lwqe_tokens[BTL_UDAPL_NUM_CONNECTION];
124 /**< number of local work queue credits available (combination of
125 posted sends and rdma writes allowed per endpoint */
126
127 int32_t endpoint_connection_seq;
128 /**< sequence number of sendrecv message for the connection est */
129
130 int32_t endpoint_connections_completed;
131 /**< count of completed connections for priv data connection est. */
132
133 opal_mutex_t endpoint_lock;
134 /**< lock for concurrent access to endpoint state */
135
136 mca_btl_udapl_addr_t endpoint_addr;
137 /**< remote address on the other side of this endpoint */
138
139 DAT_EP_HANDLE endpoint_eager;
140 DAT_EP_HANDLE endpoint_max;
141 /**< uDAPL endpoint handle */
142
143 int32_t endpoint_eager_rdma_index;
144 /**< index into array of endpoints with RDMA buffers */
145 mca_btl_udapl_eager_rdma_local_t endpoint_eager_rdma_local;
146 /**< info about local RDMA buffer */
147 mca_btl_udapl_eager_rdma_remote_t endpoint_eager_rdma_remote;
148 /**< info about remote RDMA buffer */
149 };
150
151 typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
152 typedef mca_btl_base_endpoint_t mca_btl_udapl_endpoint_t;
153
154 OBJ_CLASS_DECLARATION(mca_btl_udapl_endpoint_t);
155
156
157 /*
158 * Start sending data on an endpoint.
159 */
160
161 int mca_btl_udapl_endpoint_send(mca_btl_base_endpoint_t* endpoint,
162 mca_btl_udapl_frag_t* frag);
163
164 /*
165 * Set up OOB recv callback.
166 */
167
168 void mca_btl_udapl_endpoint_post_oob_recv(void);
169
170 /*
171 * Finish establishing a connection
172 */
173
174 int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl,
175 mca_btl_udapl_addr_t* addr,
176 int32_t* seq,
177 DAT_EP_HANDLE endpoint);
178
179 /*
180 * Send number of eager rdma credits
181 */
182 int mca_btl_udapl_endpoint_send_eager_rdma_credits(mca_btl_base_endpoint_t* endpoint);
183
184 /*
185 * Establish uDAPL endpoint parameters
186 */
187 int mca_btl_udapl_endpoint_get_params(struct mca_btl_udapl_module_t* btl,
188 DAT_EP_PARAM* ep_param);
189
190 /*
191 * Create uDAPL endpoint
192 */
193 int mca_btl_udapl_endpoint_create(struct mca_btl_udapl_module_t* btl,
194 DAT_EP_HANDLE* udapl_endpoint);
195
196 /*
197 * Send number of send recv credits
198 */
199 int mca_btl_udapl_endpoint_send_sr_credits(mca_btl_base_endpoint_t* endpoint,
200 const int connection);
201
202 /*
203 * Handle the established DAT endpoint when private data is in use
204 */
205 int mca_btl_udapl_endpoint_pd_established_conn(
206 struct mca_btl_udapl_module_t* btl,
207 DAT_EP_HANDLE established_ep);
208
209 /*
210 * Utility routine. Search list of endpoints to find one that matches
211 * the given address.
212 */
213 mca_btl_udapl_endpoint_t* mca_btl_udapl_find_endpoint_address_match(
214 struct mca_btl_udapl_module_t* btl,
215 mca_btl_udapl_addr_t addr);
216
217 END_C_DECLS
218 #endif
mca_btl_base_endpoint_t::endpoint_addr
mca_btl_udapl_addr_t endpoint_addr
remote address on the other side of this endpoint
Definition: btl_udapl_endpoint.h:136
mca_btl_base_endpoint_t::endpoint_eager_rdma_index
int32_t endpoint_eager_rdma_index
index into array of endpoints with RDMA buffers
Definition: btl_udapl_endpoint.h:143
mca_btl_base_endpoint_t::endpoint_max
DAT_EP_HANDLE endpoint_max
uDAPL endpoint handle
Definition: btl_udapl_endpoint.h:140
mca_btl_base_endpoint_t::endpoint_connection_seq
int32_t endpoint_connection_seq
sequence number of sendrecv message for the connection est
Definition: btl_udapl_endpoint.h:127
mca_btl_udapl_eager_rdma_remote_t
Definition: btl_udapl_eager_rdma.h:36
btl_udapl.h
opal_mutex_t
Definition: mutex_unix.h:53
mca_btl_udapl_addr_t
Structure used to publish uDAPL id information to peers.
Definition: btl_udapl_endpoint.h:50
opal_list.h
The opal_list_t interface is used to provide a generic doubly-linked list container for Open MPI...
mca_btl_base_endpoint_t::endpoint_eager_rdma_local
mca_btl_udapl_eager_rdma_local_t endpoint_eager_rdma_local
info about local RDMA buffer
Definition: btl_udapl_endpoint.h:145
mca_btl_udapl_frag_t
uDAPL fragment derived type.
Definition: btl_udapl_frag.h:85
opal_list_item_t
Definition: opal_list.h:98
mca_btl_base_endpoint_t::endpoint_lock
opal_mutex_t endpoint_lock
lock for concurrent access to endpoint state
Definition: btl_openib_endpoint.h:198
mca_btl_udapl_module_t
BTL Module Interface.
Definition: btl_udapl.h:106
mca_btl_base_endpoint_t::endpoint_state
mca_btl_udapl_endpoint_state_t endpoint_state
current state of the endpoint connection
Definition: btl_udapl_endpoint.h:106
mca_btl_udapl_proc_t
Represents the state of a remote process and the set of addresses that it exports.
Definition: btl_udapl_proc.h:36
mca_btl_base_endpoint_t::endpoint_max_frags
opal_list_t endpoint_max_frags
pending send frags on this endpoint
Definition: btl_udapl_endpoint.h:110
mca_btl_base_endpoint_t::endpoint_eager_rdma_remote
mca_btl_udapl_eager_rdma_remote_t endpoint_eager_rdma_remote
info about remote RDMA buffer
Definition: btl_udapl_endpoint.h:147
mca_btl_udapl_control_header_t
Definition: btl_udapl_frag.h:52
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_udapl_sr_credit_t
Definition: btl_udapl_endpoint.h:82
opal_list_t
Definition: opal_list.h:147
mca_btl_base_endpoint_t::endpoint_max_sends
int32_t endpoint_max_sends
number of sends that may be posted
Definition: btl_udapl_endpoint.h:114
mca_btl_udapl_eager_rdma_local_t
Definition: btl_udapl_eager_rdma.h:23
mca_btl_base_endpoint_t::endpoint_proc
struct mca_btl_udapl_proc_t * endpoint_proc
proc structure corresponding to endpoint
Definition: btl_udapl_endpoint.h:103
mca_btl_base_endpoint_t::endpoint_connections_completed
int32_t endpoint_connections_completed
count of completed connections for priv data connection est.
Definition: btl_udapl_endpoint.h:130
mca_btl_base_endpoint_t::endpoint_lwqe_tokens
int32_t endpoint_lwqe_tokens[BTL_UDAPL_NUM_CONNECTION]
number of local work queue credits available (combination of posted sends and rdma writes allowed per...
Definition: btl_udapl_endpoint.h:123
mca_btl_base_endpoint_t::endpoint_sr_credits
int32_t endpoint_sr_credits[BTL_UDAPL_NUM_CONNECTION]
number of recvs that are now available
Definition: btl_udapl_endpoint.h:120
mca_btl_base_endpoint_t::endpoint_btl
struct mca_btl_udapl_module_t * endpoint_btl
BTL instance that created this connection.
Definition: btl_udapl_endpoint.h:100
OBJ_CLASS_DECLARATION
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
mca_btl_base_endpoint_t::endpoint_sr_tokens
int32_t endpoint_sr_tokens[BTL_UDAPL_NUM_CONNECTION]
number of sends that may be posted
Definition: btl_udapl_endpoint.h:117