OpenMPI  0.1.1
btl_udapl_endpoint.h
1 /*
2  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2005 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2006 Sandia National Laboratories. All rights
13  * reserved.
14  * Copyright (c) 2006-2009 Sun Microsystems, Inc. All rights reserved.
15  *
16  * $COPYRIGHT$
17  *
18  * Additional copyrights may follow
19  *
20  * $HEADER$
21  */
22 
23 #ifndef MCA_BTL_UDAPL_ENDPOINT_H
24 #define MCA_BTL_UDAPL_ENDPOINT_H
25 
26 #include "opal/class/opal_list.h"
27 #include "opal/mca/event/event.h"
28 #include "ompi/mca/btl/base/btl_base_error.h"
29 #include "btl_udapl_frag.h"
30 #include "btl_udapl.h"
31 #include "btl_udapl_eager_rdma.h"
32 
33 BEGIN_C_DECLS
34 
35 
36 #define BTL_UDAPL_TOKEN_AVAIL(E, C, T) \
37 do { \
38  (T) = 0; \
39  if ( (E)->endpoint_lwqe_tokens[(C)] > 0 && \
40  ((E)->endpoint_sr_tokens[(C)] + \
41  (((C) == BTL_UDAPL_EAGER_CONNECTION)?(E)->endpoint_eager_rdma_remote.tokens:0)) \
42  ) { \
43  (T) = 1; \
44  } \
45 } while (0)
46 
47 /**
48  * Structure used to publish uDAPL id information to peers.
49  */
51  DAT_CONN_QUAL port;
52  DAT_SOCK_ADDR addr;
53  bool inuse;
54 };
56 
57 
58 /**
59  * State of uDAPL endpoint connection.
60  */
61 
62 typedef enum {
63  MCA_BTL_UDAPL_CONN_EAGER,
64  MCA_BTL_UDAPL_CONN_MAX,
65  MCA_BTL_UDAPL_CONNECTED,
66  MCA_BTL_UDAPL_CLOSED,
67  MCA_BTL_UDAPL_FAILED
68 } mca_btl_udapl_endpoint_state_t;
69 
70 /*
71  * Establish a name for the 2 connections opened per peer
72  */
73 typedef enum {
74  BTL_UDAPL_EAGER_CONNECTION,
75  BTL_UDAPL_MAX_CONNECTION,
76  BTL_UDAPL_NUM_CONNECTION
77 } mca_btl_udapl_endpoint_conn_t;
78 
79 /*
80  * Encapsulate data that describes sendrecv credit information.
81  */
84  uint32_t credits;
85  int connection; /* 0 == BTL_UDAPL_EAGER_CONNECTION;
86  1 == BTL_UDAPL_MAX_CONNECTION */
87 };
89 
90 /**
91  * An abstraction that represents a connection to a endpoint process.
92  * An instance of mca_btl_base_endpoint_t is associated w/ each process
93  * and BTL pair at startup. However, connections to the endpoint
94  * are established dynamically on an as-needed basis:
95 */
96 
98  opal_list_item_t super;
99 
101  /**< BTL instance that created this connection */
102 
104  /**< proc structure corresponding to endpoint */
105 
106  mca_btl_udapl_endpoint_state_t endpoint_state;
107  /**< current state of the endpoint connection */
108 
109  opal_list_t endpoint_eager_frags;
111  /**< pending send frags on this endpoint */
112 
113  int32_t endpoint_eager_sends;
115  /**< number of sends that may be posted */
116 
117  int32_t endpoint_sr_tokens[BTL_UDAPL_NUM_CONNECTION];
118  /**< number of sends that may be posted */
119 
120  int32_t endpoint_sr_credits[BTL_UDAPL_NUM_CONNECTION];
121  /**< number of recvs that are now available */
122 
123  int32_t endpoint_lwqe_tokens[BTL_UDAPL_NUM_CONNECTION];
124  /**< number of local work queue credits available (combination of
125  posted sends and rdma writes allowed per endpoint */
126 
128  /**< sequence number of sendrecv message for the connection est */
129 
131  /**< count of completed connections for priv data connection est. */
132 
134  /**< lock for concurrent access to endpoint state */
135 
137  /**< remote address on the other side of this endpoint */
138 
139  DAT_EP_HANDLE endpoint_eager;
140  DAT_EP_HANDLE endpoint_max;
141  /**< uDAPL endpoint handle */
142 
144  /**< index into array of endpoints with RDMA buffers */
146  /**< info about local RDMA buffer */
148  /**< info about remote RDMA buffer */
149 };
150 
153 
155 
156 
157 /*
158  * Start sending data on an endpoint.
159  */
160 
161 int mca_btl_udapl_endpoint_send(mca_btl_base_endpoint_t* endpoint,
162  mca_btl_udapl_frag_t* frag);
163 
164 /*
165  * Set up OOB recv callback.
166  */
167 
168 void mca_btl_udapl_endpoint_post_oob_recv(void);
169 
170 /*
171  * Finish establishing a connection
172  */
173 
174 int mca_btl_udapl_endpoint_finish_connect(struct mca_btl_udapl_module_t* btl,
175  mca_btl_udapl_addr_t* addr,
176  int32_t* seq,
177  DAT_EP_HANDLE endpoint);
178 
179 /*
180  * Send number of eager rdma credits
181  */
182 int mca_btl_udapl_endpoint_send_eager_rdma_credits(mca_btl_base_endpoint_t* endpoint);
183 
184 /*
185  * Establish uDAPL endpoint parameters
186  */
187 int mca_btl_udapl_endpoint_get_params(struct mca_btl_udapl_module_t* btl,
188  DAT_EP_PARAM* ep_param);
189 
190 /*
191  * Create uDAPL endpoint
192  */
193 int mca_btl_udapl_endpoint_create(struct mca_btl_udapl_module_t* btl,
194  DAT_EP_HANDLE* udapl_endpoint);
195 
196  /*
197  * Send number of send recv credits
198  */
199 int mca_btl_udapl_endpoint_send_sr_credits(mca_btl_base_endpoint_t* endpoint,
200  const int connection);
201 
202 /*
203  * Handle the established DAT endpoint when private data is in use
204  */
205 int mca_btl_udapl_endpoint_pd_established_conn(
206  struct mca_btl_udapl_module_t* btl,
207  DAT_EP_HANDLE established_ep);
208 
209 /*
210  * Utility routine. Search list of endpoints to find one that matches
211  * the given address.
212  */
213 mca_btl_udapl_endpoint_t* mca_btl_udapl_find_endpoint_address_match(
214  struct mca_btl_udapl_module_t* btl,
215  mca_btl_udapl_addr_t addr);
216 
217 END_C_DECLS
218 #endif
mca_btl_udapl_addr_t endpoint_addr
remote address on the other side of this endpoint
Definition: btl_udapl_endpoint.h:136
int32_t endpoint_eager_rdma_index
index into array of endpoints with RDMA buffers
Definition: btl_udapl_endpoint.h:143
DAT_EP_HANDLE endpoint_max
uDAPL endpoint handle
Definition: btl_udapl_endpoint.h:140
int32_t endpoint_connection_seq
sequence number of sendrecv message for the connection est
Definition: btl_udapl_endpoint.h:127
Definition: btl_udapl_eager_rdma.h:36
Definition: mutex_unix.h:53
Structure used to publish uDAPL id information to peers.
Definition: btl_udapl_endpoint.h:50
The opal_list_t interface is used to provide a generic doubly-linked list container for Open MPI...
mca_btl_udapl_eager_rdma_local_t endpoint_eager_rdma_local
info about local RDMA buffer
Definition: btl_udapl_endpoint.h:145
uDAPL fragment derived type.
Definition: btl_udapl_frag.h:85
Definition: opal_list.h:98
opal_mutex_t endpoint_lock
lock for concurrent access to endpoint state
Definition: btl_openib_endpoint.h:198
BTL Module Interface.
Definition: btl_udapl.h:106
mca_btl_udapl_endpoint_state_t endpoint_state
current state of the endpoint connection
Definition: btl_udapl_endpoint.h:106
Represents the state of a remote process and the set of addresses that it exports.
Definition: btl_udapl_proc.h:36
opal_list_t endpoint_max_frags
pending send frags on this endpoint
Definition: btl_udapl_endpoint.h:110
mca_btl_udapl_eager_rdma_remote_t endpoint_eager_rdma_remote
info about remote RDMA buffer
Definition: btl_udapl_endpoint.h:147
Definition: btl_udapl_frag.h:52
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
Definition: btl_udapl_endpoint.h:82
Definition: opal_list.h:147
int32_t endpoint_max_sends
number of sends that may be posted
Definition: btl_udapl_endpoint.h:114
Definition: btl_udapl_eager_rdma.h:23
struct mca_btl_udapl_proc_t * endpoint_proc
proc structure corresponding to endpoint
Definition: btl_udapl_endpoint.h:103
int32_t endpoint_connections_completed
count of completed connections for priv data connection est.
Definition: btl_udapl_endpoint.h:130
int32_t endpoint_lwqe_tokens[BTL_UDAPL_NUM_CONNECTION]
number of local work queue credits available (combination of posted sends and rdma writes allowed per...
Definition: btl_udapl_endpoint.h:123
int32_t endpoint_sr_credits[BTL_UDAPL_NUM_CONNECTION]
number of recvs that are now available
Definition: btl_udapl_endpoint.h:120
struct mca_btl_udapl_module_t * endpoint_btl
BTL instance that created this connection.
Definition: btl_udapl_endpoint.h:100
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
int32_t endpoint_sr_tokens[BTL_UDAPL_NUM_CONNECTION]
number of sends that may be posted
Definition: btl_udapl_endpoint.h:117