OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_ugni.h
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
4 * reserved.
5 * Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
6 * $COPYRIGHT$
7 *
8 * Additional copyrights may follow
9 *
10 * $HEADER$
11 */
12 /*
13 * The ugni btl is implemented with native Cray Gemini.
14 *
15 * Known issues with ugni:
16 * -
17 */
18
19 #ifndef MCA_BTL_UGNI_H
20 #define MCA_BTL_UGNI_H
21
22 #include "ompi_config.h"
23
24 #include "ompi/mca/mpool/mpool.h"
25 #include "ompi/mca/mpool/base/base.h"
26 #include "ompi/mca/mpool/rdma/mpool_rdma.h"
27 #include "ompi/runtime/ompi_module_exchange.h"
28 #include "opal/util/output.h"
29 #include "opal_stdint.h"
30 #include "opal/class/opal_hash_table.h"
31
32 #include "ompi/mca/btl/btl.h"
33 #include "ompi/mca/btl/base/base.h"
34 #include "ompi/mca/btl/base/btl_base_error.h"
35 #include "ompi/class/ompi_free_list.h"
36
37 #include "ompi/mca/common/ugni/common_ugni.h"
38
39 #include <errno.h>
40 #include <stdint.h>
41 #include <sys/types.h>
42 #include <assert.h>
43 #include <sys/time.h>
44 #include <gni_pub.h>
45
46 /* datagram message ids */
47 #define MCA_BTL_UGNI_CONNECT_WILDCARD_ID 0x6b69726b00000000ull
48 #define MCA_BTL_UGNI_CONNECT_DIRECTED_ID 0x6b61686e00000000ull
49 #define MCA_BTL_UGNI_DATAGRAM_MASK 0xffffffff00000000ull
50
51 typedef struct mca_btl_ugni_module_t {
52 mca_btl_base_module_t super;
53
54 ompi_common_ugni_device_t *device;
55
56 size_t endpoint_count;
57 struct mca_btl_base_endpoint_t **endpoints;
58
59 opal_list_t failed_frags;
60
61 mca_mpool_base_module_t *smsg_mpool;
62 ompi_free_list_t smsg_mboxes;
63
64 gni_ep_handle_t wildcard_ep;
65 gni_smsg_attr_t wc_remote_attr, wc_local_attr;
66
67 gni_cq_handle_t bte_local_cq;
68 gni_cq_handle_t smsg_remote_cq;
69
70 /* eager (registered) fragment list */
71 ompi_free_list_t eager_frags_send;
72 ompi_free_list_t eager_frags_recv;
73
74 /* SMSG fragment list (unregistered) */
75 ompi_free_list_t smsg_frags;
76
77 /* RDMA fragment list */
78 ompi_free_list_t rdma_frags;
79 ompi_free_list_t rdma_int_frags;
80
81 /* fragment buffer (for message if lookup) */
82 opal_hash_table_t pending_smsg_frags;
83 int32_t next_frag_id;
84 } mca_btl_ugni_module_t;
85
86 typedef struct mca_btl_ugni_component_t {
87 /* base BTL component */
88 mca_btl_base_component_2_0_0_t super;
89
90 /* maximum supported btls. hardcoded to 1 for now */
91 uint32_t ugni_max_btls;
92 /* Maximum number of entries a completion queue can hold */
93 uint32_t cq_size;
94
95 /* number of ugni modules */
96 uint32_t ugni_num_btls;
97 /* ugni modules */
98 mca_btl_ugni_module_t *modules;
99
100 size_t smsg_max_data;
101
102 /* After this message size switch to BTE protocols */
103 size_t ugni_fma_limit;
104 /* Switch to put when trying to GET at or above this size */
105 size_t ugni_get_limit;
106 /* Switch to get when sending above this size */
107 size_t ugni_smsg_limit;
108
109 /* RDMA/SMSG free list settings */
110 int ugni_free_list_num;
111 int ugni_free_list_max;
112 int ugni_free_list_inc;
113
114 /* eager free list settings */
115 int ugni_eager_num;
116 int ugni_eager_max;
117 int ugni_eager_inc;
118
119 /* number of times to retry a post */
120 int rdma_max_retries;
121
122 /* Maximum number of outstanding eager messages */
123 int smsg_max_credits;
124 /* mailbox size (computed) */
125 int smsg_mbox_size;
126 } mca_btl_ugni_component_t;
127
128 int mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
129 ompi_common_ugni_device_t *device);
130
131 /**
132 * BML->BTL notification of change in the process list.
133 *
134 * location: btl_ugni_add_procs.c
135 *
136 * @param btl (IN) BTL module
137 * @param nprocs (IN) Number of processes
138 * @param procs (IN) Array of processes
139 * @param endpoint (OUT) Array of mca_btl_base_endpoint_t structures by BTL.
140 * @param reachable (OUT) Bitmask indicating set of peer processes that are reachable by this BTL.
141 * @return OMPI_SUCCESS or error status on failure.
142 */
143 int
144 mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl,
145 size_t nprocs,
146 struct ompi_proc_t **procs,
147 struct mca_btl_base_endpoint_t **peers,
148 opal_bitmap_t *reachable);
149
150 /**
151 * Notification of change to the process list.
152 *
153 * location: btl_ugni_add_procs.c
154 *
155 * @param btl (IN) BTL module
156 * @param nprocs (IN) Number of processes
157 * @param proc (IN) Set of processes
158 * @param peer (IN) Set of peer addressing information.
159 * @return Status indicating if cleanup was successful
160 */
161 int
162 mca_btl_ugni_del_procs (struct mca_btl_base_module_t *btl,
163 size_t nprocs,
164 struct ompi_proc_t **procs,
165 struct mca_btl_base_endpoint_t **peers);
166
167 /**
168 * Initiate an asynchronous send.
169 *
170 * location: btl_ugni_send.c
171 *
172 * @param btl (IN) BTL module
173 * @param endpoint (IN) BTL addressing information
174 * @param descriptor (IN) Description of the data to be transfered
175 * @param tag (IN) The tag value used to notify the peer.
176 */
177 int
178 mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
179 struct mca_btl_base_endpoint_t *btl_peer,
180 struct mca_btl_base_descriptor_t *descriptor,
181 mca_btl_base_tag_t tag);
182
183 /**
184 * Initiate an immediate blocking send.
185 *
186 * location: btl_ugni_sendi.c
187 *
188 * @param btl (IN) BTL module
189 * @param endpoint (IN) BTL addressing information
190 * @param convertor (IN) Data type convertor
191 * @param header (IN) Pointer to header.
192 * @param header_size (IN) Size of header.
193 * @param payload_size (IN) Size of payload (from convertor).
194 * @param order (IN) The ordering tag (may be MCA_BTL_NO_ORDER)
195 * @param flags (IN) Flags.
196 * @param tag (IN) The tag value used to notify the peer.
197 * @param descriptor (OUT) The descriptor to be returned unable to be sent immediately
198 */
199 int
200 mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
201 struct mca_btl_base_endpoint_t *endpoint,
202 struct opal_convertor_t *convertor,
203 void *header, size_t header_size,
204 size_t payload_size, uint8_t order,
205 uint32_t flags, mca_btl_base_tag_t tag,
206 mca_btl_base_descriptor_t **descriptor);
207
208 /**
209 * Initiate a get operation.
210 *
211 * location: btl_ugni_get.c
212 *
213 * @param btl (IN) BTL module
214 * @param endpoint (IN) BTL addressing information
215 * @param descriptor (IN) Description of the data to be transferred
216 */
217 int
218 mca_btl_ugni_get (struct mca_btl_base_module_t *btl,
219 struct mca_btl_base_endpoint_t *endpoint,
220 struct mca_btl_base_descriptor_t *des);
221
222 /**
223 * Initiate a put operation.
224 *
225 * location: btl_ugni_put.c
226 *
227 * @param btl (IN) BTL module
228 * @param endpoint (IN) BTL addressing information
229 * @param descriptor (IN) Description of the data to be transferred
230 */
231 int
232 mca_btl_ugni_put (struct mca_btl_base_module_t *btl,
233 struct mca_btl_base_endpoint_t *endpoint,
234 struct mca_btl_base_descriptor_t *des);
235
236 mca_btl_base_descriptor_t *
237 mca_btl_ugni_alloc(struct mca_btl_base_module_t *btl,
238 struct mca_btl_base_endpoint_t *endpoint,
239 uint8_t order, size_t size, uint32_t flags);
240
241 typedef struct mca_btl_ugni_reg_t {
242 mca_mpool_base_registration_t base;
243 gni_mem_handle_t memory_hdl;
244 } mca_btl_ugni_reg_t;
245
246 /* Global structures */
247
248 OMPI_MODULE_DECLSPEC extern mca_btl_ugni_component_t mca_btl_ugni_component;
249 OMPI_MODULE_DECLSPEC extern mca_btl_ugni_module_t mca_btl_ugni_module;
250
251 #endif
opal_hash_table_t
Definition: opal_hash_table.h:42
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
output.h
OPAL output stream facility.
opal_bitmap_t
Definition: opal_bitmap.h:53
mca_btl_ugni_component_t
Definition: btl_ugni.h:86
base.h
mca_btl_ugni_module_t
Definition: btl_ugni.h:51
mca_mpool_base_registration_t
Definition: mpool.h:44
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
ompi_common_ugni_device_t
Definition: common_ugni.h:42
btl.h
Byte Transfer Layer (BTL)
ompi_free_list_t
Definition: ompi_free_list.h:39
opal_hash_table.h
A hash table that may be indexed with either fixed length (e.g.
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_base_component_2_0_0_t
BTL component descriptor.
Definition: btl.h:411
mpool.h
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
opal_convertor_t
Definition: opal_convertor.h:90
opal_list_t
Definition: opal_list.h:147
base.h
ompi_module_exchange.h
Open MPI module-related data transfer mechanism.
mpool_rdma.h
mca_btl_base_module_t
BTL module interface functions and attributes.
Definition: btl.h:786
mca_btl_ugni_reg_t
Definition: btl_ugni.h:241
mca_mpool_base_module_t
mpool module descriptor.
Definition: mpool.h:174