OpenMPI  0.1.1
btl_ugni.h
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3  * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
4  * reserved.
5  * Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
6  * $COPYRIGHT$
7  *
8  * Additional copyrights may follow
9  *
10  * $HEADER$
11  */
12 /*
13  * The ugni btl is implemented with native Cray Gemini.
14  *
15  * Known issues with ugni:
16  * -
17  */
18 
19 #ifndef MCA_BTL_UGNI_H
20 #define MCA_BTL_UGNI_H
21 
22 #include "ompi_config.h"
23 
24 #include "ompi/mca/mpool/mpool.h"
28 #include "opal/util/output.h"
29 #include "opal_stdint.h"
31 
32 #include "ompi/mca/btl/btl.h"
33 #include "ompi/mca/btl/base/base.h"
34 #include "ompi/mca/btl/base/btl_base_error.h"
35 #include "ompi/class/ompi_free_list.h"
36 
37 #include "ompi/mca/common/ugni/common_ugni.h"
38 
39 #include <errno.h>
40 #include <stdint.h>
41 #include <sys/types.h>
42 #include <assert.h>
43 #include <sys/time.h>
44 #include <gni_pub.h>
45 
46 /* datagram message ids */
47 #define MCA_BTL_UGNI_CONNECT_WILDCARD_ID 0x6b69726b00000000ull
48 #define MCA_BTL_UGNI_CONNECT_DIRECTED_ID 0x6b61686e00000000ull
49 #define MCA_BTL_UGNI_DATAGRAM_MASK 0xffffffff00000000ull
50 
51 typedef struct mca_btl_ugni_module_t {
53 
55 
56  size_t endpoint_count;
57  struct mca_btl_base_endpoint_t **endpoints;
58 
59  opal_list_t failed_frags;
60 
61  mca_mpool_base_module_t *smsg_mpool;
62  ompi_free_list_t smsg_mboxes;
63 
64  gni_ep_handle_t wildcard_ep;
65  gni_smsg_attr_t wc_remote_attr, wc_local_attr;
66 
67  gni_cq_handle_t bte_local_cq;
68  gni_cq_handle_t smsg_remote_cq;
69 
70  /* eager (registered) fragment list */
71  ompi_free_list_t eager_frags_send;
72  ompi_free_list_t eager_frags_recv;
73 
74  /* SMSG fragment list (unregistered) */
75  ompi_free_list_t smsg_frags;
76 
77  /* RDMA fragment list */
78  ompi_free_list_t rdma_frags;
79  ompi_free_list_t rdma_int_frags;
80 
81  /* fragment buffer (for message if lookup) */
82  opal_hash_table_t pending_smsg_frags;
83  int32_t next_frag_id;
85 
86 typedef struct mca_btl_ugni_component_t {
87  /* base BTL component */
89 
90  /* maximum supported btls. hardcoded to 1 for now */
91  uint32_t ugni_max_btls;
92  /* Maximum number of entries a completion queue can hold */
93  uint32_t cq_size;
94 
95  /* number of ugni modules */
96  uint32_t ugni_num_btls;
97  /* ugni modules */
98  mca_btl_ugni_module_t *modules;
99 
100  size_t smsg_max_data;
101 
102  /* After this message size switch to BTE protocols */
103  size_t ugni_fma_limit;
104  /* Switch to put when trying to GET at or above this size */
105  size_t ugni_get_limit;
106  /* Switch to get when sending above this size */
107  size_t ugni_smsg_limit;
108 
109  /* RDMA/SMSG free list settings */
110  int ugni_free_list_num;
111  int ugni_free_list_max;
112  int ugni_free_list_inc;
113 
114  /* eager free list settings */
115  int ugni_eager_num;
116  int ugni_eager_max;
117  int ugni_eager_inc;
118 
119  /* number of times to retry a post */
120  int rdma_max_retries;
121 
122  /* Maximum number of outstanding eager messages */
123  int smsg_max_credits;
124  /* mailbox size (computed) */
125  int smsg_mbox_size;
127 
128 int mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
129  ompi_common_ugni_device_t *device);
130 
131 /**
132  * BML->BTL notification of change in the process list.
133  *
134  * location: btl_ugni_add_procs.c
135  *
136  * @param btl (IN) BTL module
137  * @param nprocs (IN) Number of processes
138  * @param procs (IN) Array of processes
139  * @param endpoint (OUT) Array of mca_btl_base_endpoint_t structures by BTL.
140  * @param reachable (OUT) Bitmask indicating set of peer processes that are reachable by this BTL.
141  * @return OMPI_SUCCESS or error status on failure.
142  */
143 int
144 mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl,
145  size_t nprocs,
146  struct ompi_proc_t **procs,
147  struct mca_btl_base_endpoint_t **peers,
148  opal_bitmap_t *reachable);
149 
150 /**
151  * Notification of change to the process list.
152  *
153  * location: btl_ugni_add_procs.c
154  *
155  * @param btl (IN) BTL module
156  * @param nprocs (IN) Number of processes
157  * @param proc (IN) Set of processes
158  * @param peer (IN) Set of peer addressing information.
159  * @return Status indicating if cleanup was successful
160  */
161 int
162 mca_btl_ugni_del_procs (struct mca_btl_base_module_t *btl,
163  size_t nprocs,
164  struct ompi_proc_t **procs,
165  struct mca_btl_base_endpoint_t **peers);
166 
167 /**
168  * Initiate an asynchronous send.
169  *
170  * location: btl_ugni_send.c
171  *
172  * @param btl (IN) BTL module
173  * @param endpoint (IN) BTL addressing information
174  * @param descriptor (IN) Description of the data to be transfered
175  * @param tag (IN) The tag value used to notify the peer.
176  */
177 int
178 mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
179  struct mca_btl_base_endpoint_t *btl_peer,
180  struct mca_btl_base_descriptor_t *descriptor,
181  mca_btl_base_tag_t tag);
182 
183 /**
184  * Initiate an immediate blocking send.
185  *
186  * location: btl_ugni_sendi.c
187  *
188  * @param btl (IN) BTL module
189  * @param endpoint (IN) BTL addressing information
190  * @param convertor (IN) Data type convertor
191  * @param header (IN) Pointer to header.
192  * @param header_size (IN) Size of header.
193  * @param payload_size (IN) Size of payload (from convertor).
194  * @param order (IN) The ordering tag (may be MCA_BTL_NO_ORDER)
195  * @param flags (IN) Flags.
196  * @param tag (IN) The tag value used to notify the peer.
197  * @param descriptor (OUT) The descriptor to be returned unable to be sent immediately
198  */
199 int
200 mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
201  struct mca_btl_base_endpoint_t *endpoint,
202  struct opal_convertor_t *convertor,
203  void *header, size_t header_size,
204  size_t payload_size, uint8_t order,
205  uint32_t flags, mca_btl_base_tag_t tag,
206  mca_btl_base_descriptor_t **descriptor);
207 
208 /**
209  * Initiate a get operation.
210  *
211  * location: btl_ugni_get.c
212  *
213  * @param btl (IN) BTL module
214  * @param endpoint (IN) BTL addressing information
215  * @param descriptor (IN) Description of the data to be transferred
216  */
217 int
218 mca_btl_ugni_get (struct mca_btl_base_module_t *btl,
219  struct mca_btl_base_endpoint_t *endpoint,
220  struct mca_btl_base_descriptor_t *des);
221 
222 /**
223  * Initiate a put operation.
224  *
225  * location: btl_ugni_put.c
226  *
227  * @param btl (IN) BTL module
228  * @param endpoint (IN) BTL addressing information
229  * @param descriptor (IN) Description of the data to be transferred
230  */
231 int
232 mca_btl_ugni_put (struct mca_btl_base_module_t *btl,
233  struct mca_btl_base_endpoint_t *endpoint,
234  struct mca_btl_base_descriptor_t *des);
235 
237 mca_btl_ugni_alloc(struct mca_btl_base_module_t *btl,
238  struct mca_btl_base_endpoint_t *endpoint,
239  uint8_t order, size_t size, uint32_t flags);
240 
241 typedef struct mca_btl_ugni_reg_t {
243  gni_mem_handle_t memory_hdl;
245 
246 /* Global structures */
247 
248 OMPI_MODULE_DECLSPEC extern mca_btl_ugni_component_t mca_btl_ugni_component;
249 OMPI_MODULE_DECLSPEC extern mca_btl_ugni_module_t mca_btl_ugni_module;
250 
251 #endif
Definition: opal_hash_table.h:42
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
OPAL output stream facility.
Definition: opal_bitmap.h:53
Definition: btl_ugni.h:86
Definition: btl_ugni.h:51
Definition: mpool.h:44
Remote Open MPI process structure.
Definition: proc.h:56
Definition: common_ugni.h:42
Byte Transfer Layer (BTL)
Definition: ompi_free_list.h:39
A hash table that may be indexed with either fixed length (e.g.
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
BTL component descriptor.
Definition: btl.h:411
Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana University Research and Techno...
Definition: opal_convertor.h:90
Definition: opal_list.h:147
Open MPI module-related data transfer mechanism.
BTL module interface functions and attributes.
Definition: btl.h:786
Definition: btl_ugni.h:241
mpool module descriptor.
Definition: mpool.h:174