OpenMPI  0.1.1
btl_portals.h
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2009 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
13  * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved.
14  * $COPYRIGHT$
15  *
16  * Additional copyrights may follow
17  *
18  * $HEADER$
19  */
20 /*
21  * @file
22  */
23 #ifndef OMPI_BTL_PORTALS_H
24 #define OMPI_BTL_PORTALS_H
25 
26 #include "ompi_config.h"
27 #include "ompi/mca/common/portals/common_portals.h"
28 
29 #include "ompi/mca/btl/btl.h"
30 #include "ompi/mca/btl/base/base.h"
31 #include "ompi/class/ompi_free_list.h"
32 
33 #include "btl_portals_endpoint.h"
34 #include "btl_portals_frag.h"
35 
36 /*
37  * Portals BTL component.
38  */
40  /* base BTL component */
42 
43  /* output channel for debugging. Value settings when using
44  * output_verbose:
45  *
46  * - 0 : critical user information
47  * - 10: general execution diagnostic information
48  * - 20: initialization / shutdown diagnostic information
49  * - 30: basic debugging information
50  * - 90: useful only to developers
51  * - 100: lots and lots of performance impacting output
52  */
53  int portals_output;
54 
55  /* initial size of free lists */
56  int portals_free_list_init_num;
57  /* max size of free lists */
58  int portals_free_list_max_num;
59  /* numer of elements to grow free lists */
60  int portals_free_list_inc_num;
61 
62  /* number of eager fragments */
63  int portals_free_list_eager_max_num;
64 
65  /* shall I use portals to send to thyself? */
66  int portals_support_self;
67 
68  /* do I need a portals ACK? */
69  int portals_need_ack;
70 
71 };
73 
74 
75 #define OMPI_BTL_PORTALS_EQ_SEND 0
76 #define OMPI_BTL_PORTALS_EQ_RECV 1
77 #define OMPI_BTL_PORTALS_EQ_SIZE 2
78 
80  /* base BTL module interface */
82 
83  /* number of processes we're actively connected to. Needed to
84  know when to do activation / shutdown */
85  int32_t portals_num_procs;
86 
87  /* fragment free lists */
88  ompi_free_list_t portals_frag_eager;
89  ompi_free_list_t portals_frag_max;
90  ompi_free_list_t portals_frag_user;
91 
92  /* incoming send message receive memory descriptors */
93  int portals_recv_mds_num;
94  int portals_recv_mds_size;
95  opal_list_t portals_recv_blocks;
96 
97  /* frag for receive callbacks */
98  mca_btl_portals_frag_recv_t portals_recv_frag;
99 
100  /* event queues. Keep sends on own eq, since we can't control
101  space for the ack otherwise */
102  int portals_eq_sizes[OMPI_BTL_PORTALS_EQ_SIZE];
103  ptl_handle_eq_t portals_eq_handles[OMPI_BTL_PORTALS_EQ_SIZE];
104 
105  /* "reject" entry for recv match list */
106  ptl_handle_me_t portals_recv_reject_me_h;
107 
108  /* number outstanding sends and local rdma */
109  volatile int32_t portals_outstanding_ops;
110  int32_t portals_max_outstanding_ops;
111 
112  /* sends queued until there's time to send */
113  opal_list_t portals_queued_sends;
114 
115  /* key to use for next rdma operation */
116  volatile int64_t portals_rdma_key;
117 
118  /* our portals network interface */
119  ptl_handle_ni_t portals_ni_h;
120 
121  /* number of dropped messages */
122  ptl_sr_value_t portals_sr_dropped;
123 
124  /* descriptors for send */
125  ptl_md_t md_send;
126 };
128 
129 #if OPAL_ENABLE_FT_MPI
130 extern int mca_btl_portals_finalize_max_wait;
131 #endif /* OPAL_ENABLE_FT_MPI */
132 
133 /*
134  * Component functions (btl_portals_component.c)
135  */
136 int mca_btl_portals_component_open(void);
137 int mca_btl_portals_component_close(void);
138 
139 
140 mca_btl_base_module_t** mca_btl_portals_component_init(int *num_btls,
141  bool has_progress_threads,
142  bool has_mpi_threads);
143 
144 int mca_btl_portals_component_progress(void);
145 
146 /*
147  * Compatibility functions (btl_portals_compat_{}.c)
148  *
149  * Not part of the BTL interface. Need to be implemented for every
150  * version of Portals
151  */
152 int mca_btl_portals_init_compat(mca_btl_portals_component_t *comp);
153 
154 /* 4th argument is a ptl_peers array, as that's what we'll get back
155  from many of the access functions... */
156 int mca_btl_portals_add_procs_compat(mca_btl_portals_module_t* btl,
157  size_t nprocs, struct ompi_proc_t **procs,
158  ptl_process_id_t **ptl_peers);
159 
160 /*
161  * Module configuration functions (btl_portals.c)
162  */
163 int mca_btl_portals_finalize(struct mca_btl_base_module_t* btl_base);
164 
165 int mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
166  size_t nprocs,
167  struct ompi_proc_t **procs,
168  struct mca_btl_base_endpoint_t** peers,
169  opal_bitmap_t* reachable);
170 
171 int mca_btl_portals_del_procs(struct mca_btl_base_module_t* btl_base,
172  size_t nprocs,
173  struct ompi_proc_t **procs,
174  struct mca_btl_base_endpoint_t** peers);
175 
177 mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
178  struct mca_btl_base_endpoint_t* endpoint,
179  uint8_t order,
180  size_t size,
181  uint32_t flags);
182 
183 int mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
185 
187 mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
188  struct mca_btl_base_endpoint_t* peer,
189  mca_mpool_base_registration_t* registration,
190  struct opal_convertor_t* convertor,
191  uint8_t order,
192  size_t reserve,
193  size_t* size,
194  uint32_t flags);
195 
197 mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
198  struct mca_btl_base_endpoint_t* peer,
199  mca_mpool_base_registration_t* registration,
200  struct opal_convertor_t* convertor,
201  uint8_t order,
202  size_t reserve,
203  size_t* size,
204  uint32_t flags);
205 
206 int mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
207  struct mca_btl_base_endpoint_t* btl_peer,
208  struct mca_btl_base_descriptor_t* descriptor,
209  mca_btl_base_tag_t tag);
210 
211 
212 int mca_btl_portals_sendi(struct mca_btl_base_module_t* btl_base,
213  struct mca_btl_base_endpoint_t* endpoint,
214  struct opal_convertor_t* convertor,
215  void* header,
216  size_t header_size,
217  size_t payload_size,
218  uint8_t order,
219  uint32_t flags,
220  mca_btl_base_tag_t tag,
222 
223 int mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
224  struct mca_btl_base_endpoint_t* btl_peer,
225  struct mca_btl_base_descriptor_t* decriptor);
226 
227 
228 int mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
229  struct mca_btl_base_endpoint_t* btl_peer,
230  struct mca_btl_base_descriptor_t* decriptor);
231 
232 
233 /*
234  * global structures
235  */
236 OMPI_MODULE_DECLSPEC extern mca_btl_portals_component_t mca_btl_portals_component;
237 extern mca_btl_portals_module_t mca_btl_portals_module;
238 
239 #endif
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
Portals send fragment derived type.
Definition: btl_portals_frag.h:27
Definition: opal_bitmap.h:53
Definition: btl_portals.h:79
Definition: mpool.h:44
Remote Open MPI process structure.
Definition: proc.h:56
Definition: btl_portals.h:39
Byte Transfer Layer (BTL)
Definition: ompi_free_list.h:39
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
BTL component descriptor.
Definition: btl.h:411
Definition: opal_convertor.h:90
Definition: opal_list.h:147
BTL module interface functions and attributes.
Definition: btl.h:786