OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_portals.h
1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2009 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved.
13 * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved.
14 * $COPYRIGHT$
15 *
16 * Additional copyrights may follow
17 *
18 * $HEADER$
19 */
20 /*
21 * @file
22 */
23 #ifndef OMPI_BTL_PORTALS_H
24 #define OMPI_BTL_PORTALS_H
25
26 #include "ompi_config.h"
27 #include "ompi/mca/common/portals/common_portals.h"
28
29 #include "ompi/mca/btl/btl.h"
30 #include "ompi/mca/btl/base/base.h"
31 #include "ompi/class/ompi_free_list.h"
32
33 #include "btl_portals_endpoint.h"
34 #include "btl_portals_frag.h"
35
36 /*
37 * Portals BTL component.
38 */
39 struct mca_btl_portals_component_t {
40 /* base BTL component */
41 mca_btl_base_component_2_0_0_t super;
42
43 /* output channel for debugging. Value settings when using
44 * output_verbose:
45 *
46 * - 0 : critical user information
47 * - 10: general execution diagnostic information
48 * - 20: initialization / shutdown diagnostic information
49 * - 30: basic debugging information
50 * - 90: useful only to developers
51 * - 100: lots and lots of performance impacting output
52 */
53 int portals_output;
54
55 /* initial size of free lists */
56 int portals_free_list_init_num;
57 /* max size of free lists */
58 int portals_free_list_max_num;
59 /* numer of elements to grow free lists */
60 int portals_free_list_inc_num;
61
62 /* number of eager fragments */
63 int portals_free_list_eager_max_num;
64
65 /* shall I use portals to send to thyself? */
66 int portals_support_self;
67
68 /* do I need a portals ACK? */
69 int portals_need_ack;
70
71 };
72 typedef struct mca_btl_portals_component_t mca_btl_portals_component_t;
73
74
75 #define OMPI_BTL_PORTALS_EQ_SEND 0
76 #define OMPI_BTL_PORTALS_EQ_RECV 1
77 #define OMPI_BTL_PORTALS_EQ_SIZE 2
78
79 struct mca_btl_portals_module_t {
80 /* base BTL module interface */
81 mca_btl_base_module_t super;
82
83 /* number of processes we're actively connected to. Needed to
84 know when to do activation / shutdown */
85 int32_t portals_num_procs;
86
87 /* fragment free lists */
88 ompi_free_list_t portals_frag_eager;
89 ompi_free_list_t portals_frag_max;
90 ompi_free_list_t portals_frag_user;
91
92 /* incoming send message receive memory descriptors */
93 int portals_recv_mds_num;
94 int portals_recv_mds_size;
95 opal_list_t portals_recv_blocks;
96
97 /* frag for receive callbacks */
98 mca_btl_portals_frag_recv_t portals_recv_frag;
99
100 /* event queues. Keep sends on own eq, since we can't control
101 space for the ack otherwise */
102 int portals_eq_sizes[OMPI_BTL_PORTALS_EQ_SIZE];
103 ptl_handle_eq_t portals_eq_handles[OMPI_BTL_PORTALS_EQ_SIZE];
104
105 /* "reject" entry for recv match list */
106 ptl_handle_me_t portals_recv_reject_me_h;
107
108 /* number outstanding sends and local rdma */
109 volatile int32_t portals_outstanding_ops;
110 int32_t portals_max_outstanding_ops;
111
112 /* sends queued until there's time to send */
113 opal_list_t portals_queued_sends;
114
115 /* key to use for next rdma operation */
116 volatile int64_t portals_rdma_key;
117
118 /* our portals network interface */
119 ptl_handle_ni_t portals_ni_h;
120
121 /* number of dropped messages */
122 ptl_sr_value_t portals_sr_dropped;
123
124 /* descriptors for send */
125 ptl_md_t md_send;
126 };
127 typedef struct mca_btl_portals_module_t mca_btl_portals_module_t;
128
129 #if OPAL_ENABLE_FT_MPI
130 extern int mca_btl_portals_finalize_max_wait;
131 #endif /* OPAL_ENABLE_FT_MPI */
132
133 /*
134 * Component functions (btl_portals_component.c)
135 */
136 int mca_btl_portals_component_open(void);
137 int mca_btl_portals_component_close(void);
138
139
140 mca_btl_base_module_t** mca_btl_portals_component_init(int *num_btls,
141 bool has_progress_threads,
142 bool has_mpi_threads);
143
144 int mca_btl_portals_component_progress(void);
145
146 /*
147 * Compatibility functions (btl_portals_compat_{}.c)
148 *
149 * Not part of the BTL interface. Need to be implemented for every
150 * version of Portals
151 */
152 int mca_btl_portals_init_compat(mca_btl_portals_component_t *comp);
153
154 /* 4th argument is a ptl_peers array, as that's what we'll get back
155 from many of the access functions... */
156 int mca_btl_portals_add_procs_compat(mca_btl_portals_module_t* btl,
157 size_t nprocs, struct ompi_proc_t **procs,
158 ptl_process_id_t **ptl_peers);
159
160 /*
161 * Module configuration functions (btl_portals.c)
162 */
163 int mca_btl_portals_finalize(struct mca_btl_base_module_t* btl_base);
164
165 int mca_btl_portals_add_procs(struct mca_btl_base_module_t* btl_base,
166 size_t nprocs,
167 struct ompi_proc_t **procs,
168 struct mca_btl_base_endpoint_t** peers,
169 opal_bitmap_t* reachable);
170
171 int mca_btl_portals_del_procs(struct mca_btl_base_module_t* btl_base,
172 size_t nprocs,
173 struct ompi_proc_t **procs,
174 struct mca_btl_base_endpoint_t** peers);
175
176 mca_btl_base_descriptor_t*
177 mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
178 struct mca_btl_base_endpoint_t* endpoint,
179 uint8_t order,
180 size_t size,
181 uint32_t flags);
182
183 int mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
184 mca_btl_base_descriptor_t* des);
185
186 mca_btl_base_descriptor_t*
187 mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
188 struct mca_btl_base_endpoint_t* peer,
189 mca_mpool_base_registration_t* registration,
190 struct opal_convertor_t* convertor,
191 uint8_t order,
192 size_t reserve,
193 size_t* size,
194 uint32_t flags);
195
196 mca_btl_base_descriptor_t*
197 mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
198 struct mca_btl_base_endpoint_t* peer,
199 mca_mpool_base_registration_t* registration,
200 struct opal_convertor_t* convertor,
201 uint8_t order,
202 size_t reserve,
203 size_t* size,
204 uint32_t flags);
205
206 int mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
207 struct mca_btl_base_endpoint_t* btl_peer,
208 struct mca_btl_base_descriptor_t* descriptor,
209 mca_btl_base_tag_t tag);
210
211
212 int mca_btl_portals_sendi(struct mca_btl_base_module_t* btl_base,
213 struct mca_btl_base_endpoint_t* endpoint,
214 struct opal_convertor_t* convertor,
215 void* header,
216 size_t header_size,
217 size_t payload_size,
218 uint8_t order,
219 uint32_t flags,
220 mca_btl_base_tag_t tag,
221 mca_btl_base_descriptor_t** des);
222
223 int mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
224 struct mca_btl_base_endpoint_t* btl_peer,
225 struct mca_btl_base_descriptor_t* decriptor);
226
227
228 int mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
229 struct mca_btl_base_endpoint_t* btl_peer,
230 struct mca_btl_base_descriptor_t* decriptor);
231
232
233 /*
234 * global structures
235 */
236 OMPI_MODULE_DECLSPEC extern mca_btl_portals_component_t mca_btl_portals_component;
237 extern mca_btl_portals_module_t mca_btl_portals_module;
238
239 #endif
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
mca_btl_portals_frag_t
Portals send fragment derived type.
Definition: btl_portals_frag.h:27
opal_bitmap_t
Definition: opal_bitmap.h:53
mca_btl_portals_module_t
Definition: btl_portals.h:79
mca_mpool_base_registration_t
Definition: mpool.h:44
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
mca_btl_portals_component_t
Definition: btl_portals.h:39
btl.h
Byte Transfer Layer (BTL)
ompi_free_list_t
Definition: ompi_free_list.h:39
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_base_component_2_0_0_t
BTL component descriptor.
Definition: btl.h:411
opal_convertor_t
Definition: opal_convertor.h:90
opal_list_t
Definition: opal_list.h:147
base.h
mca_btl_base_module_t
BTL module interface functions and attributes.
Definition: btl.h:786