OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

mtl_portals4.h
1 /*
2 * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2007 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2010-2012 Sandia National Laboratories. All rights reserved.
13 * $COPYRIGHT$
14 *
15 * Additional copyrights may follow
16 *
17 * $HEADER$
18 */
19
20 #ifndef MTL_PORTALS_H_HAS_BEEN_INCLUDED
21 #define MTL_PORTALS_H_HAS_BEEN_INCLUDED
22
23 #include <portals4.h>
24
25 #include "opal/class/opal_free_list.h"
26 #include "opal/class/opal_list.h"
27 #include "opal/datatype/opal_convertor.h"
28 #include "ompi/mca/mtl/mtl.h"
29 #include "ompi/mca/mtl/base/base.h"
30
31 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
32 #include "mtl_portals4_flowctl.h"
33 #endif
34
35 BEGIN_C_DECLS
36
37 struct mca_mtl_portals4_send_request_t;
38
39 struct mca_mtl_portals4_module_t {
40 mca_mtl_base_module_t base;
41
42 /** Eager limit; messages greater than this use a rendezvous protocol */
43 size_t eager_limit;
44 /** Size of short message blocks */
45 size_t recv_short_size;
46 /** Number of short message blocks which should be created during startup */
47 int recv_short_num;
48 /** Length of both the receive and send event queues */
49 int queue_size;
50 /** Protocol for long message transfer */
51 enum { eager, rndv } protocol;
52
53 /* free list of message for matched probe */
54 opal_free_list_t fl_message;
55
56 /** Network interface handle for matched interface */
57 ptl_handle_ni_t ni_h;
58
59 /** portals index for message matching */
60 ptl_pt_index_t send_idx;
61 /** portals index for long message rendezvous */
62 ptl_pt_index_t read_idx;
63 /** portals index for flow control recovery */
64 ptl_pt_index_t flowctl_idx;
65 /** portals index for flow control recovery operatings which
66 generate full events */
67 ptl_pt_index_t flowctl_event_idx;
68
69 /** Event queue handles. See send_eq_h and recv_eq_h defines for
70 usage. Array for PtlEQPoll */
71 ptl_handle_eq_t eqs_h[2];
72
73 /** MD for zero-length sends and acks. Optimization, can be
74 reused anywhere a 0-byte ping is necessary */
75 ptl_handle_md_t zero_md_h;
76
77 /** long message receive overflow ME. Persistent ME, first in
78 overflow list on the send_idx portal table. */
79 ptl_handle_me_t long_overflow_me_h;
80
81 /** List of active short receive blocks. Active means that the ME
82 was posted to the overflow list and the UNLINK event has not
83 yet been received. */
84 opal_list_t active_recv_short_blocks;
85
86 /** List of short receive blocks waiting for FREE event. Blocks
87 are added to this list when the UNLINK event has been
88 received and removed when the FREE event is received. */
89 opal_list_t waiting_recv_short_blocks;
90
91 /** number of send-side operations started */
92 uint32_t opcount;
93
94 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
95 /** Number of event slots available for send side operations.
96 Note that this is slightly smaller than the event queue size
97 to allow space for flow control related events. */
98 uint32_t send_queue_slots;
99
100 /** free list of send items */
101 opal_free_list_t send_fl;
102 /** list of sends which are pending (either due to needing to
103 retransmit or because we're in flow control recovery */
104 opal_list_t pending_sends;
105 /** list of sends which are currently active */
106 opal_list_t active_sends;
107
108 ompi_mtl_portals4_flowctl_t flowctl;
109 #endif
110
111 #if OPAL_ENABLE_DEBUG
112 /** number of receive-side operations started. Used only for
113 debugging */
114 uint32_t recv_opcount;
115 #endif
116 };
117 typedef struct mca_mtl_portals4_module_t mca_mtl_portals4_module_t;
118
119 #define send_eq_h eqs_h[0]
120 #define recv_eq_h eqs_h[1]
121
122 extern mca_mtl_portals4_module_t ompi_mtl_portals4;
123
124 #define REQ_SEND_TABLE_ID 2
125 #define REQ_READ_TABLE_ID 3
126 #define REQ_FLOWCTL_TABLE_ID 4
127
128 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
129 #define MTL_PORTALS4_FLOWCTL_TRIGGER 0x01
130 #define MTL_PORTALS4_FLOWCTL_ALERT 0x02
131 #define MTL_PORTALS4_FLOWCTL_FANIN 0x03
132 #define MTL_PORTALS4_FLOWCTL_FANOUT 0x04
133 #endif
134
135 /* match/ignore bit manipulation
136 *
137 * 0123 4567 01234567 01234567 01234567 01234567 01234567 01234567 01234567
138 * | | |
139 * ^ | context id | source | message tag
140 * | | | |
141 * +---- protocol
142 */
143
144 #define MTL_PORTALS4_PROTOCOL_MASK 0xF000000000000000ULL
145 #define MTL_PORTALS4_CONTEXT_MASK 0x0FFF000000000000ULL
146 #define MTL_PORTALS4_SOURCE_MASK 0x0000FFFF00000000ULL
147 #define MTL_PORTALS4_TAG_MASK 0x00000000FFFFFFFFULL
148
149 #define MTL_PORTALS4_PROTOCOL_IGNR MTL_PORTALS4_PROTOCOL_MASK
150 #define MTL_PORTALS4_CONTEXT_IGNR MTL_PORTALS4_CONTEXT_MASK
151 #define MTL_PORTALS4_SOURCE_IGNR MTL_PORTALS4_SOURCE_MASK
152 #define MTL_PORTALS4_TAG_IGNR 0x000000007FFFFFFFULL
153
154 #define MTL_PORTALS4_SHORT_MSG 0x1000000000000000ULL
155 #define MTL_PORTALS4_LONG_MSG 0x2000000000000000ULL
156 #define MTL_PORTALS4_READY_MSG 0x4000000000000000ULL
157
158 /* send posting */
159 #define MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
160 { \
161 match_bits = contextid; \
162 match_bits = (match_bits << 16); \
163 match_bits |= source; \
164 match_bits = (match_bits << 32); \
165 match_bits |= (MTL_PORTALS4_TAG_MASK & tag) | type; \
166 }
167
168 /* receive posting */
169 #define MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, contextid, source, tag) \
170 { \
171 match_bits = 0; \
172 ignore_bits = MTL_PORTALS4_PROTOCOL_IGNR; \
173 \
174 match_bits = contextid; \
175 match_bits = (match_bits << 16); \
176 \
177 if (MPI_ANY_SOURCE == source) { \
178 match_bits = (match_bits << 32); \
179 ignore_bits |= MTL_PORTALS4_SOURCE_IGNR; \
180 } else { \
181 match_bits |= source; \
182 match_bits = (match_bits << 32); \
183 } \
184 \
185 if (MPI_ANY_TAG == tag) { \
186 ignore_bits |= MTL_PORTALS4_TAG_IGNR; \
187 } else { \
188 match_bits |= (MTL_PORTALS4_TAG_MASK & tag); \
189 } \
190 }
191
192 #define MTL_PORTALS4_IS_SHORT_MSG(match_bits) \
193 (0 != (MTL_PORTALS4_SHORT_MSG & match_bits))
194 #define MTL_PORTALS4_IS_LONG_MSG(match_bits) \
195 (0 != (MTL_PORTALS4_LONG_MSG & match_bits))
196 #define MTL_PORTALS4_IS_READY_MSG(match_bits) \
197 (0 != (MTL_PORTALS4_READY_MSG & match_bits))
198
199 #define MTL_PORTALS4_GET_TAG(match_bits) \
200 ((int)(match_bits & MTL_PORTALS4_TAG_MASK))
201 #define MTL_PORTALS4_GET_SOURCE(match_bits) \
202 ((int)((match_bits & MTL_PORTALS4_SOURCE_MASK) >> 32))
203
204
205 #define MTL_PORTALS4_SYNC_MSG 0x8000000000000000ULL
206
207 #define MTL_PORTALS4_SET_HDR_DATA(hdr_data, opcount, length, sync) \
208 { \
209 hdr_data = (sync) ? 1 : 0; \
210 hdr_data = (hdr_data << 15); \
211 hdr_data |= opcount & 0x7FFFULL; \
212 hdr_data = (hdr_data << 48); \
213 hdr_data |= (length & 0xFFFFFFFFFFFFULL); \
214 }
215
216 #define MTL_PORTALS4_GET_LENGTH(hdr_data) ((size_t)(hdr_data & 0xFFFFFFFFFFFFULL))
217 #define MTL_PORTALS4_IS_SYNC_MSG(hdr_data) \
218 (0 != (MTL_PORTALS4_SYNC_MSG & hdr_data))
219
220 /* MTL interface functions */
221 extern int ompi_mtl_portals4_finalize(struct mca_mtl_base_module_t *mtl);
222
223 extern int ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t* mtl,
224 size_t nprocs,
225 struct ompi_proc_t** procs,
226 struct mca_mtl_base_endpoint_t **mtl_peer_data);
227
228 extern int ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t* mtl,
229 size_t nprocs,
230 struct ompi_proc_t** procs,
231 struct mca_mtl_base_endpoint_t **mtl_peer_data);
232
233 extern int ompi_mtl_portals4_send(struct mca_mtl_base_module_t* mtl,
234 struct ompi_communicator_t* comm,
235 int dest,
236 int tag,
237 struct opal_convertor_t *convertor,
238 mca_pml_base_send_mode_t mode);
239
240 extern int ompi_mtl_portals4_isend(struct mca_mtl_base_module_t* mtl,
241 struct ompi_communicator_t* comm,
242 int dest,
243 int tag,
244 struct opal_convertor_t *convertor,
245 mca_pml_base_send_mode_t mode,
246 bool blocking,
247 mca_mtl_request_t *mtl_request);
248
249 extern int ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
250 struct ompi_communicator_t *comm,
251 int src,
252 int tag,
253 struct opal_convertor_t *convertor,
254 mca_mtl_request_t *mtl_request);
255
256 extern int ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
257 struct ompi_communicator_t *comm,
258 int src,
259 int tag,
260 int *flag,
261 struct ompi_status_public_t *status);
262
263 extern int ompi_mtl_portals4_imrecv(struct mca_mtl_base_module_t* mtl,
264 struct opal_convertor_t *convertor,
265 struct ompi_message_t **message,
266 struct mca_mtl_request_t *mtl_request);
267
268 extern int ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
269 struct ompi_communicator_t *comm,
270 int src,
271 int tag,
272 int *matched,
273 struct ompi_message_t **message,
274 struct ompi_status_public_t *status);
275
276 extern int ompi_mtl_portals4_cancel(struct mca_mtl_base_module_t* mtl,
277 mca_mtl_request_t *mtl_request,
278 int flag);
279
280 extern int ompi_mtl_portals4_add_comm(struct mca_mtl_base_module_t *mtl,
281 struct ompi_communicator_t *comm);
282
283 extern int ompi_mtl_portals4_del_comm(struct mca_mtl_base_module_t *mtl,
284 struct ompi_communicator_t *comm);
285
286 extern int ompi_mtl_portals4_progress(void);
287
288 extern int ompi_mtl_portals4_get_error(int ptl_error);
289
290 END_C_DECLS
291
292 #endif /* MTL_PORTALS_H_HAS_BEEN_INCLUDED */
mca_mtl_portals4_module_t::eqs_h
ptl_handle_eq_t eqs_h[2]
Event queue handles.
Definition: mtl_portals4.h:71
mca_mtl_portals4_module_t::recv_short_num
int recv_short_num
Number of short message blocks which should be created during startup.
Definition: mtl_portals4.h:47
mca_mtl_portals4_module_t::flowctl_event_idx
ptl_pt_index_t flowctl_event_idx
portals index for flow control recovery operatings which generate full events
Definition: mtl_portals4.h:67
mca_mtl_portals4_module_t::read_idx
ptl_pt_index_t read_idx
portals index for long message rendezvous
Definition: mtl_portals4.h:62
mca_mtl_portals4_module_t::active_recv_short_blocks
opal_list_t active_recv_short_blocks
List of active short receive blocks.
Definition: mtl_portals4.h:84
mca_mtl_portals4_module_t::opcount
uint32_t opcount
number of send-side operations started
Definition: mtl_portals4.h:92
mca_mtl_portals4_module_t::send_idx
ptl_pt_index_t send_idx
portals index for message matching
Definition: mtl_portals4.h:60
mca_mtl_portals4_module_t::flowctl_idx
ptl_pt_index_t flowctl_idx
portals index for flow control recovery
Definition: mtl_portals4.h:64
mca_mtl_portals4_module_t::eager_limit
size_t eager_limit
Eager limit; messages greater than this use a rendezvous protocol.
Definition: mtl_portals4.h:43
mca_mtl_portals4_module_t::recv_short_size
size_t recv_short_size
Size of short message blocks.
Definition: mtl_portals4.h:45
mca_mtl_request_t
Definition: mtl.h:51
opal_list.h
The opal_list_t interface is used to provide a generic doubly-linked list container for Open MPI...
ompi_proc_t
Remote Open MPI process structure.
Definition: proc.h:56
mca_mtl_portals4_module_t::protocol
enum mca_mtl_portals4_module_t::@122 protocol
Protocol for long message transfer.
mca_mtl_portals4_module_t::zero_md_h
ptl_handle_md_t zero_md_h
MD for zero-length sends and acks.
Definition: mtl_portals4.h:75
opal_free_list_t
Definition: opal_free_list.h:31
mca_mtl_portals4_module_t::ni_h
ptl_handle_ni_t ni_h
Network interface handle for matched interface.
Definition: mtl_portals4.h:57
mca_mtl_base_endpoint_t
An abstraction that represents a connection to a endpoint process.
Definition: mtl_mx_endpoint.h:50
opal_convertor_t
Definition: opal_convertor.h:90
mtl.h
Matching Transport Layer.
opal_list_t
Definition: opal_list.h:147
mca_mtl_portals4_module_t::queue_size
int queue_size
Length of both the receive and send event queues.
Definition: mtl_portals4.h:49
ompi_status_public_t
Definition: mpi.h:337
mca_mtl_base_module_t
MTL module interface functions and attributes.
Definition: mtl.h:399
mca_mtl_portals4_module_t::waiting_recv_short_blocks
opal_list_t waiting_recv_short_blocks
List of short receive blocks waiting for FREE event.
Definition: mtl_portals4.h:89
mca_mtl_portals4_module_t::long_overflow_me_h
ptl_handle_me_t long_overflow_me_h
long message receive overflow ME.
Definition: mtl_portals4.h:79
ompi_communicator_t
Definition: communicator.h:118
ompi_mtl_portals4_flowctl_t
Definition: mtl_portals4_flowctl.h:13
mca_mtl_portals4_module_t
Definition: mtl_portals4.h:39
ompi_message_t
Definition: message.h:22