OpenMPI  0.1.1
mtl_portals4.h
1 /*
2  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2007 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2010-2012 Sandia National Laboratories. All rights reserved.
13  * $COPYRIGHT$
14  *
15  * Additional copyrights may follow
16  *
17  * $HEADER$
18  */
19 
20 #ifndef MTL_PORTALS_H_HAS_BEEN_INCLUDED
21 #define MTL_PORTALS_H_HAS_BEEN_INCLUDED
22 
23 #include <portals4.h>
24 
25 #include "opal/class/opal_free_list.h"
26 #include "opal/class/opal_list.h"
27 #include "opal/datatype/opal_convertor.h"
28 #include "ompi/mca/mtl/mtl.h"
29 #include "ompi/mca/mtl/base/base.h"
30 
31 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
32 #include "mtl_portals4_flowctl.h"
33 #endif
34 
35 BEGIN_C_DECLS
36 
37 struct mca_mtl_portals4_send_request_t;
38 
41 
42  /** Eager limit; messages greater than this use a rendezvous protocol */
43  size_t eager_limit;
44  /** Size of short message blocks */
46  /** Number of short message blocks which should be created during startup */
48  /** Length of both the receive and send event queues */
50  /** Protocol for long message transfer */
51  enum { eager, rndv } protocol;
52 
53  /* free list of message for matched probe */
54  opal_free_list_t fl_message;
55 
56  /** Network interface handle for matched interface */
57  ptl_handle_ni_t ni_h;
58 
59  /** portals index for message matching */
60  ptl_pt_index_t send_idx;
61  /** portals index for long message rendezvous */
62  ptl_pt_index_t read_idx;
63  /** portals index for flow control recovery */
64  ptl_pt_index_t flowctl_idx;
65  /** portals index for flow control recovery operatings which
66  generate full events */
67  ptl_pt_index_t flowctl_event_idx;
68 
69  /** Event queue handles. See send_eq_h and recv_eq_h defines for
70  usage. Array for PtlEQPoll */
71  ptl_handle_eq_t eqs_h[2];
72 
73  /** MD for zero-length sends and acks. Optimization, can be
74  reused anywhere a 0-byte ping is necessary */
75  ptl_handle_md_t zero_md_h;
76 
77  /** long message receive overflow ME. Persistent ME, first in
78  overflow list on the send_idx portal table. */
79  ptl_handle_me_t long_overflow_me_h;
80 
81  /** List of active short receive blocks. Active means that the ME
82  was posted to the overflow list and the UNLINK event has not
83  yet been received. */
85 
86  /** List of short receive blocks waiting for FREE event. Blocks
87  are added to this list when the UNLINK event has been
88  received and removed when the FREE event is received. */
90 
91  /** number of send-side operations started */
92  uint32_t opcount;
93 
94 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
95  /** Number of event slots available for send side operations.
96  Note that this is slightly smaller than the event queue size
97  to allow space for flow control related events. */
98  uint32_t send_queue_slots;
99 
100  /** free list of send items */
101  opal_free_list_t send_fl;
102  /** list of sends which are pending (either due to needing to
103  retransmit or because we're in flow control recovery */
104  opal_list_t pending_sends;
105  /** list of sends which are currently active */
106  opal_list_t active_sends;
107 
109 #endif
110 
111 #if OPAL_ENABLE_DEBUG
112  /** number of receive-side operations started. Used only for
113  debugging */
114  uint32_t recv_opcount;
115 #endif
116 };
118 
119 #define send_eq_h eqs_h[0]
120 #define recv_eq_h eqs_h[1]
121 
122 extern mca_mtl_portals4_module_t ompi_mtl_portals4;
123 
124 #define REQ_SEND_TABLE_ID 2
125 #define REQ_READ_TABLE_ID 3
126 #define REQ_FLOWCTL_TABLE_ID 4
127 
128 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
129 #define MTL_PORTALS4_FLOWCTL_TRIGGER 0x01
130 #define MTL_PORTALS4_FLOWCTL_ALERT 0x02
131 #define MTL_PORTALS4_FLOWCTL_FANIN 0x03
132 #define MTL_PORTALS4_FLOWCTL_FANOUT 0x04
133 #endif
134 
135 /* match/ignore bit manipulation
136  *
137  * 0123 4567 01234567 01234567 01234567 01234567 01234567 01234567 01234567
138  * | | |
139  * ^ | context id | source | message tag
140  * | | | |
141  * +---- protocol
142  */
143 
144 #define MTL_PORTALS4_PROTOCOL_MASK 0xF000000000000000ULL
145 #define MTL_PORTALS4_CONTEXT_MASK 0x0FFF000000000000ULL
146 #define MTL_PORTALS4_SOURCE_MASK 0x0000FFFF00000000ULL
147 #define MTL_PORTALS4_TAG_MASK 0x00000000FFFFFFFFULL
148 
149 #define MTL_PORTALS4_PROTOCOL_IGNR MTL_PORTALS4_PROTOCOL_MASK
150 #define MTL_PORTALS4_CONTEXT_IGNR MTL_PORTALS4_CONTEXT_MASK
151 #define MTL_PORTALS4_SOURCE_IGNR MTL_PORTALS4_SOURCE_MASK
152 #define MTL_PORTALS4_TAG_IGNR 0x000000007FFFFFFFULL
153 
154 #define MTL_PORTALS4_SHORT_MSG 0x1000000000000000ULL
155 #define MTL_PORTALS4_LONG_MSG 0x2000000000000000ULL
156 #define MTL_PORTALS4_READY_MSG 0x4000000000000000ULL
157 
158 /* send posting */
159 #define MTL_PORTALS4_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
160  { \
161  match_bits = contextid; \
162  match_bits = (match_bits << 16); \
163  match_bits |= source; \
164  match_bits = (match_bits << 32); \
165  match_bits |= (MTL_PORTALS4_TAG_MASK & tag) | type; \
166  }
167 
168 /* receive posting */
169 #define MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, contextid, source, tag) \
170  { \
171  match_bits = 0; \
172  ignore_bits = MTL_PORTALS4_PROTOCOL_IGNR; \
173  \
174  match_bits = contextid; \
175  match_bits = (match_bits << 16); \
176  \
177  if (MPI_ANY_SOURCE == source) { \
178  match_bits = (match_bits << 32); \
179  ignore_bits |= MTL_PORTALS4_SOURCE_IGNR; \
180  } else { \
181  match_bits |= source; \
182  match_bits = (match_bits << 32); \
183  } \
184  \
185  if (MPI_ANY_TAG == tag) { \
186  ignore_bits |= MTL_PORTALS4_TAG_IGNR; \
187  } else { \
188  match_bits |= (MTL_PORTALS4_TAG_MASK & tag); \
189  } \
190  }
191 
192 #define MTL_PORTALS4_IS_SHORT_MSG(match_bits) \
193  (0 != (MTL_PORTALS4_SHORT_MSG & match_bits))
194 #define MTL_PORTALS4_IS_LONG_MSG(match_bits) \
195  (0 != (MTL_PORTALS4_LONG_MSG & match_bits))
196 #define MTL_PORTALS4_IS_READY_MSG(match_bits) \
197  (0 != (MTL_PORTALS4_READY_MSG & match_bits))
198 
199 #define MTL_PORTALS4_GET_TAG(match_bits) \
200  ((int)(match_bits & MTL_PORTALS4_TAG_MASK))
201 #define MTL_PORTALS4_GET_SOURCE(match_bits) \
202  ((int)((match_bits & MTL_PORTALS4_SOURCE_MASK) >> 32))
203 
204 
205 #define MTL_PORTALS4_SYNC_MSG 0x8000000000000000ULL
206 
207 #define MTL_PORTALS4_SET_HDR_DATA(hdr_data, opcount, length, sync) \
208  { \
209  hdr_data = (sync) ? 1 : 0; \
210  hdr_data = (hdr_data << 15); \
211  hdr_data |= opcount & 0x7FFFULL; \
212  hdr_data = (hdr_data << 48); \
213  hdr_data |= (length & 0xFFFFFFFFFFFFULL); \
214  }
215 
216 #define MTL_PORTALS4_GET_LENGTH(hdr_data) ((size_t)(hdr_data & 0xFFFFFFFFFFFFULL))
217 #define MTL_PORTALS4_IS_SYNC_MSG(hdr_data) \
218  (0 != (MTL_PORTALS4_SYNC_MSG & hdr_data))
219 
220 /* MTL interface functions */
221 extern int ompi_mtl_portals4_finalize(struct mca_mtl_base_module_t *mtl);
222 
223 extern int ompi_mtl_portals4_add_procs(struct mca_mtl_base_module_t* mtl,
224  size_t nprocs,
225  struct ompi_proc_t** procs,
226  struct mca_mtl_base_endpoint_t **mtl_peer_data);
227 
228 extern int ompi_mtl_portals4_del_procs(struct mca_mtl_base_module_t* mtl,
229  size_t nprocs,
230  struct ompi_proc_t** procs,
231  struct mca_mtl_base_endpoint_t **mtl_peer_data);
232 
233 extern int ompi_mtl_portals4_send(struct mca_mtl_base_module_t* mtl,
234  struct ompi_communicator_t* comm,
235  int dest,
236  int tag,
237  struct opal_convertor_t *convertor,
238  mca_pml_base_send_mode_t mode);
239 
240 extern int ompi_mtl_portals4_isend(struct mca_mtl_base_module_t* mtl,
241  struct ompi_communicator_t* comm,
242  int dest,
243  int tag,
244  struct opal_convertor_t *convertor,
245  mca_pml_base_send_mode_t mode,
246  bool blocking,
247  mca_mtl_request_t *mtl_request);
248 
249 extern int ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl,
250  struct ompi_communicator_t *comm,
251  int src,
252  int tag,
253  struct opal_convertor_t *convertor,
254  mca_mtl_request_t *mtl_request);
255 
256 extern int ompi_mtl_portals4_iprobe(struct mca_mtl_base_module_t* mtl,
257  struct ompi_communicator_t *comm,
258  int src,
259  int tag,
260  int *flag,
261  struct ompi_status_public_t *status);
262 
263 extern int ompi_mtl_portals4_imrecv(struct mca_mtl_base_module_t* mtl,
264  struct opal_convertor_t *convertor,
265  struct ompi_message_t **message,
266  struct mca_mtl_request_t *mtl_request);
267 
268 extern int ompi_mtl_portals4_improbe(struct mca_mtl_base_module_t *mtl,
269  struct ompi_communicator_t *comm,
270  int src,
271  int tag,
272  int *matched,
273  struct ompi_message_t **message,
274  struct ompi_status_public_t *status);
275 
276 extern int ompi_mtl_portals4_cancel(struct mca_mtl_base_module_t* mtl,
277  mca_mtl_request_t *mtl_request,
278  int flag);
279 
280 extern int ompi_mtl_portals4_add_comm(struct mca_mtl_base_module_t *mtl,
281  struct ompi_communicator_t *comm);
282 
283 extern int ompi_mtl_portals4_del_comm(struct mca_mtl_base_module_t *mtl,
284  struct ompi_communicator_t *comm);
285 
286 extern int ompi_mtl_portals4_progress(void);
287 
288 extern int ompi_mtl_portals4_get_error(int ptl_error);
289 
290 END_C_DECLS
291 
292 #endif /* MTL_PORTALS_H_HAS_BEEN_INCLUDED */
ptl_handle_eq_t eqs_h[2]
Event queue handles.
Definition: mtl_portals4.h:71
int recv_short_num
Number of short message blocks which should be created during startup.
Definition: mtl_portals4.h:47
ptl_pt_index_t flowctl_event_idx
portals index for flow control recovery operatings which generate full events
Definition: mtl_portals4.h:67
ptl_pt_index_t read_idx
portals index for long message rendezvous
Definition: mtl_portals4.h:62
opal_list_t active_recv_short_blocks
List of active short receive blocks.
Definition: mtl_portals4.h:84
uint32_t opcount
number of send-side operations started
Definition: mtl_portals4.h:92
ptl_pt_index_t send_idx
portals index for message matching
Definition: mtl_portals4.h:60
ptl_pt_index_t flowctl_idx
portals index for flow control recovery
Definition: mtl_portals4.h:64
size_t eager_limit
Eager limit; messages greater than this use a rendezvous protocol.
Definition: mtl_portals4.h:43
size_t recv_short_size
Size of short message blocks.
Definition: mtl_portals4.h:45
Definition: mtl.h:51
The opal_list_t interface is used to provide a generic doubly-linked list container for Open MPI...
Remote Open MPI process structure.
Definition: proc.h:56
enum mca_mtl_portals4_module_t::@122 protocol
Protocol for long message transfer.
ptl_handle_md_t zero_md_h
MD for zero-length sends and acks.
Definition: mtl_portals4.h:75
Definition: opal_free_list.h:31
ptl_handle_ni_t ni_h
Network interface handle for matched interface.
Definition: mtl_portals4.h:57
An abstraction that represents a connection to a endpoint process.
Definition: mtl_mx_endpoint.h:50
Definition: opal_convertor.h:90
Matching Transport Layer.
Definition: opal_list.h:147
int queue_size
Length of both the receive and send event queues.
Definition: mtl_portals4.h:49
Definition: mpi.h:337
MTL module interface functions and attributes.
Definition: mtl.h:399
opal_list_t waiting_recv_short_blocks
List of short receive blocks waiting for FREE event.
Definition: mtl_portals4.h:89
ptl_handle_me_t long_overflow_me_h
long message receive overflow ME.
Definition: mtl_portals4.h:79
Definition: communicator.h:118
Definition: mtl_portals4_flowctl.h:13
Definition: mtl_portals4.h:39
Definition: message.h:22