OpenMPI  0.1.1
mtl_portals.h
1 /*
2  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2007 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * $COPYRIGHT$
13  *
14  * Additional copyrights may follow
15  *
16  * $HEADER$
17  */
18 
19 #ifndef MTL_PORTALS_H_HAS_BEEN_INCLUDED
20 #define MTL_PORTALS_H_HAS_BEEN_INCLUDED
21 
22 #include "ompi_config.h"
23 #include "opal/class/opal_list.h"
24 #include "ompi/class/ompi_free_list.h"
25 #include "ompi/mca/mtl/mtl.h"
26 #include "ompi/mca/mtl/base/base.h"
27 #include "opal/datatype/opal_convertor.h"
28 
29 #include "ompi/mca/common/portals/common_portals.h"
30 
31 
32 BEGIN_C_DECLS
33 
36 
37  ptl_handle_ni_t ptl_ni_h;
38  size_t eager_limit;
39 
40  ptl_handle_eq_t ptl_eq_h;
41 
42  ptl_handle_eq_t ptl_unex_eq_h;
43 
44  /* long unex msgs - insert posted recvs before this */
45  ptl_handle_me_t ptl_unex_long_me_h;
46 
47  /* send catchall - insert short unex buffers before this */
48  ptl_handle_me_t ptl_send_catchall_me_h;
49 
50  /* catchall for ack portal */
51  ptl_handle_me_t ptl_ack_catchall_me_h;
52 
53  /* catchall for read portal */
54  ptl_handle_me_t ptl_read_catchall_me_h;
55 
56  /* for zero-length sends and acks */
57  ptl_handle_md_t ptl_zero_md_h;
58 
59  ompi_free_list_t event_fl;
60 
61  int ptl_recv_short_mds_num;
62  int ptl_recv_short_mds_size;
63 
64  opal_list_t ptl_recv_short_blocks;
65  opal_list_t unexpected_messages;
66 
67  int ptl_expected_queue_size;
68  int ptl_unexpected_queue_size;
69 
70  /* for send-side copy blocks */
71  ptl_md_t ptl_short_md;
72  ptl_handle_md_t ptl_short_md_h;
73 
74  int ptl_num_copy_blocks;
75  ptl_size_t ptl_copy_block_len;
76 
77  int *ptl_copy_block_free_list;
78  int ptl_copy_block_first_free;
79 
80  /* empty event queue for PtlMEMDPost() */
81  ptl_handle_eq_t ptl_empty_eq_h;
82 
83  /* turn off aggressive polling of the unex msg event queue */
84  bool ptl_aggressive_polling;
85 
86  /* use rendezvous for long messages */
87  bool ptl_use_rendezvous;
88 
89  /* output channel for debugging */
90  int portals_output;
91 
92 };
94 
95 extern mca_mtl_portals_module_t ompi_mtl_portals;
96 
97 OMPI_DECLSPEC mca_mtl_base_component_2_0_0_t mca_mtl_portals_component;
98 
99 
101  struct ompi_free_list_item_t super;
102  ptl_event_t ev;
103  bool is_complete;
104 };
107 
108 
109 /* match/ignore bit manipulation
110  *
111  * 0123 4567 01234567 01234567 01234567 01234567 01234567 01234567 01234567
112  * | | |
113  * ^ | context id | source | message tag
114  * | | | |
115  * +---- protocol
116  */
117 
118 #define PTL_PROTOCOL_MASK 0xF000000000000000ULL
119 #define PTL_CONTEXT_MASK 0x0FFF000000000000ULL
120 #define PTL_SOURCE_MASK 0x0000FFFF00000000ULL
121 #define PTL_TAG_MASK 0x00000000FFFFFFFFULL
122 
123 #define PTL_PROTOCOL_IGNR PTL_PROTOCOL_MASK
124 #define PTL_CONTEXT_IGNR PTL_CONTEXT_MASK
125 #define PTL_SOURCE_IGNR PTL_SOURCE_MASK
126 #define PTL_TAG_IGNR 0x000000007FFFFFFFULL
127 
128 #define PTL_SHORT_MSG 0x1000000000000000ULL
129 #define PTL_LONG_MSG 0x2000000000000000ULL
130 #define PTL_READY_MSG 0x4000000000000000ULL
131 
132 /* send posting */
133 #define PTL_SET_SEND_BITS(match_bits, contextid, source, tag, type) \
134 { \
135  match_bits = contextid; \
136  match_bits = (match_bits << 16); \
137  match_bits |= source; \
138  match_bits = (match_bits << 32); \
139  match_bits |= (PTL_TAG_MASK & tag) | type; \
140 }
141 
142 /* receive posting */
143 #define PTL_SET_RECV_BITS(match_bits, ignore_bits, contextid, source, tag) \
144 { \
145  match_bits = 0; \
146  ignore_bits = PTL_PROTOCOL_IGNR; \
147  \
148  match_bits = contextid; \
149  match_bits = (match_bits << 16); \
150  \
151  if (MPI_ANY_SOURCE == source) { \
152  match_bits = (match_bits << 32); \
153  ignore_bits |= PTL_SOURCE_IGNR; \
154  } else { \
155  match_bits |= source; \
156  match_bits = (match_bits << 32); \
157  } \
158  \
159  if (MPI_ANY_TAG == tag) { \
160  ignore_bits |= PTL_TAG_IGNR; \
161  } else { \
162  match_bits |= (PTL_TAG_MASK & tag); \
163  } \
164 }
165 
166 #define PTL_IS_SHORT_MSG(match_bits) \
167  (0 != (PTL_SHORT_MSG & match_bits))
168 #define PTL_IS_LONG_MSG(match_bits) \
169  (0 != (PTL_LONG_MSG & match_bits))
170 #define PTL_IS_READY_MSG(match_bits) \
171  (0 != (PTL_READY_MSG & match_bits))
172 #define PTL_IS_SYNC_MSG(event) \
173  (0 != event.hdr_data)
174 
175 #define PTL_GET_TAG(match_bits) ((int)(match_bits & PTL_TAG_MASK))
176 #define PTL_GET_SOURCE(match_bits) ((int)((match_bits & PTL_SOURCE_MASK) >> 32))
177 
178 /* MTL interface functions */
179 extern int ompi_mtl_portals_finalize(struct mca_mtl_base_module_t *mtl);
180 
181 extern int ompi_mtl_portals_add_procs(struct mca_mtl_base_module_t* mtl,
182  size_t nprocs,
183  struct ompi_proc_t** procs,
184  struct mca_mtl_base_endpoint_t **mtl_peer_data);
185 
186 extern int ompi_mtl_portals_del_procs(struct mca_mtl_base_module_t* mtl,
187  size_t nprocs,
188  struct ompi_proc_t** procs,
189  struct mca_mtl_base_endpoint_t **mtl_peer_data);
190 
191 extern int ompi_mtl_portals_send(struct mca_mtl_base_module_t* mtl,
192  struct ompi_communicator_t* comm,
193  int dest,
194  int tag,
195  struct opal_convertor_t *convertor,
196  mca_pml_base_send_mode_t mode);
197 
198 extern int ompi_mtl_portals_isend(struct mca_mtl_base_module_t* mtl,
199  struct ompi_communicator_t* comm,
200  int dest,
201  int tag,
202  struct opal_convertor_t *convertor,
203  mca_pml_base_send_mode_t mode,
204  bool blocking,
205  mca_mtl_request_t *mtl_request);
206 
207 extern int ompi_mtl_portals_irecv(struct mca_mtl_base_module_t* mtl,
208  struct ompi_communicator_t *comm,
209  int src,
210  int tag,
211  struct opal_convertor_t *convertor,
212  mca_mtl_request_t *mtl_request);
213 
214 extern int ompi_mtl_portals_iprobe(struct mca_mtl_base_module_t* mtl,
215  struct ompi_communicator_t *comm,
216  int src,
217  int tag,
218  int *flag,
219  struct ompi_status_public_t *status);
220 
221 extern int ompi_mtl_portals_imrecv(struct mca_mtl_base_module_t* mtl,
222  struct opal_convertor_t *convertor,
223  struct ompi_message_t **message,
224  struct mca_mtl_request_t *mtl_request);
225 
226 extern int ompi_mtl_portals_improbe(struct mca_mtl_base_module_t *mtl,
227  struct ompi_communicator_t *comm,
228  int src,
229  int tag,
230  int *matched,
231  struct ompi_message_t **message,
232  struct ompi_status_public_t *status);
233 
234 extern int ompi_mtl_portals_cancel(struct mca_mtl_base_module_t* mtl,
235  mca_mtl_request_t *mtl_request,
236  int flag);
237 
238 extern int ompi_mtl_portals_add_comm(struct mca_mtl_base_module_t *mtl,
239  struct ompi_communicator_t *comm);
240 
241 extern int ompi_mtl_portals_del_comm(struct mca_mtl_base_module_t *mtl,
242  struct ompi_communicator_t *comm);
243 
244 extern int ompi_mtl_portals_progress(void);
245 
246 
247 END_C_DECLS
248 
249 #endif /* MTL_PORTALS_H_HAS_BEEN_INCLUDED */
Definition: mtl.h:93
Definition: mtl.h:51
The opal_list_t interface is used to provide a generic doubly-linked list container for Open MPI...
Definition: mtl_portals.h:100
Remote Open MPI process structure.
Definition: proc.h:56
Definition: ompi_free_list.h:39
Definition: ompi_free_list.h:62
An abstraction that represents a connection to a endpoint process.
Definition: mtl_mx_endpoint.h:50
Definition: opal_convertor.h:90
Matching Transport Layer.
Definition: opal_list.h:147
Definition: mpi.h:337
MTL module interface functions and attributes.
Definition: mtl.h:399
Definition: mtl_portals.h:34
Definition: communicator.h:118
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236
Definition: message.h:22