OpenMPI  0.1.1
osc_pt2pt.h
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University.
3  * All rights reserved.
4  * Copyright (c) 2004-2006 The Trustees of the University of Tennessee.
5  * All rights reserved.
6  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
7  * University of Stuttgart. All rights reserved.
8  * Copyright (c) 2004-2005 The Regents of the University of California.
9  * All rights reserved.
10  * Copyright (c) 2007 Los Alamos National Security, LLC. All rights
11  * reserved.
12  * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
13  * $COPYRIGHT$
14  *
15  * Additional copyrights may follow
16  *
17  * $HEADER$
18  */
19 
20 #ifndef OMPI_OSC_PT2PT_H
21 #define OMPI_OSC_PT2PT_H
22 
23 #include "ompi_config.h"
24 #include "opal/class/opal_list.h"
25 #include "opal/class/opal_free_list.h"
27 #include "opal/threads/threads.h"
28 
29 #include "ompi/win/win.h"
30 #include "ompi/communicator/communicator.h"
31 #include "ompi/request/request.h"
32 #include "ompi/mca/osc/osc.h"
33 
34 BEGIN_C_DECLS
35 
36 #define CONTROL_MSG_TAG (-200)
37 
39  /** Extend the basic osc component interface */
41 
42  /** max size of eager message */
44 
45  /** free list of ompi_osc_pt2pt_sendreq_t structures */
47  /** free list of ompi_osc_pt2pt_replyreq_t structures */
49  /** free list of ompi_osc_pt2pt_longreq_t structures */
51  /** free list for eager / control meessages */
53 };
55 
56 
58  /** Extend the basic osc module interface */
60 
61  /** lock access to data structures in the current module */
63 
64  /** condition variable for access to current module */
66 
67  /** lock for "atomic" window updates from reductions */
69 
70  /** pointer back to window */
72 
73  /** communicator created with this window */
75 
76  /** list of ompi_osc_pt2pt_sendreq_t structures, and includes all
77  requests for this access epoch that have not already been
78  started. p2p_lock must be held when modifying this field. */
80 
81  /** list of unsigned int counters for the number of requests to a
82  particular rank in p2p_comm for this access epoc. p2p_lock
83  must be held when modifying this field */
84  unsigned int *p2p_num_pending_sendreqs;
85 
86  /** For MPI_Fence synchronization, the number of messages to send
87  in epoch. For Start/Complete, the number of updates for this
88  Complete. For lock, the number of
89  messages waiting for completion on on the origin side. Not
90  protected by p2p_lock - must use atomic counter operations. */
92 
93  /** For MPI_Fence synchronization, the number of expected incoming
94  messages. For Post/Wait, the number of expected updates from
95  complete. For lock, the number of messages on the passive side
96  we are waiting for. Not protected by p2p_lock - must use
97  atomic counter operations. */
99 
100  /** Number of "ping" messages from the remote post group we've
101  received */
103 
104  /** Number of "count" messages from the remote complete group
105  we've received */
107 
108  /** cyclic counter for a unique tag for long messages. Not
109  protected by the p2p_lock - must use create_send_tag() to
110  create a send tag */
111  volatile int32_t p2p_tag_counter;
112 
113  opal_list_t p2p_copy_pending_sendreqs;
114  unsigned int *p2p_copy_num_pending_sendreqs;
115 
116  /* ********************* FENCE data ************************ */
117  /* an array of <sizeof(p2p_comm)> ints, each containing the value
118  1. */
119  int *p2p_fence_coll_counts;
120 
121  /* ********************* PWSC data ************************ */
122  struct ompi_group_t *p2p_pw_group;
123  struct ompi_group_t *p2p_sc_group;
124  bool *p2p_sc_remote_active_ranks;
125  int *p2p_sc_remote_ranks;
126 
127  /* ********************* LOCK data ************************ */
128  int32_t p2p_lock_status; /* one of 0, MPI_LOCK_EXCLUSIVE, MPI_LOCK_SHARED */
129  int32_t p2p_shared_count;
130  opal_list_t p2p_locks_pending;
131  opal_list_t p2p_unlocks_pending;
132  int32_t p2p_lock_received_ack;
133 };
135 OMPI_MODULE_DECLSPEC extern ompi_osc_pt2pt_component_t mca_osc_pt2pt_component;
136 
137 
138 /**
139  * Helper macro for grabbing the module structure from a window instance
140  */
141 #define P2P_MODULE(win) ((ompi_osc_pt2pt_module_t*) win->w_osc_module)
142 
143 /*
144  * Component functions
145  */
146 
147 int ompi_osc_pt2pt_component_init(bool enable_progress_threads,
148  bool enable_mpi_threads);
149 
150 int ompi_osc_pt2pt_component_finalize(void);
151 
152 int ompi_osc_pt2pt_component_query(struct ompi_win_t *win,
153  struct ompi_info_t *info,
154  struct ompi_communicator_t *comm);
155 
156 int ompi_osc_pt2pt_component_select(struct ompi_win_t *win,
157  struct ompi_info_t *info,
158  struct ompi_communicator_t *comm);
159 
160 /* helper function that properly sets up request handling */
161 int ompi_osc_pt2pt_component_irecv(void *buf,
162  size_t count,
163  struct ompi_datatype_t *datatype,
164  int src,
165  int tag,
166  struct ompi_communicator_t *comm,
167  struct ompi_request_t **request,
168  ompi_request_complete_fn_t callback,
169  void *data);
170 
171 int ompi_osc_pt2pt_component_isend(void *buf,
172  size_t count,
173  struct ompi_datatype_t *datatype,
174  int dest,
175  int tag,
176  struct ompi_communicator_t *comm,
177  struct ompi_request_t **request,
178  ompi_request_complete_fn_t callback,
179  void *data);
180 
181 /*
182  * Module interface function types
183  */
184 int ompi_osc_pt2pt_module_free(struct ompi_win_t *win);
185 
186 int ompi_osc_pt2pt_module_put(void *origin_addr,
187  int origin_count,
188  struct ompi_datatype_t *origin_dt,
189  int target,
190  OPAL_PTRDIFF_TYPE target_disp,
191  int target_count,
192  struct ompi_datatype_t *target_dt,
193  struct ompi_win_t *win);
194 
195 int ompi_osc_pt2pt_module_accumulate(void *origin_addr,
196  int origin_count,
197  struct ompi_datatype_t *origin_dt,
198  int target,
199  OPAL_PTRDIFF_TYPE target_disp,
200  int target_count,
201  struct ompi_datatype_t *target_dt,
202  struct ompi_op_t *op,
203  struct ompi_win_t *win);
204 
205 int ompi_osc_pt2pt_module_get(void *origin_addr,
206  int origin_count,
207  struct ompi_datatype_t *origin_dt,
208  int target,
209  OPAL_PTRDIFF_TYPE target_disp,
210  int target_count,
211  struct ompi_datatype_t *target_dt,
212  struct ompi_win_t *win);
213 
214 int ompi_osc_pt2pt_module_fence(int assert, struct ompi_win_t *win);
215 
216 int ompi_osc_pt2pt_module_start(struct ompi_group_t *group,
217  int assert,
218  struct ompi_win_t *win);
219 int ompi_osc_pt2pt_module_complete(struct ompi_win_t *win);
220 
221 int ompi_osc_pt2pt_module_post(struct ompi_group_t *group,
222  int assert,
223  struct ompi_win_t *win);
224 
225 int ompi_osc_pt2pt_module_wait(struct ompi_win_t *win);
226 
227 int ompi_osc_pt2pt_module_test(struct ompi_win_t *win,
228  int *flag);
229 
230 int ompi_osc_pt2pt_module_lock(int lock_type,
231  int target,
232  int assert,
233  struct ompi_win_t *win);
234 
235 int ompi_osc_pt2pt_module_unlock(int target,
236  struct ompi_win_t *win);
237 
238 /*
239  * passive side sync interface functions
240  */
241 int ompi_osc_pt2pt_passive_lock(ompi_osc_pt2pt_module_t *module,
242  int32_t origin,
243  int32_t lock_type);
244 
245 int ompi_osc_pt2pt_passive_unlock(ompi_osc_pt2pt_module_t *module,
246  int32_t origin,
247  int32_t count);
248 
249 int ompi_osc_pt2pt_passive_unlock_complete(ompi_osc_pt2pt_module_t *module);
250 
251 END_C_DECLS
252 
253 #endif /* OMPI_OSC_PT2PT_H */
OSC module instance.
Definition: osc.h:269
Definition: win.h:53
volatile int32_t p2p_tag_counter
cyclic counter for a unique tag for long messages.
Definition: osc_pt2pt.h:111
size_t p2p_c_eager_size
max size of eager message
Definition: osc_pt2pt.h:43
Definition: condition.h:49
Definition: ompi_datatype.h:68
opal_condition_t p2p_cond
condition variable for access to current module
Definition: osc_pt2pt.h:65
int32_t p2p_num_complete_msgs
Number of "count" messages from the remote complete group we've received.
Definition: osc_pt2pt.h:106
Definition: mutex_unix.h:53
Definition: osc_pt2pt.h:57
ompi_osc_base_module_t super
Extend the basic osc module interface.
Definition: osc_pt2pt.h:59
The opal_list_t interface is used to provide a generic doubly-linked list container for Open MPI...
opal_mutex_t p2p_lock
lock access to data structures in the current module
Definition: osc_pt2pt.h:62
Definition: osc_pt2pt.h:38
Definition: info.h:38
ompi_win_t * p2p_win
pointer back to window
Definition: osc_pt2pt.h:71
Top-level description of requests.
One-sided Communication interface.
Definition: opal_free_list.h:31
ompi_osc_base_component_t super
Extend the basic osc component interface.
Definition: osc_pt2pt.h:40
opal_free_list_t p2p_c_buffers
free list for eager / control meessages
Definition: osc_pt2pt.h:52
OSC component interface.
Definition: osc.h:154
opal_free_list_t p2p_c_sendreqs
free list of ompi_osc_pt2pt_sendreq_t structures
Definition: osc_pt2pt.h:46
A hash table that may be indexed with either fixed length (e.g.
ompi_communicator_t * p2p_comm
communicator created with this window
Definition: osc_pt2pt.h:74
unsigned int * p2p_num_pending_sendreqs
list of unsigned int counters for the number of requests to a particular rank in p2p_comm for this ac...
Definition: osc_pt2pt.h:84
Group structure Currently we have four formats for storing the process pointers that are members of t...
Definition: group.h:79
opal_list_t p2p_pending_sendreqs
list of ompi_osc_pt2pt_sendreq_t structures, and includes all requests for this access epoch that hav...
Definition: osc_pt2pt.h:79
Definition: opal_list.h:147
int32_t p2p_num_pending_out
For MPI_Fence synchronization, the number of messages to send in epoch.
Definition: osc_pt2pt.h:91
opal_mutex_t p2p_acc_lock
lock for "atomic" window updates from reductions
Definition: osc_pt2pt.h:68
int32_t p2p_num_pending_in
For MPI_Fence synchronization, the number of expected incoming messages.
Definition: osc_pt2pt.h:98
opal_free_list_t p2p_c_replyreqs
free list of ompi_osc_pt2pt_replyreq_t structures
Definition: osc_pt2pt.h:48
int32_t p2p_num_post_msgs
Number of "ping" messages from the remote post group we've received.
Definition: osc_pt2pt.h:102
Definition: evdns.c:158
opal_free_list_t p2p_c_longreqs
free list of ompi_osc_pt2pt_longreq_t structures
Definition: osc_pt2pt.h:50
Definition: communicator.h:118
Main top-level request struct definition.
Definition: request.h:100
Back-end type of MPI_Op.
Definition: op.h:100