OpenMPI  0.1.1
communicator.h
1 /* -*- Mode: C; c-basic-offset:4 ; -*- */
2 /*
3  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4  * University Research and Technology
5  * Corporation. All rights reserved.
6  * Copyright (c) 2004-2011 The University of Tennessee and The University
7  * of Tennessee Research Foundation. All rights
8  * reserved.
9  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10  * University of Stuttgart. All rights reserved.
11  * Copyright (c) 2004-2005 The Regents of the University of California.
12  * All rights reserved.
13  * Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved.
14  * Copyright (c) 2006-2010 University of Houston. All rights reserved.
15  * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
16  * Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved.
17  *
18  * $COPYRIGHT$
19  *
20  * Additional copyrights may follow
21  *
22  * $HEADER$
23  */
24 
25 #ifndef OMPI_COMMUNICATOR_H
26 #define OMPI_COMMUNICATOR_H
27 
28 #include "ompi_config.h"
29 #include "opal/class/opal_object.h"
31 #include "opal/threads/mutex.h"
32 
33 #include "mpi.h"
34 #include "ompi/group/group.h"
35 #include "ompi/mca/coll/coll.h"
36 #include "ompi/proc/proc.h"
37 
38 BEGIN_C_DECLS
39 
41 
42 #define OMPI_COMM_INTER 0x00000001
43 #define OMPI_COMM_CART 0x00000002
44 #define OMPI_COMM_GRAPH 0x00000004
45 #define OMPI_COMM_NAMEISSET 0x00000008
46 #define OMPI_COMM_ISFREED 0x00000010
47 #define OMPI_COMM_INTRINSIC 0x00000020
48 #define OMPI_COMM_DYNAMIC 0x00000040
49 #define OMPI_COMM_INVALID 0x00000080
50 #define OMPI_COMM_PML_ADDED 0x00000100
51 #define OMPI_COMM_EXTRA_RETAIN 0x00000200
52 
53 /* some utility #defines */
54 #define OMPI_COMM_IS_INTER(comm) ((comm)->c_flags & OMPI_COMM_INTER)
55 #define OMPI_COMM_IS_INTRA(comm) (!((comm)->c_flags & OMPI_COMM_INTER))
56 #define OMPI_COMM_IS_CART(comm) ((comm)->c_flags & OMPI_COMM_CART)
57 #define OMPI_COMM_IS_GRAPH(comm) ((comm)->c_flags & OMPI_COMM_GRAPH)
58 #define OMPI_COMM_IS_INTRINSIC(comm) ((comm)->c_flags & OMPI_COMM_INTRINSIC)
59 #define OMPI_COMM_IS_FREED(comm) ((comm)->c_flags & OMPI_COMM_ISFREED)
60 #define OMPI_COMM_IS_DYNAMIC(comm) ((comm)->c_flags & OMPI_COMM_DYNAMIC)
61 #define OMPI_COMM_IS_INVALID(comm) ((comm)->c_flags & OMPI_COMM_INVALID)
62 #define OMPI_COMM_IS_PML_ADDED(comm) ((comm)->c_flags & OMPI_COMM_PML_ADDED)
63 #define OMPI_COMM_IS_EXTRA_RETAIN(comm) ((comm)->c_flags & OMPI_COMM_EXTRA_RETAIN)
64 
65 #define OMPI_COMM_SET_DYNAMIC(comm) ((comm)->c_flags |= OMPI_COMM_DYNAMIC)
66 #define OMPI_COMM_SET_INVALID(comm) ((comm)->c_flags |= OMPI_COMM_INVALID)
67 
68 #define OMPI_COMM_SET_PML_ADDED(comm) ((comm)->c_flags |= OMPI_COMM_PML_ADDED)
69 #define OMPI_COMM_SET_EXTRA_RETAIN(comm) ((comm)->c_flags |= OMPI_COMM_EXTRA_RETAIN)
70 
71 /* a set of special tags: */
72 
73 /* to recognize an MPI_Comm_join in the comm_connect_accept routine. */
74 
75 #define OMPI_COMM_ALLGATHER_TAG -31078
76 #define OMPI_COMM_BARRIER_TAG -31079
77 #define OMPI_COMM_ALLREDUCE_TAG -31080
78 
79 /**
80  * Modes required for acquiring the new comm-id.
81  * The first (INTER/INTRA) indicates whether the
82  * input comm was an inter/intra-comm, the second
83  * whether the new communicator will be an inter/intra
84  * comm
85  */
86 #define OMPI_COMM_CID_INTRA 0x00000020
87 #define OMPI_COMM_CID_INTER 0x00000040
88 #define OMPI_COMM_CID_INTRA_BRIDGE 0x00000080
89 #define OMPI_COMM_CID_INTRA_OOB 0x00000100
90 #if OPAL_ENABLE_FT_MPI
91 #define OMPI_COMM_CID_INTRA_FT 0x00000200
92 #define OMPI_COMM_CID_INTER_FT 0x00000400
93 #define OMPI_COMM_CID_INTRA_BRIDGE_FT 0x00000800
94 #define OMPI_COMM_CID_INTRA_OOB_FT 0x00001000
95 #endif /* OPAL_ENABLE_FT_MPI */
96 
97 /**
98  * The block of CIDs allocated for MPI_COMM_WORLD
99  * and other communicators
100  */
101 #define OMPI_COMM_BLOCK_WORLD 16
102 #define OMPI_COMM_BLOCK_OTHERS 8
103 
104 /* A macro comparing two CIDs */
105 #define OMPI_COMM_CID_IS_LOWER(comm1,comm2) ( ((comm1)->c_contextid < (comm2)->c_contextid)? 1:0)
106 
107 OMPI_DECLSPEC extern opal_pointer_array_t ompi_mpi_communicators;
108 #if OPAL_ENABLE_FT_MPI
109 /**
110  * This array hold the number of time each id has been used. In the case where a communicator
111  * is revoked this reference count act as a epoch, and prevent us from revoking newly created
112  * communicators, that use similar id to others communicators that are still revoked in the
113  * system.
114  */
115 OMPI_DECLSPEC extern opal_pointer_array_t ompi_mpi_comm_epoch;
116 #endif /* OPAL_ENABLE_FT_MPI */
117 
119  opal_object_t c_base;
120  opal_mutex_t c_lock; /* mutex for name and potentially
121  attributes */
122  char c_name[MPI_MAX_OBJECT_NAME];
123  uint32_t c_contextid;
124  int c_my_rank;
125  uint32_t c_flags; /* flags, e.g. intercomm,
126  topology, etc. */
127 
128  int c_id_available; /* the currently available Cid for allocation
129  to a child*/
130  int c_id_start_index; /* the starting index of the block of cids
131  allocated to this communicator*/
132  int epoch; /**< Identifier used to keep trace of the communicators revoked.
133  * This allows to avoid a race condition between the revoke-based
134  * message arriving from late peers and the creation of new communicators.
135  */
136  ompi_group_t *c_local_group;
137  ompi_group_t *c_remote_group;
138 
139  struct ompi_communicator_t *c_local_comm; /* a duplicate of the local
140  communicator in case the comm
141  is an inter-comm*/
142 
143  /* Attributes */
145 
146  /**< inscribing cube dimension */
147  int c_cube_dim;
148 
149  /* Hooks for topo module to hang things */
150  mca_base_component_t *c_topo_component;
151 
153  /**< structure of function pointers */
154 
156  /**< structure containing basic information about the topology */
157 
158  struct mca_topo_base_module_comm_t *c_topo_module;
159  /**< module specific data */
160 
161  /* index in Fortran <-> C translation array */
162 
163  int c_f_to_c_index;
164 
165 #ifdef OMPI_WANT_PERUSE
166  /*
167  * Place holder for the PERUSE events.
168  */
169  struct ompi_peruse_handle_t** c_peruse_handles;
170 #endif
171 
172  /* Error handling. This field does not have the "c_" prefix so
173  that the OMPI_ERRHDL_* macros can find it, regardless of whether
174  it's a comm, window, or file. */
175 
176  ompi_errhandler_t *error_handler;
177  ompi_errhandler_type_t errhandler_type;
178 
179  /* Hooks for PML to hang things */
180  struct mca_pml_comm_t *c_pml_comm;
181 
182  /* Collectives module interface and data */
184 
185 #if OPAL_ENABLE_FT_MPI
186  /** Are MPI_ANY_SOURCE operations enabled? - OMPI_Comm_failure_ack */
187  bool any_source_enabled;
188  /** MPI_ANY_SOURCE Failed Group Offset - OMPI_Comm_failure_get_acked */
189  int any_source_offset;
190  /** Has this communicator been revoked - OMPI_Comm_revoke() */
191  bool comm_revoked;
192  /** Force errors to collective pt2pt operations? */
193  bool collectives_force_error;
194  /** Quick lookup */
195  int num_active_local;
196  int num_active_remote;
197  int lleader;
198  int rleader;
199 #endif /* OPAL_ENABLE_FT_MPI */
200 };
202 
203 /**
204  * Padded struct to maintain back compatibiltiy.
205  *
206  * The following ompi_predefined_xxx_t structure is used to maintain
207  * backwards binary compatibility for MPI applications compiled
208  * against one version of OMPI library but dynamically linked at
209  * runtime with another. The issue is between versions the actual
210  * structure may change in size (even between debug and optimized
211  * compilation -- the structure contents change, and therefore the
212  * overall size changes).
213  *
214  * This is problematic with predefined handles because the storage of
215  * the structure ends up being located to an application's BSS. This
216  * causes problems because if one version has the predefined as size X
217  * and then the application is dynamically linked with a version that
218  * has a size of Y (where X != Y) then the application will
219  * unintentionally overrun the memory initially allocated for the
220  * structure.
221  *
222  * The solution we are using below creates a parent structure
223  * (ompi_predefined_xxx_t) that contains the base structure
224  * (ompi_xxx_t) followed by a character padding that is the size of
225  * the total size we choose to preallocate for the structure minus the
226  * amount used by the base structure. In this way, we've normalized
227  * the size of each predefined handle across multiple versions and
228  * configurations of Open MPI (e.g., MPI_COMM_WORLD will refer to a
229  * back-end struct that is X bytes long, even if we change the
230  * back-end ompi_communicator_t between version A.B and version C.D in
231  * Open MPI). When we come close to filling up the the padding we can
232  * add a pointer at the back end of the base structure to point to an
233  * extension of the type. Or we can just increase the padding and
234  * break backwards binary compatibility.
235  *
236  * The above method was decided after several failed attempts
237  * described below.
238  *
239  * - Original implementation - suffered that the base structure seemed
240  * to always change in size between Open MPI versions and/or
241  * configurations (e.g., optimized vs. debugging build).
242  *
243  * - Convert all predefined handles to run-time-assigned pointers
244  * (i.e., global variables) - This worked except in cases where an MPI
245  * application wanted to assign the predefined handle value to a
246  * global variable -- we could not guarantee to have the global
247  * variable filled until MPI_INIT was called (recall that MPI
248  * predefined handles must be assignable before MPI_INIT; e.g.,
249  * "MPI_Comm foo = MPI_COMM_WORLD").
250  *
251  * - union of struct and padding - Similar to current implementation
252  * except using a union for the parent. This worked except in cases
253  * where the compilers did not support C99 union static initalizers.
254  * It would have been a pain to convert a bunch of the code to use
255  * non-static initializers (e.g., MPI datatypes).
256  */
257 
258 /* Define for the preallocated size of the predefined handle.
259  * Note that we are using a pointer type as the base memory chunk
260  * size so when the bitness changes the size of the handle changes.
261  * This is done so we don't end up needing a structure that is
262  * incredibly larger than necessary because of the bitness.
263  */
264 #define PREDEFINED_COMMUNICATOR_PAD (sizeof(void*) * 128)
265 
267  struct ompi_communicator_t comm;
268  char padding[PREDEFINED_COMMUNICATOR_PAD - sizeof(ompi_communicator_t)];
269 };
271 
272 OMPI_DECLSPEC extern ompi_communicator_t *ompi_mpi_comm_parent;
273 OMPI_DECLSPEC extern ompi_predefined_communicator_t ompi_mpi_comm_world;
274 OMPI_DECLSPEC extern ompi_predefined_communicator_t ompi_mpi_comm_self;
275 OMPI_DECLSPEC extern ompi_predefined_communicator_t ompi_mpi_comm_null;
276 
277 /**
278  * Is this a valid communicator? This is a complicated question.
279  * :-)
280  *
281  * According to MPI-1:5.2.4 (p137):
282  *
283  * "The predefined constant MPI_COMM_NULL is the value used for
284  * invalid communicator handles."
285  *
286  * Hence, MPI_COMM_NULL is not valid. However, MPI-2:4.12.4 (p50)
287  * clearly states that the MPI_*_C2F and MPI_*_F2C functions
288  * should treat MPI_COMM_NULL as a valid communicator -- it
289  * distinctly differentiates between "invalid" handles and
290  * "MPI_*_NULL" handles. Some feel that the MPI-1 definition
291  * still holds for all other MPI functions; others feel that the
292  * MPi-2 definitions trump the MPI-1 definition. Regardless of
293  * who is right, there is ambiguity here. So we have left
294  * ompi_comm_invalid() as originally coded -- per the MPI-1
295  * definition, where MPI_COMM_NULL is an invalid communicator.
296  * The MPI_Comm_c2f() function, therefore, calls
297  * ompi_comm_invalid() but also explictily checks to see if the
298  * handle is MPI_COMM_NULL.
299  */
300 static inline int ompi_comm_invalid(ompi_communicator_t* comm)
301 {
302  if ((NULL == comm) || (MPI_COMM_NULL == comm) ||
303  (OMPI_COMM_IS_FREED(comm)) || (OMPI_COMM_IS_INVALID(comm)) )
304  return true;
305  else
306  return false;
307 }
308 
309 /**
310  * rank w/in the communicator
311  */
312 static inline int ompi_comm_rank(ompi_communicator_t* comm)
313 {
314  return comm->c_my_rank;
315 }
316 
317 /**
318  * size of the communicator
319  */
320 static inline int ompi_comm_size(ompi_communicator_t* comm)
321 {
322  return comm->c_local_group->grp_proc_count;
323 }
324 
325 /**
326  * size of the remote group for inter-communicators.
327  * returns zero for an intra-communicator
328  */
329 static inline int ompi_comm_remote_size(ompi_communicator_t* comm)
330 {
331  return (comm->c_flags & OMPI_COMM_INTER ? comm->c_remote_group->grp_proc_count : 0);
332 }
333 
334 /**
335  * Context ID for the communicator, suitable for passing to
336  * ompi_comm_lookup for getting the communicator back
337  */
338 static inline uint32_t ompi_comm_get_cid(ompi_communicator_t* comm)
339 {
340  return comm->c_contextid;
341 }
342 
343 /* return pointer to communicator associated with context id cid,
344  * No error checking is done*/
345 static inline ompi_communicator_t *ompi_comm_lookup(uint32_t cid)
346 {
347  /* array of pointers to communicators, indexed by context ID */
348  return (ompi_communicator_t*)opal_pointer_array_get_item(&ompi_mpi_communicators, cid);
349 }
350 
351 static inline struct ompi_proc_t* ompi_comm_peer_lookup(ompi_communicator_t* comm, int peer_id)
352 {
353 #if OPAL_ENABLE_DEBUG
354  if(peer_id >= comm->c_remote_group->grp_proc_count) {
355  opal_output(0, "ompi_comm_peer_lookup: invalid peer index (%d)", peer_id);
356  return (struct ompi_proc_t *) NULL;
357  }
358 #endif
359  /*return comm->c_remote_group->grp_proc_pointers[peer_id];*/
360  return ompi_group_peer_lookup(comm->c_remote_group,peer_id);
361 }
362 
363 /* Determine the rank of the specified process in this communicator
364  * @return -1 If not in communicator
365  * @return >=0 If in communicator
366  */
367 static inline int ompi_comm_peer_lookup_id(ompi_communicator_t* comm, ompi_proc_t *proc)
368 {
369 #if OPAL_ENABLE_DEBUG
370  if(NULL == proc ) {
371  opal_output(0, "ompi_comm_peer_lookup_id: invalid ompi_proc (NULL)");
372  return -1;
373  }
374 #endif
375  return ompi_group_peer_lookup_id(comm->c_remote_group, proc);
376 }
377 
378 #if OPAL_ENABLE_FT_MPI
379 #define OMPI_COMM_SET_FT(COMM, NPROCS, EPOCH) \
380  do { \
381  (COMM)->any_source_enabled = true; \
382  (COMM)->any_source_offset = 0; \
383  (COMM)->comm_revoked = false; \
384  (COMM)->collectives_force_error = false; \
385  (COMM)->num_active_local = (NPROCS); \
386  (COMM)->num_active_remote = (NPROCS); \
387  (COMM)->lleader = 0; \
388  (COMM)->rleader = 0; \
389  (COMM)->epoch = (EPOCH); \
390  } while (0)
391 
392 /*
393  * Support for MPI_ANY_SOURCE point-to-point operations
394  */
395 static inline bool ompi_comm_is_any_source_enabled(ompi_communicator_t* comm)
396 {
397  return (comm->any_source_enabled);
398 }
399 
400 /*
401  * Are collectives still active on this communicator?
402  */
403 static inline bool ompi_comm_force_error_on_collectives(ompi_communicator_t* comm)
404 {
405  return (comm->collectives_force_error);
406 }
407 
408 /*
409  * Has this communicator been revoked?
410  */
411 static inline bool ompi_comm_is_revoked(ompi_communicator_t* comm)
412 {
413  return (comm->comm_revoked);
414 }
415 
416 /*
417  * Acknowledge failures and re-enable MPI_ANY_SOURCE
418  * Related to OMPI_Comm_failure_ack() and OMPI_Comm_failure_get_acked()
419  */
420 OMPI_DECLSPEC int ompi_comm_failure_ack_internal(ompi_communicator_t* comm);
421 
422 /*
423  * Return the acknowledged group of failures
424  * Related to OMPI_Comm_failure_ack() and OMPI_Comm_failure_get_acked()
425  */
426 OMPI_DECLSPEC int ompi_comm_failure_get_acked_internal(ompi_communicator_t* comm, ompi_group_t **group );
427 
428 /*
429  * Setup/Shutdown 'revoke' handler
430  */
431 OMPI_DECLSPEC int ompi_comm_init_revoke(void);
432 OMPI_DECLSPEC int ompi_comm_finalize_revoke(void);
433 
434 /*
435  * Revoke the communicator
436  */
437 OMPI_DECLSPEC int ompi_comm_revoke_internal(ompi_communicator_t* comm);
438 
439 /*
440  * Shrink the communicator
441  */
442 OMPI_DECLSPEC int ompi_comm_shrink_internal(ompi_communicator_t* comm, ompi_communicator_t** newcomm);
443 
444 /*
445  * Process and Communicator State Accessors
446  */
447 static inline int ompi_comm_num_active_local(ompi_communicator_t* comm)
448 {
449  return (comm->num_active_local);
450 }
451 static inline int ompi_comm_num_active_remote(ompi_communicator_t* comm)
452 {
453  return (comm->num_active_remote);
454 }
455 
456 /*
457  * Check if the process is active
458  */
459 OMPI_DECLSPEC bool ompi_comm_is_proc_active(ompi_communicator_t *comm, int peer_id, bool remote);
460 
461 /*
462  * Register a new process failure
463  */
464 OMPI_DECLSPEC int ompi_comm_set_rank_failed(ompi_communicator_t *comm, int peer_id, bool remote);
465 
466 /*
467  * MPI Interface early checks
468  */
469 static inline bool ompi_comm_iface_p2p_check_proc(ompi_communicator_t *comm, int peer_id, int *err)
470 {
471  if( ompi_comm_is_revoked(comm) ) {
472  *err = MPI_ERR_REVOKED;
473  return false;
474  }
475  else if( !ompi_comm_is_proc_active(comm, peer_id, OMPI_COMM_IS_INTRA(comm)) ) {
476  *err = MPI_ERR_PROC_FAILED;
477  return false;
478  }
479  return true;
480 }
481 
482 static inline bool ompi_comm_iface_coll_check(ompi_communicator_t *comm, int *err)
483 {
484  if( ompi_comm_is_revoked(comm) ) {
485  *err = MPI_ERR_REVOKED;
486  return false;
487  }
488  else if( ompi_comm_force_error_on_collectives(comm) ) {
489  *err = MPI_ERR_PROC_FAILED;
490  return false;
491  }
492  return true;
493 }
494 
495 static inline bool ompi_comm_iface_create_check(ompi_communicator_t *comm, int *err)
496 {
497  return ompi_comm_iface_coll_check(comm, err);
498 }
499 
500 /*
501  * Communicator creation support collectives
502  * - Agreement style allreduce
503  */
504 int ompi_comm_allreduce_intra_ft( int *inbuf, int *outbuf,
505  int count, struct ompi_op_t *op,
506  ompi_communicator_t *comm,
507  ompi_communicator_t *bridgecomm,
508  void* local_leader,
509  void* remote_leader,
510  int send_first );
511 int ompi_comm_allreduce_inter_ft( int *inbuf, int *outbuf,
512  int count, struct ompi_op_t *op,
513  ompi_communicator_t *intercomm,
514  ompi_communicator_t *bridgecomm,
515  void* local_leader,
516  void* remote_leader,
517  int send_first );
518 int ompi_comm_allreduce_intra_bridge_ft(int *inbuf, int *outbuf,
519  int count, struct ompi_op_t *op,
520  ompi_communicator_t *comm,
521  ompi_communicator_t *bcomm,
522  void* lleader, void* rleader,
523  int send_first );
524 int ompi_comm_allreduce_intra_oob_ft(int *inbuf, int *outbuf,
525  int count, struct ompi_op_t *op,
526  ompi_communicator_t *comm,
527  ompi_communicator_t *bridgecomm,
528  void* lleader, void* rleader,
529  int send_first );
530 #else
531 #define OMPI_COMM_SET_FT(COMM, NPROCS, EPOCH)
532 #endif /* OPAL_ENABLE_FT_MPI */
533 
534 static inline bool ompi_comm_peer_invalid(ompi_communicator_t* comm, int peer_id)
535 {
536  if(peer_id < 0 || peer_id >= comm->c_remote_group->grp_proc_count) {
537  return true;
538  }
539  return false;
540 }
541 
542 
543 /**
544  * Initialise MPI_COMM_WORLD and MPI_COMM_SELF
545  */
546 int ompi_comm_init(void);
547 OMPI_DECLSPEC int ompi_comm_link_function(void);
548 
549 /**
550  * extract the local group from a communicator
551  */
552 OMPI_DECLSPEC int ompi_comm_group (ompi_communicator_t *comm, ompi_group_t **group);
553 
554 /**
555  * create a communicator based on a group
556  */
557 int ompi_comm_create (ompi_communicator_t* comm, ompi_group_t *group,
558  ompi_communicator_t** newcomm);
559 
560 
561 /**
562  * create a cartesian communicator
563  */
564 int ompi_topo_create (ompi_communicator_t *old_comm,
565  int ndims_or_nnodes,
566  int *dims_or_index,
567  int *periods_or_edges,
568  bool reorder,
569  ompi_communicator_t **comm_cart,
570  int cart_or_graph);
571 
572 /**
573  * split a communicator based on color and key. Parameters
574  * are identical to the MPI-counterpart of the function.
575  *
576  * @param comm: input communicator
577  * @param color
578  * @param key
579  *
580  * @
581  */
582 OMPI_DECLSPEC int ompi_comm_split (ompi_communicator_t *comm, int color, int key,
583  ompi_communicator_t** newcomm, bool pass_on_topo);
584 
585 /**
586  * split a communicator based on type and key. Parameters
587  * are identical to the MPI-counterpart of the function.
588  *
589  * @param comm: input communicator
590  * @param color
591  * @param key
592  *
593  * @
594  */
595 OMPI_DECLSPEC int ompi_comm_split_type(ompi_communicator_t *comm,
596  int split_type, int key,
597  struct ompi_info_t *info,
598  ompi_communicator_t** newcomm);
599 
600 /**
601  * dup a communicator. Parameter are identical to the MPI-counterpart
602  * of the function. It has been extracted, since we need to be able
603  * to dup a communicator internally as well.
604  *
605  * @param comm: input communicator
606  * @param newcomm: the new communicator or MPI_COMM_NULL if any error is detected.
607  */
608 OMPI_DECLSPEC int ompi_comm_dup (ompi_communicator_t *comm, ompi_communicator_t **newcomm);
609 /**
610  * compare two communicators.
611  *
612  * @param comm1,comm2: input communicators
613  *
614  */
615 int ompi_comm_compare(ompi_communicator_t *comm1, ompi_communicator_t *comm2, int *result);
616 
617 /**
618  * free a communicator
619  */
620 OMPI_DECLSPEC int ompi_comm_free (ompi_communicator_t **comm);
621 
622 /**
623  * allocate a new communicator structure
624  * @param local_group_size
625  * @param remote_group_size
626  *
627  * This routine allocates the structure, the according local and
628  * remote groups, the proc-arrays in the local and remote group.
629  * It furthermore sets the fortran index correctly,
630  * and sets all other elements to zero.
631  */
632 ompi_communicator_t* ompi_comm_allocate (int local_group_size,
633  int remote_group_size);
634 
635 /**
636  * allocate new communicator ID
637  * @param newcomm: pointer to the new communicator
638  * @param oldcomm: original comm
639  * @param bridgecomm: bridge comm for intercomm_create
640  * @param mode: combination of input
641  * OMPI_COMM_CID_INTRA: intra-comm
642  * OMPI_COMM_CID_INTER: inter-comm
643  * OMPI_COMM_CID_INTRA_BRIDGE: 2 intracomms connected by
644  * a bridge comm. local_leader
645  * and remote leader are in this
646  * case an int (rank in bridge-comm).
647  * OMPI_COMM_CID_INTRA_OOB: 2 intracomms, leaders talk
648  * through OOB. lleader and rleader
649  * are the required contact information.
650  * @param send_first: to avoid a potential deadlock for
651  * the OOB version.
652  * This routine has to be thread safe in the final version.
653  */
654 OMPI_DECLSPEC int ompi_comm_nextcid ( ompi_communicator_t* newcomm,
655  ompi_communicator_t* oldcomm,
656  ompi_communicator_t* bridgecomm,
657  void* local_leader,
658  void* remote_leader,
659  int mode,
660  int send_first);
661 
662 /**
663  * shut down the communicator infrastructure.
664  */
665 int ompi_comm_finalize (void);
666 
667 /**
668  * This is THE routine, where all the communicator stuff
669  * is really set.
670  */
671 OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
672  ompi_communicator_t* oldcomm,
673  int local_size,
674  int *local_ranks,
675  int remote_size,
676  int *remote_ranks,
677  opal_hash_table_t *attr,
678  ompi_errhandler_t *errh,
679  mca_base_component_t *topocomponent,
680  ompi_group_t *local_group,
681  ompi_group_t *remote_group );
682 /**
683  * This is a short-hand routine used in intercomm_create.
684  * The routine makes sure, that all processes have afterwards
685  * a list of ompi_proc_t pointers for the remote group.
686  */
687 struct ompi_proc_t **ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
688  ompi_communicator_t *bridge_comm,
689  int local_leader,
690  int remote_leader,
691  int tag,
692  int rsize);
693 
694 /**
695  * This routine verifies, whether local_group and remote group are overlapping
696  * in intercomm_create
697  */
698 int ompi_comm_overlapping_groups (int size, struct ompi_proc_t ** lprocs,
699  int rsize, struct ompi_proc_t ** rprocs);
700 
701 /**
702  * This is a routine determining whether the local or the
703  * remote group will be first in the new intra-comm.
704  * Just used from within MPI_Intercomm_merge.
705  */
706 int ompi_comm_determine_first ( ompi_communicator_t *intercomm,
707  int high );
708 
709 
710 OMPI_DECLSPEC int ompi_comm_activate ( ompi_communicator_t** newcomm,
711  ompi_communicator_t* comm,
712  ompi_communicator_t* bridgecomm,
713  void* local_leader,
714  void* remote_leader,
715  int mode,
716  int send_first );
717 
718 
719 /**
720  * a simple function to dump the structure
721  */
722 int ompi_comm_dump ( ompi_communicator_t *comm );
723 
724 /* setting name */
725 int ompi_comm_set_name (ompi_communicator_t *comm, char *name );
726 
727 /*
728  * these are the init and finalize functions for the comm_reg
729  * stuff. These routines are necessary for handling multi-threading
730  * scenarious in the communicator_cid allocation
731  */
732 void ompi_comm_reg_init(void);
733 void ompi_comm_reg_finalize(void);
734 
735 /* global variable to save the number od dynamic communicators */
736 extern int ompi_comm_num_dyncomm;
737 
738 
739 /* check whether any of the processes has requested support for
740  MPI_THREAD_MULTIPLE. Note, that this produces global
741  information across MPI_COMM_WORLD, in contrary to the local
742  flag ompi_mpi_thread_provided
743 */
744 OMPI_DECLSPEC int ompi_comm_cid_init ( void );
745 
746 
747 END_C_DECLS
748 
749 #endif /* OMPI_COMMUNICATOR_H */
750 
Definition: opal_hash_table.h:42
Definition: topo.h:110
Common type for all MCA components.
Definition: mca.h:250
dynamic pointer array
Definition: opal_pointer_array.h:45
Back-end type for MPI_Errorhandler.
Definition: errhandler.h:108
Definition: peruse-internal.h:30
Cached on ompi_communicator_t to hold queues/state used by the PML<->PTL interface for matching logic...
Definition: pml_bfo_comm.h:51
Definition: mutex_unix.h:53
static void * opal_pointer_array_get_item(opal_pointer_array_t *table, int element_index)
Get the value of an element in array.
Definition: opal_pointer_array.h:125
struct mca_topo_base_comm_1_0_0_t * c_topo_comm
structure containing basic information about the topology
Definition: communicator.h:155
int grp_proc_count
number of processes in group
Definition: group.h:81
Process identification structure interface.
Remote Open MPI process structure.
Definition: proc.h:56
Definition: communicator.h:266
OMPI_DECLSPEC int ompi_group_peer_lookup_id(ompi_group_t *group, ompi_proc_t *proc)
Determine the rank of the specified process in this group.
Definition: group_plist.c:305
Definition: info.h:38
Collective Communication Interface.
const struct mca_topo_base_module_1_0_0_t * c_topo
structure of function pointers
Definition: communicator.h:152
OPAL_DECLSPEC void opal_output(int output_id, const char *format,...) __opal_attribute_format__(__printf__
Main function to send output to a stream.
static struct ompi_proc_t * ompi_group_peer_lookup(ompi_group_t *group, int peer_id)
Inline function to check if sparse groups are enabled and return the direct access to the proc pointe...
Definition: group.h:333
Group structure Currently we have four formats for storing the process pointers that are members of t...
Definition: group.h:79
Base object.
Definition: opal_object.h:182
Definition: topo.h:255
struct mca_topo_base_module_comm_t * c_topo_module
module specific data
Definition: communicator.h:158
Infrastructure for MPI group support.
Mutual exclusion functions.
A simple C-language object-oriented system with single inheritance and ownership-based memory managem...
Definition: communicator.h:118
int epoch
Identifier used to keep trace of the communicators revoked.
Definition: communicator.h:132
struct opal_hash_table_t * c_keyhash
inscribing cube dimension
Definition: communicator.h:144
Back-end type of MPI_Op.
Definition: op.h:100
Collectives communicator cache structure.
Definition: coll.h:367
ompi_errhandler_type_t
Enum used to describe what kind MPI object an error handler is used for.
Definition: errhandler.h:84
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236