OpenMPI  0.1.1
btl_portals_recv.h
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2005 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * $COPYRIGHT$
13  *
14  * Additional copyrights may follow
15  *
16  * $HEADER$
17  */
18 
19 #ifndef OMPI_BTL_PORTALS_RECV_H
20 #define OMPI_BTL_PORTALS_RECV_H
21 
22 #include "btl_portals_frag.h"
23 
25  opal_list_item_t base;
26 
28 
29  void *start;
30  size_t length;
31  ptl_handle_me_t me_h;
32  ptl_handle_md_t md_h;
33 
34  volatile bool full;
35  volatile int32_t pending;
36 };
39 
40 
41 int mca_btl_portals_recv_enable(mca_btl_portals_module_t *btl);
42 
43 int mca_btl_portals_recv_disable(mca_btl_portals_module_t *btl);
44 
45 /**
46  * Create a block of memory for receiving send messages. Must call
47  * activate_block on the returned block of memory before it will be
48  * active with the POrtals library
49  *
50  * Module lock must be held before calling this function
51  */
53 mca_btl_portals_recv_block_init(mca_btl_portals_module_t *btl);
54 
55 
56 /**
57  * Free a block of memory. Will remove the match entry, then progress
58  * Portals until the pending count is returned to 0. Will then free
59  * all resources associated with block.
60  *
61  * Module lock must be held before calling this function
62  */
63 int mca_btl_portals_recv_block_free(mca_btl_portals_recv_block_t *block);
64 
65 
66 /**
67  * activate a block. Blocks that are full (have gone inactive) can be
68  * re-activated with this call. There is no need to hold the lock
69  * before calling this function
70  */
71 static inline int
72 mca_btl_portals_activate_block(mca_btl_portals_recv_block_t *block)
73 {
74  int ret;
75  ptl_process_id_t any_proc = { PTL_NID_ANY, PTL_PID_ANY };
76  ptl_md_t md;
77  uint64_t ignore_bits = ~((uint64_t) 0);
78 
79  /* if we have pending operations, something very, very, very bad
80  has happened... */
81  assert(block->pending == 0);
82 
83  if (NULL == block->start) return OMPI_ERROR;
84 
85  /* create match entry */
86  ret = PtlMEInsert(block->btl->portals_recv_reject_me_h,
87  any_proc,
88  0, /* match bits */
89  ignore_bits, /* ignore bits */
90  PTL_UNLINK,
91  PTL_INS_BEFORE,
92  &(block->me_h));
93  if (PTL_OK != ret) return OMPI_ERROR;
94 
95  /* and the memory descriptor */
96  md.start = block->start;
97  md.length = block->length;
98  /* try to throttle incoming sends so that we don't overrun the incoming
99  queue size */
100  md.threshold = mca_btl_portals_module.portals_eq_sizes[OMPI_BTL_PORTALS_EQ_RECV] /
101  (mca_btl_portals_module.portals_recv_mds_num * 2);
102  md.max_size = block->btl->super.btl_max_send_size;
103  md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE;
104  md.user_ptr = block;
105  md.eq_handle = block->btl->portals_eq_handles[OMPI_BTL_PORTALS_EQ_RECV];
106 
107  block->pending = 0;
108  block->full = false;
109  /* make sure that everyone sees the update on full value */
110  opal_atomic_mb();
111 
112  ret = PtlMDAttach(block->me_h,
113  md,
114  PTL_UNLINK,
115  &(block->md_h));
116  if (PTL_OK != ret) {
117  PtlMEUnlink(block->me_h);
118  return OMPI_ERROR;
119  }
120 
121  return OMPI_SUCCESS;
122 }
123 
124 
125 static inline void
126 mca_btl_portals_return_block_part(mca_btl_portals_module_t *btl,
128 {
129  int ret;
130 
131  OPAL_THREAD_ADD32(&(block->pending), -1);
132  if (block->full == true) {
133  if (block->pending == 0) {
134  ret = mca_btl_portals_activate_block(block);
135  if (OMPI_SUCCESS != ret) {
136  /* BWB - now what? */
137  }
138  }
139  }
140 }
141 
142 #endif /* OMPI_BTL_PORTALS_RECV_H */
#define OPAL_THREAD_ADD32(x, y)
Use an atomic operation for increment/decrement if opal_using_threads() indicates that threads are in...
Definition: mutex.h:367
Definition: btl_portals.h:79
Definition: opal_list.h:98
void opal_atomic_mb(void)
Memory barrier.
Definition: t-test2.c:52
Definition: btl_portals_recv.h:24
size_t btl_max_send_size
maximum send fragment size supported by the BTL
Definition: btl.h:792
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236