OpenMPI  0.1.1
osc_rdma_sendreq.h
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University.
3  * All rights reserved.
4  * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
5  * All rights reserved.
6  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
7  * University of Stuttgart. All rights reserved.
8  * Copyright (c) 2004-2005 The Regents of the University of California.
9  * All rights reserved.
10  * Copyright (c) 2007 Los Alamos National Security, LLC. All rights
11  * reserved.
12  * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
13  * $COPYRIGHT$
14  *
15  * Additional copyrights may follow
16  *
17  * $HEADER$
18  */
19 
20 #ifndef OMPI_OSC_RDMA_SENDREQ_H
21 #define OMPI_OSC_RDMA_SENDREQ_H
22 
23 #include "osc_rdma.h"
24 #include "osc_rdma_longreq.h"
25 
26 #include "opal/class/opal_list.h"
27 #include "ompi/datatype/ompi_datatype.h"
28 #include "opal/datatype/opal_convertor.h"
29 #include "ompi/communicator/communicator.h"
30 #include "ompi/proc/proc.h"
31 #include "ompi/memchecker.h"
32 
33 typedef enum {
34  OMPI_OSC_RDMA_GET,
35  OMPI_OSC_RDMA_ACC,
36  OMPI_OSC_RDMA_PUT
37 } ompi_osc_rdma_req_type_t;
38 
39 
41  ompi_request_t super;
42 
43  int req_refcount;
44 
45  /** type of sendreq (from ompi_osc_rdma_req_type_t) */
46  ompi_osc_rdma_req_type_t req_type;
47  /** pointer to the module that created the sendreq */
49 
50  /** Datatype for the origin side of the operation */
52  /** Convertor for the origin side of the operation. Setup for
53  either send (Put / Accumulate) or receive (Get) */
55  /** packed size of message on the origin side */
57 
58  /** rank in module's communicator for target of operation */
60  /** pointer to the proc structure for the target of the operation */
62 
63  /** displacement on target */
64  OPAL_PTRDIFF_TYPE req_target_disp;
65  /** datatype count on target */
67  /** datatype on target */
69 
70  /** op index on the target */
71  int req_op_id;
72 
73  mca_btl_base_segment_t remote_segs[1];
74 };
77 
78 
79 /** allocate and populate a sendreq structure. Both datatypes are
80  RETAINed for the life of the sendreq */
81 int
82 ompi_osc_rdma_sendreq_alloc_init(ompi_osc_rdma_req_type_t req_type,
83  void *origin_addr, int origin_count,
84  struct ompi_datatype_t *origin_dt,
85  int target, OPAL_PTRDIFF_TYPE target_disp,
86  int target_count,
87  struct ompi_datatype_t *target_datatype,
88  ompi_osc_rdma_module_t *module,
89  ompi_osc_rdma_sendreq_t **sendreq);
90 
91 static inline int
92 ompi_osc_rdma_sendreq_alloc(ompi_osc_rdma_module_t *module,
93  int target_rank,
94  ompi_osc_rdma_sendreq_t **sendreq)
95 {
96  int ret;
98  ompi_proc_t *proc = ompi_comm_peer_lookup( module->m_comm, target_rank );
99 
100  /* BWB - FIX ME - is this really the right return code? */
101  if (NULL == proc) return OMPI_ERR_OUT_OF_RESOURCE;
102 
103  OPAL_FREE_LIST_GET(&mca_osc_rdma_component.c_sendreqs,
104  item, ret);
105  if (OMPI_SUCCESS != ret) return ret;
106  *sendreq = (ompi_osc_rdma_sendreq_t*) item;
107 
108  (*sendreq)->req_module = module;
109  (*sendreq)->req_target_rank = target_rank;
110  (*sendreq)->req_target_proc = proc;
111  (*sendreq)->req_refcount = 1;
112 
113  return OMPI_SUCCESS;
114 }
115 
116 
117 static inline int
118 ompi_osc_rdma_sendreq_init_origin(ompi_osc_rdma_sendreq_t *sendreq,
119  ompi_osc_rdma_req_type_t req_type,
120  void *origin_addr,
121  int origin_count,
122  struct ompi_datatype_t *origin_dt)
123 {
124  OBJ_RETAIN(origin_dt);
125  sendreq->req_origin_datatype = origin_dt;
126  sendreq->req_type = req_type;
127 
128  if (req_type != OMPI_OSC_RDMA_GET) {
129  opal_convertor_copy_and_prepare_for_send(sendreq->req_target_proc->proc_convertor,
130  &(origin_dt->super),
131  origin_count,
132  origin_addr,
133  0,
134  &(sendreq->req_origin_convertor));
135  opal_convertor_get_packed_size(&sendreq->req_origin_convertor,
136  &sendreq->req_origin_bytes_packed);
137  } else {
138  opal_convertor_copy_and_prepare_for_recv(sendreq->req_target_proc->proc_convertor,
139  &(origin_dt->super),
140  origin_count,
141  origin_addr,
142  0,
143  &(sendreq->req_origin_convertor));
144  opal_convertor_get_packed_size(&sendreq->req_origin_convertor,
145  &sendreq->req_origin_bytes_packed);
146  }
147 
148  return OMPI_SUCCESS;
149 }
150 
151 
152 static inline int
153 ompi_osc_rdma_sendreq_init_target(ompi_osc_rdma_sendreq_t *sendreq,
154  OPAL_PTRDIFF_TYPE target_disp,
155  int target_count,
156  struct ompi_datatype_t *target_datatype)
157 {
158  OBJ_RETAIN(target_datatype);
159 
160  sendreq->req_target_disp = target_disp;
161  sendreq->req_target_count = target_count;
162  sendreq->req_target_datatype = target_datatype;
163 
164  return OMPI_SUCCESS;
165 }
166 
167 
168 static inline int
169 ompi_osc_rdma_sendreq_free(ompi_osc_rdma_sendreq_t *sendreq)
170 {
171  if (0 == (--sendreq->req_refcount)) {
172  MEMCHECKER(
173  memchecker_convertor_call(&opal_memchecker_base_mem_defined,
174  &sendreq->req_origin_convertor);
175  );
176  opal_convertor_cleanup(&sendreq->req_origin_convertor);
177 
180 
181  OPAL_FREE_LIST_RETURN(&mca_osc_rdma_component.c_sendreqs,
182  (opal_list_item_t*) sendreq);
183  }
184 
185  return OMPI_SUCCESS;
186 }
187 
188 #endif /* OMPI_OSC_RDMA_SENDREQ_H */
opal_free_list_t c_sendreqs
free list of ompi_osc_rdma_sendreq_t structures
Definition: osc_rdma.h:64
struct ompi_datatype_t * req_origin_datatype
Datatype for the origin side of the operation.
Definition: osc_rdma_sendreq.h:51
Definition: osc_rdma.h:111
int req_target_count
datatype count on target
Definition: osc_rdma_sendreq.h:66
ompi_osc_rdma_module_t * req_module
pointer to the module that created the sendreq
Definition: osc_rdma_sendreq.h:48
Definition: ompi_datatype.h:68
The opal_list_t interface is used to provide a generic doubly-linked list container for Open MPI...
Process identification structure interface.
Remote Open MPI process structure.
Definition: proc.h:56
#define OBJ_RETAIN(object)
Retain an object (by incrementing its reference count)
Definition: opal_object.h:278
Definition: opal_list.h:98
#define OBJ_RELEASE(object)
Release an object (by decrementing its reference count).
Definition: opal_object.h:324
size_t req_origin_bytes_packed
packed size of message on the origin side
Definition: osc_rdma_sendreq.h:56
Definition: opal_free_list.h:47
int req_target_rank
rank in module's communicator for target of operation
Definition: osc_rdma_sendreq.h:59
opal_datatype_t super
Base opal_datatype_t superclass.
Definition: ompi_datatype.h:69
struct ompi_datatype_t * req_target_datatype
datatype on target
Definition: osc_rdma_sendreq.h:68
Definition: opal_convertor.h:90
Definition: osc_rdma_sendreq.h:40
ompi_proc_t * req_target_proc
pointer to the proc structure for the target of the operation
Definition: osc_rdma_sendreq.h:61
struct opal_convertor_t * proc_convertor
Base convertor for the proc described by this process.
Definition: proc.h:70
ompi_osc_rdma_req_type_t req_type
type of sendreq (from ompi_osc_rdma_req_type_t)
Definition: osc_rdma_sendreq.h:46
OPAL_PTRDIFF_TYPE req_target_disp
displacement on target
Definition: osc_rdma_sendreq.h:64
opal_convertor_t req_origin_convertor
Convertor for the origin side of the operation.
Definition: osc_rdma_sendreq.h:54
int req_op_id
op index on the target
Definition: osc_rdma_sendreq.h:71
Main top-level request struct definition.
Definition: request.h:100
ompi_communicator_t * m_comm
communicator created with this window
Definition: osc_rdma.h:130
Describes a region/segment of memory that is addressable by an BTL.
Definition: btl.h:236
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236