OpenMPI  0.1.1
btl_ugni_rdma.h
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3  * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
4  * reserved.
5  * Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
6  * $COPYRIGHT$
7  *
8  * Additional copyrights may follow
9  *
10  * $HEADER$
11  */
12 
13 #if !defined(MCA_BTL_UGNI_RDMA_H)
14 #define MCA_BTL_UGNI_RDMA_H
15 
16 #include "btl_ugni.h"
17 #include "btl_ugni_frag.h"
18 
19 static inline void
20 mca_btl_ugni_post_frag_complete (ompi_common_ugni_post_desc_t *desc, int rc) {
21  mca_btl_ugni_base_frag_t *frag = MCA_BTL_UGNI_DESC_TO_FRAG(desc);
22 
23  /* always call put/get callback (if one is set) */
24  if (NULL != frag->base.des_cbfunc) {
25  frag->base.des_cbfunc(&frag->endpoint->btl->super, frag->endpoint, &frag->base, rc);
26  }
27 
28  if (OPAL_LIKELY(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
29  mca_btl_ugni_frag_return (frag);
30  }
31 }
32 
33 static inline int init_gni_post_desc(mca_btl_ugni_base_frag_t *frag,
34  gni_post_type_t op_type,
35  uint64_t lcl_addr,
36  gni_mem_handle_t *lcl_mdh,
37  uint64_t rem_addr,
38  gni_mem_handle_t *rem_mdh,
39  uint64_t bufsize,
40  gni_cq_handle_t cq_hndl) {
41  frag->post_desc.base.type = op_type;
42  frag->post_desc.base.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
43  frag->post_desc.base.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
44  frag->post_desc.base.local_addr = (uint64_t) lcl_addr;
45  frag->post_desc.base.local_mem_hndl = *lcl_mdh;
46  frag->post_desc.base.remote_addr = (uint64_t) rem_addr;
47  frag->post_desc.base.remote_mem_hndl = *rem_mdh;
48  frag->post_desc.base.length = bufsize;
49  frag->post_desc.base.rdma_mode = 0;
50  frag->post_desc.base.src_cq_hndl = cq_hndl;
51 
52  frag->post_desc.cbfunc = mca_btl_ugni_post_frag_complete;
53  frag->post_desc.endpoint = frag->endpoint->common;
54  frag->post_desc.tries = 0;
55 
56  return 0;
57 }
58 
59 static inline int mca_btl_ugni_post_fma (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
61 {
62  int rc;
63 
64  /* Post descriptor */
65  init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
66  (gni_mem_handle_t *)&lcl_seg->seg_key.key64,
67  rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
68  lcl_seg->seg_len, 0);
69 
70  rc = GNI_PostFma (frag->endpoint->common->ep_handle, &frag->post_desc.base);
71  if (GNI_RC_SUCCESS != rc) {
72 /* BTL_ERROR(("GNI_PostFma failed with rc = %d", rc)); */
73  assert(rc < 4);
74  rc = OMPI_ERR_OUT_OF_RESOURCE;
75  }
76 
77  return rc;
78 }
79 
80 static inline int mca_btl_ugni_post_bte (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
82 {
83  int rc;
84 
85  /* Post descriptor */
86  init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
87  (gni_mem_handle_t *)&lcl_seg->seg_key.key64,
88  rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
89  lcl_seg->seg_len, frag->endpoint->btl->bte_local_cq);
90 
91  rc = GNI_PostRdma (frag->endpoint->common->ep_handle, &frag->post_desc.base);
92  if (GNI_RC_SUCCESS != rc) {
93  rc = ompi_common_rc_ugni_to_ompi (rc);
94  BTL_ERROR(("GNI_PostRdma failed with rc = %d", rc));
95  }
96 
97  return rc;
98 }
99 
100 #endif /* MCA_BTL_UGNI_RDMA_H */
uint32_t des_flags
hints to BTL
Definition: btl.h:284
ompi_ptr_t seg_addr
Address of the memory.
Definition: btl.h:238
Definition: btl_ugni_frag.h:43
union mca_btl_base_segment_t::@93 seg_key
Memory segment key required by some RDMA networks.
uint32_t seg_len
Length in bytes.
Definition: btl.h:240
gni_ep_handle_t ep_handle
uGNI handle for this endpoint
Definition: common_ugni_ep.h:28
mca_btl_base_completion_fn_t des_cbfunc
local callback function
Definition: btl.h:281
Definition: common_ugni.h:81
Describes a region/segment of memory that is addressable by an BTL.
Definition: btl.h:236