OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_ugni_rdma.h
1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
4 * reserved.
5 * Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
6 * $COPYRIGHT$
7 *
8 * Additional copyrights may follow
9 *
10 * $HEADER$
11 */
12
13 #if !defined(MCA_BTL_UGNI_RDMA_H)
14 #define MCA_BTL_UGNI_RDMA_H
15
16 #include "btl_ugni.h"
17 #include "btl_ugni_frag.h"
18
19 static inline void
20 mca_btl_ugni_post_frag_complete (ompi_common_ugni_post_desc_t *desc, int rc) {
21 mca_btl_ugni_base_frag_t *frag = MCA_BTL_UGNI_DESC_TO_FRAG(desc);
22
23 /* always call put/get callback (if one is set) */
24 if (NULL != frag->base.des_cbfunc) {
25 frag->base.des_cbfunc(&frag->endpoint->btl->super, frag->endpoint, &frag->base, rc);
26 }
27
28 if (OPAL_LIKELY(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
29 mca_btl_ugni_frag_return (frag);
30 }
31 }
32
33 static inline int init_gni_post_desc(mca_btl_ugni_base_frag_t *frag,
34 gni_post_type_t op_type,
35 uint64_t lcl_addr,
36 gni_mem_handle_t *lcl_mdh,
37 uint64_t rem_addr,
38 gni_mem_handle_t *rem_mdh,
39 uint64_t bufsize,
40 gni_cq_handle_t cq_hndl) {
41 frag->post_desc.base.type = op_type;
42 frag->post_desc.base.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
43 frag->post_desc.base.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
44 frag->post_desc.base.local_addr = (uint64_t) lcl_addr;
45 frag->post_desc.base.local_mem_hndl = *lcl_mdh;
46 frag->post_desc.base.remote_addr = (uint64_t) rem_addr;
47 frag->post_desc.base.remote_mem_hndl = *rem_mdh;
48 frag->post_desc.base.length = bufsize;
49 frag->post_desc.base.rdma_mode = 0;
50 frag->post_desc.base.src_cq_hndl = cq_hndl;
51
52 frag->post_desc.cbfunc = mca_btl_ugni_post_frag_complete;
53 frag->post_desc.endpoint = frag->endpoint->common;
54 frag->post_desc.tries = 0;
55
56 return 0;
57 }
58
59 static inline int mca_btl_ugni_post_fma (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
60 mca_btl_base_segment_t *lcl_seg, mca_btl_base_segment_t *rem_seg)
61 {
62 int rc;
63
64 /* Post descriptor */
65 init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
66 (gni_mem_handle_t *)&lcl_seg->seg_key.key64,
67 rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
68 lcl_seg->seg_len, 0);
69
70 rc = GNI_PostFma (frag->endpoint->common->ep_handle, &frag->post_desc.base);
71 if (GNI_RC_SUCCESS != rc) {
72 /* BTL_ERROR(("GNI_PostFma failed with rc = %d", rc)); */
73 assert(rc < 4);
74 rc = OMPI_ERR_OUT_OF_RESOURCE;
75 }
76
77 return rc;
78 }
79
80 static inline int mca_btl_ugni_post_bte (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
81 mca_btl_base_segment_t *lcl_seg, mca_btl_base_segment_t *rem_seg)
82 {
83 int rc;
84
85 /* Post descriptor */
86 init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
87 (gni_mem_handle_t *)&lcl_seg->seg_key.key64,
88 rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
89 lcl_seg->seg_len, frag->endpoint->btl->bte_local_cq);
90
91 rc = GNI_PostRdma (frag->endpoint->common->ep_handle, &frag->post_desc.base);
92 if (GNI_RC_SUCCESS != rc) {
93 rc = ompi_common_rc_ugni_to_ompi (rc);
94 BTL_ERROR(("GNI_PostRdma failed with rc = %d", rc));
95 }
96
97 return rc;
98 }
99
100 #endif /* MCA_BTL_UGNI_RDMA_H */
mca_btl_base_descriptor_t::des_flags
uint32_t des_flags
hints to BTL
Definition: btl.h:284
mca_btl_base_segment_t::seg_addr
ompi_ptr_t seg_addr
Address of the memory.
Definition: btl.h:238
mca_btl_ugni_base_frag_t
Definition: btl_ugni_frag.h:43
mca_btl_base_segment_t::seg_key
union mca_btl_base_segment_t::@93 seg_key
Memory segment key required by some RDMA networks.
mca_btl_base_segment_t::seg_len
uint32_t seg_len
Length in bytes.
Definition: btl.h:240
ompi_common_ugni_endpoint_t::ep_handle
gni_ep_handle_t ep_handle
uGNI handle for this endpoint
Definition: common_ugni_ep.h:28
mca_btl_base_descriptor_t::des_cbfunc
mca_btl_base_completion_fn_t des_cbfunc
local callback function
Definition: btl.h:281
ompi_common_ugni_post_desc_t
Definition: common_ugni.h:81
mca_btl_base_segment_t
Describes a region/segment of memory that is addressable by an BTL.
Definition: btl.h:236