OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

btl_ofud_frag.h
1 /*
2 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2006 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2006 Sandia National Laboratories. All rights
13 * reserved.
14 * $COPYRIGHT$
15 *
16 * Additional copyrights may follow
17 *
18 * $HEADER$
19 */
20
21 #ifndef MCA_BTL_UD_FRAG_H
22 #define MCA_BTL_UD_FRAG_H
23
24 #define MCA_BTL_IB_FRAG_ALIGN (8)
25
26 #include <infiniband/verbs.h>
27
28 #include "ompi/mca/mpool/rdma/mpool_rdma.h"
29
30 #include "btl_ofud.h"
31
32
33 BEGIN_C_DECLS
34
35
36 /**
37 * Fragment types
38 */
39 typedef enum {
40 MCA_BTL_UD_FRAG_SEND,
41 MCA_BTL_UD_FRAG_USER,
42 MCA_BTL_UD_FRAG_RECV
43 } mca_btl_ud_frag_type_t;
44
45
46 struct mca_btl_ud_reg_t {
47 mca_mpool_base_registration_t base;
48 struct ibv_mr* mr;
49 };
50 typedef struct mca_btl_ud_reg_t mca_btl_ud_reg_t;
51
52
53 /* UD adds a 40 byte global routing header */
54 /* This works in strange ways - the sending side does not need to explicitly
55 include this data in sg lists. Then, on the receiving side, the extra 40
56 bytes magically appear. */
57 struct mca_btl_ud_ib_header_t {
58 uint8_t ib_grh[40];
59 };
60 typedef struct mca_btl_ud_ib_header_t mca_btl_ud_ib_header_t;
61
62 struct mca_btl_ud_header_t {
63 /*uint32_t src_qpnum;*/
64 mca_btl_base_tag_t tag;
65 };
66 typedef struct mca_btl_ud_header_t mca_btl_ud_header_t;
67
68
69 /**
70 * IB send fragment derived type.
71 */
72
73 struct mca_btl_ud_frag_t {
74 mca_btl_base_descriptor_t base;
75 mca_btl_base_segment_t segment;
76
77 struct mca_btl_base_endpoint_t* endpoint;
78
79 mca_btl_ud_frag_type_t type;
80
81 union{
82 struct ibv_recv_wr rd_desc;
83 struct ibv_send_wr sr_desc;
84 } wr_desc;
85 struct ibv_sge sg_entry;
86
87 /* When this is a send frag, hdr points right after this, as expected.
88 But when this is a receive frag, we have an extra 40 bytes provided
89 by IB, so this points 40 bytes past the end of the frag. */
90 mca_btl_ud_header_t* hdr;
91
92 mca_btl_ud_reg_t* ud_reg;
93 };
94 typedef struct mca_btl_ud_frag_t mca_btl_ud_frag_t;
95 OBJ_CLASS_DECLARATION(mca_btl_ud_frag_t);
96
97 typedef struct mca_btl_ud_frag_t mca_btl_ud_send_frag_t;
98 OBJ_CLASS_DECLARATION(mca_btl_ud_send_frag_t);
99
100 typedef struct mca_btl_ud_frag_t mca_btl_ud_user_frag_t;
101 OBJ_CLASS_DECLARATION(mca_btl_ud_user_frag_t);
102
103 typedef struct mca_btl_ud_frag_t mca_btl_ud_recv_frag_t;
104 OBJ_CLASS_DECLARATION(mca_btl_ud_recv_frag_t);
105
106
107 /*
108 * Allocate/return a UD/IB send/user fragment
109 */
110
111 #define MCA_BTL_UD_ALLOC_FRAG(btl, frag, rc) \
112 { \
113 ompi_free_list_item_t *item; \
114 OMPI_FREE_LIST_GET(&((mca_btl_ud_module_t*)btl)->send_frags, item, rc); \
115 frag = (mca_btl_ud_frag_t*) item; \
116 }
117
118 #define MCA_BTL_UD_RETURN_FRAG(btl, frag) \
119 { \
120 OMPI_FREE_LIST_RETURN( \
121 &((mca_btl_ud_module_t*)btl)->send_frags, \
122 (ompi_free_list_item_t*)(frag)); \
123 }
124
125
126 #define MCA_BTL_UD_ALLOC_USER_FRAG(btl, frag, rc) \
127 { \
128 ompi_free_list_item_t *item; \
129 OMPI_FREE_LIST_GET(&((mca_btl_ud_module_t*)btl)->user_frags, item, rc); \
130 frag = (mca_btl_ud_frag_t*) item; \
131 }
132
133 #define MCA_BTL_UD_RETURN_USER_FRAG(btl, frag) \
134 { \
135 OMPI_FREE_LIST_RETURN( \
136 &((mca_btl_ud_module_t*)btl)->user_frags, \
137 (ompi_free_list_item_t*)(frag)); \
138 }
139
140
141 struct mca_btl_ud_module_t;
142
143 END_C_DECLS
144 #endif
mca_btl_ud_reg_t
Definition: btl_ofud_frag.h:46
btl_ofud.h
mca_btl_base_descriptor_t
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
mca_btl_ud_module_t
UD/IB BTL Interface.
Definition: btl_ofud.h:126
mca_mpool_base_registration_t
Definition: mpool.h:44
mca_btl_ud_ib_header_t
Definition: btl_ofud_frag.h:57
mca_btl_base_endpoint_t
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
mca_btl_ud_frag_t
IB send fragment derived type.
Definition: btl_ofud_frag.h:73
mca_btl_ud_header_t
Definition: btl_ofud_frag.h:62
mpool_rdma.h
mca_btl_base_segment_t
Describes a region/segment of memory that is addressable by an BTL.
Definition: btl.h:236
OBJ_CLASS_DECLARATION
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236