OpenMPI  0.1.1
btl_ofud_frag.h
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2006 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2006 Sandia National Laboratories. All rights
13  * reserved.
14  * $COPYRIGHT$
15  *
16  * Additional copyrights may follow
17  *
18  * $HEADER$
19  */
20 
21 #ifndef MCA_BTL_UD_FRAG_H
22 #define MCA_BTL_UD_FRAG_H
23 
24 #define MCA_BTL_IB_FRAG_ALIGN (8)
25 
26 #include <infiniband/verbs.h>
27 
29 
30 #include "btl_ofud.h"
31 
32 
33 BEGIN_C_DECLS
34 
35 
36 /**
37  * Fragment types
38  */
39 typedef enum {
40  MCA_BTL_UD_FRAG_SEND,
41  MCA_BTL_UD_FRAG_USER,
42  MCA_BTL_UD_FRAG_RECV
43 } mca_btl_ud_frag_type_t;
44 
45 
48  struct ibv_mr* mr;
49 };
50 typedef struct mca_btl_ud_reg_t mca_btl_ud_reg_t;
51 
52 
53 /* UD adds a 40 byte global routing header */
54 /* This works in strange ways - the sending side does not need to explicitly
55  include this data in sg lists. Then, on the receiving side, the extra 40
56  bytes magically appear. */
58  uint8_t ib_grh[40];
59 };
61 
63  /*uint32_t src_qpnum;*/
64  mca_btl_base_tag_t tag;
65 };
67 
68 
69 /**
70  * IB send fragment derived type.
71  */
72 
75  mca_btl_base_segment_t segment;
76 
77  struct mca_btl_base_endpoint_t* endpoint;
78 
79  mca_btl_ud_frag_type_t type;
80 
81  union{
82  struct ibv_recv_wr rd_desc;
83  struct ibv_send_wr sr_desc;
84  } wr_desc;
85  struct ibv_sge sg_entry;
86 
87  /* When this is a send frag, hdr points right after this, as expected.
88  But when this is a receive frag, we have an extra 40 bytes provided
89  by IB, so this points 40 bytes past the end of the frag. */
91 
92  mca_btl_ud_reg_t* ud_reg;
93 };
96 
99 
102 
105 
106 
107 /*
108  * Allocate/return a UD/IB send/user fragment
109  */
110 
111 #define MCA_BTL_UD_ALLOC_FRAG(btl, frag, rc) \
112 { \
113  ompi_free_list_item_t *item; \
114  OMPI_FREE_LIST_GET(&((mca_btl_ud_module_t*)btl)->send_frags, item, rc); \
115  frag = (mca_btl_ud_frag_t*) item; \
116 }
117 
118 #define MCA_BTL_UD_RETURN_FRAG(btl, frag) \
119 { \
120  OMPI_FREE_LIST_RETURN( \
121  &((mca_btl_ud_module_t*)btl)->send_frags, \
122  (ompi_free_list_item_t*)(frag)); \
123 }
124 
125 
126 #define MCA_BTL_UD_ALLOC_USER_FRAG(btl, frag, rc) \
127 { \
128  ompi_free_list_item_t *item; \
129  OMPI_FREE_LIST_GET(&((mca_btl_ud_module_t*)btl)->user_frags, item, rc); \
130  frag = (mca_btl_ud_frag_t*) item; \
131 }
132 
133 #define MCA_BTL_UD_RETURN_USER_FRAG(btl, frag) \
134 { \
135  OMPI_FREE_LIST_RETURN( \
136  &((mca_btl_ud_module_t*)btl)->user_frags, \
137  (ompi_free_list_item_t*)(frag)); \
138 }
139 
140 
141 struct mca_btl_ud_module_t;
142 
143 END_C_DECLS
144 #endif
Definition: btl_ofud_frag.h:46
A descriptor that holds the parameters to a send/put/get operation along w/ a callback routine that i...
Definition: btl.h:275
UD/IB BTL Interface.
Definition: btl_ofud.h:126
Definition: mpool.h:44
Definition: btl_ofud_frag.h:57
State of ELAN endpoint connection.
Definition: btl_elan_endpoint.h:33
IB send fragment derived type.
Definition: btl_ofud_frag.h:73
Definition: btl_ofud_frag.h:62
Describes a region/segment of memory that is addressable by an BTL.
Definition: btl.h:236
#define OBJ_CLASS_DECLARATION(NAME)
Declaration for class descriptor.
Definition: opal_object.h:236