14 #if !defined(MEM_OP_NAME)
22 # define STRINGIFY_(arg) #arg
23 # define STRINGIFY(arg) STRINGIFY_(arg)
27 # define DT__CONCAT(a, b) a##b
28 # define DT_CONCAT(a, b) DT__CONCAT(a, b)
32 #define _predefined_data DT_CONCAT(MEM_OP_NAME,_predefined_data)
33 #define _contiguous_loop DT_CONCAT(MEM_OP_NAME,_contiguous_loop)
34 #define _copy_content_same_ddt DT_CONCAT(MEM_OP_NAME,_copy_content_same_ddt)
38 unsigned char* SOURCE_BASE,
41 unsigned char* SOURCE,
42 unsigned char* DESTINATION,
45 uint32_t _copy_count = (COUNT);
48 unsigned char* _source = (SOURCE) + _elem->
disp;
49 unsigned char* _destination = (DESTINATION) + _elem->
disp;
51 _copy_blength = opal_datatype_basicDatatypes[_elem->
common.
type]->
size;
53 if( _copy_blength == (uint32_t)_elem->
extent ) {
54 _copy_blength *= _copy_count;
55 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
56 (DATATYPE), (TOTAL_COUNT) );
58 DO_DEBUG(
opal_output( 0,
"copy 1. %s( %p, %p, %lu ) => space %lu\n",
59 STRINGIFY(MEM_OP_NAME), _destination, _source, (
unsigned long)_copy_blength, (
unsigned long)(*(SPACE)) ); );
60 MEM_OP( _destination, _source, _copy_blength );
61 _source += _copy_blength;
62 _destination += _copy_blength;
65 for( _i = 0; _i < _copy_count; _i++ ) {
66 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
67 (DATATYPE), (TOTAL_COUNT) );
68 DO_DEBUG(
opal_output( 0,
"copy 2. %s( %p, %p, %lu ) => space %lu\n",
69 STRINGIFY(MEM_OP_NAME), _destination, _source, (
unsigned long)_copy_blength, (
unsigned long)(*(SPACE) - (_i * _copy_blength)) ); );
70 MEM_OP( _destination, _source, _copy_blength );
72 _destination += _elem->
extent;
74 _copy_blength *= _copy_count;
76 *(SPACE) -= _copy_blength;
81 unsigned char* SOURCE_BASE,
84 unsigned char* SOURCE,
85 unsigned char* DESTINATION,
91 unsigned char* _destination = (DESTINATION) + _end_loop->
first_elem_disp;
92 size_t _copy_loops = (COUNT);
95 if( _loop->
extent == (OPAL_PTRDIFF_TYPE)_end_loop->
size ) {
96 _copy_loops *= _end_loop->
size;
97 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_loops, (SOURCE_BASE),
98 (DATATYPE), (TOTAL_COUNT) );
99 MEM_OP( _destination, _source, _copy_loops );
101 for( _i = 0; _i < _copy_loops; _i++ ) {
102 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _end_loop->
size, (SOURCE_BASE),
103 (DATATYPE), (TOTAL_COUNT) );
104 DO_DEBUG(
opal_output( 0,
"copy 3. %s( %p, %p, %lu ) => space %lu\n",
105 STRINGIFY(MEM_OP_NAME), _destination, _source, (
unsigned long)_end_loop->
size, (
unsigned long)(*(SPACE) - _i * _end_loop->
size) ); );
106 MEM_OP( _destination, _source, _end_loop->
size );
108 _destination += _loop->
extent;
110 _copy_loops *= _end_loop->
size;
112 *(SPACE) -= _copy_loops;
115 static inline int32_t _copy_content_same_ddt(
const opal_datatype_t* datatype, int32_t count,
116 char* destination_base,
char* source_base )
124 size_t iov_len_local;
125 unsigned char *source = (
unsigned char*)source_base,
126 *destination = (
unsigned char*)destination_base;
128 DO_DEBUG(
opal_output( 0,
"_copy_content_same_ddt( %p, %d, dst %p, src %p )\n",
129 (
void*)datatype, count, destination_base, source_base ); );
131 iov_len_local = count * datatype->
size;
136 if( datatype->
flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
137 OPAL_PTRDIFF_TYPE extent = (datatype->
ub - datatype->
lb);
141 destination += datatype->
true_lb;
143 if( (OPAL_PTRDIFF_TYPE)datatype->
size == extent ) {
144 size_t total_length = iov_len_local;
145 size_t memop_chunk = opal_datatype_memop_block_size;
146 while( total_length > 0 ) {
147 if( memop_chunk > total_length ) memop_chunk = total_length;
148 OPAL_DATATYPE_SAFEGUARD_POINTER( destination, memop_chunk,
149 (
unsigned char*)destination_base, datatype, count );
150 OPAL_DATATYPE_SAFEGUARD_POINTER( source, memop_chunk,
151 (
unsigned char*)source_base, datatype, count );
152 DO_DEBUG(
opal_output( 0,
"copy c1. %s( %p, %p, %lu ) => space %lu\n",
153 STRINGIFY(MEM_OP_NAME), destination, source, (
unsigned long)memop_chunk, (
unsigned long)total_length ); );
154 MEM_OP( destination, source, memop_chunk );
155 destination += memop_chunk;
156 source += memop_chunk;
157 total_length -= memop_chunk;
161 for( pos_desc = 0; (int32_t)pos_desc < count; pos_desc++ ) {
162 OPAL_DATATYPE_SAFEGUARD_POINTER( destination, datatype->
size,
163 (
unsigned char*)destination_base, datatype, count );
164 OPAL_DATATYPE_SAFEGUARD_POINTER( source, datatype->
size,
165 (
unsigned char*)source_base, datatype, count );
166 DO_DEBUG(
opal_output( 0,
"copy c2. %s( %p, %p, %lu ) => space %lu\n",
167 STRINGIFY(MEM_OP_NAME), destination, source, (
unsigned long)datatype->
size,
168 (
unsigned long)(iov_len_local - (pos_desc * datatype->
size)) ); );
169 MEM_OP( destination, source, datatype->
size );
170 destination += extent;
177 pStack->
count = count;
183 if( datatype->
opt_desc.desc != NULL ) {
184 description = datatype->
opt_desc.desc;
186 description = datatype->
desc.desc;
189 if( description[0].elem.common.type == OPAL_DATATYPE_LOOP )
190 count_desc = description[0].loop.
loops;
192 count_desc = description[0].elem.
count;
193 pElem = &(description[pos_desc]);
196 while( OPAL_LIKELY(pElem->elem.
common.
flags & OPAL_DATATYPE_FLAG_DATA) ) {
198 _predefined_data( pElem, datatype, (
unsigned char*)source_base, count, count_desc,
199 source, destination, &iov_len_local );
201 UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
203 if( OPAL_DATATYPE_END_LOOP == pElem->elem.
common.
type ) {
204 DO_DEBUG(
opal_output( 0,
"copy end_loop count %d stack_pos %d pos_desc %d disp %ld space %lu\n",
205 (
int)pStack->
count, stack_pos, pos_desc, (
long)pStack->
disp, (
unsigned long)iov_len_local ); );
206 if( --(pStack->
count) == 0 ) {
207 if( stack_pos == 0 ) {
208 assert( iov_len_local == 0 );
215 pos_desc = pStack->
index + 1;
216 if( pStack->
index == -1 ) {
217 pStack->
disp += (datatype->
ub - datatype->
lb);
219 assert( OPAL_DATATYPE_LOOP == description[pStack->
index].loop.
common.
type );
223 source = (
unsigned char*)source_base + pStack->
disp;
224 destination = (
unsigned char*)destination_base + pStack->
disp;
225 UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
226 DO_DEBUG(
opal_output( 0,
"copy new_loop count %d stack_pos %d pos_desc %d disp %ld space %lu\n",
227 (
int)pStack->
count, stack_pos, pos_desc, (
long)pStack->
disp, (
unsigned long)iov_len_local ); );
229 if( OPAL_DATATYPE_LOOP == pElem->elem.
common.
type ) {
230 OPAL_PTRDIFF_TYPE local_disp = (OPAL_PTRDIFF_TYPE)source;
231 if( pElem->loop.
common.
flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
232 _contiguous_loop( pElem, datatype, (
unsigned char*)source_base, count, count_desc,
233 source, destination, &iov_len_local );
234 pos_desc += pElem->loop.
items + 1;
235 goto update_loop_description;
237 local_disp = (OPAL_PTRDIFF_TYPE)source - local_disp;
238 PUSH_STACK( pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
239 pStack->
disp + local_disp);
241 update_loop_description:
242 source = (
unsigned char*)source_base + pStack->
disp;
243 destination = (
unsigned char*)destination_base + pStack->
disp;
244 UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
245 DDT_DUMP_STACK( pStack, stack_pos, pElem,
"advance loop" );
uint16_t flags
the flags
Definition: opal_datatype.h:105
OPAL_PTRDIFF_TYPE extent
extent of each block (in bytes)
Definition: opal_datatype_internal.h:161
Definition: opal_datatype_internal.h:175
OPAL_PTRDIFF_TYPE first_elem_disp
the displacement of the first block in the loop
Definition: opal_datatype_internal.h:180
uint32_t loops
number of elements
Definition: opal_datatype_internal.h:168
Definition: opal_datatype_internal.h:184
Definition: opal_datatype_internal.h:166
size_t count
number of times we still have to do it
Definition: opal_convertor.h:80
dt_type_desc_t desc
the data description
Definition: opal_datatype.h:121
OPAL_PTRDIFF_TYPE lb
lower bound in memory
Definition: opal_datatype.h:112
OPAL_PTRDIFF_TYPE ub
upper bound in memory
Definition: opal_datatype.h:113
uint32_t items
number of items in the loop
Definition: opal_datatype_internal.h:169
ddt_elem_id_description common
basic data description and flags
Definition: opal_datatype_internal.h:158
uint16_t type
the basic data type id
Definition: opal_datatype_internal.h:150
uint32_t count
number of blocks
Definition: opal_datatype_internal.h:159
OPAL_DECLSPEC void opal_output(int output_id, const char *format,...) __opal_attribute_format__(__printf__
Main function to send output to a stream.
Definition: opal_convertor.h:77
size_t size
real size of the data in the loop
Definition: opal_datatype_internal.h:179
size_t size
total size in bytes of the memory used by the data if the data is put on a contiguous buffer ...
Definition: opal_datatype.h:108
OPAL_PTRDIFF_TYPE extent
extent of the whole loop
Definition: opal_datatype_internal.h:171
Definition: opal_datatype.h:103
ddt_elem_id_description common
basic data description and flags
Definition: opal_datatype_internal.h:167
uint16_t flags
flags for the record
Definition: opal_datatype_internal.h:149
uint32_t btypes[OPAL_DATATYPE_MAX_SUPPORTED]
basic elements count used to compute the size of the datatype for remote nodes.
Definition: opal_datatype.h:125
dt_type_desc_t opt_desc
short description of the data used when conversion is useless or in the send case (without conversion...
Definition: opal_datatype.h:122
Definition: opal_datatype_internal.h:157
OPAL_PTRDIFF_TYPE true_lb
the true lb of the data without user defined lb and ub
Definition: opal_datatype.h:110
int32_t index
index in the element description
Definition: opal_convertor.h:78
OPAL_PTRDIFF_TYPE disp
actual displacement depending on the count field
Definition: opal_convertor.h:81
OPAL_PTRDIFF_TYPE disp
displacement of the first block
Definition: opal_datatype_internal.h:162