OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

Data Structures | Macros | Typedefs | Functions | Variables
btl_gm.h File Reference
`#include "ompi_config.h"`
`#include <sys/types.h>`
`#include <string.h>`
`#include <gm.h>`
`#include "opal/mca/event/event.h"`
`#include "ompi/class/ompi_free_list.h"`
`#include "ompi/mca/btl/btl.h"`
`#include "ompi/mca/btl/base/base.h"`
`#include "ompi/mca/mpool/mpool.h"`
`#include "btl_gm_endpoint.h"`

Go to the source code of this file.

## Data Structures
struct | mca_btl_gm_component_t
| Myrinet (GM) BTL component. More...
struct | mca_btl_gm_module_t
| BTL Module Interface. More...

## Macros
#define | GM_BUFFER_SIZE 7
#define | GM_BUFFER_LENGTH gm_max_length_for_size(GM_BUFFER_SIZE)
#define | MCA_BTL_GM_ACQUIRE_TOKEN_NL(btl, frag)
| Acquire a send token - queue the fragment if none available. More...
#define | MCA_BTL_GM_ACQUIRE_TOKEN(btl, frag)
#define | MCA_BTL_GM_RETURN_TOKEN(btl)
| Return send token and dequeue and pending fragments mca_btl_gm_component.gm_lock is already held. More...

## Typedefs
typedef struct mca_btl_gm_component_t | mca_btl_gm_component_t
typedef struct mca_btl_gm_module_t | mca_btl_gm_module_t

## Functions
int | mca_btl_gm_component_open (void)
| Register GM component parameters with the MCA framework.
int | mca_btl_gm_component_close (void)
| Any final cleanup before being unloaded.
mca_btl_base_module_t ** | mca_btl_gm_component_init (int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
| GM component initialization. More...
int | mca_btl_gm_component_progress (void)
| GM component progress.
int | mca_btl_gm_finalize (struct mca_btl_base_module_t *btl)
| Cleanup any resources held by the BTL. More...
int | mca_btl_gm_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
| PML->BTL notification of change in the process list. More...
int | mca_btl_gm_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
| PML->BTL notification of change in the process list. More...
int | mca_btl_gm_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
| Initiate an asynchronous send. More...
int | mca_btl_gm_put (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
| Initiate an asynchronous put. More...
int | mca_btl_gm_get (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
| Initiate an asynchronous get. More...
int | mca_btl_gm_register_error_cb (struct mca_btl_base_module_t *btl, mca_btl_base_module_error_cb_fn_t cbfunc)
| Register a callback function that is called on error. More...
mca_btl_base_descriptor_t * | mca_btl_gm_alloc (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
| Allocate a descriptor with a segment of the requested size. More...
int | mca_btl_gm_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
| Return a segment allocated by this BTL. More...
mca_btl_base_descriptor_t * | mca_btl_gm_prepare_src (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
| Prepare a descriptor for send/rdma using the supplied convertor. More...
mca_btl_base_descriptor_t * | mca_btl_gm_prepare_dst (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
| Prepare a descriptor for send/rdma using the supplied convertor. More...
int | mca_btl_gm_ft_event (int state)
| Fault Tolerance Event Notification Function. More...

## Variables
OMPI_MODULE_DECLSPEC mca_btl_gm_component_t | mca_btl_gm_component
mca_btl_gm_module_t | mca_btl_gm_module

## Macro Definition Documentation

#define MCA_BTL_GM_ACQUIRE_TOKEN | ( |  | btl,
|  |  | frag
| ) |  |

Value:
do { \
/* queue the descriptor if there are no send tokens */ \
if(OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, -1) < 0) { \
opal_list_append(&gm_btl->gm_pending, (opal_list_item_t*)frag); \
OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, 1); \
OPAL_THREAD_UNLOCK(&mca_btl_gm_component.gm_lock); \
return OMPI_SUCCESS; \
} \
} while (0) \
OPAL_THREAD_ADD32
#define OPAL_THREAD_ADD32(x, y)
Use an atomic operation for increment/decrement if opal_using_threads() indicates that threads are in...
Definition: mutex.h:367
mca_btl_gm_component_t::gm_lock
opal_mutex_t gm_lock
lock for accessing module state
Definition: btl_gm.h:67
opal_list_item_t
Definition: opal_list.h:98
OPAL_THREAD_UNLOCK
#define OPAL_THREAD_UNLOCK(mutex)
Unlock a mutex if opal_using_threads() says that multiple threads may be active in the process...
Definition: mutex.h:309
opal_list_append
#define opal_list_append(l, i)
Append an item to the end of the list.
Definition: opal_list.h:410

#define MCA_BTL_GM_ACQUIRE_TOKEN_NL | ( |  | btl,
|  |  | frag
| ) |  |

Value:
do { \
/* queue the descriptor if there are no send tokens */ \
if(OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, -1) < 0) { \
opal_list_append(&gm_btl->gm_pending, (opal_list_item_t*)frag); \
OPAL_THREAD_ADD32(&gm_btl->gm_num_send_tokens, 1); \
return OMPI_SUCCESS; \
} \
} while (0) \
OPAL_THREAD_ADD32
#define OPAL_THREAD_ADD32(x, y)
Use an atomic operation for increment/decrement if opal_using_threads() indicates that threads are in...
Definition: mutex.h:367
opal_list_item_t
Definition: opal_list.h:98
opal_list_append
#define opal_list_append(l, i)
Append an item to the end of the list.
Definition: opal_list.h:410

Acquire a send token - queue the fragment if none available.

#define MCA_BTL_GM_RETURN_TOKEN | ( |  | btl | ) |

Value:
do { \
OPAL_THREAD_ADD32( &btl->gm_num_send_tokens, 1 ); \
if(opal_list_get_size(&btl->gm_pending)) { \
mca_btl_gm_frag_t* frag; \
frag = (mca_btl_gm_frag_t*)opal_list_remove_first(&btl->gm_pending); \
if(NULL != frag) { \
switch(frag->type) { \
case MCA_BTL_GM_SEND: \
case MCA_BTL_GM_EAGER: \
mca_btl_gm_send_nl(&btl->super, frag->endpoint, &frag->base, frag->hdr->tag); \
break; \
case MCA_BTL_GM_PUT: \
mca_btl_gm_put_nl(&btl->super, frag->endpoint, &frag->base); \
break; \
case MCA_BTL_GM_GET: \
mca_btl_gm_get_nl(&btl->super, frag->endpoint, &frag->base); \
break; \
} \
} \
} \
} while (0)
OPAL_THREAD_ADD32
#define OPAL_THREAD_ADD32(x, y)
Use an atomic operation for increment/decrement if opal_using_threads() indicates that threads are in...
Definition: mutex.h:367
opal_list_get_size
static size_t opal_list_get_size(opal_list_t *list)
Return the number of items in a list.
Definition: opal_list.h:299
mca_btl_gm_frag_t
GM send fragment derived type.
Definition: btl_gm_frag.h:42
opal_list_remove_first
static opal_list_item_t * opal_list_remove_first(opal_list_t *list)
Remove the first item from the list and return it.
Definition: opal_list.h:522

Return send token and dequeue and pending fragments mca_btl_gm_component.gm_lock is already held.

## Function Documentation

int mca_btl_gm_add_procs | ( | struct mca_btl_base_module_t * | btl,
|  | size_t | nprocs,
|  | struct ompi_proc_t ** | procs,
|  | struct mca_btl_base_endpoint_t ** | peers,
|  | opal_bitmap_t * | reachable
| ) |  |

PML->BTL notification of change in the process list.

Parameters

  btl | (IN)
  nprocs | (IN) Number of processes
  procs | (IN) Set of processes
  peers | (OUT) Set of (optional) peer addressing info.
  peers | (IN/OUT) Set of processes that are reachable via this BTL.

Returns
  OMPI_SUCCESS or error status on failure.

References mca_btl_base_endpoint_t::endpoint_btl, OBJ_RELEASE, ompi_proc_local(), opal_bitmap_set_bit(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, and mca_btl_gm_proc_t::proc_lock.

mca_btl_base_descriptor_t* mca_btl_gm_alloc | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | uint8_t | order,
|  | size_t | size,
|  | uint32_t | flags
| ) |  |

Allocate a descriptor with a segment of the requested size.

Note that the BTL layer may choose to return a smaller size if it cannot support the request.

Parameters

  btl | (IN) BTL module
  size | (IN) Request segment size.

Allocate a descriptor with a segment of the requested size.

Parameters

  btl | (IN) BTL module
  size | (IN) Request segment size.

References mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_base_descriptor_t::order, and mca_btl_base_segment_t::seg_len.

mca_btl_base_module_t** mca_btl_gm_component_init | ( | int * | num_btl_modules,
|  | bool | allow_multi_user_threads,
|  | bool | have_hidden_threads
| ) |  |

GM component initialization.

Parameters

  num_btl_modules | (OUT) Number of BTLs returned in BTL array.
  allow_multi_user_threads | (OUT) Flag indicating wether BTL supports user threads (TRUE)
  have_hidden_threads | (OUT) Flag indicating wether BTL uses threads (TRUE)

References mca_btl_gm_component_t::gm_btls, mca_btl_gm_component_t::gm_lock, mca_btl_gm_component_t::gm_max_btls, mca_btl_gm_component_t::gm_num_btls, ompi_mpi_thread_multiple, opal_output(), OPAL_THREAD_LOCK, and OPAL_THREAD_UNLOCK.

int mca_btl_gm_del_procs | ( | struct mca_btl_base_module_t * | btl,
|  | size_t | nprocs,
|  | struct ompi_proc_t ** | procs,
|  | struct mca_btl_base_endpoint_t ** | peers
| ) |  |

PML->BTL notification of change in the process list.

Parameters

  btl | (IN) BTL instance
  nproc | (IN) Number of processes.
  procs | (IN) Set of processes.
  peers | (IN) Set of peer data structures.

Returns
  Status indicating if cleanup was successful

int mca_btl_gm_finalize | ( | struct mca_btl_base_module_t * | btl | ) |

Cleanup any resources held by the BTL.

Parameters

  btl | BTL instance.

Returns
  OMPI_SUCCESS or error status on failure.

References mca_btl_gm_component_t::gm_lock, OBJ_DESTRUCT, OPAL_THREAD_LOCK, and OPAL_THREAD_UNLOCK.

int mca_btl_gm_free | ( | struct mca_btl_base_module_t * | btl,
|  | mca_btl_base_descriptor_t * | des
| ) |  |

Return a segment allocated by this BTL.

Parameters

  btl | (IN) BTL module
  descriptor | (IN) Allocated descriptor.

Return a segment allocated by this BTL.

References mca_btl_base_module_t::btl_mpool, and mca_mpool_base_module_t::mpool_deregister.

int mca_btl_gm_ft_event | ( | int | state | ) |

Fault Tolerance Event Notification Function.

Parameters

  state | Checkpoint Stae

Returns
  OMPI_SUCCESS or failure status

int mca_btl_gm_get | ( | mca_btl_base_module_t * | btl,
|  | mca_btl_base_endpoint_t * | endpoint,
|  | mca_btl_base_descriptor_t * | des
| ) |  |

Initiate an asynchronous get.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transferred

References mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_src, mca_btl_gm_component_t::gm_lock, OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

mca_btl_base_descriptor_t* mca_btl_gm_prepare_dst | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | struct mca_mpool_base_registration_t * | registration,
|  | struct opal_convertor_t * | convertor,
|  | uint8_t | order,
|  | size_t | reserve,
|  | size_t * | size,
|  | uint32_t | flags
| ) |  |

Prepare a descriptor for send/rdma using the supplied convertor.

If the convertor references data that is contigous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL peer addressing
  convertor | (IN) Data type convertor
  reserve | (IN) Additional bytes requested by upper layer to precede user data
  size | (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)

References mca_btl_base_module_t::btl_mpool, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_mpool_base_module_t::mpool_register, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

mca_btl_base_descriptor_t* mca_btl_gm_prepare_src | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | struct mca_mpool_base_registration_t * | registration,
|  | struct opal_convertor_t * | convertor,
|  | uint8_t | order,
|  | size_t | reserve,
|  | size_t * | size,
|  | uint32_t | flags
| ) |  |

Prepare a descriptor for send/rdma using the supplied convertor.

If the convertor references data that is contigous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL peer addressing
  convertor | (IN) Data type convertor
  reserve | (IN) Additional bytes requested by upper layer to precede user data
  size | (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)

Prepare a descriptor for send/rdma using the supplied convertor.

Parameters

  btl | (IN) BTL module
  peer | (IN) BTL peer addressing

References mca_btl_base_module_t::btl_max_send_size, mca_btl_base_module_t::btl_mpool, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_mpool_base_module_t::mpool_register, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

int mca_btl_gm_put | ( | mca_btl_base_module_t * | btl,
|  | mca_btl_base_endpoint_t * | endpoint,
|  | mca_btl_base_descriptor_t * | des
| ) |  |

Initiate an asynchronous put.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transferred

References mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_src, mca_btl_gm_component_t::gm_lock, OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

int mca_btl_gm_register_error_cb | ( | struct mca_btl_base_module_t * | btl,
|  | mca_btl_base_module_error_cb_fn_t | cbfunc
| ) |  |

Register a callback function that is called on error.

Parameters

  btl | (IN) BTL module

Returns
  Status indicating if registration was successful

int mca_btl_gm_send | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | struct mca_btl_base_descriptor_t * | des,
|  | mca_btl_base_tag_t | tag
| ) |  |

Initiate an asynchronous send.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transfered
  tag | (IN) The tag value used to notify the peer.

References mca_btl_base_descriptor_t::des_flags, mca_btl_gm_component_t::gm_lock, mca_btl_gm_module_t::gm_repost, opal_list_get_size(), opal_list_remove_first(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, and mca_btl_base_segment_t::seg_len.