OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

Data Structures | Macros | Typedefs | Functions | Variables
btl_udapl.h File Reference
`#include "ompi_config.h"`
`#include <sys/types.h>`
`#include <string.h>`
`#include <dat/udat.h>`
`#include "opal/class/opal_pointer_array.h"`
`#include "opal/mca/event/event.h"`
`#include "ompi/class/ompi_free_list.h"`
`#include "ompi/mca/btl/btl.h"`
`#include "ompi/mca/btl/base/base.h"`
`#include "ompi/mca/btl/base/btl_base_error.h"`
`#include "orte/util/show_help.h"`
`#include "ompi/mca/mpool/mpool.h"`
`#include "btl_udapl_endpoint.h"`

Go to the source code of this file.

## Data Structures
struct | mca_btl_udapl_component_t
| uDAPL BTL component. More...
struct | mca_btl_udapl_module_t
| BTL Module Interface. More...
struct | mca_btl_udapl_reg_t

## Macros
#define | VERBOSE_CRITICAL 0
| Report Out from uDAPL BTL. More...
#define | VERBOSE_SHOW_HELP 10
#define | VERBOSE_INFORM 20
#define | VERBOSE_DIAGNOSE 30
#define | VERBOSE_DEVELOPER 90
#define | BTL_UDAPL_VERBOSE_OUTPUT(verbose_level, args)
#define | BTL_UDAPL_VERBOSE_HELP(verbose_level, args)
#define | MCA_BTL_UDAPL_ERROR(ret, str)

## Typedefs
typedef struct mca_btl_udapl_component_t | mca_btl_udapl_component_t
typedef struct mca_btl_udapl_module_t | mca_btl_udapl_module_t
typedef struct mca_btl_udapl_reg_t | mca_btl_udapl_reg_t

## Functions
int | mca_btl_udapl_component_open (void)
| Register uDAPL component parameters with the MCA framework. More...
int | mca_btl_udapl_component_close (void)
| Any final cleanup before being unloaded.
mca_btl_base_module_t ** | mca_btl_udapl_component_init (int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
| uDAPL component initialization. More...
int | mca_btl_udapl_component_progress (void)
| uDAPL component progress.
int | mca_btl_udapl_init (DAT_NAME_PTR ia_name, struct mca_btl_udapl_module_t *btl)
| Initialize resources for a new BTL/uDAPL IA. More...
int | mca_btl_udapl_finalize (struct mca_btl_base_module_t *btl)
| Cleanup any resources held by the BTL. More...
int | mca_btl_udapl_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
| PML->BTL notification of change in the process list. More...
int | mca_btl_udapl_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
| PML->BTL notification of change in the process list. More...
int | mca_btl_udapl_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
| Initiate an asynchronous send. More...
int | mca_btl_udapl_put (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
| Initiate an asynchronous put. More...
int | mca_btl_udapl_get (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *decriptor)
| Initiate an asynchronous get. More...
mca_btl_base_descriptor_t * | mca_btl_udapl_alloc (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
| Allocate a descriptor with a segment of the requested size. More...
int | mca_btl_udapl_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
| Return a segment allocated by this BTL. More...
mca_btl_base_descriptor_t * | mca_btl_udapl_prepare_src (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
| Prepare a descriptor for send/rdma using the supplied convertor. More...
mca_btl_base_descriptor_t * | mca_btl_udapl_prepare_dst (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, struct mca_mpool_base_registration_t *, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
| Prepare a descriptor for send/rdma using the supplied convertor. More...
int | mca_btl_udapl_ft_event (int state)
| Fault Tolerance Event Notification Function. More...

## Variables
OMPI_MODULE_DECLSPEC mca_btl_udapl_component_t | mca_btl_udapl_component
mca_btl_udapl_module_t | mca_btl_udapl_module

## Macro Definition Documentation

#define BTL_UDAPL_VERBOSE_HELP | ( |  | verbose_level,
|  |  | args
| ) |  |

Value:
do { \
if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
orte_show_help args; \
} \
} while(0);
orte_show_help
ORTE_DECLSPEC int orte_show_help(const char *filename, const char *topic, bool want_error_header,...)
Show help.
Definition: show_help.c:636
mca_btl_udapl_component_t::udapl_verbosity
int32_t udapl_verbosity
report out level, see "Report Out from uDAPL BTL" below for details.
Definition: btl_udapl.h:55

#define BTL_UDAPL_VERBOSE_OUTPUT | ( |  | verbose_level,
|  |  | args
| ) |  |

Value:
do { \
if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
BTL_OUTPUT(args); \
} \
} while(0);
mca_btl_udapl_component_t::udapl_verbosity
int32_t udapl_verbosity
report out level, see "Report Out from uDAPL BTL" below for details.
Definition: btl_udapl.h:55

#define VERBOSE_CRITICAL 0

Report Out from uDAPL BTL.

- BTL_ERROR() : Use to report out errors from uDAPL BTL. These are critical errors which will most likely cause the program to fail so this message should always be reported to the user. Defined in btl/base/btl_base_error.h. Example: dat_strerror(rc, (const char**)&major, (const char**)&minor); BTL_ERROR(("ERROR: %s %s %s\n", "dat_cr_accept", major, minor));
- BTL_UDAPL_VERBOSE_OUTPUT() : Use to output different levels of verbosity to the user. See Note below. Example: BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_DIAGNOSE, ("WARNING: don't %s", "jump"));
- BTL_UDAPL_VERBOSE_HELP() : Use output information as defined in uDAPL BTL help file (help-mpi-btl-udapl.txt). See Note below. Example: BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP, ("help-mpi-btl-udapl.txt", "invalid num rdma segments", true, 22));

Note : - Verbose levels are defined below. These levels are controlled by the mca parameter "btl_udapl_verbose". The verbose level is set to 10 by default so that critical error and useful help information will appear. Which ever value this param is set to, those messages as well as any lower level verbose messages will be reported.

- Setting "btl_udapl_verbose" to "-1" will turn off all messages reported by the use of BTL_UDAPL_VERBOSE_*().
- These macros should not be used in a critical path as they are always included in the compiled code.
- These macros rely on the use of paranthesis around the "args" value.

Values used with BTL_UDAPL_VERBOSE_*():

- 0: critical user information; should always be reported; on by default
- 10: useful help messages that would be reported from "help-mpi-btl-udapl.txt"; accessed from BTL_UDAPL_VERBOSE_HELP(); on by default
- 20: general execution diagnostic information; may be useful to user or btl developer
- 30: basic debugging/diagnostic information
- 90: useful only to developers
- 100: other components do not appear to go beyond 100 for verbose levels so noting here as the max for future reference

Referenced by mca_btl_udapl_component_progress().

## Function Documentation

int mca_btl_udapl_add_procs | ( | struct mca_btl_base_module_t * | btl,
|  | size_t | nprocs,
|  | struct ompi_proc_t ** | procs,
|  | struct mca_btl_base_endpoint_t ** | peers,
|  | opal_bitmap_t * | reachable
| ) |  |

PML->BTL notification of change in the process list.

Parameters

  btl | (IN)
  nprocs | (IN) Number of processes
  procs | (IN) Set of processes
  peers | (OUT) Set of (optional) peer addressing info.
  peers | (IN/OUT) Set of processes that are reachable via this BTL.

Returns
  OMPI_SUCCESS or error status on failure.

References mca_btl_base_endpoint_t::endpoint_btl, OBJ_RELEASE, ompi_proc_local(), opal_bitmap_set_bit(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, and mca_btl_udapl_proc_t::proc_lock.

mca_btl_base_descriptor_t* mca_btl_udapl_alloc | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | uint8_t | order,
|  | size_t | size,
|  | uint32_t | flags
| ) |  |

Allocate a descriptor with a segment of the requested size.

Note that the BTL layer may choose to return a smaller size if it cannot support the request.

Parameters

  btl | (IN) BTL module
  size | (IN) Request segment size.

Allocate a descriptor with a segment of the requested size.

Parameters

  btl | (IN) BTL module
  size | (IN) Request segment size.

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

mca_btl_base_module_t** mca_btl_udapl_component_init | ( | int * | num_btl_modules,
|  | bool | allow_multi_user_threads,
|  | bool | have_hidden_threads
| ) |  |

uDAPL component initialization.

Parameters

  num_btl_modules | (OUT) Number of BTLs returned in BTL array.
  allow_multi_user_threads | (OUT) Flag indicating wether BTL supports user threads (TRUE)
  have_hidden_threads | (OUT) Flag indicating wether BTL uses threads (TRUE)

References mca_btl_udapl_init(), ompi_mpi_thread_multiple, opal_argv_copy(), opal_argv_split(), mca_btl_udapl_component_t::udapl_btls, mca_btl_udapl_component_t::udapl_max_btls, and mca_btl_udapl_component_t::udapl_num_btls.

int mca_btl_udapl_component_open | ( | void |  | ) |

Register uDAPL component parameters with the MCA framework.

Register uDAPL component parameters with the MCA framework.

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_module_t::btl_max_send_size, OBJ_CONSTRUCT, mca_btl_udapl_module_t::super, mca_btl_udapl_component_t::udapl_btls, mca_btl_udapl_component_t::udapl_lock, mca_btl_udapl_component_t::udapl_num_btls, and mca_btl_udapl_component_t::udapl_procs.

int mca_btl_udapl_del_procs | ( | struct mca_btl_base_module_t * | btl,
|  | size_t | nprocs,
|  | struct ompi_proc_t ** | procs,
|  | struct mca_btl_base_endpoint_t ** | peers
| ) |  |

PML->BTL notification of change in the process list.

Parameters

  btl | (IN) BTL instance
  nproc | (IN) Number of processes.
  procs | (IN) Set of processes.
  peers | (IN) Set of peer data structures.

Returns
  Status indicating if cleanup was successful

int mca_btl_udapl_finalize | ( | struct mca_btl_base_module_t * | btl | ) |

Cleanup any resources held by the BTL.

Parameters

  btl | BTL instance.

Returns
  OMPI_SUCCESS or error status on failure.

References mca_btl_base_module_t::btl_mpool, OBJ_DESTRUCT, opal_pointer_array_get_item(), and mca_btl_udapl_module_t::super.

int mca_btl_udapl_free | ( | struct mca_btl_base_module_t * | btl,
|  | mca_btl_base_descriptor_t * | des
| ) |  |

Return a segment allocated by this BTL.

Parameters

  btl | (IN) BTL module
  descriptor | (IN) Allocated descriptor.

Return a segment allocated by this BTL.

References mca_btl_base_module_t::btl_mpool, and mca_mpool_base_module_t::mpool_deregister.

Referenced by mca_btl_udapl_component_progress().

int mca_btl_udapl_ft_event | ( | int | state | ) |

Fault Tolerance Event Notification Function.

Parameters

  state | Checkpoint Stae

Returns
  OMPI_SUCCESS or failure status

int mca_btl_udapl_get | ( | mca_btl_base_module_t * | btl,
|  | mca_btl_base_endpoint_t * | endpoint,
|  | mca_btl_base_descriptor_t * | des
| ) |  |

Initiate an asynchronous get.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transferred

int mca_btl_udapl_init | ( | DAT_NAME_PTR | ia_name,
|  | mca_btl_udapl_module_t * | btl
| ) |  |

Initialize resources for a new BTL/uDAPL IA.

Parameters

  ia_name | Name of uDAPL interface adapter
  btl | BTL instance.

Returns
  OMPI_SUCCESS or error status on failure.

Initialize resources for a new BTL/uDAPL IA.

References mca_btl_base_module_t::btl_mpool, OBJ_CLASS, OBJ_CONSTRUCT, opal_pointer_array_init(), mca_btl_udapl_module_t::super, mca_btl_udapl_component_t::udapl_buffer_alignment, mca_btl_udapl_component_t::udapl_free_list_inc, mca_btl_udapl_component_t::udapl_free_list_max, mca_btl_udapl_component_t::udapl_free_list_num, mca_btl_udapl_component_t::udapl_max_eager_rdma_peers, and mca_btl_udapl_component_t::udapl_mpool_name.

Referenced by mca_btl_udapl_component_init().

mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | struct mca_mpool_base_registration_t * | registration,
|  | struct opal_convertor_t * | convertor,
|  | uint8_t | order,
|  | size_t | reserve,
|  | size_t * | size,
|  | uint32_t | flags
| ) |  |

Prepare a descriptor for send/rdma using the supplied convertor.

If the convertor references data that is contiguous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL peer addressing
  convertor | (IN) Data type convertor
  reserve | (IN) Additional bytes requested by upper layer to precede user data
  size | (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)

References mca_btl_base_module_t::btl_mpool, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_mpool_base_module_t::mpool_register, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_key, and mca_btl_base_segment_t::seg_len.

mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | struct mca_mpool_base_registration_t * | registration,
|  | struct opal_convertor_t * | convertor,
|  | uint8_t | order,
|  | size_t | reserve,
|  | size_t * | size,
|  | uint32_t | flags
| ) |  |

Prepare a descriptor for send/rdma using the supplied convertor.

If the convertor references data that is contigous, the descriptor may simply point to the user buffer. Otherwise, this routine is responsible for allocating buffer space and packing if required.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL peer addressing
  convertor | (IN) Data type convertor
  reserve | (IN) Additional bytes requested by upper layer to precede user data
  size | (IN/OUT) Number of bytes to prepare (IN), number of bytes actually prepared (OUT)

Prepare a descriptor for send/rdma using the supplied convertor.

Parameters

  btl | (IN) BTL module
  peer | (IN) BTL peer addressing

References mca_btl_base_module_t::btl_max_send_size, mca_btl_base_module_t::btl_mpool, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_mpool_base_module_t::mpool_register, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

int mca_btl_udapl_put | ( | mca_btl_base_module_t * | btl,
|  | mca_btl_base_endpoint_t * | endpoint,
|  | mca_btl_base_descriptor_t * | des
| ) |  |

Initiate an asynchronous put.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transferred

References mca_btl_base_descriptor_t::des_dst, mca_btl_base_endpoint_t::endpoint_lock, mca_btl_base_endpoint_t::endpoint_lwqe_tokens, mca_btl_base_endpoint_t::endpoint_max, mca_btl_base_endpoint_t::endpoint_max_frags, mca_btl_base_endpoint_t::endpoint_sr_tokens, opal_list_append, opal_progress(), OPAL_THREAD_ADD32, OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, mca_btl_base_segment_t::seg_addr, mca_btl_base_segment_t::seg_key, and mca_btl_base_segment_t::seg_len.

int mca_btl_udapl_send | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | struct mca_btl_base_descriptor_t * | des,
|  | mca_btl_base_tag_t | tag
| ) |  |

Initiate an asynchronous send.

Parameters

  btl | (IN) BTL module
  endpoint | (IN) BTL addressing information
  descriptor | (IN) Description of the data to be transferred
  tag | (IN) The tag value used to notify the peer.

References mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.