OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

Data Structures | Macros | Typedefs | Functions | Variables
btl_ofud.h File Reference
`#include "ompi_config.h"`
`#include <sys/types.h>`
`#include <infiniband/verbs.h>`
`#include "opal/class/opal_hash_table.h"`
`#include "ompi/class/ompi_free_list.h"`
`#include "ompi/mca/btl/btl.h"`
`#include "ompi/mca/btl/base/btl_base_error.h"`
`#include "ompi/mca/btl/base/base.h"`
`#include "ompi/mca/mpool/mpool.h"`
`#include "btl_ofud_endpoint.h"`

Go to the source code of this file.

## Data Structures
struct | mca_btl_ud_component_t
| UD Infiniband (IB) BTL component. More...
struct | mca_btl_ud_module_t
| UD/IB BTL Interface. More...

## Macros
#define | MCA_BTL_UD_NUM_QP 4
#define | MCA_BTL_UD_ENABLE_PROFILE 0
| Profiling variables.
#define | MCA_BTL_UD_START_TIME(var)
#define | MCA_BTL_UD_END_TIME(var)
#define | MCA_BTL_UD_SHOW_TIME(var)

## Typedefs
typedef struct mca_btl_ud_component_t | mca_btl_ud_component_t
typedef mca_btl_base_recv_reg_t | mca_btl_ud_recv_reg_t
typedef struct mca_btl_ud_module_t | mca_btl_ud_module_t

## Functions
mca_btl_base_module_t ** | mca_btl_ud_component_init (int *num_btl_modules, bool allow_multi_user_threads, bool have_hidden_threads)
| IB component initialization. More...
int | mca_btl_ud_component_progress (void)
| UD/IB component progress.
int | mca_btl_ud_finalize (struct mca_btl_base_module_t *btl)
| Cleanup any resources held by the BTL. More...
int | mca_btl_ud_add_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers, opal_bitmap_t *reachable)
| PML->BTL notification of change in the process list. More...
int | mca_btl_ud_del_procs (struct mca_btl_base_module_t *btl, size_t nprocs, struct ompi_proc_t **procs, struct mca_btl_base_endpoint_t **peers)
| PML->BTL notification of change in the process list. More...
int | mca_btl_ud_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *btl_peer, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag)
| PML->BTL Initiate a send of the specified size. More...
mca_btl_base_descriptor_t * | mca_btl_ud_alloc (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags)
| Allocate a descriptor. More...
int | mca_btl_ud_free (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *des)
| Return a segment allocated by this BTL. More...
mca_btl_base_descriptor_t * | mca_btl_ud_prepare_src (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *peer, mca_mpool_base_registration_t *registration, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags)
| Pack data and return a descriptor that can be used for send/put. More...
int | mca_btl_ud_module_init (mca_btl_ud_module_t *ud_btl)
int | mca_btl_ud_ft_event (int state)
| Fault Tolerance Event Notification Function. More...

## Variables
OMPI_MODULE_DECLSPEC mca_btl_ud_component_t | mca_btl_ofud_component
mca_btl_ud_module_t | mca_btl_ofud_module

## Function Documentation

int mca_btl_ud_add_procs | ( | struct mca_btl_base_module_t * | btl,
|  | size_t | nprocs,
|  | struct ompi_proc_t ** | procs,
|  | struct mca_btl_base_endpoint_t ** | peers,
|  | opal_bitmap_t * | reachable
| ) |  |

PML->BTL notification of change in the process list.

Parameters

  btl | (IN)
  nprocs | (IN) Number of processes
  procs | (IN) Set of processes
  peers | (OUT) Set of (optional) peer addressing info.
  peers | (IN/OUT) Set of processes that are reachable via this BTL.

Returns
  OMPI_SUCCESS or error status on failure.

References OBJ_CONSTRUCT, OBJ_RELEASE, opal_bitmap_set_bit(), opal_hash_table_init(), opal_hash_table_set_value_uint64(), OPAL_THREAD_LOCK, OPAL_THREAD_UNLOCK, mca_btl_ud_proc_t::proc_lock, mca_btl_base_endpoint_t::rem_addr, and mca_btl_base_endpoint_t::rmt_ah.

mca_btl_base_descriptor_t* mca_btl_ud_alloc | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | uint8_t | order,
|  | size_t | size,
|  | uint32_t | flags
| ) |  |

Allocate a descriptor.

Parameters

  btl | (IN) BTL module
  size | (IN) Requested descriptor size.

Allocate a descriptor.

Parameters

  btl | (IN) BTL module
  size | (IN) Request segment size.

When allocating a segment we pull a pre-alllocated segment from one of two free lists, an eager list and a max list

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::order, and mca_btl_base_segment_t::seg_len.

mca_btl_base_module_t** mca_btl_ud_component_init | ( | int * | num_btl_modules,
|  | bool | allow_multi_user_threads,
|  | bool | have_hidden_threads
| ) |  |

IB component initialization.

Parameters

  num_btl_modules | (OUT) Number of BTLs returned in BTL array.
  allow_multi_user_threads | (OUT) Flag indicating wether BTL supports user threads (TRUE)
  have_hidden_threads | (OUT) Flag indicating whether BTL uses threads (TRUE)

(1) read interface list from verbs and compare against component parameters then create a BTL instance for selected interfaces (2) publish BTL addressing info

We must loop through all the hca id's, get their handles and for each hca we query the number of ports on the hca and set up a distinct btl module for each hca port

References mca_btl_ud_module_t::addr, mca_btl_ud_component_t::max_btls, mca_base_param_find(), mca_base_param_lookup_string(), mca_btl_ud_component_t::num_btls, OBJ_CONSTRUCT, OBJ_DESTRUCT, ompi_mpi_thread_multiple, opal_argv_copy(), opal_argv_free(), opal_argv_split(), opal_list_append, opal_list_remove_first(), ORTE_PROC_MY_NAME, orte_show_help(), and mca_btl_ud_component_t::ud_btls.

int mca_btl_ud_del_procs | ( | struct mca_btl_base_module_t * | btl,
|  | size_t | nprocs,
|  | struct ompi_proc_t ** | procs,
|  | struct mca_btl_base_endpoint_t ** | peers
| ) |  |

PML->BTL notification of change in the process list.

Parameters

  btl | (IN) BTL instance
  nproc | (IN) Number of processes.
  procs | (IN) Set of processes.
  peers | (IN) Set of peer data structures.

Returns
  Status indicating if cleanup was successful

References OBJ_RELEASE, opal_hash_table_remove_value_uint64(), and mca_btl_base_endpoint_t::rem_addr.

int mca_btl_ud_finalize | ( | struct mca_btl_base_module_t * | btl | ) |

Cleanup any resources held by the BTL.

Parameters

  btl | BTL instance.

Returns
  OMPI_SUCCESS or error status on failure.

References mca_btl_base_module_t::btl_mpool, mca_btl_ud_module_t::ib_qp, OBJ_DESTRUCT, mca_btl_ud_module_t::pending_frags, mca_btl_ud_module_t::recv_frags, mca_btl_ud_module_t::send_frags, mca_btl_ud_module_t::ud_lock, and mca_btl_ud_module_t::user_frags.

int mca_btl_ud_free | ( | struct mca_btl_base_module_t * | btl,
|  | mca_btl_base_descriptor_t * | des
| ) |  |

Return a segment allocated by this BTL.

Parameters

  btl | (IN) BTL module
  descriptor | (IN) Allocated descriptor.

Return a segment allocated by this BTL.

Return the segment to the appropriate preallocated segment list

References mca_btl_base_module_t::btl_mpool, and mca_mpool_base_module_t::mpool_deregister.

Referenced by mca_btl_ud_component_progress().

int mca_btl_ud_ft_event | ( | int | state | ) |

Fault Tolerance Event Notification Function.

Parameters

  state | Checkpoint State

Returns
  OMPI_SUCCESS or failure status

mca_btl_base_descriptor_t* mca_btl_ud_prepare_src | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | endpoint,
|  | mca_mpool_base_registration_t * | registration,
|  | struct opal_convertor_t * | convertor,
|  | uint8_t | order,
|  | size_t | reserve,
|  | size_t * | size,
|  | uint32_t | flags
| ) |  |

Pack data and return a descriptor that can be used for send/put.

Parameters

  btl | (IN) BTL module
  peer | (IN) BTL peer addressing

Pack data and return a descriptor that can be used for send/put.

Parameters

  btl | (IN) BTL module
  peer | (IN) BTL peer addressing

prepare source's behavior depends on the following: Has a valid memory registration been passed to prepare_src? if so we attempt to use the pre-registred user-buffer, if the memory registration is to small (only a portion of the user buffer) then we must reregister the user buffer Has the user requested the memory to be left pinned? if so we insert the memory registration into a memory tree for later lookup, we may also remove a previous registration if a MRU (most recently used) list of registions is full, this prevents resources from being exhausted. Is the requested size larger than the btl's max send size? if so and we aren't asked to leave the registration pinned then we register the memory if the user's buffer is contiguous. Otherwise we choose from two free lists of pre-registered memory in which to pack the data into.

References mca_btl_base_module_t::btl_eager_limit, mca_btl_base_module_t::btl_max_send_size, mca_btl_base_module_t::btl_mpool, mca_btl_base_descriptor_t::des_dst, mca_btl_base_descriptor_t::des_dst_cnt, mca_btl_base_descriptor_t::des_flags, mca_btl_base_descriptor_t::des_src, mca_btl_base_descriptor_t::des_src_cnt, mca_mpool_base_module_t::mpool_register, mca_btl_base_descriptor_t::order, mca_btl_base_segment_t::seg_addr, and mca_btl_base_segment_t::seg_len.

int mca_btl_ud_send | ( | struct mca_btl_base_module_t * | btl,
|  | struct mca_btl_base_endpoint_t * | btl_peer,
|  | struct mca_btl_base_descriptor_t * | descriptor,
|  | mca_btl_base_tag_t | tag
| ) |  |

PML->BTL Initiate a send of the specified size.

Parameters

  btl | (IN) BTL instance
  btl_base_peer | (IN) BTL peer addressing
  send_request | (IN/OUT) Send request (allocated by PML via mca_btl_base_request_alloc_fn_t)
  size | (IN) Number of bytes PML is requesting BTL to deliver
  flags | (IN) Flags that should be passed to the peer via the message header.
  request | (OUT) OMPI_SUCCESS if the BTL was able to queue one or more fragments