\hypertarget{pml__bfo__failover_8c}{\section{ompi/mca/pml/bfo/pml\+\_\+bfo\+\_\+failover.c File Reference}
\label{pml__bfo__failover_8c}\index{ompi/mca/pml/bfo/pml\+\_\+bfo\+\_\+failover.\+c@{ompi/mca/pml/bfo/pml\+\_\+bfo\+\_\+failover.\+c}}
}


Functions that implement failover capabilities.  


{\ttfamily \#include \char`\"{}ompi\+\_\+config.\+h\char`\"{}}\\*
{\ttfamily \#include $<$stdlib.\+h$>$}\\*
{\ttfamily \#include $<$string.\+h$>$}\\*
{\ttfamily \#include \char`\"{}opal/class/opal\+\_\+bitmap.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/pml/pml.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/pml/base/base.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/btl/btl.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/btl/base/base.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+component.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+comm.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+hdr.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+recvfrag.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+sendreq.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+recvreq.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+rdmafrag.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}pml\+\_\+bfo\+\_\+failover.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/bml/base/base.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}orte/mca/errmgr/errmgr.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}orte/mca/grpcomm/grpcomm.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}orte/util/show\+\_\+help.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}orte/mca/notifier/notifier.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/runtime/ompi\+\_\+cr.\+h\char`\"{}}\\*
\subsection*{Functions}
\begin{DoxyCompactItemize}
\item 
static void \hyperlink{pml__bfo__failover_8c_a37a6e43b438b451f6a236db20898f620}{mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, \hyperlink{structmca__bml__base__endpoint__t}{mca\+\_\+bml\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$ep)
\begin{DoxyCompactList}\small\item\em This function is called since when we are mapping out a B\+M\+L. \end{DoxyCompactList}\item 
bool \hyperlink{pml__bfo__failover_8c_a61619e0f5a4b44e12fd69427e3059e73}{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+msg} (\hyperlink{structmca__pml__bfo__comm__proc__t}{mca\+\_\+pml\+\_\+bfo\+\_\+comm\+\_\+proc\+\_\+t} $\ast$proc, \hyperlink{structmca__pml__bfo__match__hdr__t}{mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t} $\ast$hdr)
\begin{DoxyCompactList}\small\item\em When running with failover enabled, check the P\+M\+L sequence numbers to see if we have received a duplicate message. \end{DoxyCompactList}\item 
bool \hyperlink{pml__bfo__failover_8c_af9cda77a04361a27ce190da460c60ef1}{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+fin} (\hyperlink{unionmca__pml__bfo__hdr__t}{mca\+\_\+pml\+\_\+bfo\+\_\+hdr\+\_\+t} $\ast$hdr, \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$rdma, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl)
\begin{DoxyCompactList}\small\item\em This function checks to see if we have received a duplicate F\+I\+N message. \end{DoxyCompactList}\item 
\hypertarget{pml__bfo__failover_8c_a18398ad70cc534b84115962c152ef386}{void \hyperlink{pml__bfo__failover_8c_a18398ad70cc534b84115962c152ef386}{mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+fin} (struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des)}\label{pml__bfo__failover_8c_a18398ad70cc534b84115962c152ef386}

\begin{DoxyCompactList}\small\item\em Repost a F\+I\+N message if we get an error on the completion event. \end{DoxyCompactList}\item 
\hyperlink{structmca__pml__bfo__recv__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$ \hyperlink{pml__bfo__failover_8c_a5a8c8e42899718ae7cbe164f7900a97a}{mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request} (\hyperlink{structmca__pml__bfo__match__hdr__t}{mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t} $\ast$hdr)
\begin{DoxyCompactList}\small\item\em This function is called when a R\+N\+D\+V or R\+G\+E\+T is received with the F\+L\+A\+G\+S\+\_\+\+R\+E\+S\+T\+A\+R\+T flag set. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_ad27dc653cc5d4a5be33c422a7e0b4992}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag, \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, void $\ast$cbdata)
\begin{DoxyCompactList}\small\item\em Callback for when a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message is received. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_a1fde523b6826518993369460d6c3a6b9}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag, \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, void $\ast$cbdata)
\begin{DoxyCompactList}\small\item\em Callback for when a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K message is received. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_a2f7e207b3bd8fbacf2b32448e43a7b19}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag, \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, void $\ast$cbdata)
\begin{DoxyCompactList}\small\item\em Callback for when a R\+E\+C\+V\+E\+R\+R\+N\+O\+T\+I\+F\+Y message is received. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_ad9b6b294d13f87e143e194ecc65b7f72}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnack} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag, \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, void $\ast$cbdata)
\begin{DoxyCompactList}\small\item\em Callback for when a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+A\+C\+K message is received. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_a36efc5fb4674dc619cd24813bdcebbe8}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify} (\hyperlink{structmca__pml__bfo__send__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$sendreq, bool repost, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag, int status, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl)
\begin{DoxyCompactList}\small\item\em This function gets called when failover is enabled and an error occurs during the rendezvous protocol. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_aec55643d673280be8f685cf83565adb2}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart} (\hyperlink{structmca__pml__bfo__send__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$sendreq, bool repost, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag)
\begin{DoxyCompactList}\small\item\em This function restarts a R\+N\+D\+V send request. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_aecf95c1315b590837bfd6ad15950051c}{mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+match\+\_\+fragment} (struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des)
\begin{DoxyCompactList}\small\item\em This function will repost a match fragment. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_ac260ba077b2f5daa53ac0ec79cda856b}{mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, struct \hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$ep, struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, int status)
\begin{DoxyCompactList}\small\item\em Completion callback for rndvrestartnotify completion event. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_a2303d9a603d0042a5055d91d2c7b888a}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify} (\hyperlink{structmca__pml__bfo__recv__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$recvreq, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag, int status)
\begin{DoxyCompactList}\small\item\em This function is called when an error is detected on a completion event on the receiving side. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_aee082d267b4d638b8859da7d2e355241}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack} (\hyperlink{structmca__pml__bfo__recv__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$recvreq, mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t tag, int status, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl)
\begin{DoxyCompactList}\small\item\em This function is called when it may be time to send a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K message back to the sending side. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_a18aa312796513ad6d6ae0ca8bfdb34f5}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack} (\hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$olddes, \hyperlink{structompi__proc__t}{ompi\+\_\+proc\+\_\+t} $\ast$ompi\+\_\+proc, bool repost)
\begin{DoxyCompactList}\small\item\em Called after the receipt of a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message to a request that no longer matches. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_ab22055c7e6dac0939ff8bddb5613f728}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset} (\hyperlink{structmca__pml__bfo__recv__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$match)
\begin{DoxyCompactList}\small\item\em Reset all the receive request fields to match what a request looks like when it is first started. \end{DoxyCompactList}\item 
\hypertarget{pml__bfo__failover_8c_a5125897eaac1177cc8eafe7170cf5ba1}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+restart\+\_\+completion} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, struct \hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$ep, struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, int status)}\label{pml__bfo__failover_8c_a5125897eaac1177cc8eafe7170cf5ba1}

\item 
\hypertarget{pml__bfo__failover_8c_a6a6bf9d5e1e816c4ec4e4cda623c2e29}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+map\+\_\+out\+\_\+btl} (struct \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, \hyperlink{structompi__proc__t}{ompi\+\_\+proc\+\_\+t} $\ast$errproc, char $\ast$btlname)}\label{pml__bfo__failover_8c_a6a6bf9d5e1e816c4ec4e4cda623c2e29}

\item 
\hypertarget{pml__bfo__failover_8c_ac0d83cb2facb14eae36ff3a790d29abc}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+failover\+\_\+error\+\_\+handler} (struct \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, int32\+\_\+t flags, \hyperlink{structompi__proc__t}{ompi\+\_\+proc\+\_\+t} $\ast$errproc, char $\ast$btlname)}\label{pml__bfo__failover_8c_ac0d83cb2facb14eae36ff3a790d29abc}

\item 
void \hyperlink{pml__bfo__failover_8c_ae313feda12bd9b44d1bb9299b74a7ed1}{mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, int status)
\begin{DoxyCompactList}\small\item\em Call each time we get a completion event on A\+C\+K or P\+U\+T message. \end{DoxyCompactList}\item 
\hypertarget{pml__bfo__failover_8c_a0c8b1a804b64637e38da353c32222b2f}{int \hyperlink{pml__bfo__failover_8c_a0c8b1a804b64637e38da353c32222b2f}{mca\+\_\+pml\+\_\+bfo\+\_\+register\+\_\+callbacks} (void)}\label{pml__bfo__failover_8c_a0c8b1a804b64637e38da353c32222b2f}

\begin{DoxyCompactList}\small\item\em Register four functions to handle extra P\+M\+L message types that are utilized when a failover occurs. \end{DoxyCompactList}\item 
\hypertarget{pml__bfo__failover_8c_aa0500831e4b18eb571bfa11af5b7e859}{void \hyperlink{pml__bfo__failover_8c_aa0500831e4b18eb571bfa11af5b7e859}{mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+rndv\+\_\+fields} (\hyperlink{unionmca__pml__bfo__hdr__t}{mca\+\_\+pml\+\_\+bfo\+\_\+hdr\+\_\+t} $\ast$hdr, \hyperlink{structmca__pml__bfo__send__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$sendreq, char $\ast$type)}\label{pml__bfo__failover_8c_aa0500831e4b18eb571bfa11af5b7e859}

\begin{DoxyCompactList}\small\item\em Update a few fields when we are restarting either a R\+N\+D\+V or R\+G\+E\+T type message. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_af3f7bb6e1bb957a64da2b154918a64c8}{mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+eager\+\_\+bml\+\_\+btl\+\_\+recv\+\_\+ctl} (\hyperlink{structmca__bml__base__btl__t}{mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t} $\ast$$\ast$bml\+\_\+btl, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des)
\begin{DoxyCompactList}\small\item\em The following set of functions are all called when it is determined that the cached bml\+\_\+btl-\/$>$btl does not match the btl handed back by the callback function. \end{DoxyCompactList}\item 
\hypertarget{pml__bfo__failover_8c_a3c32b48da6f3fb3bf8be2caa323fc3b1}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+find\+\_\+sendreq\+\_\+eager\+\_\+bml\+\_\+btl} (\hyperlink{structmca__bml__base__btl__t}{mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t} $\ast$$\ast$bml\+\_\+btl, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, \hyperlink{structmca__pml__bfo__send__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$sendreq, char $\ast$type)}\label{pml__bfo__failover_8c_a3c32b48da6f3fb3bf8be2caa323fc3b1}

\item 
\hypertarget{pml__bfo__failover_8c_aeb78578caebfe1eae4be46bb7ea1882d}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+find\+\_\+sendreq\+\_\+rdma\+\_\+bml\+\_\+btl} (\hyperlink{structmca__bml__base__btl__t}{mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t} $\ast$$\ast$bml\+\_\+btl, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, \hyperlink{structmca__pml__bfo__send__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$sendreq, char $\ast$type)}\label{pml__bfo__failover_8c_aeb78578caebfe1eae4be46bb7ea1882d}

\item 
\hypertarget{pml__bfo__failover_8c_a60d9c91f825ddec6fab6b0375bdc05f6}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+find\+\_\+recvreq\+\_\+eager\+\_\+bml\+\_\+btl} (\hyperlink{structmca__bml__base__btl__t}{mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t} $\ast$$\ast$bml\+\_\+btl, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, \hyperlink{structmca__pml__bfo__recv__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$recvreq, char $\ast$type)}\label{pml__bfo__failover_8c_a60d9c91f825ddec6fab6b0375bdc05f6}

\item 
\hypertarget{pml__bfo__failover_8c_afd7a828c5bdeac1af68b817468163567}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+find\+\_\+recvreq\+\_\+rdma\+\_\+bml\+\_\+btl} (\hyperlink{structmca__bml__base__btl__t}{mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t} $\ast$$\ast$bml\+\_\+btl, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, \hyperlink{structmca__pml__bfo__recv__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$recvreq, char $\ast$type)}\label{pml__bfo__failover_8c_afd7a828c5bdeac1af68b817468163567}

\item 
bool \hyperlink{pml__bfo__failover_8c_a666ae8e0664c80db2cbad141a4255e22}{mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error} (struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, \hyperlink{structmca__pml__bfo__send__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$sendreq)
\begin{DoxyCompactList}\small\item\em The completion event for the R\+N\+D\+V message has returned with an error. \end{DoxyCompactList}\item 
void \hyperlink{pml__bfo__failover_8c_ac4bc3bc36c7f89e47311a829a4e02a5c}{mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error} (\hyperlink{structmca__pml__bfo__send__request__t}{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$sendreq, int status, \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, int type, char $\ast$description)
\begin{DoxyCompactList}\small\item\em Check to see if an error has occurred on this send request. \end{DoxyCompactList}\item 
\hypertarget{pml__bfo__failover_8c_a9e05b4ce9a451c722320b9d5f4bfaf8c}{void {\bfseries mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+ctl\+\_\+completion\+\_\+status\+\_\+error} (struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des)}\label{pml__bfo__failover_8c_a9e05b4ce9a451c722320b9d5f4bfaf8c}

\end{DoxyCompactItemize}


\subsection{Detailed Description}
Functions that implement failover capabilities. 

To utilize the failover feature, one needs to configure the library with --enable-\/openib-\/failover. Then the system that is being used must have two or more openib B\+T\+Ls in use. When an error occurs, the B\+T\+L will call into this P\+M\+L to map out the offending B\+T\+L and continue using the one that is still working. Most of the differences between the ob1 P\+M\+L and the bfo P\+M\+L are contained in this file. 

\subsection{Function Documentation}
\hypertarget{pml__bfo__failover_8c_ae313feda12bd9b44d1bb9299b74a7ed1}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status@{mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status@{mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{struct {\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{int}]{status}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_ae313feda12bd9b44d1bb9299b74a7ed1}


Call each time we get a completion event on A\+C\+K or P\+U\+T message. 

These types of messages are receive control type messages. This function is only called if the underlying B\+T\+L supports failover. Otherwise, there is no need for this check. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbdata, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+rdma\+\_\+hdr\+\_\+t\+::hdr\+\_\+des, mca\+\_\+pml\+\_\+bfo\+\_\+common\+\_\+hdr\+\_\+t\+::hdr\+\_\+type, mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify(), mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack(), opal\+\_\+output\+\_\+verbose(), mca\+\_\+pml\+\_\+base\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+ompi, ompi\+\_\+request\+\_\+t\+::req\+\_\+status, and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.

\hypertarget{pml__bfo__failover_8c_ac4bc3bc36c7f89e47311a829a4e02a5c}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error@{mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error@{mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$}]{sendreq, }
\item[{int}]{status, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{int}]{type, }
\item[{char $\ast$}]{description}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_ac4bc3bc36c7f89e47311a829a4e02a5c}


Check to see if an error has occurred on this send request. 

If it has and there are no outstanding events, then we can start the restart dance. 

References mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify(), opal\+\_\+output\+\_\+verbose(), mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+peer, and mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+sequence.

\hypertarget{pml__bfo__failover_8c_a37a6e43b438b451f6a236db20898f620}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets@{mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets@{mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets}]{\setlength{\rightskip}{0pt plus 5cm}static void mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{{\bf mca\+\_\+bml\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$}]{ep}
\end{DoxyParamCaption}
)\hspace{0.3cm}{\ttfamily [static]}}}\label{pml__bfo__failover_8c_a37a6e43b438b451f6a236db20898f620}


This function is called since when we are mapping out a B\+M\+L. 

This will walk through the four P\+M\+L lists and dispatch with the fragments/requests. There are four different lists and each one is handled slighty differently. In all cases, we first see if the message is associated with the endpoint that is being mapped out. If not, then just leave it alone and put it back on the list. If it is associated with the endpoint, then a each list handles it slighlty differently. Also, in some cases, we actually adjust the pointers to the B\+M\+Ls in the messages as they may have changed when the B\+M\+L is mapped out. That is because this is called after we have mapped out the offending B\+M\+L and adjusted the array of available B\+M\+Ls. 

References mca\+\_\+bml\+\_\+base\+\_\+endpoint\+\_\+t\+::btl\+\_\+eager, mca\+\_\+bml\+\_\+base\+\_\+endpoint\+\_\+t\+::btl\+\_\+rdma, mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+find(), mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+next(), mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify(), mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack(), mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify(), opal\+\_\+list\+\_\+append, opal\+\_\+list\+\_\+get\+\_\+size(), opal\+\_\+list\+\_\+remove\+\_\+first(), opal\+\_\+output(), opal\+\_\+output\+\_\+verbose(), O\+P\+A\+L\+\_\+\+T\+H\+R\+E\+A\+D\+\_\+\+L\+O\+C\+K, O\+P\+A\+L\+\_\+\+T\+H\+R\+E\+A\+D\+\_\+\+U\+N\+L\+O\+C\+K, ompi\+\_\+proc\+\_\+t\+::proc\+\_\+bml, mca\+\_\+pml\+\_\+base\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+base, and mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+proc.

\hypertarget{pml__bfo__failover_8c_a5a8c8e42899718ae7cbe164f7900a97a}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request@{mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request@{mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request}]{\setlength{\rightskip}{0pt plus 5cm}{\bf mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t}$\ast$ mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t} $\ast$}]{hdr}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a5a8c8e42899718ae7cbe164f7900a97a}


This function is called when a R\+N\+D\+V or R\+G\+E\+T is received with the F\+L\+A\+G\+S\+\_\+\+R\+E\+S\+T\+A\+R\+T flag set. 

This means this message already has a receive request already associated with it. 

References mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+common, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+ctx, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+seq, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+common\+\_\+hdr\+\_\+t\+::hdr\+\_\+type, mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset(), opal\+\_\+output\+\_\+verbose(), mca\+\_\+pml\+\_\+base\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+comm, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+ompi, and ompi\+\_\+request\+\_\+t\+::req\+\_\+status.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+match().

\hypertarget{pml__bfo__failover_8c_af9cda77a04361a27ce190da460c60ef1}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+fin@{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+fin}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+fin@{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+fin}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+fin}]{\setlength{\rightskip}{0pt plus 5cm}bool mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+fin (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+hdr\+\_\+t} $\ast$}]{hdr, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{rdma, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_af9cda77a04361a27ce190da460c60ef1}


This function checks to see if we have received a duplicate F\+I\+N message. 

This is done by first pulling the pointer of the request that the F\+I\+N message is pointing to from the message. We then check the various fields in the request to the fields in the header and make sure they match. If they do not, then the request must have been recycled already and this is a duplicate F\+I\+N message. We have to do this check on every F\+I\+N message that we receive. 

References mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t\+::btl\+\_\+flags, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbdata, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+flags, mca\+\_\+pml\+\_\+bfo\+\_\+fin\+\_\+hdr\+\_\+t\+::hdr\+\_\+common, mca\+\_\+pml\+\_\+bfo\+\_\+common\+\_\+hdr\+\_\+t\+::hdr\+\_\+flags, opal\+\_\+output\+\_\+verbose(), mca\+\_\+pml\+\_\+base\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+comm, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+ompi, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+peer, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+sequence, ompi\+\_\+request\+\_\+t\+::req\+\_\+status, and mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+type.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+fin().

\hypertarget{pml__bfo__failover_8c_a61619e0f5a4b44e12fd69427e3059e73}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+msg@{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+msg}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+msg@{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+msg}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+msg}]{\setlength{\rightskip}{0pt plus 5cm}bool mca\+\_\+pml\+\_\+bfo\+\_\+is\+\_\+duplicate\+\_\+msg (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+comm\+\_\+proc\+\_\+t} $\ast$}]{proc, }
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t} $\ast$}]{hdr}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a61619e0f5a4b44e12fd69427e3059e73}


When running with failover enabled, check the P\+M\+L sequence numbers to see if we have received a duplicate message. 

This check is done for for all M\+A\+T\+C\+H fragments. It is also done for R\+N\+D\+V and R\+G\+E\+T fragments that do not have the M\+C\+A\+\_\+\+P\+M\+L\+\_\+\+B\+F\+O\+\_\+\+H\+D\+R\+\_\+\+F\+L\+A\+G\+S\+\_\+\+R\+E\+S\+T\+A\+R\+T flag set. We set the window size to half the total range of sequence numbers. We only enter this code when the seq\+\_\+num is not the expected one. A few more notes on the algorithm used here. In normal operation, the expected value will either be equal to or less than the sequence number of the header. This is because we are using this sequence number to detect packets arriving prior to them being expected. If we determine that expected is less than header, then make sure this is not a rollover case. We do that by adding the maxnum to the expected. 
\begin{DoxyParams}{Parameters}
{\em proc} & Pointer to proc from where message came \\
\hline
{\em hdr} & Pointer to header of message \\
\hline
\end{DoxyParams}


References mca\+\_\+pml\+\_\+bfo\+\_\+comm\+\_\+proc\+\_\+t\+::expected\+\_\+sequence, mca\+\_\+pml\+\_\+bfo\+\_\+comm\+\_\+proc\+\_\+t\+::frags\+\_\+cant\+\_\+match, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+common, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+seq, mca\+\_\+pml\+\_\+bfo\+\_\+common\+\_\+hdr\+\_\+t\+::hdr\+\_\+type, opal\+\_\+list\+\_\+get\+\_\+end(), opal\+\_\+list\+\_\+get\+\_\+first(), opal\+\_\+list\+\_\+get\+\_\+next, opal\+\_\+list\+\_\+get\+\_\+size(), opal\+\_\+output(), and opal\+\_\+output\+\_\+verbose().



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+match(), and mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+match().

\hypertarget{pml__bfo__failover_8c_a2f7e207b3bd8fbacf2b32448e43a7b19}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{void $\ast$}]{cbdata}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a2f7e207b3bd8fbacf2b32448e43a7b19}


Callback for when a R\+E\+C\+V\+E\+R\+R\+N\+O\+T\+I\+F\+Y message is received. 

This message is sent from the receiver to the sender and tells the sender that the receiver has seen an error. This will trigger the sender to start the request restart sequence. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+dst, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+ctx, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+seq, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify(), opal\+\_\+output\+\_\+verbose(), and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+register\+\_\+callbacks().

\hypertarget{pml__bfo__failover_8c_a1fde523b6826518993369460d6c3a6b9}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{void $\ast$}]{cbdata}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a1fde523b6826518993369460d6c3a6b9}


Callback for when a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K message is received. 

This message is sent from the receiver to the sender to acknowledge the receipt of the R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message. At this point, the sender can reset the send request and restart the message. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+dst, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+ctx, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+seq, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart(), opal\+\_\+output\+\_\+verbose(), and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+register\+\_\+callbacks().

\hypertarget{pml__bfo__failover_8c_ad9b6b294d13f87e143e194ecc65b7f72}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnack}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnack}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnack}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnack (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{void $\ast$}]{cbdata}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_ad9b6b294d13f87e143e194ecc65b7f72}


Callback for when a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+A\+C\+K message is received. 

This message is sent from the receiver to the sender and tells the sender that the receiver has already completed the message and there is nothing else to be done. The sender should then just make the send request complete. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+dst, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+ctx, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+seq, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+src, opal\+\_\+output\+\_\+verbose(), and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+register\+\_\+callbacks().

\hypertarget{pml__bfo__failover_8c_ad27dc653cc5d4a5be33c422a7e0b4992}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{void $\ast$}]{cbdata}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_ad27dc653cc5d4a5be33c422a7e0b4992}


Callback for when a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message is received. 

Four new callbacks for the four new message types.

A R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message is sent from the sender to the receiver telling the receiver that the message is going to be started over. The receiver first makes sure that the request being pointed to is still valid. If it is not, that means the receiver must have completed the request and therefore we need to send a N\+A\+C\+K back to the sender. The receiver then makes sure this is not a duplicate message. If it is a duplicate, it will just drop it. Otherwise, it will then send a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K message if there are no outstanding events on the receiver. Otherwise, it will just change the state of the request and wait for another event to send the R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K to the sender. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+dst, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+ctx, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+seq, mca\+\_\+pml\+\_\+bfo\+\_\+match\+\_\+hdr\+\_\+t\+::hdr\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack(), mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack(), ompi\+\_\+proc\+\_\+find(), opal\+\_\+output\+\_\+verbose(), and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+register\+\_\+callbacks().

\hypertarget{pml__bfo__failover_8c_a2303d9a603d0042a5055d91d2c7b888a}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+recverrnotify (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$}]{recvreq, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag, }
\item[{int}]{status}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a2303d9a603d0042a5055d91d2c7b888a}


This function is called when an error is detected on a completion event on the receiving side. 

This can come from a A\+C\+K, P\+U\+T, R\+D\+M\+A read (G\+E\+T) or R\+E\+C\+V\+E\+R\+R\+N\+O\+T\+I\+F\+Y completion event. When this happens, check the state of the request and decide if the sender needs be notified that a problem was seen. If no R\+E\+C\+V\+E\+R\+R\+N\+O\+T\+I\+F\+Y message has been sent and no R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y has been received from the sender, then send a message telling the sender an error was seen. 

References mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t\+::btl, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbfunc, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+next(), opal\+\_\+output(), opal\+\_\+output\+\_\+verbose(), ompi\+\_\+proc\+\_\+t\+::proc\+\_\+bml, mca\+\_\+pml\+\_\+base\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+comm, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+ompi, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+proc, ompi\+\_\+request\+\_\+t\+::req\+\_\+status, and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status(), and mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets().

\hypertarget{pml__bfo__failover_8c_ab22055c7e6dac0939ff8bddb5613f728}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+reset (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$}]{match}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_ab22055c7e6dac0939ff8bddb5613f728}


Reset all the receive request fields to match what a request looks like when it is first started. 

This gets called when the rendezvous/rget message is being restarted. 

References mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t\+::mpool\+\_\+deregister, mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+ack\+\_\+sent, mca\+\_\+pml\+\_\+base\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+bytes\+\_\+received, ompi\+\_\+request\+\_\+t\+::req\+\_\+complete, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+convertor, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+ompi, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+pml\+\_\+complete, and ompi\+\_\+request\+\_\+t\+::req\+\_\+state.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+get\+\_\+request().

\hypertarget{pml__bfo__failover_8c_aee082d267b4d638b8859da7d2e355241}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartack (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+t} $\ast$}]{recvreq, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag, }
\item[{int}]{status, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_aee082d267b4d638b8859da7d2e355241}


This function is called when it may be time to send a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K message back to the sending side. 

This can happen because we received a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message from the sender. This can also happen if we have noticed that the request has received the R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message, but has not yet sent out the R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K because there were still some pending receive events on the request. That means we can enter this routine from a completion event on a A\+C\+K, P\+U\+T, or R\+D\+M\+A read as well as from the receipt of a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message. If all is good, we sent the R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K message back to the sender. Then sometime later a message will arrive telling us to reset and restart the receive request. 

References mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t\+::btl, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbdata, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbfunc, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+next(), opal\+\_\+output(), opal\+\_\+output\+\_\+verbose(), ompi\+\_\+proc\+\_\+t\+::proc\+\_\+bml, mca\+\_\+pml\+\_\+base\+\_\+recv\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+comm, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+ompi, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+proc, ompi\+\_\+request\+\_\+t\+::req\+\_\+status, and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+check\+\_\+recv\+\_\+ctl\+\_\+completion\+\_\+status(), mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets(), and mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify().

\hypertarget{pml__bfo__failover_8c_a18aa312796513ad6d6ae0ca8bfdb34f5}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack@{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+request\+\_\+rndvrestartnack (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{olddes, }
\item[{{\bf ompi\+\_\+proc\+\_\+t} $\ast$}]{ompi\+\_\+proc, }
\item[{bool}]{repost}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a18aa312796513ad6d6ae0ca8bfdb34f5}


Called after the receipt of a R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+N\+O\+T\+I\+F\+Y message to a request that no longer matches. 

This can happen if the sender detected an error, but the receiver actually received all the data. Therefore send a N\+A\+C\+K back instead of the A\+C\+K so that the sender can complete its request. This happens very rarely. Note that we need to make use of the hdr\+\_\+dst\+\_\+rank that we received from the notify message. This is so the sending side make sure the message matches a valid request on the sending side. 

References mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+t\+::arr\+\_\+size, mca\+\_\+bml\+\_\+base\+\_\+endpoint\+\_\+t\+::btl\+\_\+eager, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbdata, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbfunc, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+dst, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+next(), opal\+\_\+output(), opal\+\_\+output\+\_\+verbose(), ompi\+\_\+proc\+\_\+t\+::proc\+\_\+bml, ompi\+\_\+proc\+\_\+t\+::proc\+\_\+name, mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr, and orte\+\_\+process\+\_\+name\+\_\+t\+::vpid.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartnotify().

\hypertarget{pml__bfo__failover_8c_aecf95c1315b590837bfd6ad15950051c}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+match\+\_\+fragment@{mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+match\+\_\+fragment}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+match\+\_\+fragment@{mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+match\+\_\+fragment}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+match\+\_\+fragment}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+repost\+\_\+match\+\_\+fragment (
\begin{DoxyParamCaption}
\item[{struct {\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_aecf95c1315b590837bfd6ad15950051c}


This function will repost a match fragment. 

This function has to handle the case where there may not be a request associated with the fragment and just use the information in the fragment to repost the send. 

References mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+t\+::arr\+\_\+size, mca\+\_\+bml\+\_\+base\+\_\+endpoint\+\_\+t\+::btl\+\_\+eager, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbdata, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+context, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+flags, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+next(), opal\+\_\+output(), opal\+\_\+output\+\_\+verbose(), mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+bytes\+\_\+packed, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+convertor, mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr, and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+len.

\hypertarget{pml__bfo__failover_8c_a666ae8e0664c80db2cbad141a4255e22}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error@{mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error@{mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error}]{\setlength{\rightskip}{0pt plus 5cm}bool mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error (
\begin{DoxyParamCaption}
\item[{struct {\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$}]{sendreq}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a666ae8e0664c80db2cbad141a4255e22}


The completion event for the R\+N\+D\+V message has returned with an error. 

We know that the send request we are looking at is valid because it cannot be completed until the sendreq-\/$>$req\+\_\+state value reaches 0. And for the sendreq-\/$>$req\+\_\+state to reach 0, the completion event on the R\+N\+D\+V message must occur. So, we do not bother checking whether the send request is valid, because we know it is, but we put a few asserts in for good measure. We then check a few fields in the request to decide what to do. If the sendreq-\/$>$req\+\_\+error is set, that means that something has happend already to the request and we do not want to restart it. Presumably, we may have received a R\+E\+C\+V\+E\+R\+R\+N\+O\+T\+I\+F\+Y message from the receiver. We also check the sendreq-\/$>$req\+\_\+acked field to see if it has been acked. If it has, then again we do not restart everything because obviously the R\+N\+D\+V message has made it to the other side. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart(), and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.

\hypertarget{pml__bfo__failover_8c_ac260ba077b2f5daa53ac0ec79cda856b}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion@{mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion@{mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{struct {\bf mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$}]{ep, }
\item[{struct {\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{int}]{status}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_ac260ba077b2f5daa53ac0ec79cda856b}


Completion callback for rndvrestartnotify completion event. 

If the R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K has already been received, then reset and restart. Otherwise, just update the state and let the R\+N\+D\+V\+R\+E\+S\+T\+A\+R\+T\+A\+C\+K trigger the reset and restart. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart(), mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify(), opal\+\_\+output\+\_\+verbose(), mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+comm, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+peer, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+sequence, and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify().

\hypertarget{pml__bfo__failover_8c_aec55643d673280be8f685cf83565adb2}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart@{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart@{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+restart (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$}]{sendreq, }
\item[{bool}]{repost, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_aec55643d673280be8f685cf83565adb2}


This function restarts a R\+N\+D\+V send request. 

When this is called, all the fields in the send request are reset and the send is started over. The sendreq-\/$>$req\+\_\+restartseq will be non-\/zero which will trigger a special flag in the R\+N\+D\+V header which indicates the match has already happened on the receiving side. 

References mca\+\_\+bml\+\_\+base\+\_\+endpoint\+\_\+t\+::btl\+\_\+eager, mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+next(), mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+size(), M\+C\+A\+\_\+\+P\+M\+L\+\_\+\+B\+A\+S\+E\+\_\+\+S\+E\+N\+D\+\_\+\+S\+T\+A\+R\+T, opal\+\_\+list\+\_\+get\+\_\+begin(), opal\+\_\+list\+\_\+get\+\_\+last(), opal\+\_\+list\+\_\+remove\+\_\+item(), opal\+\_\+output\+\_\+verbose(), O\+P\+A\+L\+\_\+\+T\+H\+R\+E\+A\+D\+\_\+\+L\+O\+C\+K, O\+P\+A\+L\+\_\+\+T\+H\+R\+E\+A\+D\+\_\+\+U\+N\+L\+O\+C\+K, ompi\+\_\+proc\+\_\+t\+::proc\+\_\+bml, mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+addr, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+addr, mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+comm, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+convertor, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+peer, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+proc, mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+send\+\_\+mode, and mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+sequence.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+rndvrestartack(), mca\+\_\+pml\+\_\+bfo\+\_\+rndv\+\_\+completion\+\_\+status\+\_\+error(), and mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion().

\hypertarget{pml__bfo__failover_8c_a36efc5fb4674dc619cd24813bdcebbe8}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify@{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+rndvrestartnotify (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+pml\+\_\+bfo\+\_\+send\+\_\+request\+\_\+t} $\ast$}]{sendreq, }
\item[{bool}]{repost, }
\item[{mca\+\_\+btl\+\_\+base\+\_\+tag\+\_\+t}]{tag, }
\item[{int}]{status, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_a36efc5fb4674dc619cd24813bdcebbe8}


This function gets called when failover is enabled and an error occurs during the rendezvous protocol. 

A message is sent to the receiving side notifying the request that the communication is going to be starting over. However, none of the information in the send request is reset yet, so that any in flight fragments can still find a home. Information in the send request gets reset when the completion event for this send occurs A\+N\+D an A\+C\+K has been received back from the receiver. 

References mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t\+::btl, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbfunc, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+array\+\_\+get\+\_\+next(), mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion(), opal\+\_\+output(), opal\+\_\+output\+\_\+verbose(), O\+R\+T\+E\+\_\+\+P\+R\+O\+C\+\_\+\+M\+Y\+\_\+\+N\+A\+M\+E, ompi\+\_\+proc\+\_\+t\+::proc\+\_\+bml, mca\+\_\+pml\+\_\+base\+\_\+send\+\_\+request\+\_\+t\+::req\+\_\+base, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+comm, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+peer, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+proc, mca\+\_\+pml\+\_\+base\+\_\+request\+\_\+t\+::req\+\_\+sequence, and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.



Referenced by mca\+\_\+pml\+\_\+bfo\+\_\+completion\+\_\+sendreq\+\_\+has\+\_\+error(), mca\+\_\+pml\+\_\+bfo\+\_\+error\+\_\+pending\+\_\+packets(), mca\+\_\+pml\+\_\+bfo\+\_\+recv\+\_\+frag\+\_\+callback\+\_\+recverrnotify(), and mca\+\_\+pml\+\_\+bfo\+\_\+rndvrestartnotify\+\_\+completion().

\hypertarget{pml__bfo__failover_8c_af3f7bb6e1bb957a64da2b154918a64c8}{\index{pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}!mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+eager\+\_\+bml\+\_\+btl\+\_\+recv\+\_\+ctl@{mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+eager\+\_\+bml\+\_\+btl\+\_\+recv\+\_\+ctl}}
\index{mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+eager\+\_\+bml\+\_\+btl\+\_\+recv\+\_\+ctl@{mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+eager\+\_\+bml\+\_\+btl\+\_\+recv\+\_\+ctl}!pml\+\_\+bfo\+\_\+failover.\+c@{pml\+\_\+bfo\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+eager\+\_\+bml\+\_\+btl\+\_\+recv\+\_\+ctl}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+pml\+\_\+bfo\+\_\+update\+\_\+eager\+\_\+bml\+\_\+btl\+\_\+recv\+\_\+ctl (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+bml\+\_\+base\+\_\+btl\+\_\+t} $\ast$$\ast$}]{bml\+\_\+btl, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{btl, }
\item[{struct {\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des}
\end{DoxyParamCaption}
)}}\label{pml__bfo__failover_8c_af3f7bb6e1bb957a64da2b154918a64c8}


The following set of functions are all called when it is determined that the cached bml\+\_\+btl-\/$>$btl does not match the btl handed back by the callback function. 

This means that the bml\+\_\+btl array has been shuffled and the bml\+\_\+btl matching the btl has to be found back. If it cannot be found, then just find a different one to use. 

References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbdata, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+pml\+\_\+bfo\+\_\+ack\+\_\+hdr\+\_\+t\+::hdr\+\_\+dst\+\_\+req, mca\+\_\+pml\+\_\+bfo\+\_\+common\+\_\+hdr\+\_\+t\+::hdr\+\_\+type, opal\+\_\+output(), and mca\+\_\+btl\+\_\+base\+\_\+segment\+\_\+t\+::seg\+\_\+addr.

