\hypertarget{btl__openib__failover_8c}{\section{ompi/mca/btl/openib/btl\+\_\+openib\+\_\+failover.c File Reference}
\label{btl__openib__failover_8c}\index{ompi/mca/btl/openib/btl\+\_\+openib\+\_\+failover.\+c@{ompi/mca/btl/openib/btl\+\_\+openib\+\_\+failover.\+c}}
}


Functions specific to implementing failover support.  


{\ttfamily \#include \char`\"{}ompi\+\_\+config.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}opal\+\_\+stdint.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}btl\+\_\+openib.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}btl\+\_\+openib\+\_\+endpoint.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}btl\+\_\+openib\+\_\+proc.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}btl\+\_\+openib\+\_\+failover.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}opal/util/opal\+\_\+sos.\+h\char`\"{}}\\*
\subsection*{Functions}
\begin{DoxyCompactItemize}
\item 
static void \hyperlink{btl__openib__failover_8c_af670b31acf6cfbeda6c6a9af412c0cee}{error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags} (\hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$ep, struct \hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$module, bool errout)
\begin{DoxyCompactList}\small\item\em This function will find all the pending fragments on an endpoint and call the callback function with O\+M\+P\+I\+\_\+\+E\+R\+R\+O\+R. \end{DoxyCompactList}\item 
static void \hyperlink{btl__openib__failover_8c_a7a49606caf8140c7d93f369641592789}{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify} (\hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$endpoint, uint8\+\_\+t type, int index)
\begin{DoxyCompactList}\small\item\em This function is used to send a message to the remote side indicating the endpoint is broken and telling the remote side to brings its endpoint down as well. \end{DoxyCompactList}\item 
\hypertarget{btl__openib__failover_8c_a9b41bd772bbcfe39d6dbbd546a53f1da}{void {\bfseries mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+local\+\_\+rdma\+\_\+frags} (\hyperlink{structmca__btl__openib__device__t}{mca\+\_\+btl\+\_\+openib\+\_\+device\+\_\+t} $\ast$device)}\label{btl__openib__failover_8c_a9b41bd772bbcfe39d6dbbd546a53f1da}

\item 
void \hyperlink{btl__openib__failover_8c_a1f3ddf816ad5b4ee3a5cad4d22f98bda}{mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues} (bool errout)
\begin{DoxyCompactList}\small\item\em This function is a debugging tool. \end{DoxyCompactList}\item 
\hypertarget{btl__openib__failover_8c_afdd254add17fffe8425e1686e98ae850}{static void {\bfseries dump\+\_\+local\+\_\+rdma\+\_\+frags} (\hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+t} $\ast$endpoint)}\label{btl__openib__failover_8c_afdd254add17fffe8425e1686e98ae850}

\item 
void \hyperlink{btl__openib__failover_8c_a8d24bd6bef69af6d68d19f35352dfdb2}{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error} (\hyperlink{structmca__btl__openib__module__t}{mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t} $\ast$openib\+\_\+btl, \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$des, int qp, \hyperlink{structompi__proc__t}{ompi\+\_\+proc\+\_\+t} $\ast$remote\+\_\+proc, \hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+t} $\ast$endpoint)
\begin{DoxyCompactList}\small\item\em This function is called when we get an error on the completion event of a fragment. \end{DoxyCompactList}\item 
void \hyperlink{btl__openib__failover_8c_a4027610aedc70570559b30d125592eac}{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error} (\hyperlink{structmca__btl__openib__module__t}{mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t} $\ast$openib\+\_\+btl)
\begin{DoxyCompactList}\small\item\em This functions allows an error to map out the entire B\+T\+L. \end{DoxyCompactList}\item 
void \hyperlink{btl__openib__failover_8c_ac7e75cc42802b695c3451f2a0ba58183}{btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages} (\hyperlink{structmca__btl__openib__control__header__t}{mca\+\_\+btl\+\_\+openib\+\_\+control\+\_\+header\+\_\+t} $\ast$ctl\+\_\+hdr, \hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+t} $\ast$ep)
\begin{DoxyCompactList}\small\item\em This function gets called when a control message is received that is one of the following types\+: M\+C\+A\+\_\+\+B\+T\+L\+\_\+\+O\+P\+E\+N\+I\+B\+\_\+\+C\+O\+N\+T\+R\+O\+L\+\_\+\+E\+P\+\_\+\+B\+R\+O\+K\+E\+N M\+C\+A\+\_\+\+B\+T\+L\+\_\+\+O\+P\+E\+N\+I\+B\+\_\+\+C\+O\+N\+T\+R\+O\+L\+\_\+\+E\+P\+\_\+\+E\+A\+G\+E\+R\+\_\+\+R\+D\+M\+A\+\_\+\+E\+R\+R\+O\+R message Note that we are using the working connection to send information about the broken connection. \end{DoxyCompactList}\item 
\hypertarget{btl__openib__failover_8c_a2a67e4960e579a2597de18d0bc284923}{static void {\bfseries mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify\+\_\+cb} (\hyperlink{structmca__btl__base__module__t}{mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$btl, struct \hyperlink{structmca__btl__base__endpoint__t}{mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$endpoint, struct \hyperlink{structmca__btl__base__descriptor__t}{mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$descriptor, int status)}\label{btl__openib__failover_8c_a2a67e4960e579a2597de18d0bc284923}

\end{DoxyCompactItemize}


\subsection{Detailed Description}
Functions specific to implementing failover support. 

This file is conditionally copiled into the B\+T\+L when one configures it in with --enable-\/openib-\/failover. When this file is compiled in, the multi-\/\+B\+T\+L configurations can handle errors. The requirement is that there needs to be more than one openib B\+T\+L in use so that all the traffic can move to the other B\+T\+L. This does not support failing over to a different B\+T\+L like T\+C\+P. 

\subsection{Function Documentation}
\hypertarget{btl__openib__failover_8c_ac7e75cc42802b695c3451f2a0ba58183}{\index{btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}!btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages@{btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages}}
\index{btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages@{btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages}!btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}}
\subsubsection[{btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages}]{\setlength{\rightskip}{0pt plus 5cm}void btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+openib\+\_\+control\+\_\+header\+\_\+t} $\ast$}]{ctl\+\_\+hdr, }
\item[{{\bf mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+t} $\ast$}]{ep}
\end{DoxyParamCaption}
)}}\label{btl__openib__failover_8c_ac7e75cc42802b695c3451f2a0ba58183}


This function gets called when a control message is received that is one of the following types\+: M\+C\+A\+\_\+\+B\+T\+L\+\_\+\+O\+P\+E\+N\+I\+B\+\_\+\+C\+O\+N\+T\+R\+O\+L\+\_\+\+E\+P\+\_\+\+B\+R\+O\+K\+E\+N M\+C\+A\+\_\+\+B\+T\+L\+\_\+\+O\+P\+E\+N\+I\+B\+\_\+\+C\+O\+N\+T\+R\+O\+L\+\_\+\+E\+P\+\_\+\+E\+A\+G\+E\+R\+\_\+\+R\+D\+M\+A\+\_\+\+E\+R\+R\+O\+R message Note that we are using the working connection to send information about the broken connection. 

That is why we have to look at the various information in the control message to figure out which endpoint is broken. It is (obviously) not the one the message was received on, because we would not have received the message in that case. In the case of the B\+R\+O\+K\+E\+N message, that means the remote side is notifying us that it has brought down its half of the connection. Therefore, we need to bring out half down. This is done because it has been observed that there are cases where only one side of the connection actually sees the error. This means we can be left in a state where one side believes it has two B\+T\+Ls, but the other side believes it only has one. This can cause problems. In the case of the E\+A\+G\+E\+R\+\_\+\+R\+D\+M\+A\+\_\+\+E\+R\+R\+O\+R, see elsewhere in the code what we are doing. 
\begin{DoxyParams}{Parameters}
{\em ctl\+\_\+hdr} & Pointer control header that was received \\
\hline
\end{DoxyParams}


References mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::eager\+\_\+rdma\+\_\+local, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::endpoint\+\_\+proc, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::endpoint\+\_\+state, mca\+\_\+btl\+\_\+openib\+\_\+device\+\_\+t\+::endpoints, mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t\+::error\+\_\+cb, error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags(), mca\+\_\+btl\+\_\+openib\+\_\+eager\+\_\+rdma\+\_\+local\+\_\+t\+::head, mca\+\_\+btl\+\_\+openib\+\_\+component\+\_\+t\+::ib\+\_\+num\+\_\+btls, mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t\+::lid, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::nbo, opal\+\_\+output\+\_\+verbose(), opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+item(), opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+size(), mca\+\_\+btl\+\_\+openib\+\_\+component\+\_\+t\+::openib\+\_\+btls, O\+R\+T\+E\+\_\+\+P\+R\+O\+C\+\_\+\+M\+Y\+\_\+\+N\+A\+M\+E, ompi\+\_\+proc\+\_\+t\+::proc\+\_\+name, mca\+\_\+btl\+\_\+elan\+\_\+proc\+\_\+t\+::proc\+\_\+ompi, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::rem\+\_\+info, and orte\+\_\+process\+\_\+name\+\_\+t\+::vpid.

\hypertarget{btl__openib__failover_8c_af670b31acf6cfbeda6c6a9af412c0cee}{\index{btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}!error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags@{error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags}}
\index{error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags@{error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags}!btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}}
\subsubsection[{error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags}]{\setlength{\rightskip}{0pt plus 5cm}static void error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$}]{ep, }
\item[{struct {\bf mca\+\_\+btl\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{module, }
\item[{bool}]{errout}
\end{DoxyParamCaption}
)\hspace{0.3cm}{\ttfamily [static]}}}\label{btl__openib__failover_8c_af670b31acf6cfbeda6c6a9af412c0cee}


This function will find all the pending fragments on an endpoint and call the callback function with O\+M\+P\+I\+\_\+\+E\+R\+R\+O\+R. 

It walks through each qp with each priority and looks for both no\+\_\+credits\+\_\+pending\+\_\+frags and no\+\_\+wqe\+\_\+pending\+\_\+frags. It then looks for any pending\+\_\+lazy\+\_\+frags, pending\+\_\+put\+\_\+frags, and pending\+\_\+get\+\_\+frags. This function is only called when running with failover support enabled. Note that the errout parameter allows the function to also be used as a debugging tool to see if there are any fragments on any of the queues. 
\begin{DoxyParams}{Parameters}
{\em ep} & Pointer to endpoint that had error \\
\hline
{\em module} & Pointer to module that had error \\
\hline
{\em errout} & Boolean which says whether to error them out or not \\
\hline
\end{DoxyParams}


References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbfunc, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+flags, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::endpoint\+\_\+btl, mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+qp\+\_\+t\+::no\+\_\+credits\+\_\+pending\+\_\+frags, mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+qp\+\_\+t\+::no\+\_\+wqe\+\_\+pending\+\_\+frags, mca\+\_\+btl\+\_\+openib\+\_\+component\+\_\+t\+::num\+\_\+qps, opal\+\_\+list\+\_\+get\+\_\+size(), opal\+\_\+list\+\_\+remove\+\_\+first(), opal\+\_\+output\+\_\+verbose(), mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::pending\+\_\+get\+\_\+frags, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::pending\+\_\+lazy\+\_\+frags, and mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::pending\+\_\+put\+\_\+frags.



Referenced by btl\+\_\+openib\+\_\+handle\+\_\+failover\+\_\+control\+\_\+messages(), mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues(), mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error(), and mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error().

\hypertarget{btl__openib__failover_8c_a1f3ddf816ad5b4ee3a5cad4d22f98bda}{\index{btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}!mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues@{mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues}}
\index{mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues@{mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues}!btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+btl\+\_\+openib\+\_\+dump\+\_\+all\+\_\+internal\+\_\+queues (
\begin{DoxyParamCaption}
\item[{bool}]{errout}
\end{DoxyParamCaption}
)}}\label{btl__openib__failover_8c_a1f3ddf816ad5b4ee3a5cad4d22f98bda}


This function is a debugging tool. 

If you notify a hang, you can call this function from a debugger and see if there are any messages stuck in any of the queues. If you call it with errout=true, then it will error them out. Otherwise, it will just print out the size of the queues with data in them. 

References mca\+\_\+btl\+\_\+openib\+\_\+device\+\_\+t\+::endpoints, error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags(), mca\+\_\+btl\+\_\+openib\+\_\+component\+\_\+t\+::ib\+\_\+num\+\_\+btls, opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+item(), opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+size(), and mca\+\_\+btl\+\_\+openib\+\_\+component\+\_\+t\+::openib\+\_\+btls.

\hypertarget{btl__openib__failover_8c_a7a49606caf8140c7d93f369641592789}{\index{btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}!mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify@{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify}}
\index{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify@{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify}!btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify}]{\setlength{\rightskip}{0pt plus 5cm}static void mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t} $\ast$}]{endpoint, }
\item[{uint8\+\_\+t}]{type, }
\item[{int}]{index}
\end{DoxyParamCaption}
)\hspace{0.3cm}{\ttfamily [static]}}}\label{btl__openib__failover_8c_a7a49606caf8140c7d93f369641592789}


This function is used to send a message to the remote side indicating the endpoint is broken and telling the remote side to brings its endpoint down as well. 

This is needed because there are cases where only one side of the connection determines that the there was a problem. 
\begin{DoxyParams}{Parameters}
{\em endpoint} & Pointer to endpoint with error \\
\hline
{\em type} & Type of message to be sent, can be one of two types \\
\hline
{\em index} & When sending R\+D\+M\+A error message, index is non zero \\
\hline
\end{DoxyParams}


References mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::endpoint\+\_\+btl, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::endpoint\+\_\+proc, mca\+\_\+btl\+\_\+openib\+\_\+device\+\_\+t\+::endpoints, mca\+\_\+btl\+\_\+openib\+\_\+component\+\_\+t\+::ib\+\_\+num\+\_\+btls, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::nbo, opal\+\_\+output\+\_\+verbose(), opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+item(), opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+size(), mca\+\_\+btl\+\_\+openib\+\_\+component\+\_\+t\+::openib\+\_\+btls, O\+R\+T\+E\+\_\+\+P\+R\+O\+C\+\_\+\+M\+Y\+\_\+\+N\+A\+M\+E, and mca\+\_\+btl\+\_\+elan\+\_\+proc\+\_\+t\+::proc\+\_\+ompi.



Referenced by mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error(), and mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error().

\hypertarget{btl__openib__failover_8c_a4027610aedc70570559b30d125592eac}{\index{btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}!mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error@{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error}}
\index{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error@{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error}!btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+btl\+\_\+error (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t} $\ast$}]{openib\+\_\+btl}
\end{DoxyParamCaption}
)}}\label{btl__openib__failover_8c_a4027610aedc70570559b30d125592eac}


This functions allows an error to map out the entire B\+T\+L. 

First a call is made up to the P\+M\+L to map out all connections from this B\+T\+L. Then a message is sent to all the endpoints connected to this B\+T\+L. This function is enabled by the btl\+\_\+openib\+\_\+port\+\_\+error\+\_\+failover M\+C\+A parameter. If that parameter is not set, then this function does not do anything. 
\begin{DoxyParams}{Parameters}
{\em openib\+\_\+btl} & Pointer to B\+T\+L that had the error \\
\hline
\end{DoxyParams}


References mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::endpoint\+\_\+state, mca\+\_\+btl\+\_\+openib\+\_\+device\+\_\+t\+::endpoints, mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t\+::error\+\_\+cb, error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags(), mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t\+::lid, mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify(), opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+item(), and opal\+\_\+pointer\+\_\+array\+\_\+get\+\_\+size().

\hypertarget{btl__openib__failover_8c_a8d24bd6bef69af6d68d19f35352dfdb2}{\index{btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}!mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error@{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error}}
\index{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error@{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error}!btl\+\_\+openib\+\_\+failover.\+c@{btl\+\_\+openib\+\_\+failover.\+c}}
\subsubsection[{mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+btl\+\_\+openib\+\_\+handle\+\_\+endpoint\+\_\+error (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t} $\ast$}]{openib\+\_\+btl, }
\item[{{\bf mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t} $\ast$}]{des, }
\item[{int}]{qp, }
\item[{{\bf ompi\+\_\+proc\+\_\+t} $\ast$}]{remote\+\_\+proc, }
\item[{{\bf mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+t} $\ast$}]{endpoint}
\end{DoxyParamCaption}
)}}\label{btl__openib__failover_8c_a8d24bd6bef69af6d68d19f35352dfdb2}


This function is called when we get an error on the completion event of a fragment. 

We check to see what type of fragment it is and act accordingly. In most cases, we first call up into the P\+M\+L and have it map out this connection for any future communication. In addition, this function will possibly send some control messages over the other openib B\+T\+L. The first control message will tell the remote side to also map out this connection. The second control message makes sure the eager R\+D\+M\+A connection remains in a sane state. See that function for more details. 
\begin{DoxyParams}{Parameters}
{\em openib\+\_\+btl} & Pointer to B\+T\+L that had the error \\
\hline
{\em des} & Pointer to descriptor that had the error \\
\hline
{\em qp} & Queue pair that had the error \\
\hline
{\em remote\+\_\+proc} & Pointer to process that had the error \\
\hline
{\em endpoint} & Pointer to endpoint that had the error \\
\hline
\end{DoxyParams}


References mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+cbfunc, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+flags, mca\+\_\+btl\+\_\+base\+\_\+descriptor\+\_\+t\+::des\+\_\+src, mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::endpoint\+\_\+state, mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t\+::error\+\_\+cb, error\+\_\+out\+\_\+all\+\_\+pending\+\_\+frags(), mca\+\_\+btl\+\_\+base\+\_\+endpoint\+\_\+t\+::get\+\_\+tokens, mca\+\_\+btl\+\_\+openib\+\_\+module\+\_\+t\+::lid, mca\+\_\+btl\+\_\+openib\+\_\+endpoint\+\_\+notify(), opal\+\_\+list\+\_\+remove\+\_\+first(), opal\+\_\+output\+\_\+verbose(), and O\+P\+A\+L\+\_\+\+T\+H\+R\+E\+A\+D\+\_\+\+A\+D\+D32.

