\hypertarget{mpool__rgpusm__module_8c}{\section{ompi/mca/mpool/rgpusm/mpool\+\_\+rgpusm\+\_\+module.c File Reference}
\label{mpool__rgpusm__module_8c}\index{ompi/mca/mpool/rgpusm/mpool\+\_\+rgpusm\+\_\+module.\+c@{ompi/mca/mpool/rgpusm/mpool\+\_\+rgpusm\+\_\+module.\+c}}
}


This memory pool is used for getting the memory handle of remote G\+P\+U memory when using C\+U\+D\+A.  


{\ttfamily \#include \char`\"{}ompi\+\_\+config.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}opal/align.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}orte/util/name\+\_\+fns.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}orte/runtime/orte\+\_\+globals.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/mpool/rgpusm/mpool\+\_\+rgpusm.\+h\char`\"{}}\\*
{\ttfamily \#include $<$errno.\+h$>$}\\*
{\ttfamily \#include $<$string.\+h$>$}\\*
{\ttfamily \#include \char`\"{}ompi/mca/rcache/rcache.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/rcache/base/base.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/mpool/base/base.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/runtime/params.\+h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}ompi/mca/common/cuda/common\+\_\+cuda.\+h\char`\"{}}\\*
\subsection*{Macros}
\begin{DoxyCompactItemize}
\item 
\hypertarget{mpool__rgpusm__module_8c_a9171a80404a7ee4d2fc74e6915871255}{\#define {\bfseries O\+P\+A\+L\+\_\+\+D\+I\+S\+A\+B\+L\+E\+\_\+\+E\+N\+A\+B\+L\+E\+\_\+\+M\+E\+M\+\_\+\+D\+E\+B\+U\+G}~1}\label{mpool__rgpusm__module_8c_a9171a80404a7ee4d2fc74e6915871255}

\item 
\#define {\bfseries S\+E\+T\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T\+\_\+\+T\+O\+\_\+\+Z\+E\+R\+O}()
\item 
\hypertarget{mpool__rgpusm__module_8c_a1edb31de85a1708e162f42592a923203}{\#define {\bfseries R\+E\+S\+T\+O\+R\+E\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T}()~mca\+\_\+mpool\+\_\+base\+\_\+page\+\_\+size\+\_\+log = saved\+\_\+page\+\_\+size;}\label{mpool__rgpusm__module_8c_a1edb31de85a1708e162f42592a923203}

\item 
\hypertarget{mpool__rgpusm__module_8c_a903599aebde23a9f4ff9273920ade58e}{\#define {\bfseries R\+G\+P\+U\+S\+M\+\_\+\+M\+P\+O\+O\+L\+\_\+\+N\+R\+E\+G\+S}~100}\label{mpool__rgpusm__module_8c_a903599aebde23a9f4ff9273920ade58e}

\end{DoxyCompactItemize}
\subsection*{Functions}
\begin{DoxyCompactItemize}
\item 
\hypertarget{mpool__rgpusm__module_8c_a511cdceebd2e76e0aec4b0d24477f273}{static bool {\bfseries mca\+\_\+mpool\+\_\+rgpusm\+\_\+deregister\+\_\+lru} (\hyperlink{structmca__mpool__base__module__t}{mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t} $\ast$mpool)}\label{mpool__rgpusm__module_8c_a511cdceebd2e76e0aec4b0d24477f273}

\item 
\hypertarget{mpool__rgpusm__module_8c_aab506d3bfac7d40d1ed33dcd30f0ece3}{void {\bfseries mca\+\_\+mpool\+\_\+rgpusm\+\_\+module\+\_\+init} (\hyperlink{structmca__mpool__rgpusm__module__t}{mca\+\_\+mpool\+\_\+rgpusm\+\_\+module\+\_\+t} $\ast$mpool)}\label{mpool__rgpusm__module_8c_aab506d3bfac7d40d1ed33dcd30f0ece3}

\item 
\hypertarget{mpool__rgpusm__module_8c_a60cb6600ccb066249ed9634330628c39}{int \hyperlink{mpool__rgpusm__module_8c_a60cb6600ccb066249ed9634330628c39}{mca\+\_\+mpool\+\_\+rgpusm\+\_\+register} (\hyperlink{structmca__mpool__base__module__t}{mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t} $\ast$mpool, void $\ast$addr, size\+\_\+t size, uint32\+\_\+t flags, \hyperlink{structmca__mpool__base__registration__t}{mca\+\_\+mpool\+\_\+base\+\_\+registration\+\_\+t} $\ast$$\ast$reg)}\label{mpool__rgpusm__module_8c_a60cb6600ccb066249ed9634330628c39}

\begin{DoxyCompactList}\small\item\em register block of memory \end{DoxyCompactList}\item 
void \hyperlink{mpool__rgpusm__module_8c_ae5fe510e9266a1946cb8208d0f52e7ba}{mca\+\_\+mpool\+\_\+rgpusm\+\_\+free} (\hyperlink{structmca__mpool__base__module__t}{mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t} $\ast$mpool, void $\ast$addr, \hyperlink{structmca__mpool__base__registration__t}{mca\+\_\+mpool\+\_\+base\+\_\+registration\+\_\+t} $\ast$registration)
\begin{DoxyCompactList}\small\item\em free function \end{DoxyCompactList}\item 
\hypertarget{mpool__rgpusm__module_8c_a5f7b2cd791ea5b000f5c31280ec689d0}{int \hyperlink{mpool__rgpusm__module_8c_a5f7b2cd791ea5b000f5c31280ec689d0}{mca\+\_\+mpool\+\_\+rgpusm\+\_\+find} (struct \hyperlink{structmca__mpool__base__module__t}{mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t} $\ast$mpool, void $\ast$addr, size\+\_\+t size, \hyperlink{structmca__mpool__base__registration__t}{mca\+\_\+mpool\+\_\+base\+\_\+registration\+\_\+t} $\ast$$\ast$reg)}\label{mpool__rgpusm__module_8c_a5f7b2cd791ea5b000f5c31280ec689d0}

\begin{DoxyCompactList}\small\item\em find registration for a given block of memory \end{DoxyCompactList}\item 
\hypertarget{mpool__rgpusm__module_8c_a2985b97ceda7af4ed1236fb2be4c51d2}{static bool {\bfseries registration\+\_\+is\+\_\+cachebale} (\hyperlink{structmca__mpool__base__registration__t}{mca\+\_\+mpool\+\_\+base\+\_\+registration\+\_\+t} $\ast$reg)}\label{mpool__rgpusm__module_8c_a2985b97ceda7af4ed1236fb2be4c51d2}

\item 
\hypertarget{mpool__rgpusm__module_8c_ae7a08586c4476f2e38ba3119a91ba452}{int \hyperlink{mpool__rgpusm__module_8c_ae7a08586c4476f2e38ba3119a91ba452}{mca\+\_\+mpool\+\_\+rgpusm\+\_\+deregister} (struct \hyperlink{structmca__mpool__base__module__t}{mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t} $\ast$mpool, \hyperlink{structmca__mpool__base__registration__t}{mca\+\_\+mpool\+\_\+base\+\_\+registration\+\_\+t} $\ast$reg)}\label{mpool__rgpusm__module_8c_ae7a08586c4476f2e38ba3119a91ba452}

\begin{DoxyCompactList}\small\item\em deregister memory \end{DoxyCompactList}\item 
\hypertarget{mpool__rgpusm__module_8c_ae2ab3464a2bccbe9a2bd28a1a227a44e}{void \hyperlink{mpool__rgpusm__module_8c_ae2ab3464a2bccbe9a2bd28a1a227a44e}{mca\+\_\+mpool\+\_\+rgpusm\+\_\+finalize} (struct \hyperlink{structmca__mpool__base__module__t}{mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t} $\ast$mpool)}\label{mpool__rgpusm__module_8c_ae2ab3464a2bccbe9a2bd28a1a227a44e}

\begin{DoxyCompactList}\small\item\em finalize mpool \end{DoxyCompactList}\item 
int \hyperlink{mpool__rgpusm__module_8c_a454183a37d32d9fb95d8cb0d8e411c8c}{mca\+\_\+mpool\+\_\+rgpusm\+\_\+ft\+\_\+event} (int state)
\begin{DoxyCompactList}\small\item\em Fault Tolerance Event Notification \hyperlink{structFunction}{Function}. \end{DoxyCompactList}\end{DoxyCompactItemize}
\subsection*{Variables}
\begin{DoxyCompactItemize}
\item 
\hypertarget{mpool__rgpusm__module_8c_a2e7e936496b33489e2e617308f797eec}{static size\+\_\+t {\bfseries saved\+\_\+page\+\_\+size}}\label{mpool__rgpusm__module_8c_a2e7e936496b33489e2e617308f797eec}

\end{DoxyCompactItemize}


\subsection{Detailed Description}
This memory pool is used for getting the memory handle of remote G\+P\+U memory when using C\+U\+D\+A. 

Hence, the name is \char`\"{}rgpusm\char`\"{} for \char`\"{}remote
\+C\+U\+D\+A\char`\"{} G\+P\+U memory. There is a cache that can be used to store the remote handles in case they are reused to save on the registration cost as that can be expensive, on the order of 100 usecs. The cache can also be used just to track how many handles are in use at a time. It is best to look at this with the three different scenarios that are possible.
\begin{DoxyEnumerate}
\item mpool\+\_\+rgpusm\+\_\+leave\+\_\+pinned=0, cache\+\_\+size=unlimited
\item mpool\+\_\+rgpusm\+\_\+leave\+\_\+pinned=0, cache\+\_\+size=limited
\item mpool\+\_\+rgpusm\+\_\+leave\+\_\+pinned=1, cache\+\_\+size=unlimited (default)
\item mpool\+\_\+rgpusm\+\_\+leave\+\_\+pinned=1, cache\+\_\+size=limited.
\end{DoxyEnumerate}

Case 1\+: The cache is unused and remote memory is registered and unregistered for each transaction. The amount of outstanding registered memory is unlimited. Case 2\+: The cache keeps track of how much memory is registered at a time. Since leave pinned is 0, any memory that is registered is in use. If the amount to register exceeds the amount, we will error out. This could be handled more gracefully, but this is not a common way to run, so we will leave as is. Case 3\+: The cache is needed to track current and past transactions. However, there is no limit on the number that can be stored. Therefore, once memory enters the cache, and gets registered, it stays that way forever. Case 4\+: The cache is needed to track current and past transactions. In addition, a list of most recently used (but no longer in use) registrations is stored so that it can be used to evict registrations from the cache. In addition, these registrations are deregistered.

I also want to capture how we can run into the case where we do not find something in the cache, but when we try to register it, we get an error back from the C\+U\+D\+A library saying the memory is in use. This can happen in the following scenario. The application mallocs a buffer of size 32\+K. The library loads this in the cache and registers it. The application then frees the buffer. It then mallocs a buffer of size 64\+K. This malloc returns the same base address as the first 32\+K allocation. The library searches the cache, but since the size is larger than the original allocation it does not find the registration. It then attempts to register this. The C\+U\+D\+A library returns an error saying it is already mapped. To handle this, we return an error of O\+M\+P\+I\+\_\+\+E\+R\+R\+\_\+\+W\+O\+U\+L\+D\+\_\+\+B\+L\+O\+C\+K to the memory pool. The memory pool then looks for the registration based on the base address and a size of 4. We use the small size to make sure that we find the registration. This registration is evicted, and we try to register again. 

\subsection{Macro Definition Documentation}
\hypertarget{mpool__rgpusm__module_8c_a3646caacd4489b71c4802fed0685e0e5}{\index{mpool\+\_\+rgpusm\+\_\+module.\+c@{mpool\+\_\+rgpusm\+\_\+module.\+c}!S\+E\+T\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T\+\_\+\+T\+O\+\_\+\+Z\+E\+R\+O@{S\+E\+T\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T\+\_\+\+T\+O\+\_\+\+Z\+E\+R\+O}}
\index{S\+E\+T\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T\+\_\+\+T\+O\+\_\+\+Z\+E\+R\+O@{S\+E\+T\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T\+\_\+\+T\+O\+\_\+\+Z\+E\+R\+O}!mpool\+\_\+rgpusm\+\_\+module.\+c@{mpool\+\_\+rgpusm\+\_\+module.\+c}}
\subsubsection[{S\+E\+T\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T\+\_\+\+T\+O\+\_\+\+Z\+E\+R\+O}]{\setlength{\rightskip}{0pt plus 5cm}\#define S\+E\+T\+\_\+\+P\+A\+G\+E\+\_\+\+A\+L\+I\+G\+N\+M\+E\+N\+T\+\_\+\+T\+O\+\_\+\+Z\+E\+R\+O(
\begin{DoxyParamCaption}
{}
\end{DoxyParamCaption}
)}}\label{mpool__rgpusm__module_8c_a3646caacd4489b71c4802fed0685e0e5}
{\bfseries Value\+:}
\begin{DoxyCode}
saved\_page\_size = mca\_mpool\_base\_page\_size\_log; \(\backslash\)
    mca\_mpool\_base\_page\_size\_log = 0;
\end{DoxyCode}


\subsection{Function Documentation}
\hypertarget{mpool__rgpusm__module_8c_ae5fe510e9266a1946cb8208d0f52e7ba}{\index{mpool\+\_\+rgpusm\+\_\+module.\+c@{mpool\+\_\+rgpusm\+\_\+module.\+c}!mca\+\_\+mpool\+\_\+rgpusm\+\_\+free@{mca\+\_\+mpool\+\_\+rgpusm\+\_\+free}}
\index{mca\+\_\+mpool\+\_\+rgpusm\+\_\+free@{mca\+\_\+mpool\+\_\+rgpusm\+\_\+free}!mpool\+\_\+rgpusm\+\_\+module.\+c@{mpool\+\_\+rgpusm\+\_\+module.\+c}}
\subsubsection[{mca\+\_\+mpool\+\_\+rgpusm\+\_\+free}]{\setlength{\rightskip}{0pt plus 5cm}void mca\+\_\+mpool\+\_\+rgpusm\+\_\+free (
\begin{DoxyParamCaption}
\item[{{\bf mca\+\_\+mpool\+\_\+base\+\_\+module\+\_\+t} $\ast$}]{mpool, }
\item[{void $\ast$}]{addr, }
\item[{{\bf mca\+\_\+mpool\+\_\+base\+\_\+registration\+\_\+t} $\ast$}]{registration}
\end{DoxyParamCaption}
)}}\label{mpool__rgpusm__module_8c_ae5fe510e9266a1946cb8208d0f52e7ba}


free function 

free memory allocated by alloc function 

References mca\+\_\+mpool\+\_\+rgpusm\+\_\+deregister().

\hypertarget{mpool__rgpusm__module_8c_a454183a37d32d9fb95d8cb0d8e411c8c}{\index{mpool\+\_\+rgpusm\+\_\+module.\+c@{mpool\+\_\+rgpusm\+\_\+module.\+c}!mca\+\_\+mpool\+\_\+rgpusm\+\_\+ft\+\_\+event@{mca\+\_\+mpool\+\_\+rgpusm\+\_\+ft\+\_\+event}}
\index{mca\+\_\+mpool\+\_\+rgpusm\+\_\+ft\+\_\+event@{mca\+\_\+mpool\+\_\+rgpusm\+\_\+ft\+\_\+event}!mpool\+\_\+rgpusm\+\_\+module.\+c@{mpool\+\_\+rgpusm\+\_\+module.\+c}}
\subsubsection[{mca\+\_\+mpool\+\_\+rgpusm\+\_\+ft\+\_\+event}]{\setlength{\rightskip}{0pt plus 5cm}int mca\+\_\+mpool\+\_\+rgpusm\+\_\+ft\+\_\+event (
\begin{DoxyParamCaption}
\item[{int}]{state}
\end{DoxyParamCaption}
)}}\label{mpool__rgpusm__module_8c_a454183a37d32d9fb95d8cb0d8e411c8c}


Fault Tolerance Event Notification \hyperlink{structFunction}{Function}. 


\begin{DoxyParams}{Parameters}
{\em state} & Checkpoint Stae \\
\hline
\end{DoxyParams}
\begin{DoxyReturn}{Returns}
O\+M\+P\+I\+\_\+\+S\+U\+C\+C\+E\+S\+S or failure status 
\end{DoxyReturn}
