OpenMPI  0.1.1
paffinity.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2005 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved.
13  * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
14  * Copyright (c) 2010 IBM Corporation. All rights reserved.
15  * $COPYRIGHT$
16  *
17  * Additional copyrights may follow
18  *
19  * $HEADER$
20  */
21 /**
22  * @file
23  *
24  * paffinity (processor affinity) framework component interface
25  * definitions.
26  *
27  * Intent
28  *
29  * This framework is used to support the OS-specific API for placement
30  * of processes on processors. It does *not* decide scheduling issues
31  * -- it is simply for assigning the current process it to a specific
32  * processor set. As such, the components are likely to be extremely
33  * short/simple -- there will likely be one component for each OS/API
34  * that we support (e.g., Linux, IRIX, etc.). As a direct
35  * consequence, there will likely only be one component that is
36  * useable on a given platform (making selection easy).
37  *
38  * It is *not* an error if there is no paffinity component available;
39  * processor affinity services are simply not available. Hence,
40  * paffinity component functions are invoked through short wrapper
41  * functions in paffinity/base (that check to see if there is a
42  * selected component before invoking function pointers). If there is
43  * no selected component, they return an appropriate error code.
44  *
45  * In the paffinity interface, we make the distinction between LOGICAL
46  * and PHYSICAL processors. LOGICAL processors are defined to have
47  * some corresponding PHYSICAL processor that both exists and is
48  * currently online. LOGICAL processors numbered countiguously
49  * starting with 0. PHYSICAL processors are numbered according to the
50  * underlying operating system; they are represented by integers, but
51  * no guarantees are made about their values.
52  *
53  * Hence, LOGICAL processor IDs are convenient for humans and are in
54  * the range of [0,N-1] (assuming N processors are currently online).
55  * Each LOGICAL processor has a 1:1 relationship with a PHYSICAL
56  * processor, but the PHYSICAL processor's ID can be any unique
57  * integer value.
58  *
59  * ***NOTE*** Obtaining information about socket/core IDs is not well
60  * supported in many OS's. Programmers using this paffinity interface
61  * should fully expect to sometimes get OPAL_ERR_NOT_SUPPORTED back
62  * when calling such functions.
63 
64  * General scheme
65  *
66  * The component has one function: query(). It simply returns a
67  * priority (for the unlikely event where there are multiple
68  * components available on a given platform).
69  *
70  * Note that the only real paffinity module that matters these days is
71  * hwloc. Its module functions will return OPAL_ERR_NOT_INITIALIZED
72  * if the underlying hwloc system has not yet been initialized (which
73  * may not have occured yet since we tend to only initialize hwloc
74  * if/when necessary -- it's not nice to have N processes on a server
75  * all initialize hwloc simultaneously, because they might all
76  * simultaneously bang on /proc and /syst, etc.).
77  *
78  * The module has the following functions:
79  *
80  * - module_init: initialze the module
81  * - set: set this process's affinity to a specific processor set
82  * - get: get this process's processor affinity set
83  * - map physical (socket ID, core ID) -> physical processor ID
84  * - map physical processor ID -> physical (socket ID, core ID)
85  * - get the number of logical processors
86  * - get the number of logical sockets
87  * - get the number of logical cores on a specific socket
88  * - map logical processor ID -> physical processor ID
89  * - map logical socket ID -> physical socket ID
90  * - map physical socket ID, logical core ID -> physical core ID
91  * - module_finalize: finalize the module
92  */
93 
94 #ifndef OPAL_PAFFINITY_H
95 #define OPAL_PAFFINITY_H
96 
97 #include "opal_config.h"
98 
99 #ifdef HAVE_STRING_H
100 #include <string.h>
101 #endif
102 
103 #include "opal/mca/mca.h"
104 #include "opal/mca/base/base.h"
105 
106 /* ******************************************************************** */
107 typedef uint16_t opal_paffinity_locality_t;
108 
109 /** Process locality definitions */
110 #define OPAL_PROC_LOCALITY_UNKNOWN 0x0000
111 #define OPAL_PROC_NON_LOCAL 0x8000
112 #define OPAL_PROC_ON_CLUSTER 0x0400
113 #define OPAL_PROC_ON_CU 0x0200
114 #define OPAL_PROC_ON_NODE 0x0100
115 #define OPAL_PROC_ON_BOARD 0x0080
116 #define OPAL_PROC_ON_NUMA 0x0040
117 #define OPAL_PROC_ON_SOCKET 0x0020
118 #define OPAL_PROC_ON_L3CACHE 0x0010
119 #define OPAL_PROC_ON_L2CACHE 0x0008
120 #define OPAL_PROC_ON_L1CACHE 0x0004
121 #define OPAL_PROC_ON_CORE 0x0002
122 #define OPAL_PROC_ON_HWTHREAD 0x0001
123 #define OPAL_PROC_ALL_LOCAL 0x0fff
124 
125 /** Process locality macros */
126 #define OPAL_PROC_ON_LOCAL_HWTHREAD(n) ((n) & OPAL_PROC_ON_HWTHREAD)
127 #define OPAL_PROC_ON_LOCAL_CORE(n) ((n) & OPAL_PROC_ON_CORE)
128 #define OPAL_PROC_ON_LOCAL_L1CACHE(n) ((n) & OPAL_PROC_ON_L1CACHE)
129 #define OPAL_PROC_ON_LOCAL_L2CACHE(n) ((n) & OPAL_PROC_ON_L2CACHE)
130 #define OPAL_PROC_ON_LOCAL_L3CACHE(n) ((n) & OPAL_PROC_ON_L3CACHE)
131 #define OPAL_PROC_ON_LOCAL_SOCKET(n) ((n) & OPAL_PROC_ON_SOCKET)
132 #define OPAL_PROC_ON_LOCAL_NUMA(n) ((n) & OPAL_PROC_ON_NUMA)
133 #define OPAL_PROC_ON_LOCAL_BOARD(n) ((n) & OPAL_PROC_ON_BOARD)
134 #define OPAL_PROC_ON_LOCAL_NODE(n) ((n) & OPAL_PROC_ON_NODE)
135 #define OPAL_PROC_ON_LOCAL_CU(n) ((n) & OPAL_PROC_ON_CU)
136 #define OPAL_PROC_ON_LOCAL_CLUSTER(n) ((n) & OPAL_PROC_ON_CLUSTER)
137 
138 /* Process binding modes */
139 #define OPAL_PAFFINITY_DO_NOT_BIND 0x01
140 #define OPAL_PAFFINITY_BIND_TO_CORE 0x02
141 #define OPAL_PAFFINITY_BIND_TO_SOCKET 0x04
142 #define OPAL_PAFFINITY_BIND_TO_BOARD 0x08
143 #define OPAL_PAFFINITY_BIND_IF_SUPPORTED 0x80
144 /* ******************************************************************** */
145 
146 
147 /**
148  * Buffer type for paffinity processor masks.
149  * Copied almost directly from PLPA.
150  */
151 
152 /**
153  * \internal
154  * Internal type used for the underlying bitmask unit
155  */
156 typedef unsigned long int opal_paffinity_base_bitmask_t;
157 
158 /**
159  * \internal
160  * Number of bits in opal_paffinity_base_bitmask_t
161  */
162 #define OPAL_PAFFINITY_BITMASK_T_NUM_BITS (sizeof(opal_paffinity_base_bitmask_t) * 8)
163 /**
164  * \internal
165  * How many bits we want
166  */
167 #define OPAL_PAFFINITY_BITMASK_CPU_MAX 1024
168 /**
169  * \internal
170  * How many opal_paffinity_base_bitmask_t's we need
171  */
172 #define OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS (OPAL_PAFFINITY_BITMASK_CPU_MAX / OPAL_PAFFINITY_BITMASK_T_NUM_BITS)
173 
174 /**
175  * \internal
176  * How many bytes in a cpu set
177  */
178 #define OPAL_PAFFINITY_CPU_SET_NUM_BYTES (OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS * sizeof(opal_paffinity_base_bitmask_t))
179 
180 /**
181  * Public processor bitmask type
182  */
184  opal_paffinity_base_bitmask_t bitmask[OPAL_PAFFINITY_BITMASK_NUM_ELEMENTS];
186 
187 /***************************************************************************/
188 
189 /**
190  * \internal
191  * Internal macro for identifying the byte in a bitmask array
192  */
193 #define OPAL_PAFFINITY_CPU_BYTE(num) ((num) / OPAL_PAFFINITY_BITMASK_T_NUM_BITS)
194 
195 /**
196  * \internal
197  * Internal macro for identifying the bit in a bitmask array
198  */
199 #define OPAL_PAFFINITY_CPU_BIT(num) ((num) % OPAL_PAFFINITY_BITMASK_T_NUM_BITS)
200 
201 /***************************************************************************/
202 
203 /**
204  * Public macro to zero out a OPAL_PAFFINITY cpu set
205  */
206 #define OPAL_PAFFINITY_CPU_ZERO(cpuset) \
207  memset(&(cpuset), 0, sizeof(opal_paffinity_base_cpu_set_t))
208 
209 /**
210  * Public macro to set a bit in a OPAL_PAFFINITY cpu set
211  */
212 #define OPAL_PAFFINITY_CPU_SET(num, cpuset) \
213  (cpuset).bitmask[OPAL_PAFFINITY_CPU_BYTE(num)] |= ((opal_paffinity_base_bitmask_t) 1 << OPAL_PAFFINITY_CPU_BIT(num))
214 
215 /**
216  * Public macro to clear a bit in a OPAL_PAFFINITY cpu set
217  */
218 #define OPAL_PAFFINITY_CPU_CLR(num, cpuset) \
219  (cpuset).bitmask[OPAL_PAFFINITY_CPU_BYTE(num)] &= ~((opal_paffinity_base_bitmask_t) 1 << OPAL_PAFFINITY_CPU_BIT(num))
220 
221 /**
222  * Public macro to test if a bit is set in a OPAL_PAFFINITY cpu set
223  */
224 #define OPAL_PAFFINITY_CPU_ISSET(num, cpuset) \
225  (0 != (((cpuset).bitmask[OPAL_PAFFINITY_CPU_BYTE(num)]) & ((opal_paffinity_base_bitmask_t) 1 << OPAL_PAFFINITY_CPU_BIT(num))))
226 
227 /**
228  * Public macro to test if a process is bound anywhere
229  */
230 #define OPAL_PAFFINITY_PROCESS_IS_BOUND(cpuset, bound) \
231  do { \
232  int i, num_processors, num_bound; \
233  *(bound) = false; \
234  if (OPAL_SUCCESS == \
235  opal_paffinity_base_get_processor_info(&num_processors)) { \
236  num_bound = 0; \
237  for (i = 0; i < OPAL_PAFFINITY_BITMASK_CPU_MAX; i++) { \
238  if (OPAL_PAFFINITY_CPU_ISSET(i, (cpuset))) { \
239  num_bound++; \
240  } \
241  } \
242  if (0 < num_bound && (1 == num_processors || \
243  num_bound < num_processors)) { \
244  *(bound) = true; \
245  } \
246  } \
247  } while(0);
248 
249 /***************************************************************************/
250 
251 /**
252  * Module initialization function. Should return OPAL_SUCCESS.
253  */
255 
256 /**
257  * Module function to set this process' affinity to a specific set of
258  * PHYSICAL CPUs.
259  */
261 
262 
263 /**
264  * Module function to get this process' affinity to a specific set of
265  * PHYSICAL CPUs. Returns any binding in the cpumask. This function -only-
266  * returns something other than OPAL_SUCCESS if an actual error is encountered.
267  * You will need to check the mask to find out if this process is actually
268  * bound somewhere specific - a macro for that purpose is provided above
269  */
271 
272 /**
273  * Returns mapping of PHYSICAL socket:core -> PHYSICAL processor id.
274  *
275  * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
276  * supported
277  */
279  int physical_core,
280  int *physical_processor_id);
281 
282 /**
283  * Provides mapping of PHYSICAL processor id -> PHYSICAL socket:core.
284  *
285  * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
286  * supported
287  */
288 typedef int (*opal_paffinity_base_module_get_map_to_socket_core_fn_t)(int physical_processor_id,
289  int *physical_socket,
290  int *physical_core);
291 
292 /**
293  * Provides number of LOGICAL processors in a host.
294  *
295  * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
296  * supported
297  */
298 typedef int (*opal_paffinity_base_module_get_processor_info_fn_t)(int *num_processors);
299 
300 /**
301  * Provides the number of LOGICAL sockets in a host.
302  *
303  * Return OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
304  * supported
305  */
306 typedef int (*opal_paffinity_base_module_get_socket_info_fn_t)(int *num_sockets);
307 
308 /**
309  * Provides the number of LOGICAL cores in a PHYSICAL socket.
310  *
311  * Returns OPAL_SUCCESS or OPAL_ERR_NOT_SUPPORTED if not
312  * supported.
313  */
314 typedef int (*opal_paffinity_base_module_get_core_info_fn_t)(int physical_socket, int *num_cores);
315 
316 /**
317  * Provide the PHYSICAL processor ID that corresponds to the
318  * given LOGICAL processor ID.
319  *
320  * Return OPAL_ERR_NOT_SUPPORTED if not supported.
321  */
322 typedef int (*opal_paffinity_base_module_get_physical_processor_id_fn_t)(int logical_processor_id, int *physical_processor_id);
323 
324 /**
325  * Provide the PHYSICAL socket ID that corresponds to the given
326  * LOGICAL socket ID.
327  *
328  * Return OPAL_ERR_NOT_SUPPORTED if not supported.
329  */
330 typedef int (*opal_paffinity_base_module_get_physical_socket_id_fn_t)(int logical_socket_id, int *physical_socket_id);
331 
332 /**
333  * Provide the PHYSICAL core ID that corresponds to the given LOGICAL
334  * core ID on the given PHYSICAL socket ID.
335  *
336  * Return OPAL_ERR_NOT_SUPPORTED if not supported.
337  */
338 typedef int (*opal_paffinity_base_module_get_physical_core_id_fn_t)(int physical_socket_id, int logical_core_id, int *physical_core_id);
339 
340 
341 /**
342  * Module finalize function. Invoked by the base on the selected
343  * module when the paffinity framework is being shut down.
344  */
346 
347 
348 /**
349  * Structure for paffinity components.
350  */
352  /** MCA base component */
354  /** MCA base data */
356 };
357 /**
358  * Convenience typedef
359  */
362 
363 
364 /**
365  * Structure for paffinity modules
366  */
368  /** Module initialization function */
370 
371  /** Set this process' affinity */
373 
374  /** Get this process' affinity */
376 
377  /** Map socket:core to processor ID */
379 
380  /** Map processor ID to socket:core */
382 
383  /** Return the max processor ID */
385 
386  /** Return the max socket number */
388 
389  /** Return the max core number */
391 
392  /* Return physical processor id */
393  opal_paffinity_base_module_get_physical_processor_id_fn_t paff_get_physical_processor_id;
394 
395  /* Return physical socket id */
396  opal_paffinity_base_module_get_physical_socket_id_fn_t paff_get_physical_socket_id;
397 
398  /* Return physical core id */
400 
401  /** Shut down this module */
403 };
404 /**
405  * Convenience typedef
406  */
409 
410 
411 /*
412  * Macro for use in components that are of type paffinity
413  */
414 #define OPAL_PAFFINITY_BASE_VERSION_2_0_1 \
415  MCA_BASE_VERSION_2_0_0, \
416  "paffinity", 2, 0, 1
417 
418 #endif /* OPAL_PAFFINITY_H */
opal_paffinity_base_module_get_map_to_socket_core_fn_t paff_get_map_to_socket_core
Map processor ID to socket:core.
Definition: paffinity.h:381
Common type for all MCA components.
Definition: mca.h:250
opal_paffinity_base_module_get_processor_info_fn_t paff_get_processor_info
Return the max processor ID.
Definition: paffinity.h:384
Public processor bitmask type.
Definition: paffinity.h:183
opal_paffinity_base_module_finalize_fn_t paff_module_finalize
Shut down this module.
Definition: paffinity.h:402
int(* opal_paffinity_base_module_get_processor_info_fn_t)(int *num_processors)
Provides number of LOGICAL processors in a host.
Definition: paffinity.h:298
opal_paffinity_base_module_get_fn_t paff_module_get
Get this process' affinity.
Definition: paffinity.h:375
int(* opal_paffinity_base_module_finalize_fn_t)(void)
Module finalize function.
Definition: paffinity.h:345
int(* opal_paffinity_base_module_get_physical_core_id_fn_t)(int physical_socket_id, int logical_core_id, int *physical_core_id)
Provide the PHYSICAL core ID that corresponds to the given LOGICAL core ID on the given PHYSICAL sock...
Definition: paffinity.h:338
mca_base_component_data_t base_data
MCA base data.
Definition: paffinity.h:355
int(* opal_paffinity_base_module_get_physical_processor_id_fn_t)(int logical_processor_id, int *physical_processor_id)
Provide the PHYSICAL processor ID that corresponds to the given LOGICAL processor ID...
Definition: paffinity.h:322
Top-level interface for all MCA components.
int(* opal_paffinity_base_module_get_physical_socket_id_fn_t)(int logical_socket_id, int *physical_socket_id)
Provide the PHYSICAL socket ID that corresponds to the given LOGICAL socket ID.
Definition: paffinity.h:330
int(* opal_paffinity_base_module_get_map_to_socket_core_fn_t)(int physical_processor_id, int *physical_socket, int *physical_core)
Provides mapping of PHYSICAL processor id -> PHYSICAL socket:core.
Definition: paffinity.h:288
opal_paffinity_base_module_init_1_1_0_fn_t paff_module_init
Module initialization function.
Definition: paffinity.h:369
Meta data for MCA v2.0.0 components.
Definition: mca.h:309
opal_paffinity_base_module_get_socket_info_fn_t paff_get_socket_info
Return the max socket number.
Definition: paffinity.h:387
int(* opal_paffinity_base_module_init_1_1_0_fn_t)(void)
Module initialization function.
Definition: paffinity.h:254
int(* opal_paffinity_base_module_get_socket_info_fn_t)(int *num_sockets)
Provides the number of LOGICAL sockets in a host.
Definition: paffinity.h:306
opal_paffinity_base_module_get_map_to_processor_id_fn_t paff_get_map_to_processor_id
Map socket:core to processor ID.
Definition: paffinity.h:378
opal_paffinity_base_module_set_fn_t paff_module_set
Set this process' affinity.
Definition: paffinity.h:372
mca_base_component_t base_version
MCA base component.
Definition: paffinity.h:353
int(* opal_paffinity_base_module_set_fn_t)(opal_paffinity_base_cpu_set_t cpumask)
Module function to set this process' affinity to a specific set of PHYSICAL CPUs. ...
Definition: paffinity.h:260
unsigned long int opal_paffinity_base_bitmask_t
Buffer type for paffinity processor masks.
Definition: paffinity.h:156
opal_paffinity_base_module_get_core_info_fn_t paff_get_core_info
Return the max core number.
Definition: paffinity.h:390
Structure for paffinity modules.
Definition: paffinity.h:367
struct opal_paffinity_base_cpu_set_t opal_paffinity_base_cpu_set_t
Public processor bitmask type.
int(* opal_paffinity_base_module_get_map_to_processor_id_fn_t)(int physical_socket, int physical_core, int *physical_processor_id)
Returns mapping of PHYSICAL socket:core -> PHYSICAL processor id.
Definition: paffinity.h:278
Structure for paffinity components.
Definition: paffinity.h:351
int(* opal_paffinity_base_module_get_fn_t)(opal_paffinity_base_cpu_set_t *cpumask)
Module function to get this process' affinity to a specific set of PHYSICAL CPUs. ...
Definition: paffinity.h:270
int(* opal_paffinity_base_module_get_core_info_fn_t)(int physical_socket, int *num_cores)
Provides the number of LOGICAL cores in a PHYSICAL socket.
Definition: paffinity.h:314