OpenMPI 0.1.1

- Main Page
- Related Pages
- Modules
- Data Structures
- Files

- File List
- Globals

opal_cr.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2005 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
13 * $COPYRIGHT$
14 *
15 * Additional copyrights may follow
16 *
17 * $HEADER$
18 */
19
20 /**
21 * @file
22 *
23 * Checkpoint functionality for Open MPI
24 */
25
26 #include "opal_config.h"
27 #include "opal/mca/crs/crs.h"
28 #include "opal/mca/event/event.h"
29 #include "opal/util/output.h"
30 #include "opal/prefetch.h"
31
32 #ifndef OPAL_CR_H
33 #define OPAL_CR_H
34
35
36 BEGIN_C_DECLS
37
38 /*
39 * Some defines shared with opal-[checkpoint|restart] commands
40 */
41 #define OPAL_CR_DONE ((char) 0)
42 #define OPAL_CR_ACK ((char) 1)
43 #define OPAL_CR_CHECKPOINT ((char) 2)
44 #define OPAL_CR_NAMED_PROG_R ("opal_cr_prog_read")
45 #define OPAL_CR_NAMED_PROG_W ("opal_cr_prog_write")
46 #define OPAL_CR_BASE_ENV_NAME ("opal_cr_restart-env")
47
48 /*
49 * Possible responses to a checkpoint request from opal-checkpoint
50 */
51 enum opal_cr_ckpt_cmd_state_t {
52 OPAL_CHECKPOINT_CMD_START, /* Checkpoint is starting on this request */
53 OPAL_CHECKPOINT_CMD_IN_PROGRESS, /* Checkpoint is currently running */
54 OPAL_CHECKPOINT_CMD_NULL, /* Checkpoint cannot be started because it is not supported */
55 OPAL_CHECKPOINT_CMD_ERROR, /* An error occurred such that the checkpoint cannot be completed */
56 /* State of the checkpoint operation */
57 OPAL_CR_STATUS_NONE, /* No checkpoint in progress */
58 OPAL_CR_STATUS_REQUESTED, /* Checkpoint has been requested */
59 OPAL_CR_STATUS_RUNNING, /* Checkpoint is currently running */
60 OPAL_CR_STATUS_TERM, /* Checkpoint is running and will terminate process upon completion */
61 /* State of the continue operation */
62 OPAL_CR_STATUS_CONTINUE,
63 /* State of the restart operation */
64 OPAL_CR_STATUS_RESTART_PRE,
65 OPAL_CR_STATUS_RESTART_POST
66 };
67 typedef enum opal_cr_ckpt_cmd_state_t opal_cr_ckpt_cmd_state_t;
68
69 /* An output handle to be used by the cr runtime
70 * functionality as an argument to opal_output() */
71 OPAL_DECLSPEC extern int opal_cr_output;
72
73 /* Directory containing the named pipes for communication
74 * with the opal-checkpoint tool */
75 OPAL_DECLSPEC extern char * opal_cr_pipe_dir;
76
77 /* Signal that opal-checkpoint uses to contact the
78 * application process */
79 OPAL_DECLSPEC extern int opal_cr_entry_point_signal;
80
81 /* If Checkpointing is enabled in this application */
82 OPAL_DECLSPEC extern bool opal_cr_is_enabled;
83
84 /* If the application running is a tool
85 * (e.g., opal-checkpoint, orted, ...) */
86 OPAL_DECLSPEC extern bool opal_cr_is_tool;
87
88 /* If a checkpoint has been requested */
89 OPAL_DECLSPEC extern int opal_cr_checkpoint_request;
90
91 /* The current state of a checkpoint operation */
92 OPAL_DECLSPEC extern int opal_cr_checkpointing_state;
93
94 #if OPAL_ENABLE_CRDEBUG == 1
95 /* Whether or not C/R Debugging is enabled for this process */
96 OPAL_DECLSPEC extern int MPIR_debug_with_checkpoint;
97
98 /*
99 * Set/clear the current thread id for the checkpointing thread
100 */
101 OPAL_DECLSPEC int opal_cr_debug_set_current_ckpt_thread_self(void);
102 OPAL_DECLSPEC int opal_cr_debug_clear_current_ckpt_thread(void);
103
104 /*
105 * This MPI Debugger function needs to be accessed here and have a specific
106 * name. Thus we are breaking the traditional naming conventions to provide this functionality.
107 */
108 OPAL_DECLSPEC int MPIR_checkpoint_debugger_detach(void);
109
110 /**
111 * A tight loop to wait for debugger to release this process from the
112 * breakpoint.
113 */
114 OPAL_DECLSPEC void *MPIR_checkpoint_debugger_breakpoint(void);
115
116 /**
117 * A function for the debugger or CRS to force all threads into
118 */
119 OPAL_DECLSPEC void *MPIR_checkpoint_debugger_waitpoint(void);
120
121 /**
122 * A signal handler to force all threads to wait when debugger detaches
123 */
124 OPAL_DECLSPEC void MPIR_checkpoint_debugger_signal_handler(int signo);
125 #endif
126
127 /*
128 * Refresh environment variables after a restart
129 */
130 OPAL_DECLSPEC int opal_cr_refresh_environ(int prev_pid);
131
132 /*
133 * If this is an application that doesn't want to have
134 * a notification callback installed, set this to false.
135 * To see the effect, this must be called before opal_cr_init().
136 * Default: Enabled
137 */
138 OPAL_DECLSPEC int opal_cr_set_enabled(bool);
139
140 /**
141 * Initialize the notification and coordination
142 * elements.
143 */
144 OPAL_DECLSPEC int opal_cr_init(void);
145
146 /**
147 * Finalize the notification and coordination
148 * elements.
149 */
150 OPAL_DECLSPEC int opal_cr_finalize(void);
151
152 /*************************************************
153 * Check to see if a checkpoint has been requested
154 *
155 * When the checkpoint thread is disabled:
156 * This will be checked whenever the MPI Library
157 * is entered by the application. It will stop
158 * the application for the duration of the entire
159 * checkpoint.
160 * When the checkpoint thread is enabled:
161 * The request is handled in the thread parallel
162 * with the execution of the program regardless
163 * of where the program is in exection.
164 * The problem with this method is that it
165 * requires the support of progress threads
166 * which is currently not working properly :/
167 *
168 *************************************************/
169 OPAL_DECLSPEC void opal_cr_test_if_checkpoint_ready(void);
170
171 /* If the checkpoint operation should be stalled to
172 * wait for another sevice to complete before
173 * continuing with the checkpoint */
174 OPAL_DECLSPEC extern bool opal_cr_stall_check;
175 OPAL_DECLSPEC extern bool opal_cr_currently_stalled;
176
177 #if OPAL_ENABLE_FT_THREAD == 1
178 /* Some thread functions */
179 OPAL_DECLSPEC void opal_cr_thread_init_library(void);
180 OPAL_DECLSPEC void opal_cr_thread_finalize_library(void);
181 OPAL_DECLSPEC void opal_cr_thread_abort_library(void);
182 OPAL_DECLSPEC void opal_cr_thread_enter_library(void);
183 OPAL_DECLSPEC void opal_cr_thread_exit_library(void);
184 OPAL_DECLSPEC void opal_cr_thread_noop_progress(void);
185 #endif /* OPAL_ENABLE_FT_THREAD == 1 */
186
187 /*
188 * If not using FT then make the #defines noops
189 */
190 #if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0
191 #define OPAL_CR_TEST_CHECKPOINT_READY() ;
192 #define OPAL_CR_TEST_CHECKPOINT_READY_STALL() ;
193 #define OPAL_CR_INIT_LIBRARY() ;
194 #define OPAL_CR_FINALIZE_LIBRARY() ;
195 #define OPAL_CR_ABORT_LIBRARY() ;
196 #define OPAL_CR_ENTER_LIBRARY() ;
197 #define OPAL_CR_EXIT_LIBRARY() ;
198 #define OPAL_CR_NOOP_PROGRESS() ;
199 #endif /* #if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0 */
200
201 /*
202 * If using FT
203 */
204 #if OPAL_ENABLE_FT_CR == 1
205 #define OPAL_CR_TEST_CHECKPOINT_READY() \
206 { \
207 if(OPAL_UNLIKELY(opal_cr_is_enabled) ) { \
208 opal_cr_test_if_checkpoint_ready(); \
209 } \
210 }
211
212 #define OPAL_CR_TEST_CHECKPOINT_READY_STALL() \
213 { \
214 if(OPAL_UNLIKELY(opal_cr_is_enabled && !opal_cr_stall_check)) { \
215 opal_cr_test_if_checkpoint_ready(); \
216 } \
217 }
218
219 /* If *not* using FT thread */
220 #if OPAL_ENABLE_FT_THREAD == 0
221 #define OPAL_CR_INIT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
222 #define OPAL_CR_FINALIZE_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
223 #define OPAL_CR_ABORT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
224 #define OPAL_CR_ENTER_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
225 #define OPAL_CR_EXIT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
226 #define OPAL_CR_NOOP_PROGRESS() OPAL_CR_TEST_CHECKPOINT_READY();
227 #endif /* OPAL_ENABLE_FT_THREAD == 0 */
228
229 /* If using FT thread */
230 #if OPAL_ENABLE_FT_THREAD == 1
231 #define OPAL_CR_INIT_LIBRARY() \
232 { \
233 opal_cr_thread_init_library(); \
234 }
235 #define OPAL_CR_FINALIZE_LIBRARY() \
236 { \
237 opal_cr_thread_finalize_library(); \
238 }
239 #define OPAL_CR_ABORT_LIBRARY() \
240 { \
241 opal_cr_thread_abort_library(); \
242 }
243 #define OPAL_CR_ENTER_LIBRARY() \
244 { \
245 opal_cr_thread_enter_library(); \
246 }
247 #define OPAL_CR_EXIT_LIBRARY() \
248 { \
249 opal_cr_thread_exit_library(); \
250 }
251 #define OPAL_CR_NOOP_PROGRESS() \
252 { \
253 opal_cr_thread_noop_progress(); \
254 }
255 #endif /* OPAL_ENABLE_FT_THREAD == 1 */
256
257 #endif /* OPAL_ENABLE_FT_CR == 1 */
258
259 /*******************************
260 * Notification Routines
261 *******************************/
262 /*******************************
263 * Notification Routines
264 *******************************/
265 /**
266 * A function to respond to the async checkpoint request
267 * this is useful when figuring out who should respond
268 * when stalling.
269 */
270 typedef int (*opal_cr_notify_callback_fn_t) (opal_cr_ckpt_cmd_state_t);
271
272 OPAL_DECLSPEC int opal_cr_reg_notify_callback
273 (opal_cr_notify_callback_fn_t new_func,
274 opal_cr_notify_callback_fn_t *prev_func);
275
276 /**
277 * Function to go through the INC
278 * - Call Registered INC_Coord(CHECKPOINT)
279 * - Call the CRS.checkpoint()
280 * - Call Registered INC_Coord(state)
281 */
282 OPAL_DECLSPEC int opal_cr_inc_core(pid_t pid,
283 opal_crs_base_snapshot_t *snapshot,
284 opal_crs_base_ckpt_options_t *options,
285 int *state);
286
287 OPAL_DECLSPEC int opal_cr_inc_core_prep(void);
288 OPAL_DECLSPEC int opal_cr_inc_core_ckpt(pid_t pid,
289 opal_crs_base_snapshot_t *snapshot,
290 opal_crs_base_ckpt_options_t *options,
291 int *state);
292 OPAL_DECLSPEC int opal_cr_inc_core_recover(int state);
293
294
295 /*******************************
296 * User Coordination Routines
297 *******************************/
298 typedef enum {
299 OMPI_CR_INC_PRE_CRS_PRE_MPI = 0,
300 OMPI_CR_INC_PRE_CRS_POST_MPI = 1,
301 OMPI_CR_INC_CRS_PRE_CKPT = 2,
302 OMPI_CR_INC_CRS_POST_CKPT = 3,
303 OMPI_CR_INC_POST_CRS_PRE_MPI = 4,
304 OMPI_CR_INC_POST_CRS_POST_MPI = 5,
305 OMPI_CR_INC_MAX = 6
306 } opal_cr_user_inc_callback_event_t;
307
308 typedef enum {
309 OMPI_CR_INC_STATE_PREPARE = 0,
310 OMPI_CR_INC_STATE_CONTINUE = 1,
311 OMPI_CR_INC_STATE_RESTART = 2,
312 OMPI_CR_INC_STATE_ERROR = 3
313 } opal_cr_user_inc_callback_state_t;
314
315 /**
316 * User coordination callback routine
317 */
318 typedef int (*opal_cr_user_inc_callback_fn_t)(opal_cr_user_inc_callback_event_t event,
319 opal_cr_user_inc_callback_state_t state);
320
321 OPAL_DECLSPEC int opal_cr_user_inc_register_callback
322 (opal_cr_user_inc_callback_event_t event,
323 opal_cr_user_inc_callback_fn_t function,
324 opal_cr_user_inc_callback_fn_t *prev_function);
325
326 OPAL_DECLSPEC int trigger_user_inc_callback(opal_cr_user_inc_callback_event_t event,
327 opal_cr_user_inc_callback_state_t state);
328
329
330 /*******************************
331 * Coordination Routines
332 *******************************/
333 /**
334 * Coordination callback routine signature
335 */
336 typedef int (*opal_cr_coord_callback_fn_t) (int);
337
338 /**
339 * Register a checkpoint coodination routine
340 * for a higher level.
341 */
342 OPAL_DECLSPEC int opal_cr_reg_coord_callback
343 (opal_cr_coord_callback_fn_t new_func,
344 opal_cr_coord_callback_fn_t *prev_func);
345
346 /**
347 * OPAL Checkpoint Coordination Routine
348 */
349 OPAL_DECLSPEC int opal_cr_coord(int state);
350
351 /**
352 * Checkpoint life-cycle timing
353 */
354 OPAL_DECLSPEC void opal_cr_set_time(int idx);
355 OPAL_DECLSPEC void opal_cr_display_all_timers(void);
356 OPAL_DECLSPEC void opal_cr_clear_timers(void);
357
358 OPAL_DECLSPEC extern bool opal_cr_timing_enabled;
359 OPAL_DECLSPEC extern bool opal_cr_timing_barrier_enabled;
360 OPAL_DECLSPEC extern int opal_cr_timing_my_rank;
361 OPAL_DECLSPEC extern int opal_cr_timing_target_rank;
362
363
364 #define OPAL_CR_TIMER_ENTRY0 0
365 #define OPAL_CR_TIMER_ENTRY1 1
366 #define OPAL_CR_TIMER_ENTRY2 2
367 #define OPAL_CR_TIMER_CRCPBR0 3
368 #define OPAL_CR_TIMER_CRCP0 4
369 #define OPAL_CR_TIMER_CRCPBR1 5
370 #define OPAL_CR_TIMER_P2P0 6
371 #define OPAL_CR_TIMER_P2P1 7
372 #define OPAL_CR_TIMER_P2PBR0 8
373 #define OPAL_CR_TIMER_CORE0 9
374 #define OPAL_CR_TIMER_CORE1 10
375 #define OPAL_CR_TIMER_COREBR0 11
376 #define OPAL_CR_TIMER_P2P2 12
377 #define OPAL_CR_TIMER_P2PBR1 13
378 #define OPAL_CR_TIMER_P2P3 14
379 #define OPAL_CR_TIMER_P2PBR2 15
380 #define OPAL_CR_TIMER_CRCP1 16
381 #define OPAL_CR_TIMER_COREBR1 17
382 #define OPAL_CR_TIMER_CORE2 18
383 #define OPAL_CR_TIMER_ENTRY3 19
384 #define OPAL_CR_TIMER_ENTRY4 20
385 #define OPAL_CR_TIMER_MAX 21
386
387
388 #define OPAL_CR_CLEAR_TIMERS() \
389 { \
390 if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
391 opal_cr_clear_timers(); \
392 } \
393 }
394
395 #define OPAL_CR_SET_TIMER(idx) \
396 { \
397 if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
398 opal_cr_set_time(idx); \
399 } \
400 }
401
402 #define OPAL_CR_DISPLAY_ALL_TIMERS() \
403 { \
404 if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
405 opal_cr_display_all_timers(); \
406 } \
407 }
408
409 END_C_DECLS
410
411 #endif /* OPAL_CR_H */
412
opal_cr_set_time
OPAL_DECLSPEC void opal_cr_set_time(int idx)
Checkpoint life-cycle timing.
Definition: opal_cr.c:1195
opal_cr_init
OPAL_DECLSPEC int opal_cr_init(void)
Initialize the notification and coordination elements.
Definition: opal_cr.c:197
output.h
OPAL output stream facility.
opal_cr_finalize
OPAL_DECLSPEC int opal_cr_finalize(void)
Finalize the notification and coordination elements.
Definition: opal_cr.c:468
event
Structure to represent a single event.
Definition: event_struct.h:87
opal_cr_notify_callback_fn_t
int(* opal_cr_notify_callback_fn_t)(opal_cr_ckpt_cmd_state_t)
A function to respond to the async checkpoint request this is useful when figuring out who should res...
Definition: opal_cr.h:270
opal_crs_base_snapshot_1_0_0_t
Structure for Single process snapshot Each component is assumed to have extened this definition in th...
Definition: crs.h:107
opal_crs_base_ckpt_options_1_0_0_t
Definition: crs.h:76
opal_cr_coord
OPAL_DECLSPEC int opal_cr_coord(int state)
OPAL Checkpoint Coordination Routine.
Definition: opal_cr.c:774
opal_cr_coord_callback_fn_t
int(* opal_cr_coord_callback_fn_t)(int)
Coordination callback routine signature.
Definition: opal_cr.h:336
prefetch.h
Compiler-specific prefetch functions.
opal_cr_inc_core
OPAL_DECLSPEC int opal_cr_inc_core(pid_t pid, opal_crs_base_snapshot_t *snapshot, opal_crs_base_ckpt_options_t *options, int *state)
Function to go through the INC.
Definition: opal_cr.c:736
opal_cr_user_inc_callback_fn_t
int(* opal_cr_user_inc_callback_fn_t)(opal_cr_user_inc_callback_event_t event, opal_cr_user_inc_callback_state_t state)
User coordination callback routine.
Definition: opal_cr.h:318
crs.h
Checkpoint and Restart Service (CRS) Interface.
opal_cr_reg_coord_callback
OPAL_DECLSPEC int opal_cr_reg_coord_callback(opal_cr_coord_callback_fn_t new_func, opal_cr_coord_callback_fn_t *prev_func)
Register a checkpoint coodination routine for a higher level.
Definition: opal_cr.c:874