OpenMPI  0.1.1
opal_cr.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2005 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
13  * $COPYRIGHT$
14  *
15  * Additional copyrights may follow
16  *
17  * $HEADER$
18  */
19 
20 /**
21  * @file
22  *
23  * Checkpoint functionality for Open MPI
24  */
25 
26 #include "opal_config.h"
27 #include "opal/mca/crs/crs.h"
28 #include "opal/mca/event/event.h"
29 #include "opal/util/output.h"
30 #include "opal/prefetch.h"
31 
32 #ifndef OPAL_CR_H
33 #define OPAL_CR_H
34 
35 
36 BEGIN_C_DECLS
37 
38 /*
39  * Some defines shared with opal-[checkpoint|restart] commands
40  */
41 #define OPAL_CR_DONE ((char) 0)
42 #define OPAL_CR_ACK ((char) 1)
43 #define OPAL_CR_CHECKPOINT ((char) 2)
44 #define OPAL_CR_NAMED_PROG_R ("opal_cr_prog_read")
45 #define OPAL_CR_NAMED_PROG_W ("opal_cr_prog_write")
46 #define OPAL_CR_BASE_ENV_NAME ("opal_cr_restart-env")
47 
48 /*
49  * Possible responses to a checkpoint request from opal-checkpoint
50  */
51 enum opal_cr_ckpt_cmd_state_t {
52  OPAL_CHECKPOINT_CMD_START, /* Checkpoint is starting on this request */
53  OPAL_CHECKPOINT_CMD_IN_PROGRESS, /* Checkpoint is currently running */
54  OPAL_CHECKPOINT_CMD_NULL, /* Checkpoint cannot be started because it is not supported */
55  OPAL_CHECKPOINT_CMD_ERROR, /* An error occurred such that the checkpoint cannot be completed */
56  /* State of the checkpoint operation */
57  OPAL_CR_STATUS_NONE, /* No checkpoint in progress */
58  OPAL_CR_STATUS_REQUESTED, /* Checkpoint has been requested */
59  OPAL_CR_STATUS_RUNNING, /* Checkpoint is currently running */
60  OPAL_CR_STATUS_TERM, /* Checkpoint is running and will terminate process upon completion */
61  /* State of the continue operation */
62  OPAL_CR_STATUS_CONTINUE,
63  /* State of the restart operation */
64  OPAL_CR_STATUS_RESTART_PRE,
65  OPAL_CR_STATUS_RESTART_POST
66 };
67 typedef enum opal_cr_ckpt_cmd_state_t opal_cr_ckpt_cmd_state_t;
68 
69  /* An output handle to be used by the cr runtime
70  * functionality as an argument to opal_output() */
71  OPAL_DECLSPEC extern int opal_cr_output;
72 
73  /* Directory containing the named pipes for communication
74  * with the opal-checkpoint tool */
75  OPAL_DECLSPEC extern char * opal_cr_pipe_dir;
76 
77  /* Signal that opal-checkpoint uses to contact the
78  * application process */
79  OPAL_DECLSPEC extern int opal_cr_entry_point_signal;
80 
81  /* If Checkpointing is enabled in this application */
82  OPAL_DECLSPEC extern bool opal_cr_is_enabled;
83 
84  /* If the application running is a tool
85  * (e.g., opal-checkpoint, orted, ...) */
86  OPAL_DECLSPEC extern bool opal_cr_is_tool;
87 
88  /* If a checkpoint has been requested */
89  OPAL_DECLSPEC extern int opal_cr_checkpoint_request;
90 
91  /* The current state of a checkpoint operation */
92  OPAL_DECLSPEC extern int opal_cr_checkpointing_state;
93 
94 #if OPAL_ENABLE_CRDEBUG == 1
95  /* Whether or not C/R Debugging is enabled for this process */
96  OPAL_DECLSPEC extern int MPIR_debug_with_checkpoint;
97 
98  /*
99  * Set/clear the current thread id for the checkpointing thread
100  */
101  OPAL_DECLSPEC int opal_cr_debug_set_current_ckpt_thread_self(void);
102  OPAL_DECLSPEC int opal_cr_debug_clear_current_ckpt_thread(void);
103 
104  /*
105  * This MPI Debugger function needs to be accessed here and have a specific
106  * name. Thus we are breaking the traditional naming conventions to provide this functionality.
107  */
108  OPAL_DECLSPEC int MPIR_checkpoint_debugger_detach(void);
109 
110  /**
111  * A tight loop to wait for debugger to release this process from the
112  * breakpoint.
113  */
114  OPAL_DECLSPEC void *MPIR_checkpoint_debugger_breakpoint(void);
115 
116  /**
117  * A function for the debugger or CRS to force all threads into
118  */
119  OPAL_DECLSPEC void *MPIR_checkpoint_debugger_waitpoint(void);
120 
121  /**
122  * A signal handler to force all threads to wait when debugger detaches
123  */
124  OPAL_DECLSPEC void MPIR_checkpoint_debugger_signal_handler(int signo);
125 #endif
126 
127  /*
128  * Refresh environment variables after a restart
129  */
130  OPAL_DECLSPEC int opal_cr_refresh_environ(int prev_pid);
131 
132  /*
133  * If this is an application that doesn't want to have
134  * a notification callback installed, set this to false.
135  * To see the effect, this must be called before opal_cr_init().
136  * Default: Enabled
137  */
138  OPAL_DECLSPEC int opal_cr_set_enabled(bool);
139 
140  /**
141  * Initialize the notification and coordination
142  * elements.
143  */
144  OPAL_DECLSPEC int opal_cr_init(void);
145 
146  /**
147  * Finalize the notification and coordination
148  * elements.
149  */
150  OPAL_DECLSPEC int opal_cr_finalize(void);
151 
152  /*************************************************
153  * Check to see if a checkpoint has been requested
154  *
155  * When the checkpoint thread is disabled:
156  * This will be checked whenever the MPI Library
157  * is entered by the application. It will stop
158  * the application for the duration of the entire
159  * checkpoint.
160  * When the checkpoint thread is enabled:
161  * The request is handled in the thread parallel
162  * with the execution of the program regardless
163  * of where the program is in exection.
164  * The problem with this method is that it
165  * requires the support of progress threads
166  * which is currently not working properly :/
167  *
168  *************************************************/
169  OPAL_DECLSPEC void opal_cr_test_if_checkpoint_ready(void);
170 
171  /* If the checkpoint operation should be stalled to
172  * wait for another sevice to complete before
173  * continuing with the checkpoint */
174  OPAL_DECLSPEC extern bool opal_cr_stall_check;
175  OPAL_DECLSPEC extern bool opal_cr_currently_stalled;
176 
177 #if OPAL_ENABLE_FT_THREAD == 1
178  /* Some thread functions */
179  OPAL_DECLSPEC void opal_cr_thread_init_library(void);
180  OPAL_DECLSPEC void opal_cr_thread_finalize_library(void);
181  OPAL_DECLSPEC void opal_cr_thread_abort_library(void);
182  OPAL_DECLSPEC void opal_cr_thread_enter_library(void);
183  OPAL_DECLSPEC void opal_cr_thread_exit_library(void);
184  OPAL_DECLSPEC void opal_cr_thread_noop_progress(void);
185 #endif /* OPAL_ENABLE_FT_THREAD == 1 */
186 
187  /*
188  * If not using FT then make the #defines noops
189  */
190 #if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0
191 #define OPAL_CR_TEST_CHECKPOINT_READY() ;
192 #define OPAL_CR_TEST_CHECKPOINT_READY_STALL() ;
193 #define OPAL_CR_INIT_LIBRARY() ;
194 #define OPAL_CR_FINALIZE_LIBRARY() ;
195 #define OPAL_CR_ABORT_LIBRARY() ;
196 #define OPAL_CR_ENTER_LIBRARY() ;
197 #define OPAL_CR_EXIT_LIBRARY() ;
198 #define OPAL_CR_NOOP_PROGRESS() ;
199 #endif /* #if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0 */
200 
201  /*
202  * If using FT
203  */
204 #if OPAL_ENABLE_FT_CR == 1
205 #define OPAL_CR_TEST_CHECKPOINT_READY() \
206  { \
207  if(OPAL_UNLIKELY(opal_cr_is_enabled) ) { \
208  opal_cr_test_if_checkpoint_ready(); \
209  } \
210  }
211 
212 #define OPAL_CR_TEST_CHECKPOINT_READY_STALL() \
213  { \
214  if(OPAL_UNLIKELY(opal_cr_is_enabled && !opal_cr_stall_check)) { \
215  opal_cr_test_if_checkpoint_ready(); \
216  } \
217  }
218 
219 /* If *not* using FT thread */
220 #if OPAL_ENABLE_FT_THREAD == 0
221 #define OPAL_CR_INIT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
222 #define OPAL_CR_FINALIZE_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
223 #define OPAL_CR_ABORT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
224 #define OPAL_CR_ENTER_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
225 #define OPAL_CR_EXIT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
226 #define OPAL_CR_NOOP_PROGRESS() OPAL_CR_TEST_CHECKPOINT_READY();
227 #endif /* OPAL_ENABLE_FT_THREAD == 0 */
228 
229 /* If using FT thread */
230 #if OPAL_ENABLE_FT_THREAD == 1
231 #define OPAL_CR_INIT_LIBRARY() \
232  { \
233  opal_cr_thread_init_library(); \
234  }
235 #define OPAL_CR_FINALIZE_LIBRARY() \
236  { \
237  opal_cr_thread_finalize_library(); \
238  }
239 #define OPAL_CR_ABORT_LIBRARY() \
240  { \
241  opal_cr_thread_abort_library(); \
242  }
243 #define OPAL_CR_ENTER_LIBRARY() \
244  { \
245  opal_cr_thread_enter_library(); \
246  }
247 #define OPAL_CR_EXIT_LIBRARY() \
248  { \
249  opal_cr_thread_exit_library(); \
250  }
251 #define OPAL_CR_NOOP_PROGRESS() \
252  { \
253  opal_cr_thread_noop_progress(); \
254  }
255 #endif /* OPAL_ENABLE_FT_THREAD == 1 */
256 
257 #endif /* OPAL_ENABLE_FT_CR == 1 */
258 
259  /*******************************
260  * Notification Routines
261  *******************************/
262  /*******************************
263  * Notification Routines
264  *******************************/
265  /**
266  * A function to respond to the async checkpoint request
267  * this is useful when figuring out who should respond
268  * when stalling.
269  */
270  typedef int (*opal_cr_notify_callback_fn_t) (opal_cr_ckpt_cmd_state_t);
271 
272  OPAL_DECLSPEC int opal_cr_reg_notify_callback
274  opal_cr_notify_callback_fn_t *prev_func);
275 
276  /**
277  * Function to go through the INC
278  * - Call Registered INC_Coord(CHECKPOINT)
279  * - Call the CRS.checkpoint()
280  * - Call Registered INC_Coord(state)
281  */
282  OPAL_DECLSPEC int opal_cr_inc_core(pid_t pid,
283  opal_crs_base_snapshot_t *snapshot,
285  int *state);
286 
287  OPAL_DECLSPEC int opal_cr_inc_core_prep(void);
288  OPAL_DECLSPEC int opal_cr_inc_core_ckpt(pid_t pid,
289  opal_crs_base_snapshot_t *snapshot,
291  int *state);
292  OPAL_DECLSPEC int opal_cr_inc_core_recover(int state);
293 
294 
295  /*******************************
296  * User Coordination Routines
297  *******************************/
298  typedef enum {
299  OMPI_CR_INC_PRE_CRS_PRE_MPI = 0,
300  OMPI_CR_INC_PRE_CRS_POST_MPI = 1,
301  OMPI_CR_INC_CRS_PRE_CKPT = 2,
302  OMPI_CR_INC_CRS_POST_CKPT = 3,
303  OMPI_CR_INC_POST_CRS_PRE_MPI = 4,
304  OMPI_CR_INC_POST_CRS_POST_MPI = 5,
305  OMPI_CR_INC_MAX = 6
306  } opal_cr_user_inc_callback_event_t;
307 
308  typedef enum {
309  OMPI_CR_INC_STATE_PREPARE = 0,
310  OMPI_CR_INC_STATE_CONTINUE = 1,
311  OMPI_CR_INC_STATE_RESTART = 2,
312  OMPI_CR_INC_STATE_ERROR = 3
313  } opal_cr_user_inc_callback_state_t;
314 
315  /**
316  * User coordination callback routine
317  */
318  typedef int (*opal_cr_user_inc_callback_fn_t)(opal_cr_user_inc_callback_event_t event,
319  opal_cr_user_inc_callback_state_t state);
320 
321  OPAL_DECLSPEC int opal_cr_user_inc_register_callback
322  (opal_cr_user_inc_callback_event_t event,
324  opal_cr_user_inc_callback_fn_t *prev_function);
325 
326  OPAL_DECLSPEC int trigger_user_inc_callback(opal_cr_user_inc_callback_event_t event,
327  opal_cr_user_inc_callback_state_t state);
328 
329 
330  /*******************************
331  * Coordination Routines
332  *******************************/
333  /**
334  * Coordination callback routine signature
335  */
336  typedef int (*opal_cr_coord_callback_fn_t) (int);
337 
338  /**
339  * Register a checkpoint coodination routine
340  * for a higher level.
341  */
342  OPAL_DECLSPEC int opal_cr_reg_coord_callback
343  (opal_cr_coord_callback_fn_t new_func,
344  opal_cr_coord_callback_fn_t *prev_func);
345 
346  /**
347  * OPAL Checkpoint Coordination Routine
348  */
349  OPAL_DECLSPEC int opal_cr_coord(int state);
350 
351  /**
352  * Checkpoint life-cycle timing
353  */
354  OPAL_DECLSPEC void opal_cr_set_time(int idx);
355  OPAL_DECLSPEC void opal_cr_display_all_timers(void);
356  OPAL_DECLSPEC void opal_cr_clear_timers(void);
357 
358  OPAL_DECLSPEC extern bool opal_cr_timing_enabled;
359  OPAL_DECLSPEC extern bool opal_cr_timing_barrier_enabled;
360  OPAL_DECLSPEC extern int opal_cr_timing_my_rank;
361  OPAL_DECLSPEC extern int opal_cr_timing_target_rank;
362 
363 
364 #define OPAL_CR_TIMER_ENTRY0 0
365 #define OPAL_CR_TIMER_ENTRY1 1
366 #define OPAL_CR_TIMER_ENTRY2 2
367 #define OPAL_CR_TIMER_CRCPBR0 3
368 #define OPAL_CR_TIMER_CRCP0 4
369 #define OPAL_CR_TIMER_CRCPBR1 5
370 #define OPAL_CR_TIMER_P2P0 6
371 #define OPAL_CR_TIMER_P2P1 7
372 #define OPAL_CR_TIMER_P2PBR0 8
373 #define OPAL_CR_TIMER_CORE0 9
374 #define OPAL_CR_TIMER_CORE1 10
375 #define OPAL_CR_TIMER_COREBR0 11
376 #define OPAL_CR_TIMER_P2P2 12
377 #define OPAL_CR_TIMER_P2PBR1 13
378 #define OPAL_CR_TIMER_P2P3 14
379 #define OPAL_CR_TIMER_P2PBR2 15
380 #define OPAL_CR_TIMER_CRCP1 16
381 #define OPAL_CR_TIMER_COREBR1 17
382 #define OPAL_CR_TIMER_CORE2 18
383 #define OPAL_CR_TIMER_ENTRY3 19
384 #define OPAL_CR_TIMER_ENTRY4 20
385 #define OPAL_CR_TIMER_MAX 21
386 
387 
388 #define OPAL_CR_CLEAR_TIMERS() \
389  { \
390  if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
391  opal_cr_clear_timers(); \
392  } \
393  }
394 
395 #define OPAL_CR_SET_TIMER(idx) \
396  { \
397  if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
398  opal_cr_set_time(idx); \
399  } \
400  }
401 
402 #define OPAL_CR_DISPLAY_ALL_TIMERS() \
403  { \
404  if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
405  opal_cr_display_all_timers(); \
406  } \
407  }
408 
409 END_C_DECLS
410 
411 #endif /* OPAL_CR_H */
412 
OPAL_DECLSPEC void opal_cr_set_time(int idx)
Checkpoint life-cycle timing.
Definition: opal_cr.c:1195
OPAL_DECLSPEC int opal_cr_init(void)
Initialize the notification and coordination elements.
Definition: opal_cr.c:197
OPAL output stream facility.
OPAL_DECLSPEC int opal_cr_finalize(void)
Finalize the notification and coordination elements.
Definition: opal_cr.c:468
Structure to represent a single event.
Definition: event_struct.h:87
int(* opal_cr_notify_callback_fn_t)(opal_cr_ckpt_cmd_state_t)
A function to respond to the async checkpoint request this is useful when figuring out who should res...
Definition: opal_cr.h:270
Structure for Single process snapshot Each component is assumed to have extened this definition in th...
Definition: crs.h:107
OPAL_DECLSPEC int opal_cr_coord(int state)
OPAL Checkpoint Coordination Routine.
Definition: opal_cr.c:774
int(* opal_cr_coord_callback_fn_t)(int)
Coordination callback routine signature.
Definition: opal_cr.h:336
Compiler-specific prefetch functions.
OPAL_DECLSPEC int opal_cr_inc_core(pid_t pid, opal_crs_base_snapshot_t *snapshot, opal_crs_base_ckpt_options_t *options, int *state)
Function to go through the INC.
Definition: opal_cr.c:736
int(* opal_cr_user_inc_callback_fn_t)(opal_cr_user_inc_callback_event_t event, opal_cr_user_inc_callback_state_t state)
User coordination callback routine.
Definition: opal_cr.h:318
Checkpoint and Restart Service (CRS) Interface.
OPAL_DECLSPEC int opal_cr_reg_coord_callback(opal_cr_coord_callback_fn_t new_func, opal_cr_coord_callback_fn_t *prev_func)
Register a checkpoint coodination routine for a higher level.
Definition: opal_cr.c:874