New schedule stuff.

IMPORTANT:  if you don't have a src/schedule directory, please
do an "update -d"  or your code won't compile!

You can now create schedule groups and schedule items  (other groups
or functions) IN a group.  You may then schedule a group AT a
particular CCTK entry point, or inside another group.

There is a new keyword WHILE in the schedule specification.  The
argument(s) to this are the full names of integer scalars, and the
item which is scheduled WHILE will be executed while this integer, or
these integers, are non-zero.  Note that currently there is little
checking of the type of these scalars, 'though this will be coming
soon.

BEFORE, AFTER, or WHILE can take a list of arguments of the form
(a,b,c) to stop you having to do something like

BEFORE a BEFORE b BEFORE c

So the full format of a schedule statement is

SCHEDULE [GROUP] <name> <AT entrypoint | IN group> [BEFORE|AFTER <item | (item...)>] [WHILE <integer gridscalar>]
{
[LANG: <language>]
[COMM: <group(s)>]
[STORAGE: <groups(s)]
[TRIGGERS: <groups(s)]
} "<description>"

Note that even for a group you need to provide
{
} "<description>"

'though that may be relaxed soon.

Also TRIGGERS are now on a group basis, not on a variable basis.

So, for example, to test this I had

--------------------------------------

SCHEDULE TEST1_InitialData AT INITIAL
{
  LANG: Fortran
} "Initialise"

SCHEDULE TEST1_InitialData AT EVOL
{
  LANG: Fortran
} "Initialise"

SCHEDULE TEST1_Evolve IN testit
{
  LANG: Fortran
} "Evolution routine"

SCHEDULE GROUP testit WHILE test1::foo_int_scalar AT EVOL AFTER TEST1_InitialData
{
} "Test a group within a group. "

---------------------------------------

where the initialisation routine set test1::foo_int_scalar to 4 and
the evolution routine decremented it.  The group "testit" is redundant here
as only one thing is scheduled in it, the WHILE and AFTER could have gone
directly onto the scheduling of TEST1_Evolve.


Apart from the checking of the while stuff, the perl does a fair amount of
checking that groups exist.  It doesn't yet check that a group you
are scheduling at or in exists, and neither does the C, so please check the
screen output of the final schedule tree.  Checks for this an for
unreachable groups will go in, but perhaps not in the next week.

There's also some tidying up which needs to go in, but this stuff
passes all the tests that the unmodified code does.

I have also tried to make the output of the CST a bit more useful, 'though
now it's a wee bit long-winded.

Please send me comments if you want more output of various things, or would
like the current output, or how the WHILE stuff works, changed.


Tom


IMPORTANT:  if you don't have a src/schedule directory, please
do an "update -d"  or your code won't compile!


git-svn-id: http://svn.cactuscode.org/flesh/trunk@966 17b73243-c579-4c4c-a9d2-2d5706c11dac

10 files changed, 406 insertions, 307 deletions

diff --git a/src/include/cctk_GroupsOnGH.h b/src/include/cctk_GroupsOnGH.h
index fadd7c0b..603365e4 100644
--- a/src/include/cctk_GroupsOnGH.h
+++ b/src/include/cctk_GroupsOnGH.h
@@ -23,6 +23,11 @@ void *CCTK_VarDataPtrI(cGH *GH, int timelevel, int varindex);
 
 void *CCTK_VarDataPtrB(cGH *GH, int timelevel, int varindex, char *fullvarname);
 
+int CCTK_DisableGroupStorageI(void *GH, int group);
+int CCTK_DisableGroupCommI(void *GH, int group);
+int CCTK_EnableGroupStorageI(void *GH, int group);
+int CCTK_EnableGroupCommI(void *GH, int group);
+
 #ifdef __cplusplus 
 }
 #endif
diff --git a/src/main/CactusDefaultInitialise.c b/src/main/CactusDefaultInitialise.c
index 854e3697..410f3b11 100644
--- a/src/main/CactusDefaultInitialise.c
+++ b/src/main/CactusDefaultInitialise.c
@@ -115,21 +115,8 @@ int CactusInitialiseGH(cGH *GH)
 
   GH->rfr_top = NULL;
 
-  rfrInitTree(&(GH->rfr_top), 
-	      CCTKi_rfrStorageOn,
-	      CCTKi_rfrStorageOff,
-	      CCTKi_rfrCommunicationOn,
-	      CCTKi_rfrCommunicationOff,
-	      CCTKi_rfrTriggerable,
-	      CCTKi_rfrTriggerSaysGo,
-	      CCTKi_rfrTriggerAction,
-	      CCTKi_rfrCallFunc);
-
   /* Do the rfr initialisation on this GH */
-  CCTKi_BindingsScheduleRegister("RFRINIT", (void *)GH);
-
-  /* Report the rfr tree */
-  CCTKi_rfrPrintTree(GH,GH->rfr_top);
+  CCTK_ScheduleGHInit((void *)GH);
 
   /* Initialise all the extensions. */
   CCTKi_InitGHExtensions(GH);
diff --git a/src/main/CallStartupFunctions.c b/src/main/CallStartupFunctions.c
index 7db2e5ee..4d921fa3 100644
--- a/src/main/CallStartupFunctions.c
+++ b/src/main/CallStartupFunctions.c
@@ -10,6 +10,7 @@
 #include <stdio.h>
 
 #include "cctk_Flesh.h"
+#include "cctk_schedule.h"
 
 static char *rcsid = "$Id$";
 
@@ -32,7 +33,7 @@ int dummy(tFleshConfig *);
 int CallStartupFunctions(tFleshConfig *ConfigData)
 {
 
-  CCTKi_BindingsScheduleRegister("STARTUP", NULL);
+  CCTK_ScheduleTraverse("CCTK_STARTUP", NULL);
 
   return 0;
 }
diff --git a/src/main/GroupsOnGH.c b/src/main/GroupsOnGH.c
index aefa2a6c..caae5cba 100644
--- a/src/main/GroupsOnGH.c
+++ b/src/main/GroupsOnGH.c
@@ -17,6 +17,7 @@
 #include "cctk_Groups.h"
 #include "cctk_WarnLevel.h"
 #include "cctk_GroupsOnGH.h"
+#include "cctk_Comm.h"
 
 /*#define DEBUG_GROUPS*/
 
@@ -195,3 +196,150 @@ void *CCTK_VarDataPtrB(cGH *GH, int timelevel, int varindex, char *fullvarname)
   }
 }
 
+ /*@@
+   @routine    CCTK_EnableGroupCommI
+   @date       Sat Feb 13 17:06:30 1999
+   @author     Tom Goodale
+   @desc 
+   Enables communication for a group based upon its name.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
+int CCTK_EnableGroupCommI(void *GH, int group)
+{
+  int retcode;
+  char *group_name;
+
+  group_name = CCTK_GroupName(group);
+  if(group_name)
+  {
+#ifdef DEBUG
+    printf("Turning on comm in %s for group %s (%d)\n",__FILE__group_name,group);
+#endif
+    retcode = CCTK_EnableGroupComm(GH, group_name);
+
+    free(group_name);
+  }
+  else
+  {
+    retcode = 0;
+  }
+
+  return retcode;
+}
+
+ /*@@
+   @routine    CCTK_EnableGroupStorageI
+   @date       Sat Feb 13 17:06:30 1999
+   @author     Tom Goodale
+   @desc 
+   Enables storage for a group based upon its name.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
+int CCTK_EnableGroupStorageI(void *GH, int group)
+{
+  int retcode;
+  char *group_name;
+
+  group_name = CCTK_GroupName(group);
+  if(group_name)
+  {
+#ifdef DEBUG
+    printf("Turning on storage in %s for group %s (%d)\n",__FILE__group_name,group);
+#endif
+    retcode = CCTK_EnableGroupStorage(GH, group_name);
+
+    free(group_name);
+  }
+  else
+  {
+    retcode = 0;
+  }
+
+  return retcode;
+}
+
+ /*@@
+   @routine    CCTK_DisableGroupCommI
+   @date       Sat Feb 13 17:06:30 1999
+   @author     Tom Goodale
+   @desc 
+   Routine to switch communication off for a group based upon its index
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
+int CCTK_DisableGroupCommI(void *GH, int group)
+{
+  int retcode;
+  char *group_name;
+
+  group_name = CCTK_GroupName(group);
+  if(group_name)
+  {
+#ifdef DEBUG
+    printf("Turning off comm in %s for group %s (%d)\n",__FILE__,group_name,group);
+#endif
+    retcode = CCTK_DisableGroupComm(GH, group_name);
+
+    free(group_name);
+  }
+  else
+  {
+    retcode = 0;
+  }
+
+  return retcode;
+}
+
+ /*@@
+   @routine    CCTK_DisableGroupStorageI
+   @date       Sat Feb 13 17:06:30 1999
+   @author     Tom Goodale
+   @desc 
+   Routine to switch storage off for a group based upon its index
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
+int CCTK_DisableGroupStorageI(void *GH, int group)
+{
+  int retcode;
+  char *group_name;
+
+  group_name = CCTK_GroupName(group);
+  if(group_name)
+  {
+#ifdef DEBUG
+    printf("Turning off storage in %s for group %s (%d)\n",__FILE__,group_name,group);
+#endif
+    retcode = CCTK_DisableGroupStorage(GH, group_name);
+
+    free(group_name);
+  }
+  else
+  {
+    retcode = 0;
+  }
+
+  return retcode;
+}
diff --git a/src/main/InitialiseCactus.c b/src/main/InitialiseCactus.c
index e5767288..b90211be 100644
--- a/src/main/InitialiseCactus.c
+++ b/src/main/InitialiseCactus.c
@@ -9,7 +9,9 @@
 
 #include <stdio.h>
 
+#include "cctk.h"
 #include "cctk_Flesh.h"
+#include "cctki_schedule.h"
 
 int ProcessCommandLine(int *inargc, char ***inargv, tFleshConfig *ConfigData);
 int ProcessEnvironment(int *argc, char ***argv,tFleshConfig *ConfigData);
@@ -65,12 +67,41 @@ int InitialiseCactus(int *argc, char ***argv, tFleshConfig *ConfigData)
 
   ProcessParameterDatabase(ConfigData);
 
+  InitialiseScheduler(ConfigData);
+
   CallStartupFunctions(ConfigData);
 
   return 0;
 }
 
+ /*@@
+   @routine    InitialiseScheduler
+   @date       Fri Sep 17 19:34:55 1999
+   @author     Tom Goodale
+   @desc 
+   Initialise all scheduled items
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
+int InitialiseScheduler(tFleshConfig *ConfigData)
+{
+  int retcode;
+  
+  CCTKi_BindingsScheduleInitialise();
 
+  retcode = CCTKi_ScheduleSortAllGroups();
+
+  CCTK_PRINTSEPARATOR
+  CCTK_SchedulePrint(NULL);
+  CCTK_PRINTSEPARATOR
+
+  return retcode;
+}
 
 
 
diff --git a/src/main/InitialiseDataStructures.c b/src/main/InitialiseDataStructures.c
index c153cc4b..449d427d 100644
--- a/src/main/InitialiseDataStructures.c
+++ b/src/main/InitialiseDataStructures.c
@@ -52,7 +52,7 @@ int InitialiseDataStructures(tFleshConfig *ConfigData)
   CCTKi_BindingsImplementationsInitialise();
   /*  CCTKi_BindingsParametersInitialise();*/
   CCTKi_BindingsVariablesInitialise();
-  CCTKi_BindingsScheduleInitialise();
+  /*CCTKi_BindingsScheduleInitialise();*/
 
   CCTKi_ActivateThorn("Cactus");
 
diff --git a/src/main/ScheduleInterface.c b/src/main/ScheduleInterface.c
index 8d410c98..5001450f 100644
--- a/src/main/ScheduleInterface.c
+++ b/src/main/ScheduleInterface.c
@@ -1,4 +1,3 @@
-#define DEBUG
  /*@@
    @file      ScheduleInterface.c
    @date      Thu Sep 16 14:06:21 1999
@@ -70,7 +69,9 @@ typedef struct
 {
   cGH *GH;
   t_schedpoint schedpoint;
-  
+
+  int whiling;
+
 } t_sched_data;
 
 
@@ -180,9 +181,8 @@ int CCTK_ScheduleFunction(void *function,
   }
   else
   {
-#ifdef DEBUG
-    fprintf(stderr, "Failed to schedule %s at %s!!!\n", name, where);
-#endif
+    fprintf(stderr, "Internal error: Failed to schedule %s at %s!!!\n", name, where);
+    exit(2);
     retcode = -1;
   }
 
@@ -429,32 +429,33 @@ int CCTK_SchedulePrint(const char *where)
 
   data.GH = NULL;
   data.schedpoint = schedpoint_misc;
+  data.whiling = 0;
 
   if(!where)
   {
-    printf ("Startup routines\n");
+    printf ("  Startup routines\n");
     SchedulePrint("CCTK_STARTUP");
     printf("\n");
-    printf ("Parameter checking routines\n");
+    printf ("  Parameter checking routines\n");
     SchedulePrint("CCTK_PARAMCHECK");
     printf("\n");
-    printf("Initialisation\n");
+    printf("  Initialisation\n");
     SchedulePrint("CCTK_INITIAL");
     SchedulePrint("CCTK_POSTINITIAL");
     SchedulePrint("CCTK_POSTSTEP");
     printf("\n");
-    printf ("do loop over timesteps\n");
+    printf ("  do loop over timesteps\n");
     SchedulePrint("CCTK_PRESTEP");
     SchedulePrint("CCTK_EVOL");
     SchedulePrint("CCTK_BOUND");
-    printf ("  t = t+dt\n");
+    printf ("    t = t+dt\n");
     SchedulePrint("CCTK_POSTSTEP");
-    printf ("  if (analysis)\n");
+    printf ("    if (analysis)\n");
     indent_level +=2;
     SchedulePrint("CCTK_ANALYSIS");
     indent_level -=2;
-    printf ("  endif\n");
-    printf ("enddo\n");
+    printf ("    endif\n");
+    printf ("  enddo\n");
   }
   else
   {
@@ -707,28 +708,113 @@ static int SchedulePrint(const char *where)
  ********************************************************************/
 
 
+ /*@@
+   @routine    CCTKi_SchedulePrintEntry
+   @date       Sun Sep 19 13:31:23 1999
+   @author     Tom Goodale
+   @desc 
+   Routine called on entry to a group when traversing for printing.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_SchedulePrintEntry(t_attribute *attribute, 
                                     t_sched_data *data)
 {
-  indent_level++;
+  indent_level += 2;
   return 1;
 }
 
+ /*@@
+   @routine    CCTKi_SchedulePrintEntry
+   @date       Sun Sep 19 13:31:23 1999
+   @author     Tom Goodale
+   @desc 
+   Routine called on exit to a group when traversing for printing.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_SchedulePrintExit(t_attribute *attribute, 
                                    t_sched_data *data)
 {
-  indent_level--;
+  indent_level -=2;
   return 1;
 }
 
+ /*@@
+   @routine    CCTKi_SchedulePrintEntry
+   @date       Sun Sep 19 13:31:23 1999
+   @author     Tom Goodale
+   @desc 
+   Routine called for while ofo a group when traversing for printing.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_SchedulePrintWhile(int n_whiles, 
                                     char **whiles, 
                                     t_attribute *attribute, 
                                     t_sched_data *data)
 {
-  return 0;
+  int i;
+
+  if(!data->whiling)
+  {
+    for(i=0; i < indent_level+2; i++) printf(" ");
+
+    printf("while (");
+  
+    for(i = 0; i < n_whiles; i++)
+    {
+      if(i > 0)
+      {
+        printf(" && ");
+      }
+
+      printf("%s", whiles[i]);
+    }
+    
+    printf(")\n");
+  }
+  else
+  {
+    for(i=0; i < indent_level; i++) printf(" ");
+
+    printf("end while\n");
+  }
+
+  data->whiling = !data->whiling;
+
+  return data->whiling;
 }
 
+ /*@@
+   @routine    CCTKi_SchedulePrintFunction
+   @date       Sun Sep 19 13:36:25 1999
+   @author     Tom Goodale
+   @desc 
+   Function which actually prints out data about a group or a function.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_SchedulePrintFunction(void *function, 
                                        t_attribute *attribute, 
                                        t_sched_data *data)
@@ -747,6 +833,20 @@ static int CCTKi_SchedulePrintFunction(void *function,
  ********************************************************************/
 
 
+ /*@@
+   @routine    CCTKi_ScheduleCallEntry
+   @date       Sun Sep 19 13:24:06 1999
+   @author     Tom Goodale
+   @desc 
+   Routine called when a schedule group is entered.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_ScheduleCallEntry(t_attribute *attribute, 
                                    t_sched_data *data)
 {
@@ -761,6 +861,7 @@ static int CCTKi_ScheduleCallEntry(t_attribute *attribute,
 
     if(data->schedpoint == schedpoint_analysis)
     {
+      /* In analysis, so check triggers */
       for (i = 0; i < attribute->n_trigger_groups ; i++) 
       { 
         index = CCTK_FirstVarIndexI(attribute->trigger_groups[i]);
@@ -778,17 +879,20 @@ static int CCTKi_ScheduleCallEntry(t_attribute *attribute,
 
     if(go)
     {
+      /* Switch on storage for groups */
       for(i = 0; i < attribute->n_mem_groups; i++)
       {
         attribute->StorageOnEntry[i] = CCTK_EnableGroupStorageI(data->GH,attribute->mem_groups[i]);
       }
 
+      /* Switch on communication for groups. */
       for(i = 0; i < attribute->n_comm_groups; i++)
       {
         attribute->CommOnEntry[i] = CCTK_EnableGroupCommI(data->GH,attribute->comm_groups[i]);
       }
     }
 
+    /* Remember if we have switched on storage and comm or not. */
     attribute->done_entry = go;
   }
   else
@@ -799,6 +903,20 @@ static int CCTKi_ScheduleCallEntry(t_attribute *attribute,
   return go;
 }
 
+ /*@@
+   @routine    CCTKi_ScheduleCallExit
+   @date       Sun Sep 19 13:25:24 1999
+   @author     Tom Goodale
+   @desc 
+   Routine called on exit from a schedule group.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_ScheduleCallExit(t_attribute *attribute, 
                                   t_sched_data *data)
 {
@@ -806,11 +924,13 @@ static int CCTKi_ScheduleCallExit(t_attribute *attribute,
   int index;
   int last;
 
+  /* Only do this if the entry routine did stuff. */
   if(attribute && attribute->done_entry)
   {
 
     if(data->schedpoint == schedpoint_analysis)
     {
+      /* In analysis, so do any trigger actions. */
       for (i = 0; i < attribute->n_trigger_groups ; i++) 
       { 
         index = CCTK_FirstVarIndexI(attribute->trigger_groups[i]);
@@ -822,12 +942,13 @@ static int CCTKi_ScheduleCallExit(t_attribute *attribute,
       }
     }
 
-
+    /* Switch off storage if it was switched on in entry. */
     for(i = 0; i < attribute->n_mem_groups; i++)
     {
       if(!attribute->StorageOnEntry[i]) CCTK_DisableGroupStorageI(data->GH,attribute->mem_groups[i]);
     }
 
+    /* Switch off communication if it was done in entry. */
     for(i = 0; i < attribute->n_comm_groups; i++)
     {
       if(!attribute->CommOnEntry[i]) CCTK_DisableGroupCommI(data->GH,attribute->comm_groups[i]);
@@ -837,14 +958,53 @@ static int CCTKi_ScheduleCallExit(t_attribute *attribute,
   return 1;
 }
 
+ /*@@
+   @routine    CCTKi_ScheduleCallWhile
+   @date       Sun Sep 19 13:27:53 1999
+   @author     Tom Goodale
+   @desc 
+   Routine called to check variables to see if a group or function should be executed.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_ScheduleCallWhile(int n_whiles, 
                                    char **whiles, 
                                    t_attribute *attribute, 
                                    t_sched_data *data)
 {
-  return 0;
+  int i;
+  int retcode;
+
+  retcode = 1;
+
+  /* FIXME - should do a lot of validation either here or on registration */
+  for(i = 0; i < n_whiles; i++)
+  {
+    retcode = retcode && *((int *)CCTK_VarDataPtr(data->GH, 0, whiles[i]));
+  }
+
+  return retcode;
 }
 
+ /*@@
+   @routine    CCTKi_ScheduleCallFunction
+   @date       Sun Sep 19 13:29:14 1999
+   @author     Tom Goodale
+   @desc 
+   The routine which actually calls a function.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_ScheduleCallFunction(void *function, 
                                       t_attribute *attribute, 
                                       t_sched_data *data)
@@ -873,6 +1033,21 @@ static int CCTKi_ScheduleCallFunction(void *function,
  ***************  Specialised Startup Routines   ********************
  ********************************************************************/
 
+ /*@@
+   @routine    CCTKi_ScheduleStartupFunction
+   @date       Sun Sep 19 13:30:00 1999
+   @author     Tom Goodale
+   @desc 
+   Startup routines take no arguments, so use this calling function instead
+   of the one generally used.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+@@*/
 static int CCTKi_ScheduleStartupFunction(void *function, 
                                          t_attribute *attribute, 
                                          t_sched_data *data)
diff --git a/src/main/make.code.defn b/src/main/make.code.defn
index efdda784..a45a32da 100644
--- a/src/main/make.code.defn
+++ b/src/main/make.code.defn
@@ -29,6 +29,8 @@ GHExtensions.c\
 WarnLevel.c\
 ActiveThorns.c\
 Parameters.c\
-Subsystems.c
+Subsystems.c\
+ScheduleInterface.c\
+FortranWrappers.c
 
 
diff --git a/src/main/rfrInterface.c b/src/main/rfrInterface.c
index 1c8a05a0..0e57f8f2 100644
--- a/src/main/rfrInterface.c
+++ b/src/main/rfrInterface.c
@@ -47,281 +47,31 @@ int CCTK_rfrTraverse(cGH *GH, int rfrpoint)
   return 0;
 }
 
+#include "cctk_schedule.h"
 
- /*@@
-   @routine    rfrPrintTree
-   @date       14 Mar 1999
-   @author     Gabrielle Allen
-   @desc 
-               Prints the order of the RFR calling tree
-   @enddesc 
-   @calls      
-   @history 
-   @endhistory
- 
-   @var        
-   @vdesc      
-   @vtype      
-   @vio        
-   @vcomment 
-   @endvar 
-
-   @returntype int
-   @returndesc
-      0 = routine did not print rfr tree
-      1 = routine did print rfr tree
-   @endreturndesc
-
-   @version    $Header$
-@@*/
-
-int CCTKi_rfrPrintTree(cGH *GH,void *rfr_top)
-{
-  DECLARE_CCTK_PARAMETERS
-
-  static int showed_tree = 0;
-
-  /* Return if the users doesn't want to see the tree */
-  if (!cctk_show_rfr_tree || cctk_brief_output) return 0;
-
-  if (!showed_tree) 
-  {
-
-    showed_tree = 1;
-
-    CCTK_PRINTSEPARATOR
-    printf ("Evolution tree from the RFR\n");
-
-    rfrPrintDescs(rfr_top,GH,CCTK_BASEGRID,"CCTK_BASEGRID","");
-    rfrPrintDescs(rfr_top,GH,CCTK_INITIAL, "CCTK_INITIAL","");
-    rfrPrintDescs(rfr_top,GH,CCTK_POSTINITIAL, "CCTK_POSTINITIAL","");
-    rfrPrintDescs(rfr_top,GH,CCTK_POSTSTEP,"CCTK_POSTSTEP","");
-    printf ("\n   do loop over timesteps\n");
-    rfrPrintDescs(rfr_top,GH,CCTK_PRESTEP,"CCTK_PRESTEP","   ");
-    rfrPrintDescs(rfr_top,GH,CCTK_EVOL,"CCTK_EVOL","   ");
-    rfrPrintDescs(rfr_top,GH,CCTK_BOUND,"CCTK_BOUND","   ");
-    
-    printf ("     t = t+dt\n");
-    rfrPrintDescs(rfr_top,GH,CCTK_POSTSTEP,"CCTK_POSTSTEP","   ");
-    
-    printf ("     if (analysis)\n");
-    rfrPrintDescs(rfr_top,GH,CCTK_ANALYSIS,"CCTK_ANALYSIS","      ");
-    printf ("     endif\n");
-    printf ("   enddo\n");
-    CCTK_PRINTSEPARATOR
-
-  }
-
-  return 1;
-
-}
-
- /*@@
-   @routine    CCTKi_rfrStorageOn
-   @date       Sat Feb 13 17:06:30 1999
-   @author     Tom Goodale
-   @desc 
-   Routine called by the rfr to switch storage on for a group
-   @enddesc 
-   @calls     
-   @calledby   
-   @history 
- 
-   @endhistory 
-
-@@*/
-int CCTKi_rfrStorageOn(void *GH, int group)
-{
-  int retcode;
-  char *group_name;
-
-  group_name = CCTK_GroupName(group);
-
-  if(group_name)
-  {
-#ifdef RFRDEBUG
-    printf("Turning on storage in rfrInterface.c for group %s (%d)\n",group_name,group);
-#endif
-    retcode = CCTK_EnableGroupStorage(GH, group_name);
-    free(group_name);
-  }
-  else
-  {
-    retcode = 0;
-  }
-
-  return retcode;
-}
-
- /*@@
-   @routine    CCTKi_rfrStorageOff
-   @date       Sat Feb 13 17:06:30 1999
-   @author     Tom Goodale
-   @desc 
-   Routine called by the rfr to switch storage off for a group
-   @enddesc 
-   @calls     
-   @calledby   
-   @history 
- 
-   @endhistory 
+#define SCHEDULE(x) case CCTK_ ## x : CCTK_ScheduleTraverse("CCTK_" #x, data); break
 
-@@*/
-int CCTKi_rfrStorageOff(void *GH, int group)
+void rfrTraverse(void *rfr_top, void *data, int when) 
 {
-  int retcode;
-  char *group_name;
-
-  group_name = CCTK_GroupName(group);
-  if(group_name)
+  switch(when)
   {
-#ifdef RFRDEBUG
-    printf("Turning off storage in rfrInterface.c for group %s (%d)\n",group_name,group);
-#endif
-    retcode = CCTK_DisableGroupStorage(GH, group_name);
-
-    free(group_name);
+    SCHEDULE(PARAMCHECK);
+    SCHEDULE(BASEGRID);
+    SCHEDULE(RECOVER);
+    SCHEDULE(INITIAL);
+    SCHEDULE(POSTINITIAL);
+    SCHEDULE(CPINITIAL);
+    SCHEDULE(PRESTEP);
+    SCHEDULE(POSTSTEP);
+    SCHEDULE(EVOL);
+    SCHEDULE(BOUND);
+    SCHEDULE(CHECKPOINT);
+    SCHEDULE(ANALYSIS);
+    SCHEDULE(TERMINATE);
+    SCHEDULE(CONVERGENCE);
+    default :
+      CCTK_WARN(0, "Unknown scheduling point"); 
   }
-  else
-  {
-    retcode = 0;
-  }
-
-  return retcode;
-}
 
- /*@@
-   @routine    CCTKi_rfrCommunicationOn
-   @date       Sat Feb 13 17:06:30 1999
-   @author     Tom Goodale
-   @desc 
-   Routine called by the rfr to switch communication on for a group
-   @enddesc 
-   @calls     
-   @calledby   
-   @history 
- 
-   @endhistory 
-
-@@*/
-int CCTKi_rfrCommunicationOn(void *GH, int group)
-{
-  int retcode;
-  char *group_name;
-
-  group_name = CCTK_GroupName(group);
-  if(group_name)
-  {
-#ifdef RFRDEBUG
-    printf("Turning on comm in rfrInterface.c for group %s (%d)\n",group_name,group);
-#endif
-    retcode = CCTK_EnableGroupComm(GH, group_name);
-
-    free(group_name);
-  }
-  else
-  {
-    retcode = 0;
-  }
-
-  return retcode;
-}
-
- /*@@
-   @routine    CCTKi_rfrCommunicationOff
-   @date       Sat Feb 13 17:06:30 1999
-   @author     Tom Goodale
-   @desc 
-   Routine called by the rfr to switch communication off for a group
-   @enddesc 
-   @calls     
-   @calledby   
-   @history 
- 
-   @endhistory 
-
-@@*/
-int CCTKi_rfrCommunicationOff(void *GH, int group)
-{
-  int retcode;
-  char *group_name;
-
-  group_name = CCTK_GroupName(group);
-  if(group_name)
-  {
-#ifdef RFRDEBUG
-    printf("Turning off comm in rfrInterface.c for group %s (%d)\n",group_name,group);
-#endif
-    retcode = CCTK_DisableGroupComm(GH, group_name);
-
-    free(group_name);
-  }
-  else
-  {
-    retcode = 0;
-  }
-
-  return retcode;
-}
-
- /*@@
-   @routine    CCTKi_rfrTriggerable
-   @date       Sat March 6 1999
-   @author     Gabrielle Allen
-   @desc 
-   Returns true if this rfr entry point should be triggerable
-   for some event, otherwise returns false in which case none 
-   of the triggers stuff is done.
-   @enddesc 
-   @calls     
-   @calledby   
-   @history 
- 
-   @endhistory 
-   @var     entrypoint 
-   @vdesc   describes the entrypoint of the RFR
-   @vtype   int
-   @vio     in
-   @vcomment RFR entrypoint macros are in src/include/rfr_constants.h
-   @endvar 
-@@*/
-
-int CCTKi_rfrTriggerable(int entrypoint)
-{
-  if (entrypoint == CCTK_ANALYSIS)
-    {
-      return 1;
-    }
-  else 
-    return 0;
-}
-
-
-
- /*@@
-   @routine    CCTKi_rfrCallFunc
-   @date       Sat Feb 13 17:08:39 1999
-   @author     Tom Goodale
-   @desc 
-   Routine called by the rfr to call a function.
-   @enddesc 
-   @calls     
-   @calledby   
-   @history 
- 
-   @endhistory 
-
-@@*/
-int CCTKi_rfrCallFunc(void *GH, int language, void *function)
-{
-
-  void (*calledfunc)(void *);
-
-  calledfunc = (void (*)(void *))function;
-
-  /* Call the function. */
-  
-  calledfunc(GH);
-
-  return 0;
+  return;
 }
diff --git a/src/make.code.defn b/src/make.code.defn
index b268651e..795936d5 100644
--- a/src/make.code.defn
+++ b/src/make.code.defn
@@ -1,2 +1,2 @@
 SRCS =
-SUBDIRS = IO comm rfr util main
+SUBDIRS = IO comm util main schedule