path: root/src/Storage.c

 /*@@
   @file    Storage.c
   @date    Tue Jun 13 2000
   @author  Thomas Radke
   @desc 
            Pugh storage functions
   @enddesc 
   @version $Header$
 @@*/

/*#define DEBUG_PUGH*/

#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h> 
#include <string.h>

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#include "cctk.h"
#include "cctk_Parameters.h"

#include "pugh.h"
#include "pughi.h"

static char *rcsid="$Header$";

CCTK_FILEVERSION(CactusPUGH_PUGH_Storage_c)

static float totalstorage = 0; /* Storage for GAs in Bytes */
static int totalnumber = 0;  /* Number of stored GAs */


/*@@
  @routine PUGH_ArrayGroupSize
  @author  Gabrielle Allen
  @date    11 Apr 1999
  @desc
  Returns size of arrays in a group, in a given direction
  @enddesc
  @calls   
  @history
 
  @endhistory 
  @var     GH
  @vdesc   Pointer to CCTK grid hierarchy
  @vtype   cGH
  @vio     in
  @endvar 
  @var     groupname
  @vdesc   name of the group to be synchronized
  @vtype   const char *
  @vio     in
  @vcomment either index or groupname must be present
  @endvar 
  @var     group
  @vdesc   index of group
  @vtype   int
  @vio     in
  @vcomment either index or groupname must be given
  @endvar
  @var      dir
  @vdesc    direction
  @vtype    int
  @vio      in
  @@*/

const int *PUGH_ArrayGroupSize(cGH *GH, int dir, int group, const char *groupname)
{
  int *sizep;
  pGA *GA;

  if (groupname)
  {
    group = CCTK_GroupIndex(groupname);
    
    if (group >= 0)
    {

      /* get pointer to pGA for a timelevel of first variable in this group */
      GA = (pGA *) PUGH_pGH(GH)->variables[CCTK_FirstVarIndexI(group)][0];

      if (GA->storage == PUGH_STORAGE)
      {
	if (dir >= 0 && dir < GA->extras->dim)
	{
	  sizep = &GA->extras->lnsize[dir];
	}
	else
	{
	  sizep = NULL;
	  CCTK_WARN(1, "Wrong value for dir");
	}
      }
      else
      {
	sizep = &_cctk_one;
      }
    }
  }

  return (sizep);
}


/*@@
  @routine PUGH_QueryGroupStorage
  @author  Gabrielle Allen
  @date    13 Apr 1999
  @desc
  Returns true if group has storage assigned, false otherwise
  @enddesc
  @calls   CCTK_DecomposeName
  @history
 
  @endhistory 
  @var     GH
  @vdesc   Pointer to CCTK grid hierarchy
  @vtype   cGH
  @vio     in
  @endvar 
  @var     groupname
  @vdesc   name of the group to be queried
  @vtype   const char *
  @vio     in
  @vcomment either index or groupname must be present
  @endvar 
  @var     group
  @vdesc   index of group
  @vtype   int
  @vio     in
  @vcomment either index or groupname must be given
  @endvar
  @@*/

int PUGH_QueryGroupStorage(cGH *GH, int group, const char *groupname)
{

  int first;
  int storage=PUGH_UNDEFINEDSTORAGE;
  int retval=0;
  int grouptype;

  if (groupname)
  {
    group = CCTK_GroupIndex(groupname);
  }

  /* get first variable in group */
  first = CCTK_FirstVarIndexI(group);
  if (first < 0) 
  {
    CCTK_VWarn(1, __LINE__, __FILE__, CCTK_THORNSTRING,
               "Invalid group ID %d for %s in PUGH_QueryGroupStorage", 
	       group, groupname);
  }
  else
  {
    grouptype = CCTK_GroupTypeI(group);

    if (grouptype == CCTK_SCALAR)
    {
      storage = PUGH_STORAGE;
      retval = 1;
    }
    else if (grouptype == CCTK_GF || grouptype == CCTK_ARRAY)
    {
      storage = ((pGA *) PUGH_pGH(GH)->variables[first][0])->storage;  
      if (storage == PUGH_STORAGE)
      {
	retval = 1;
      }
      else if (storage == PUGH_NOSTORAGE)
      {
	retval = 0;
      }
      else
      {
	CCTK_WARN(1,"Inconsistency in PUGH_QueryGroupStorage");
      }
    }
    else
    {
      CCTK_WARN(1,"Unknown group type in PUGH_QueryGroupStorage");
    }

  }

  return (retval);
}


/*@@
  @routine PUGH_EnableGroupStorage
  @author  Tom Goodale
  @date    30 Mar 1999
  @desc
  Enables storage for all variables in the group indicated by groupname.
  @enddesc
  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
           PUGH_EnableGArrayGroupStorage
  @history
 
  @endhistory 
  @var     GH
  @vdesc   Pointer to CCTK grid hierarchy
  @vtype   cGH
  @vio     in
  @endvar 
  @var     groupname
  @vdesc   name of the group to be synchronized
  @vtype   const char *
  @vio     in
  @endvar 
  @@*/

int PUGH_EnableGroupStorage(cGH *GH, const char *groupname)
{

  DECLARE_CCTK_PARAMETERS

  int group;               /* group index */
  int first_var;           /* first variable's index */
  cGroup pgroup;           /* group information */
  int rc;                  /* return code */
  pGA *GA;
  pGH *pughGH;

#ifdef DEBUG_PUGH
  printf(" PUGH_EnableGroupStorage: request for group '%s'\n", groupname);
  fflush(stdout);
#endif

  pughGH = PUGH_pGH(GH);
  group = CCTK_GroupIndex(groupname);

  if (group > -1 && pughGH)
  {
    first_var = CCTK_FirstVarIndexI(group);

    /* get the group info from its index */
    CCTK_GroupData(group, &pgroup);

    /* SCALAR types have always switched on memory */
    if (pgroup.grouptype == CCTK_SCALAR)
    {
      rc = 1;
    }
    else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
    {
      rc = PUGH_EnableGArrayGroupStorage(pughGH,
                                         first_var,
                                         pgroup.numvars,
                                         pgroup.numtimelevels);
    }
    else
    {
      CCTK_WARN(1, "Unknown group type in PUGH_EnableGroupStorage");
      rc = -1;
    }

    if (rc == 0)
    {
      /* get GA pointer of first var in group */
      GA = (pGA *) pughGH->variables[first_var][0];
      totalnumber  += pgroup.numvars*pgroup.numtimelevels;
      totalstorage += ((float)(GA->extras->npoints * GA->varsize *
        pgroup.numtimelevels * pgroup.numvars))/(1024*1024);
    }

    /* Report on memory usage */
    if (storage_verbose)
    {
      if (rc == 0)
      {
        /* Memory toggled */
        printf("Switched memory on for %s \n  "
               "  [Num Arrays: %d Total Size: %6.2fMB]\n",
               groupname, totalnumber, totalstorage);
      }
      else if (rc == 1)
      {
        /* Memory already on */
        printf("Memory already on for %s\n  "
               "  [Num Arrays: %d Total Size: %6.2fMB]\n",
               groupname, totalnumber, totalstorage);
      }
    }
  }
  else
  {

    if (!pughGH)
    {
      CCTK_VWarn(1,__LINE__,__FILE__,CCTK_THORNSTRING,
		 "PUGH_EnableGroupStorage: Error with cctkGH pointer "
		 "for group %s",groupname);
      rc = -2;
    }
    else
    {
      CCTK_VWarn(1,__LINE__,__FILE__,CCTK_THORNSTRING,
		 "PUGH_EnableGroupStorage: Invalid group %s",groupname);
      rc = -3;
    }

  }

  return (rc);

}


/*@@
  @routine PUGH_DisableGroupStorage
  @author  Tom Goodale
  @date    30 Mar 1999
  @desc
  Disables storage for all variables in the group indicated by groupname.
  @enddesc
  @calls   CCTK_DecomposeName CCTK_GroupIndex CCTK_GroupData CCTK_WARN
  @history
 
  @endhistory 
  @var     GH
  @vdesc   Pointer to CCTK grid hierarchy
  @vtype   cGH
  @vio     in
  @endvar 
  @var     groupname
  @vdesc   name of the group to be synchronized
  @vtype   const char *
  @vio     in
  @endvar 
  @@*/

int PUGH_DisableGroupStorage(cGH *GH, const char *groupname)
{
  DECLARE_CCTK_PARAMETERS

  int group;               /* group index */
  cGroup pgroup;           /* group information */
  int rc;                  /* return code */
  pGH *pughGH;             /* PUGH extension reference */
  pGA *GA;
  int level;
  int first_var,var;
  int unchanged;           /* count how many aren't toggled */

  /* get PUGH extension handle */
  pughGH = PUGH_pGH(GH);
  
#ifdef DEBUG_PUGH
  printf(" PUGH_DisableGroupStorage: request for group '%s'\n", groupname);
  fflush(stdout);
#endif

  /* get the group info from its index */
  group = CCTK_GroupIndex(groupname);
  CCTK_GroupData(group, &pgroup);

  /* get global index of first variable in group */
  first_var = CCTK_FirstVarIndexI(group);

  unchanged = 0;

  if (pgroup.grouptype == CCTK_SCALAR)
  {
    rc = 1;
  }
  else if (pgroup.grouptype == CCTK_GF || pgroup.grouptype == CCTK_ARRAY)
  {
    for (var = first_var; var < first_var+pgroup.numvars; var++)
    {
      for (level = 0; level < pgroup.numtimelevels; level++)
      {
        unchanged = unchanged + 
          PUGH_DisableGArrayDataStorage((pGA *)(pughGH->variables[var][level]));
      }
    }

    rc = 1;
  }
  else
  {
    CCTK_WARN(1, "Unknown group type in PUGH_DisableGroupStorage");
    rc = 0;
  }

  /* Report on memory usage */
  if (storage_verbose)
  {
    if (unchanged == 0)
    {
      GA = (pGA *) pughGH->variables[first_var][0];

      /* Memory toggled */
      totalnumber  -= pgroup.numvars;
      totalstorage -= (float) (GA->extras->npoints * GA->varsize *
                      pgroup.numtimelevels * pgroup.numvars) / (1024 * 1024);
      printf("Switched memory off for %s \n"
             "  [Num Arrays: %d Total Size: %6.2fMB]\n",
             groupname, totalnumber, totalstorage);
    }
    else if (unchanged == pgroup.numvars)
    {
      /* Memory already off */
      printf("Memory already off for %s\n",groupname);
    }
    else
    {
      CCTK_WARN(1,"Inconsistency in group memory assignment");
    }
  }

  return (rc);
}


/*****************************************************************************/
/* local functions                                                           */
/*****************************************************************************/

int PUGH_EnableGArrayGroupStorage(pGH *pughGH,
                                  int first_var,
                                  int n_variables,
                                  int n_timelevels)
{
  DECLARE_CCTK_PARAMETERS
  int nstorage;   /* Number of Arrays for which storage was set */
  int nnostorage; /* Number of Arrays for which no storage was set */
  int retval;
  int var;
  pGA *GA;
  int level;

  nstorage = 0;
  nnostorage = 0;

  for (var = first_var; var < first_var + n_variables; var++)
  {
    for (level = 0; level < n_timelevels; level++)
    {
      GA = (pGA *) pughGH->variables[var][level];

      if (! GA->storage)
      {
#ifdef DEBUG_PUGH
        printf(" PUGH_EnableGArrayGroupStorage: request for var '%s' "
               "timelevel %d\n", GA->name, level);
        fflush(stdout);
#endif
        PUGH_EnableGArrayDataStorage(GA, 
                                     pughGH->myproc,
                                     zero_memory, 
                                     padding_active,
                                     padding_cacheline_bits,
                                     padding_size,
                                     padding_address_spacing);

        ((cGH *) pughGH->callerid)->data[var][level] = GA->data;

        nnostorage++;
      }
      else
      {
        ((cGH *) pughGH->callerid)->data[var][level] = GA->data;
        nstorage++;
      }
    }
  }

  if (nstorage > 0 && nnostorage > 0)
  {
    CCTK_WARN(0, "Group storage violation in PUGH_EnableGArrayGroupStorage");
    retval = -1;
  }
  else if (nstorage > 0)
  {
    retval = 1;
  }
  else if (nnostorage > 0)
  {
    retval = 0;
  }
  else
  {
    CCTK_WARN(0, "Problem in PUGH_EnableGArrayGroupStorage");
    retval = -1;
  }

  return retval;

}


int PUGH_EnableGArrayDataStorage(pGA *GA,
                                 int this_proc,
                                 int zero_memory,
                                 int padding_active,
                                 int padding_cacheline_bits,
                                 int padding_size,
                                 int padding_address_spacing)
{

  if(GA->storage == PUGH_NOSTORAGE)
  {
#ifdef DEBUG_PUGH
        printf(" PUGH_EnableGArrayDataStorage: allocating storage "
               "for var '%s'\n", GA->name);
        fflush(stdout);
#endif

    /* Now assign memory for the variable itself */
    if(GA->padddata)
    {
      free(GA->padddata);
    }

    if(!padding_active)
    {
      /* Easy case. */
      if(zero_memory)
      {
        GA->padddata = calloc(GA->extras->npoints, GA->varsize);
      }
      else
      {
        GA->padddata = malloc(GA->extras->npoints * GA->varsize);
      }

      GA->data = GA->padddata;
    }
    else
    {

      /* Use the Cactus Cache alignment function */
      GA->data = Util_CacheMalloc(GA->arrayid,
                                  GA->extras->npoints * GA->varsize,
                                  &(GA->padddata));

      /* Zero the memory if desired. */
      if(GA->data && zero_memory)
      {
        memset(GA->data, 0, GA->extras->npoints * GA->varsize);
      }

    }
  }

  if (!GA->padddata)
  {
    CCTK_VWarn (0, __LINE__, __FILE__, CCTK_THORNSTRING,
                "FATAL ERROR: Cannot allocate data for %s [%d]\n",
                GA->name, GA->id);
  }

  GA->storage = PUGH_STORAGE;

  return 0;

}


int PUGH_DisableGArrayDataStorage(pGA *GA)
{
  int retval;

  if (GA->storage == PUGH_STORAGE)
  {
#ifdef DEBUG_PUGH
        printf(" PUGH_DisableGArrayDataStorage: freeing storage for var '%s'\n",
               GA->name);
        fflush(stdout);
#endif
    if (GA->padddata)
    {
      free(GA->padddata);
      GA->padddata = NULL;
      GA->data = NULL;
    }

    GA->padddata = calloc(1, GA->varsize);
    GA->data = GA->padddata;

    GA->storage = PUGH_NOSTORAGE;
    retval = 0;
  }
  else
  {
    retval = 1;
  }

  return retval;
}