New cache alignment stuff.

Tom


git-svn-id: http://svn.cactuscode.org/flesh/trunk@1178 17b73243-c579-4c4c-a9d2-2d5706c11dac

10 files changed, 393 insertions, 1 deletions

diff --git a/lib/make/config.h.in b/lib/make/config.h.in
index c70e2466..bc2c898f 100644
--- a/lib/make/config.h.in
+++ b/lib/make/config.h.in
@@ -78,6 +78,11 @@
 #undef HAVE_TIME_GETRUSAGE
 #undef HAVE_TIME__FTIME
 
+/* Cache stuff */
+
+#undef CCTK_L2_CACHELINE_BYTES
+#undef CCTK_L2_CACHE_SIZE
+
 /******************************************************************************/
 
 #ifdef CCODE
diff --git a/lib/make/configure b/lib/make/configure
index a839133a..78080a7e 100755
--- a/lib/make/configure
+++ b/lib/make/configure
@@ -2822,6 +2822,21 @@ LDFLAGS="$KNOWN_LDFLAGS $LDFLAGS"
 
 : ${PERL_BACKUP_NECESSARY=""}
 
+# Cache stuff - this will probably change 
+
+: ${L2_CACHELINE_BYTES="0"}
+: ${L2_CACHE_SIZE="0"}
+
+cat >> confdefs.h <<EOF
+#define CCTK_CACHELINE_BYTES $L2_CACHELINE_BYTES
+EOF
+ 
+cat >> confdefs.h <<EOF
+#define CCTK_CACHE_SIZE $L2_CACHE_SIZE
+EOF
+ 
+
+
 
  
 
diff --git a/lib/make/configure.in b/lib/make/configure.in
index 1f16df40..ab23f0cf 100644
--- a/lib/make/configure.in
+++ b/lib/make/configure.in
@@ -459,6 +459,15 @@ AC_SUBST(PERL_BACKUP_NECESSARY)
 
 : ${PERL_BACKUP_NECESSARY=""}
 
+# Cache stuff - this will probably change 
+
+: ${L2_CACHELINE_BYTES="0"}
+: ${L2_CACHE_SIZE="0"}
+
+AC_DEFINE_UNQUOTED(CCTK_L2_CACHELINE_BYTES, $L2_CACHELINE_BYTES) 
+AC_DEFINE_UNQUOTED(CCTK_L2_CACHE_SIZE, $L2_CACHE_SIZE) 
+
+
 dnl Run any other scripts necessary
 
 AC_OUTPUT_COMMANDS(,$PERL -s $PERL_CONFIGURE_SCRIPT -compiler="$F77" -opts="$F77FLAGS" .) 
diff --git a/src/include/cctk_Cache.h b/src/include/cctk_Cache.h
new file mode 100644
index 00000000..1790bf63
--- /dev/null
+++ b/src/include/cctk_Cache.h
@@ -0,0 +1,27 @@
+ /*@@
+   @header    cctk_Cache.h
+   @date      Tue Nov 30 09:31:45 1999
+   @author    Tom Goodale
+   @desc 
+   Prototypes for cache routines.
+   @enddesc 
+   @version $Header$
+ @@*/
+
+#ifndef _CCTK_CACHE_H_
+#define _CCTK_CACHE_H_
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+void *CCTK_CacheMalloc(unsigned index, 
+                       unsigned long size, 
+                       void **realstart);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _CCTK_CACHE_H_ */
diff --git a/src/include/cctki_Cache.h b/src/include/cctki_Cache.h
new file mode 100644
index 00000000..c924f3bf
--- /dev/null
+++ b/src/include/cctki_Cache.h
@@ -0,0 +1,28 @@
+ /*@@
+   @header    cctki_Cache.h
+   @date      Tue Nov 30 09:44:46 1999
+   @author    Tom Goodale
+   @desc 
+   CCTK internal cache stuff.   
+   @enddesc 
+ @@*/
+
+#ifndef _CCTKI_CACHE_H_
+#define _CCTKI_CACHE_H_
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+int CCTKi_CacheDataSet(unsigned long cacheline_bytes,
+                       unsigned long cache_size);
+
+int CCTKi_CacheDataGet(unsigned long *cacheline_bytes,
+                       unsigned long *cache_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _CCTKI_CACHE_H_ */
diff --git a/src/main/SetupCache.c b/src/main/SetupCache.c
new file mode 100644
index 00000000..cf4e1cff
--- /dev/null
+++ b/src/main/SetupCache.c
@@ -0,0 +1,66 @@
+ /*@@
+   @file      SetupCache.c
+   @date      Tue Nov 30 10:30:09 1999
+   @author    Tom Goodale
+   @desc 
+   Sets up cache stuff for the CCTK
+   @enddesc 
+ @@*/
+
+static char *rcsid = "$Header$";
+
+#include "cctk.h"
+#include "cctki_Cache.h"
+#include "cctk_parameters.h"
+
+/********************************************************************
+ *********************   CCTK Local Routines   **********************
+ ********************************************************************/
+
+ /*@@
+   @routine    CCTKi_SetupCache
+   @date       Tue Nov 30 10:50:02 1999
+   @author     Tom Goodale
+   @desc 
+   Sets the cache information.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+   @returntype int
+   @returndesc
+   0
+   @endreturndesc
+
+@@*/
+int CCTKi_SetupCache(void)
+{
+  DECLARE_CCTK_PARAMETERS
+
+  unsigned long cache_size;
+  unsigned long cacheline_bytes;
+
+  if(manual_cache_setup)
+  {
+    cache_size = manual_cache_size;
+    cacheline_bytes = manual_cacheline_bytes;
+  }
+  else
+  {
+    /* FIXME:  Remove this check for release */
+#ifdef CCTK_L2_CACHE_SIZE
+    cache_size      = CCTK_L2_CACHE_SIZE;
+    cacheline_bytes = CCTK_L2_CACHELINE_BYTES;
+#else
+    cache_size      = 0;
+    cacheline_bytes = 0;
+#endif
+  }
+
+  CCTKi_CacheDataSet(cacheline_bytes, cache_size);
+
+  return 0;
+}
diff --git a/src/main/make.code.defn b/src/main/make.code.defn
index 8546d4cc..b527fa41 100644
--- a/src/main/make.code.defn
+++ b/src/main/make.code.defn
@@ -29,6 +29,7 @@ ProcessParameterDatabase.c\
 rfrInterface.c\
 ScheduleInterface.c\
 SetParams.c\
+SetupCache.c\
 ShutdownCactus.c\
 Subsystems.c\
 WarnLevel.c
diff --git a/src/param.ccl b/src/param.ccl
index 622886be..5da2918b 100644
--- a/src/param.ccl
+++ b/src/param.ccl
@@ -30,6 +30,20 @@ BOOLEAN cctk_brief_output "Give only brief output"
 {
 } "no"
 
+BOOLEAN manual_cache_setup "Set the cache size manually"
+{
+} "no"
+
+INT manual_cache_size "The size to set the cache to if not done automatically (bytes)"
+{
+  0: :: "Any whole number"
+} 0
+
+INT manual_cacheline_bytes "The size of a cacheline if not set automatically (bytes)"
+{
+  0: :: "Any whole number"
+} 0
+
 restricted:
 
 REAL cctk_initial_time "Initial time for evolution"
diff --git a/src/util/Cache.c b/src/util/Cache.c
new file mode 100644
index 00000000..42a99a4d
--- /dev/null
+++ b/src/util/Cache.c
@@ -0,0 +1,226 @@
+ /*@@
+   @file      Cache.c
+   @date      Tue Nov 30 08:07:12 1999
+   @author    Tom Goodale
+   @desc 
+   Routines dealing with cache alignment.
+   @enddesc 
+ @@*/
+
+static char *rcsid = "$Header$";
+
+#include <stdlib.h>
+
+#include "cctk_Cache.h"
+#include "cctki_Cache.h"
+
+/********************************************************************
+ *********************     Local Data   *****************************
+ ********************************************************************/
+
+/* Structure to hold cache information. */
+static struct s_cache_data
+{
+  unsigned long cacheline_bytes;
+  unsigned long cache_size;
+} cache_data;
+
+/* The number of times the size of the cache has been set. */
+static int cache_set = 0;
+
+/********************************************************************
+ *********************     External Routines   **********************
+ ********************************************************************/
+
+ /*@@
+   @routine    CCTK_CacheMalloc
+   @date       Tue Nov 30 08:13:03 1999
+   @author     Tom Goodale
+   @desc 
+   Allocates memory aligned on the 'index'ed cache line.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+   @var     index
+   @vdesc   cache line index
+   @vtype   int
+   @vio     in
+   @vcomment 
+   The cache line to align on
+   @endvar 
+   @var     size
+   @vdesc   the amount of memory to allocate
+   @vtype   unsigned long
+   @vio     in
+   @vcomment 
+   This is the size (in bytes) of memory to allocate.
+   @endvar 
+   @var     realstart
+   @vdesc   The real starting address of the memory
+   @vtype   void **
+   @vio     out
+   @vcomment 
+   On return holds the actual starting address of the memory.
+   @endvar 
+
+   @returntype void *
+   @returndesc
+   The cache-aligned memory address.
+   @endreturndesc
+
+@@*/
+void *CCTK_CacheMalloc(unsigned index, 
+                       unsigned long size, 
+                       void **realstart)
+{
+  char *retval;
+  char *data;
+  unsigned long cacheline_bytes;
+  unsigned long cache_size;
+  unsigned long offset;
+  unsigned long initial_index;
+  unsigned long pad;
+
+  if(CCTKi_CacheDataGet(&cacheline_bytes, &cache_size))
+  {
+    cacheline_bytes = 0;
+    cache_size = 0;
+  }
+
+  data = malloc(size+cache_size);
+
+  if(data)
+  {
+    if(cache_size)
+    {
+      /* Find how far this memory is into a cache line */
+      offset = ((unsigned long)data%cache_size)%cacheline_bytes;
+      /* Find which cache line in the cache it is in */
+      initial_index = ((unsigned long)data%cache_size)/cacheline_bytes;
+      
+      pad = ((index-initial_index)*cacheline_bytes + cacheline_bytes - offset)%cache_size;
+    }
+    else
+    {
+      pad = 0;
+    }
+
+    retval = data + pad;
+  }
+  else
+  {
+    retval = NULL;
+  }
+
+  *realstart = (void *)data;
+
+  return (void *)retval;
+}
+
+/********************************************************************
+ *********************   CCTK Local Routines   **********************
+ ********************************************************************/
+
+ /*@@
+   @routine    CCTKi_CacheDataSet
+   @date       Tue Nov 30 08:12:05 1999
+   @author     Tom Goodale
+   @desc 
+   Stores the sizes of the cache.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+   @var     cacheline_bytes
+   @vdesc   The size of a cacheline
+   @vtype   unsigned long
+   @vio     in
+   @vcomment 
+   Number of bytes in a cache line.
+   @endvar 
+   @var     cache_size
+   @vdesc   Size of the cache
+   @vtype   unsigned long
+   @vio     in
+   @vcomment 
+   The size (in bytes) of the cache.
+   @endvar 
+
+   @returntype int
+   @returndesc
+   The number of times this function has been called before.
+   @endreturndesc
+
+@@*/
+int CCTKi_CacheDataSet(unsigned long cacheline_bytes,
+                       unsigned long cache_size)
+{
+  cache_data.cacheline_bytes = cacheline_bytes;
+  cache_data.cache_size      = cache_size;
+
+  cache_set++;
+
+  return cache_set-1;
+}
+
+ /*@@
+   @routine    CCTKi_CacheDataGet
+   @date       Tue Nov 30 08:12:41 1999
+   @author     Tom Goodale
+   @desc 
+   Gets the sizes of the cache.
+   @enddesc 
+   @calls     
+   @calledby   
+   @history 
+ 
+   @endhistory 
+
+   @var     cacheline_bytes
+   @vdesc   The size of a cacheline
+   @vtype   unsigned long *
+   @vio     out
+   @vcomment 
+   Number of bytes in a cache line.
+   @endvar 
+   @var     cache_size
+   @vdesc   Size of the cache
+   @vtype   unsigned long *
+   @vio     out
+   @vcomment 
+   The size (in bytes) of the cache.
+   @endvar 
+
+   @returntype int
+   @returndesc
+   0  - success
+   -1 - cache sizes have not been set yet.
+   @endreturndesc
+
+@@*/
+int CCTKi_CacheDataGet(unsigned long *cacheline_bytes,
+                       unsigned long *cache_size)
+{
+  int retcode;
+
+  if(cache_set)
+  {
+    *cacheline_bytes = cache_data.cacheline_bytes;
+    *cache_size      = cache_data.cache_size;
+    retcode = 0;
+  }
+  else
+  {
+    retcode = -1;
+  }
+
+  return retcode; 
+}
+
diff --git a/src/util/make.code.defn b/src/util/make.code.defn
index 80219d8a..bab3e659 100644
--- a/src/util/make.code.defn
+++ b/src/util/make.code.defn
@@ -12,7 +12,8 @@ getopt1.c\
 StoreHandledData.c\
 gnu_regex.c\
 SKBinTree.c\
-Hash.c
+Hash.c\
+Cache.c