aboutsummaryrefslogtreecommitdiff
path: root/Carpet/doc
diff options
context:
space:
mode:
authoreschnett <>2001-03-01 11:40:00 +0000
committereschnett <>2001-03-01 11:40:00 +0000
commit310f0ea48d18866b773136aed11200b6eda6378b (patch)
tree445d3e34ce8b89812994b6614f7bc9f4acbc7fe2 /Carpet/doc
Initial revision
darcs-hash:20010301114010-f6438-12fb8a9ffcc80e86c0a97e37b5b0dae0dbc59b79.gz
Diffstat (limited to 'Carpet/doc')
-rw-r--r--Carpet/doc/Grid1.eps188
-rw-r--r--Carpet/doc/Grid1.fig51
-rw-r--r--Carpet/doc/Makefile29
-rw-r--r--Carpet/doc/Periodic1.eps201
-rw-r--r--Carpet/doc/Periodic1.fig39
-rw-r--r--Carpet/doc/Periodic2.eps262
-rw-r--r--Carpet/doc/Periodic2.fig99
-rw-r--r--Carpet/doc/carpet.bib50
-rw-r--r--Carpet/doc/documentation.tex892
-rw-r--r--Carpet/doc/first-steps.tex583
-rw-r--r--Carpet/doc/internals.tex777
11 files changed, 3171 insertions, 0 deletions
diff --git a/Carpet/doc/Grid1.eps b/Carpet/doc/Grid1.eps
new file mode 100644
index 000000000..257a57dfd
--- /dev/null
+++ b/Carpet/doc/Grid1.eps
@@ -0,0 +1,188 @@
+%!PS-Adobe-2.0 EPSF-2.0
+%%Title: Grid1.eps
+%%Creator: fig2dev Version 3.2 Patchlevel 3d
+%%CreationDate: Fri May 2 11:09:16 2003
+%%For: hawke@xeon06.aei-potsdam.mpg.de (Ian Hawke)
+%%BoundingBox: 0 0 612 461
+%%Magnification: 1.0000
+%%EndComments
+/$F2psDict 200 dict def
+$F2psDict begin
+$F2psDict /mtrx matrix put
+/col-1 {0 setgray} bind def
+/col0 {0.000 0.000 0.000 srgb} bind def
+/col1 {0.000 0.000 1.000 srgb} bind def
+/col2 {0.000 1.000 0.000 srgb} bind def
+/col3 {0.000 1.000 1.000 srgb} bind def
+/col4 {1.000 0.000 0.000 srgb} bind def
+/col5 {1.000 0.000 1.000 srgb} bind def
+/col6 {1.000 1.000 0.000 srgb} bind def
+/col7 {1.000 1.000 1.000 srgb} bind def
+/col8 {0.000 0.000 0.560 srgb} bind def
+/col9 {0.000 0.000 0.690 srgb} bind def
+/col10 {0.000 0.000 0.820 srgb} bind def
+/col11 {0.530 0.810 1.000 srgb} bind def
+/col12 {0.000 0.560 0.000 srgb} bind def
+/col13 {0.000 0.690 0.000 srgb} bind def
+/col14 {0.000 0.820 0.000 srgb} bind def
+/col15 {0.000 0.560 0.560 srgb} bind def
+/col16 {0.000 0.690 0.690 srgb} bind def
+/col17 {0.000 0.820 0.820 srgb} bind def
+/col18 {0.560 0.000 0.000 srgb} bind def
+/col19 {0.690 0.000 0.000 srgb} bind def
+/col20 {0.820 0.000 0.000 srgb} bind def
+/col21 {0.560 0.000 0.560 srgb} bind def
+/col22 {0.690 0.000 0.690 srgb} bind def
+/col23 {0.820 0.000 0.820 srgb} bind def
+/col24 {0.500 0.190 0.000 srgb} bind def
+/col25 {0.630 0.250 0.000 srgb} bind def
+/col26 {0.750 0.380 0.000 srgb} bind def
+/col27 {1.000 0.500 0.500 srgb} bind def
+/col28 {1.000 0.630 0.630 srgb} bind def
+/col29 {1.000 0.750 0.750 srgb} bind def
+/col30 {1.000 0.880 0.880 srgb} bind def
+/col31 {1.000 0.840 0.000 srgb} bind def
+
+end
+save
+newpath 0 461 moveto 0 0 lineto 612 0 lineto 612 461 lineto closepath clip newpath
+-49.5 526.5 translate
+1 -1 scale
+
+/cp {closepath} bind def
+/ef {eofill} bind def
+/gr {grestore} bind def
+/gs {gsave} bind def
+/sa {save} bind def
+/rs {restore} bind def
+/l {lineto} bind def
+/m {moveto} bind def
+/rm {rmoveto} bind def
+/n {newpath} bind def
+/s {stroke} bind def
+/sh {show} bind def
+/slc {setlinecap} bind def
+/slj {setlinejoin} bind def
+/slw {setlinewidth} bind def
+/srgb {setrgbcolor} bind def
+/rot {rotate} bind def
+/sc {scale} bind def
+/sd {setdash} bind def
+/ff {findfont} bind def
+/sf {setfont} bind def
+/scf {scalefont} bind def
+/sw {stringwidth} bind def
+/tr {translate} bind def
+/tnt {dup dup currentrgbcolor
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb}
+ bind def
+/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul
+ 4 -2 roll mul srgb} bind def
+/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def
+/$F2psEnd {$F2psEnteredState restore end} def
+
+$F2psBegin
+10 setmiterlimit
+ 0.06000 0.06000 sc
+%
+% Fig objects follow
+%
+% Polyline
+7.500 slw
+n 1200 1200 m 10800 1200 l 10800 8400 l 1200 8400 l
+ cp gs col0 s gr
+% Polyline
+n 3600 1200 m
+ 3600 8400 l gs col0 s gr
+% Polyline
+n 6000 1200 m
+ 6000 8400 l gs col0 s gr
+% Polyline
+n 8400 1200 m
+ 8400 8400 l gs col0 s gr
+% Polyline
+n 1200 3600 m
+ 10800 3600 l gs col0 s gr
+% Polyline
+n 1200 6000 m
+ 10800 6000 l gs col0 s gr
+% Polyline
+15.000 slw
+ [60] 0 sd
+n 3600 3600 m 8400 3600 l 8400 6000 l 3600 6000 l
+ cp gs col1 s gr [] 0 sd
+% Polyline
+ [90] 0 sd
+n 4800 3600 m
+ 4800 6000 l gs col1 s gr [] 0 sd
+% Polyline
+ [90] 0 sd
+n 7200 3600 m
+ 7200 6000 l gs col1 s gr [] 0 sd
+% Polyline
+ [90] 0 sd
+n 6000 3600 m
+ 6000 6000 l gs col1 s gr [] 0 sd
+% Polyline
+ [90] 0 sd
+n 3600 4800 m
+ 8400 4800 l gs col1 s gr [] 0 sd
+% Polyline
+30.000 slw
+ [15 90] 90 sd
+n 6000 4800 m 7200 4800 l 7200 6000 l 6000 6000 l
+ cp gs col2 s gr [] 0 sd
+% Polyline
+ [15 90] 90 sd
+n 6600 4800 m
+ 6600 6000 l gs col2 s gr [] 0 sd
+% Polyline
+ [15 90] 90 sd
+n 6000 5400 m
+ 7200 5400 l gs col2 s gr [] 0 sd
+/Times-Roman ff 360.00 scf sf
+3525 8775 m
+gs 1 -1 sc (4) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+8250 8775 m
+gs 1 -1 sc (12) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+10650 8775 m
+gs 1 -1 sc (16) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+4725 6375 m
+gs 1 -1 sc (6) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+6525 6375 m
+gs 1 -1 sc (9) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+1125 8775 m
+gs 1 -1 sc (0) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+7125 6375 m
+gs 1 -1 sc (10) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+825 8550 m
+gs 1 -1 sc (0) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+825 6150 m
+gs 1 -1 sc (4) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+825 3750 m
+gs 1 -1 sc (8) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+825 1350 m
+gs 1 -1 sc (12) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+3300 4950 m
+gs 1 -1 sc (6) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+5700 5550 m
+gs 1 -1 sc (5) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+5925 8775 m
+gs 1 -1 sc (8) col0 sh gr
+$F2psEnd
+rs
diff --git a/Carpet/doc/Grid1.fig b/Carpet/doc/Grid1.fig
new file mode 100644
index 000000000..42763db47
--- /dev/null
+++ b/Carpet/doc/Grid1.fig
@@ -0,0 +1,51 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 1200 1200 10800 1200 10800 8400 1200 8400 1200 1200
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 3600 1200 3600 8400
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 6000 1200 6000 8400
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 8400 1200 8400 8400
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 1200 3600 10800 3600
+2 1 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 1200 6000 10800 6000
+2 2 1 2 1 7 50 0 -1 4.000 0 0 -1 0 0 5
+ 3600 3600 8400 3600 8400 6000 3600 6000 3600 3600
+2 1 1 2 1 7 50 0 -1 6.000 0 0 -1 0 0 2
+ 4800 3600 4800 6000
+2 1 1 2 1 7 50 0 -1 6.000 0 0 -1 0 0 2
+ 7200 3600 7200 6000
+2 1 1 2 1 7 50 0 -1 6.000 0 0 -1 0 0 2
+ 6000 3600 6000 6000
+2 1 1 2 1 7 50 0 -1 6.000 0 0 -1 0 0 2
+ 3600 4800 8400 4800
+2 2 2 3 2 7 50 0 -1 6.000 0 0 -1 0 0 5
+ 6000 4800 7200 4800 7200 6000 6000 6000 6000 4800
+2 1 2 3 2 7 50 0 -1 6.000 0 0 -1 0 0 2
+ 6600 4800 6600 6000
+2 1 2 3 2 7 50 0 -1 6.000 0 0 -1 0 0 2
+ 6000 5400 7200 5400
+4 0 0 50 0 0 24 0.0000 4 255 180 3525 8775 4\001
+4 0 0 50 0 0 24 0.0000 4 255 360 8250 8775 12\001
+4 0 0 50 0 0 24 0.0000 4 255 360 10650 8775 16\001
+4 0 0 50 0 0 24 0.0000 4 255 180 4725 6375 6\001
+4 0 0 50 0 0 24 0.0000 4 255 180 6525 6375 9\001
+4 0 0 50 0 0 24 0.0000 4 255 180 1125 8775 0\001
+4 0 0 50 0 0 24 0.0000 4 255 360 7125 6375 10\001
+4 0 0 50 0 0 24 0.0000 4 255 180 825 8550 0\001
+4 0 0 50 0 0 24 0.0000 4 255 180 825 6150 4\001
+4 0 0 50 0 0 24 0.0000 4 255 180 825 3750 8\001
+4 0 0 50 0 0 24 0.0000 4 255 360 825 1350 12\001
+4 0 0 50 0 0 24 0.0000 4 255 180 3300 4950 6\001
+4 0 0 50 0 0 24 0.0000 4 255 180 5700 5550 5\001
+4 0 0 50 0 0 24 0.0000 4 255 180 5925 8775 8\001
diff --git a/Carpet/doc/Makefile b/Carpet/doc/Makefile
new file mode 100644
index 000000000..8c402b013
--- /dev/null
+++ b/Carpet/doc/Makefile
@@ -0,0 +1,29 @@
+# $Header: /home/eschnett/C/carpet/Carpet/Carpet/doc/Makefile,v 1.2 2002/11/29 16:48:35 schnetter Exp $
+
+NAME=documentation
+
+all: ${NAME}.dvi ${NAME}.ps ${NAME}.ps.gz ${NAME}.pdf
+
+${NAME}.dvi: carpet.bib
+
+%.dvi: %.tex
+ latex $*
+ bibtex $*
+ latex $*
+ latex $*
+
+%.ps: %.dvi
+ dvips $*
+
+%.pdf: %.dvi
+ pdflatex $*
+ thumbpdf $*
+ pdflatex $*
+
+%.gz: %
+ gzip --best -c $* > $*.gz
+
+clean:
+ $(RM) ${NAME}.aux ${NAME}.bbl ${NAME}.blg ${NAME}.dvi ${NAME}.log ${NAME}.out ${NAME}.pdf ${NAME}.ps ${NAME}.ps.gz ${NAME}.tpt
+
+.PSEUDO: all clean
diff --git a/Carpet/doc/Periodic1.eps b/Carpet/doc/Periodic1.eps
new file mode 100644
index 000000000..ed463e2c5
--- /dev/null
+++ b/Carpet/doc/Periodic1.eps
@@ -0,0 +1,201 @@
+%!PS-Adobe-2.0 EPSF-2.0
+%%Title: Periodic1.eps
+%%Creator: fig2dev Version 3.2 Patchlevel 3d
+%%CreationDate: Wed Jul 16 14:12:03 2003
+%%For: admin@nbdell08.aei-potsdam.mpg.de ()
+%%BoundingBox: 0 0 239 132
+%%Magnification: 1.0000
+%%EndComments
+/$F2psDict 200 dict def
+$F2psDict begin
+$F2psDict /mtrx matrix put
+/col-1 {0 setgray} bind def
+/col0 {0.000 0.000 0.000 srgb} bind def
+/col1 {0.000 0.000 1.000 srgb} bind def
+/col2 {0.000 1.000 0.000 srgb} bind def
+/col3 {0.000 1.000 1.000 srgb} bind def
+/col4 {1.000 0.000 0.000 srgb} bind def
+/col5 {1.000 0.000 1.000 srgb} bind def
+/col6 {1.000 1.000 0.000 srgb} bind def
+/col7 {1.000 1.000 1.000 srgb} bind def
+/col8 {0.000 0.000 0.560 srgb} bind def
+/col9 {0.000 0.000 0.690 srgb} bind def
+/col10 {0.000 0.000 0.820 srgb} bind def
+/col11 {0.530 0.810 1.000 srgb} bind def
+/col12 {0.000 0.560 0.000 srgb} bind def
+/col13 {0.000 0.690 0.000 srgb} bind def
+/col14 {0.000 0.820 0.000 srgb} bind def
+/col15 {0.000 0.560 0.560 srgb} bind def
+/col16 {0.000 0.690 0.690 srgb} bind def
+/col17 {0.000 0.820 0.820 srgb} bind def
+/col18 {0.560 0.000 0.000 srgb} bind def
+/col19 {0.690 0.000 0.000 srgb} bind def
+/col20 {0.820 0.000 0.000 srgb} bind def
+/col21 {0.560 0.000 0.560 srgb} bind def
+/col22 {0.690 0.000 0.690 srgb} bind def
+/col23 {0.820 0.000 0.820 srgb} bind def
+/col24 {0.500 0.190 0.000 srgb} bind def
+/col25 {0.630 0.250 0.000 srgb} bind def
+/col26 {0.750 0.380 0.000 srgb} bind def
+/col27 {1.000 0.500 0.500 srgb} bind def
+/col28 {1.000 0.630 0.630 srgb} bind def
+/col29 {1.000 0.750 0.750 srgb} bind def
+/col30 {1.000 0.880 0.880 srgb} bind def
+/col31 {1.000 0.840 0.000 srgb} bind def
+
+end
+save
+newpath 0 132 moveto 0 0 lineto 239 0 lineto 239 132 lineto closepath clip newpath
+-174.8 292.5 translate
+1 -1 scale
+
+/cp {closepath} bind def
+/ef {eofill} bind def
+/gr {grestore} bind def
+/gs {gsave} bind def
+/sa {save} bind def
+/rs {restore} bind def
+/l {lineto} bind def
+/m {moveto} bind def
+/rm {rmoveto} bind def
+/n {newpath} bind def
+/s {stroke} bind def
+/sh {show} bind def
+/slc {setlinecap} bind def
+/slj {setlinejoin} bind def
+/slw {setlinewidth} bind def
+/srgb {setrgbcolor} bind def
+/rot {rotate} bind def
+/sc {scale} bind def
+/sd {setdash} bind def
+/ff {findfont} bind def
+/sf {setfont} bind def
+/scf {scalefont} bind def
+/sw {stringwidth} bind def
+/tr {translate} bind def
+/tnt {dup dup currentrgbcolor
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb}
+ bind def
+/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul
+ 4 -2 roll mul srgb} bind def
+ /DrawEllipse {
+ /endangle exch def
+ /startangle exch def
+ /yrad exch def
+ /xrad exch def
+ /y exch def
+ /x exch def
+ /savematrix mtrx currentmatrix def
+ x y tr xrad yrad sc 0 0 1 startangle endangle arc
+ closepath
+ savematrix setmatrix
+ } def
+
+/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def
+/$F2psEnd {$F2psEnteredState restore end} def
+
+$F2psBegin
+10 setmiterlimit
+ 0.06000 0.06000 sc
+%
+% Fig objects follow
+%
+% Arc
+7.500 slw
+gs clippath
+5377 3399 m 5433 3378 l 5380 3237 l 5394 3360 l 5324 3257 l cp
+3566 3378 m 3622 3399 l 3675 3257 l 3606 3360 l 3619 3237 l cp
+eoclip
+n 4500.0 3637.5 937.5 -163.7 -16.3 arc
+gs col0 s gr
+ gr
+
+% arrowhead
+n 5324 3257 m 5394 3360 l 5380 3237 l 5324 3257 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 3619 3237 m 3606 3360 l 3675 3257 l 3619 3237 l cp gs 0.00 setgray ef gr col0 s
+% Arc
+gs clippath
+4777 3399 m 4833 3378 l 4780 3237 l 4794 3360 l 4724 3257 l cp
+2966 3378 m 3022 3399 l 3075 3257 l 3006 3360 l 3019 3237 l cp
+eoclip
+n 3900.0 3637.5 937.5 -163.7 -16.3 arc
+gs col0 s gr
+ gr
+
+% arrowhead
+n 4724 3257 m 4794 3360 l 4780 3237 l 4724 3257 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 3019 3237 m 3006 3360 l 3075 3257 l 3019 3237 l cp gs 0.00 setgray ef gr col0 s
+% Arc
+gs clippath
+6033 3821 m 5977 3800 l 5924 3942 l 5994 3840 l 5980 3962 l cp
+4222 3800 m 4166 3821 l 4219 3962 l 4206 3840 l 4275 3942 l cp
+eoclip
+n 5100.0 3562.5 937.5 163.7 16.3 arcn
+gs col0 s gr
+ gr
+
+% arrowhead
+n 5980 3962 m 5994 3840 l 5924 3942 l 5980 3962 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 4275 3942 m 4206 3840 l 4219 3962 l 4275 3942 l cp gs 0.00 setgray ef gr col0 s
+% Arc
+gs clippath
+6633 3821 m 6577 3800 l 6524 3942 l 6594 3840 l 6580 3962 l cp
+4822 3800 m 4766 3821 l 4819 3962 l 4806 3840 l 4875 3942 l cp
+eoclip
+n 5700.0 3562.5 937.5 163.7 16.3 arcn
+gs col0 s gr
+ gr
+
+% arrowhead
+n 6580 3962 m 6594 3840 l 6524 3942 l 6580 3962 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 4875 3942 m 4806 3840 l 4819 3962 l 4875 3942 l cp gs 0.00 setgray ef gr col0 s
+% Ellipse
+n 4200 3600 75 75 0 360 DrawEllipse gs col0 s gr
+
+% Ellipse
+n 4800 3600 75 75 0 360 DrawEllipse gs col0 s gr
+
+% Ellipse
+n 5400 3600 75 75 0 360 DrawEllipse gs col0 s gr
+
+% Polyline
+n 2925 3525 m 3075 3525 l 3075 3675 l 2925 3675 l
+ cp gs col0 s gr
+% Polyline
+n 3525 3525 m 3675 3525 l 3675 3675 l 3525 3675 l
+ cp gs col0 s gr
+% Polyline
+n 5925 3525 m 6075 3525 l 6075 3675 l 5925 3675 l
+ cp gs col0 s gr
+% Polyline
+n 6525 3525 m 6675 3525 l 6675 3675 l 6525 3675 l
+ cp gs col0 s gr
+/Times-Roman ff 360.00 scf sf
+2925 4875 m
+gs 1 -1 sc (0) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+3450 4875 m
+gs 1 -1 sc (2) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+4125 4875 m
+gs 1 -1 sc (4) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+4725 4875 m
+gs 1 -1 sc (6) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+5325 4875 m
+gs 1 -1 sc (8) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+5925 4875 m
+gs 1 -1 sc (10) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+6525 4875 m
+gs 1 -1 sc (12) col0 sh gr
+$F2psEnd
+rs
diff --git a/Carpet/doc/Periodic1.fig b/Carpet/doc/Periodic1.fig
new file mode 100644
index 000000000..b99903fab
--- /dev/null
+++ b/Carpet/doc/Periodic1.fig
@@ -0,0 +1,39 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+5 1 0 1 0 7 50 0 -1 0.000 0 0 1 1 4500.000 3637.500 3600 3375 4500 2700 5400 3375
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+5 1 0 1 0 7 50 0 -1 0.000 0 0 1 1 3900.000 3637.500 3000 3375 3900 2700 4800 3375
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+5 1 0 1 0 7 50 0 -1 0.000 0 1 1 1 5100.000 3562.500 4200 3825 5100 4500 6000 3825
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+5 1 0 1 0 7 50 0 -1 0.000 0 1 1 1 5700.000 3562.500 4800 3825 5700 4500 6600 3825
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+1 3 0 1 0 7 50 0 -1 0.000 1 0.0000 4200 3600 75 75 4200 3600 4125 3600
+1 3 0 1 0 7 50 0 -1 0.000 1 0.0000 4800 3600 75 75 4800 3600 4725 3600
+1 3 0 1 0 7 50 0 -1 0.000 1 0.0000 5400 3600 75 75 5400 3600 5325 3600
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 2925 3525 3075 3525 3075 3675 2925 3675 2925 3525
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 3525 3525 3675 3525 3675 3675 3525 3675 3525 3525
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 5925 3525 6075 3525 6075 3675 5925 3675 5925 3525
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 6525 3525 6675 3525 6675 3675 6525 3675 6525 3525
+4 0 0 50 0 0 24 0.0000 4 255 180 2925 4875 0\001
+4 0 0 50 0 0 24 0.0000 4 255 180 3450 4875 2\001
+4 0 0 50 0 0 24 0.0000 4 255 180 4125 4875 4\001
+4 0 0 50 0 0 24 0.0000 4 255 180 4725 4875 6\001
+4 0 0 50 0 0 24 0.0000 4 255 180 5325 4875 8\001
+4 0 0 50 0 0 24 0.0000 4 255 360 5925 4875 10\001
+4 0 0 50 0 0 24 0.0000 4 255 360 6525 4875 12\001
diff --git a/Carpet/doc/Periodic2.eps b/Carpet/doc/Periodic2.eps
new file mode 100644
index 000000000..067fdcbcf
--- /dev/null
+++ b/Carpet/doc/Periodic2.eps
@@ -0,0 +1,262 @@
+%!PS-Adobe-2.0 EPSF-2.0
+%%Title: Periodic2.eps
+%%Creator: fig2dev Version 3.2 Patchlevel 3d
+%%CreationDate: Wed Jul 16 14:10:26 2003
+%%For: admin@nbdell08.aei-potsdam.mpg.de ()
+%%BoundingBox: 0 0 239 132
+%%Magnification: 1.0000
+%%EndComments
+/$F2psDict 200 dict def
+$F2psDict begin
+$F2psDict /mtrx matrix put
+/col-1 {0 setgray} bind def
+/col0 {0.000 0.000 0.000 srgb} bind def
+/col1 {0.000 0.000 1.000 srgb} bind def
+/col2 {0.000 1.000 0.000 srgb} bind def
+/col3 {0.000 1.000 1.000 srgb} bind def
+/col4 {1.000 0.000 0.000 srgb} bind def
+/col5 {1.000 0.000 1.000 srgb} bind def
+/col6 {1.000 1.000 0.000 srgb} bind def
+/col7 {1.000 1.000 1.000 srgb} bind def
+/col8 {0.000 0.000 0.560 srgb} bind def
+/col9 {0.000 0.000 0.690 srgb} bind def
+/col10 {0.000 0.000 0.820 srgb} bind def
+/col11 {0.530 0.810 1.000 srgb} bind def
+/col12 {0.000 0.560 0.000 srgb} bind def
+/col13 {0.000 0.690 0.000 srgb} bind def
+/col14 {0.000 0.820 0.000 srgb} bind def
+/col15 {0.000 0.560 0.560 srgb} bind def
+/col16 {0.000 0.690 0.690 srgb} bind def
+/col17 {0.000 0.820 0.820 srgb} bind def
+/col18 {0.560 0.000 0.000 srgb} bind def
+/col19 {0.690 0.000 0.000 srgb} bind def
+/col20 {0.820 0.000 0.000 srgb} bind def
+/col21 {0.560 0.000 0.560 srgb} bind def
+/col22 {0.690 0.000 0.690 srgb} bind def
+/col23 {0.820 0.000 0.820 srgb} bind def
+/col24 {0.500 0.190 0.000 srgb} bind def
+/col25 {0.630 0.250 0.000 srgb} bind def
+/col26 {0.750 0.380 0.000 srgb} bind def
+/col27 {1.000 0.500 0.500 srgb} bind def
+/col28 {1.000 0.630 0.630 srgb} bind def
+/col29 {1.000 0.750 0.750 srgb} bind def
+/col30 {1.000 0.880 0.880 srgb} bind def
+/col31 {1.000 0.840 0.000 srgb} bind def
+
+end
+save
+newpath 0 132 moveto 0 0 lineto 239 0 lineto 239 132 lineto closepath clip newpath
+-174.8 292.5 translate
+1 -1 scale
+
+/cp {closepath} bind def
+/ef {eofill} bind def
+/gr {grestore} bind def
+/gs {gsave} bind def
+/sa {save} bind def
+/rs {restore} bind def
+/l {lineto} bind def
+/m {moveto} bind def
+/rm {rmoveto} bind def
+/n {newpath} bind def
+/s {stroke} bind def
+/sh {show} bind def
+/slc {setlinecap} bind def
+/slj {setlinejoin} bind def
+/slw {setlinewidth} bind def
+/srgb {setrgbcolor} bind def
+/rot {rotate} bind def
+/sc {scale} bind def
+/sd {setdash} bind def
+/ff {findfont} bind def
+/sf {setfont} bind def
+/scf {scalefont} bind def
+/sw {stringwidth} bind def
+/tr {translate} bind def
+/tnt {dup dup currentrgbcolor
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add
+ 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb}
+ bind def
+/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul
+ 4 -2 roll mul srgb} bind def
+ /DrawEllipse {
+ /endangle exch def
+ /startangle exch def
+ /yrad exch def
+ /xrad exch def
+ /y exch def
+ /x exch def
+ /savematrix mtrx currentmatrix def
+ x y tr xrad yrad sc 0 0 1 startangle endangle arc
+ closepath
+ savematrix setmatrix
+ } def
+
+/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def
+/$F2psEnd {$F2psEnteredState restore end} def
+
+$F2psBegin
+10 setmiterlimit
+ 0.06000 0.06000 sc
+%
+% Fig objects follow
+%
+% Polyline
+30.000 slw
+n 4125 3525 m
+ 4275 3675 l gs col1 s gr
+% Polyline
+n 4125 3675 m
+ 4275 3525 l gs col1 s gr
+% Polyline
+n 4425 3525 m
+ 4575 3675 l gs col1 s gr
+% Polyline
+n 4425 3675 m
+ 4575 3525 l gs col1 s gr
+% Polyline
+n 4725 3525 m
+ 4875 3675 l gs col1 s gr
+% Polyline
+n 4725 3675 m
+ 4875 3525 l gs col1 s gr
+% Polyline
+n 5025 3525 m
+ 5175 3675 l gs col1 s gr
+% Polyline
+n 5025 3675 m
+ 5175 3525 l gs col1 s gr
+% Polyline
+n 5325 3525 m
+ 5475 3675 l gs col1 s gr
+% Polyline
+n 5325 3675 m
+ 5475 3525 l gs col1 s gr
+% Polyline
+n 5625 3525 m
+ 5775 3675 l gs col1 s gr
+% Polyline
+n 5625 3675 m
+ 5775 3525 l gs col1 s gr
+% Polyline
+n 3900 3525 m
+ 3900 3675 l gs col1 0.00 shd ef gr gs col1 s gr
+% Polyline
+n 3825 3600 m
+ 3975 3600 l gs col1 0.00 shd ef gr gs col1 s gr
+% Polyline
+n 3600 3525 m
+ 3600 3675 l gs col1 0.00 shd ef gr gs col1 s gr
+% Polyline
+n 3525 3600 m
+ 3675 3600 l gs col1 0.00 shd ef gr gs col1 s gr
+% Polyline
+n 6000 3525 m
+ 6000 3675 l gs col1 0.00 shd ef gr gs col1 s gr
+% Polyline
+n 5925 3600 m
+ 6075 3600 l gs col1 0.00 shd ef gr gs col1 s gr
+% Polyline
+n 6300 3525 m
+ 6300 3675 l gs col1 0.00 shd ef gr gs col1 s gr
+% Polyline
+n 6225 3600 m
+ 6375 3600 l gs col1 0.00 shd ef gr gs col1 s gr
+% Arc
+7.500 slw
+gs clippath
+5377 3399 m 5433 3378 l 5380 3237 l 5394 3360 l 5324 3257 l cp
+3566 3378 m 3622 3399 l 3675 3257 l 3606 3360 l 3619 3237 l cp
+eoclip
+n 4500.0 3637.5 937.5 -163.7 -16.3 arc
+gs col0 s gr
+ gr
+
+% arrowhead
+n 5324 3257 m 5394 3360 l 5380 3237 l 5324 3257 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 3619 3237 m 3606 3360 l 3675 3257 l 3619 3237 l cp gs 0.00 setgray ef gr col0 s
+% Arc
+gs clippath
+5677 3399 m 5733 3378 l 5680 3237 l 5694 3360 l 5624 3257 l cp
+3866 3378 m 3922 3399 l 3975 3257 l 3906 3360 l 3919 3237 l cp
+eoclip
+n 4800.0 3637.5 937.5 -163.7 -16.3 arc
+gs col0 s gr
+ gr
+
+% arrowhead
+n 5624 3257 m 5694 3360 l 5680 3237 l 5624 3257 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 3919 3237 m 3906 3360 l 3975 3257 l 3919 3237 l cp gs 0.00 setgray ef gr col0 s
+% Arc
+gs clippath
+6033 3821 m 5977 3800 l 5924 3942 l 5994 3840 l 5980 3962 l cp
+4222 3800 m 4166 3821 l 4219 3962 l 4206 3840 l 4275 3942 l cp
+eoclip
+n 5100.0 3562.5 937.5 163.7 16.3 arcn
+gs col0 s gr
+ gr
+
+% arrowhead
+n 5980 3962 m 5994 3840 l 5924 3942 l 5980 3962 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 4275 3942 m 4206 3840 l 4219 3962 l 4275 3942 l cp gs 0.00 setgray ef gr col0 s
+% Arc
+gs clippath
+6333 3821 m 6277 3800 l 6224 3942 l 6294 3840 l 6280 3962 l cp
+4522 3800 m 4466 3821 l 4519 3962 l 4506 3840 l 4575 3942 l cp
+eoclip
+n 5400.0 3562.5 937.5 163.7 16.3 arcn
+gs col0 s gr
+ gr
+
+% arrowhead
+n 6280 3962 m 6294 3840 l 6224 3942 l 6280 3962 l cp gs 0.00 setgray ef gr col0 s
+% arrowhead
+n 4575 3942 m 4506 3840 l 4519 3962 l 4575 3942 l cp gs 0.00 setgray ef gr col0 s
+% Ellipse
+n 4200 3600 75 75 0 360 DrawEllipse gs col0 s gr
+
+% Ellipse
+n 4800 3600 75 75 0 360 DrawEllipse gs col0 s gr
+
+% Ellipse
+n 5400 3600 75 75 0 360 DrawEllipse gs col0 s gr
+
+% Polyline
+n 2925 3525 m 3075 3525 l 3075 3675 l 2925 3675 l
+ cp gs col0 s gr
+% Polyline
+n 3525 3525 m 3675 3525 l 3675 3675 l 3525 3675 l
+ cp gs col0 s gr
+% Polyline
+n 5925 3525 m 6075 3525 l 6075 3675 l 5925 3675 l
+ cp gs col0 s gr
+% Polyline
+n 6525 3525 m 6675 3525 l 6675 3675 l 6525 3675 l
+ cp gs col0 s gr
+/Times-Roman ff 360.00 scf sf
+2925 4875 m
+gs 1 -1 sc (0) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+3450 4875 m
+gs 1 -1 sc (2) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+4125 4875 m
+gs 1 -1 sc (4) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+4725 4875 m
+gs 1 -1 sc (6) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+5325 4875 m
+gs 1 -1 sc (8) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+5925 4875 m
+gs 1 -1 sc (10) col0 sh gr
+/Times-Roman ff 360.00 scf sf
+6525 4875 m
+gs 1 -1 sc (12) col0 sh gr
+$F2psEnd
+rs
diff --git a/Carpet/doc/Periodic2.fig b/Carpet/doc/Periodic2.fig
new file mode 100644
index 000000000..f8fc9e532
--- /dev/null
+++ b/Carpet/doc/Periodic2.fig
@@ -0,0 +1,99 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter
+100.00
+Single
+-2
+1200 2
+5 1 0 1 0 7 50 0 -1 0.000 0 0 1 1 4500.000 3637.500 3600 3375 4500 2700 5400 3375
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+5 1 0 1 0 7 50 0 -1 0.000 0 0 1 1 4800.000 3637.500 3900 3375 4800 2700 5700 3375
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+5 1 0 1 0 7 50 0 -1 0.000 0 1 1 1 5100.000 3562.500 4200 3825 5100 4500 6000 3825
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+5 1 0 1 0 7 50 0 -1 0.000 0 1 1 1 5400.000 3562.500 4500 3825 5400 4500 6300 3825
+ 1 1 1.00 60.00 120.00
+ 1 1 1.00 60.00 120.00
+6 4050 3450 4350 3750
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 4125 3525 4275 3675
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 4125 3675 4275 3525
+-6
+6 4350 3450 4650 3750
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 4425 3525 4575 3675
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 4425 3675 4575 3525
+-6
+6 4650 3450 4950 3750
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 4725 3525 4875 3675
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 4725 3675 4875 3525
+-6
+6 4950 3450 5250 3750
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 5025 3525 5175 3675
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 5025 3675 5175 3525
+-6
+6 5250 3450 5550 3750
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 5325 3525 5475 3675
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 5325 3675 5475 3525
+-6
+6 5550 3450 5850 3750
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 5625 3525 5775 3675
+2 1 0 3 1 7 50 0 -1 0.000 0 0 -1 0 0 2
+ 5625 3675 5775 3525
+-6
+6 3750 3450 4050 3750
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 3900 3525 3900 3675
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 3825 3600 3975 3600
+-6
+6 3450 3450 3750 3750
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 3600 3525 3600 3675
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 3525 3600 3675 3600
+-6
+6 5850 3450 6150 3750
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 6000 3525 6000 3675
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 5925 3600 6075 3600
+-6
+6 6150 3450 6450 3750
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 6300 3525 6300 3675
+2 1 0 3 1 1 50 0 0 0.000 0 0 -1 0 0 2
+ 6225 3600 6375 3600
+-6
+1 3 0 1 0 7 50 0 -1 0.000 1 0.0000 4200 3600 75 75 4200 3600 4125 3600
+1 3 0 1 0 7 50 0 -1 0.000 1 0.0000 4800 3600 75 75 4800 3600 4725 3600
+1 3 0 1 0 7 50 0 -1 0.000 1 0.0000 5400 3600 75 75 5400 3600 5325 3600
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 2925 3525 3075 3525 3075 3675 2925 3675 2925 3525
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 3525 3525 3675 3525 3675 3675 3525 3675 3525 3525
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 5925 3525 6075 3525 6075 3675 5925 3675 5925 3525
+2 2 0 1 0 7 50 0 -1 0.000 0 0 -1 0 0 5
+ 6525 3525 6675 3525 6675 3675 6525 3675 6525 3525
+4 0 0 50 0 0 24 0.0000 4 255 180 2925 4875 0\001
+4 0 0 50 0 0 24 0.0000 4 255 180 3450 4875 2\001
+4 0 0 50 0 0 24 0.0000 4 255 180 4125 4875 4\001
+4 0 0 50 0 0 24 0.0000 4 255 180 4725 4875 6\001
+4 0 0 50 0 0 24 0.0000 4 255 180 5325 4875 8\001
+4 0 0 50 0 0 24 0.0000 4 255 360 5925 4875 10\001
+4 0 0 50 0 0 24 0.0000 4 255 360 6525 4875 12\001
diff --git a/Carpet/doc/carpet.bib b/Carpet/doc/carpet.bib
new file mode 100644
index 000000000..f95b7da84
--- /dev/null
+++ b/Carpet/doc/carpet.bib
@@ -0,0 +1,50 @@
+% $Header: /home/eschnett/C/carpet/Carpet/Carpet/doc/carpet.bib,v 1.2 2003/08/15 15:36:05 schnetter Exp $
+
+@Misc{Carpet__erik-schnetter,
+ author = {Erik Schnetter},
+ title = {\href{mailto:schnetter@uni-tuebingen.de} {\textless
+ schnetter@uni-tuebingen.de\textgreater}}
+}
+
+@Misc{Carpet__astro-psu-edu,
+ author = {{Department for} Astronomy and Astrophysics},
+ title = {\href{http://www.astro.psu.edu/}
+ {http://www.astro.psu.edu/}}
+}
+
+@Misc{Carpet__psu-edu,
+ author = {{Penn State University}},
+ title = {\href{http://www.psu.edu/} {http://www.psu.edu/}}
+}
+
+@Misc{Carpet__cactuscode-org,
+ author = {{Cactus web pages}},
+ title = {\href{http://www.cactuscode.org/}
+ {http://www.cactuscode.org/}}
+}
+
+@Misc{Carpet__gnuplot-info,
+ author = {gnuplot},
+ title = {\href{http://www.gnuplot.info/}
+ {http://www.gnuplot.info/}}
+}
+
+@Misc{Carpet__FlexIO,
+ author = {John Shalf},
+ title = {{FlexIO} library:
+ \href{http://zeus.ncsa.uiuc.edu/~jshalf/FlexIO/}
+ {http://zeus.ncsa.uiuc.edu/\textasciitilde
+ jshalf/FlexIO/}}
+}
+
+@Misc{Carpet__HDF,
+ author = {HDF},
+ title = {\href{http://hdf.ncsa.uiuc.edu/}
+ {http://hdf.ncsa.uiuc.edu/}}
+}
+
+@Misc{Carpet__CVS,
+ author = {CVS},
+ title = {\href{http://www.cvshome.org/}
+ {http://www.cvshome.org/}}
+}
diff --git a/Carpet/doc/documentation.tex b/Carpet/doc/documentation.tex
new file mode 100644
index 000000000..546b93d4d
--- /dev/null
+++ b/Carpet/doc/documentation.tex
@@ -0,0 +1,892 @@
+% *======================================================================*
+% Cactus Thorn template for ThornGuide documentation
+% Author: Ian Kelley
+% Date: Sun Jun 02, 2002
+% $Header: /home/eschnett/C/carpet/Carpet/Carpet/doc/documentation.tex,v 1.10 2003/07/21 18:48:08 schnetter Exp $
+%
+% Thorn documentation in the latex file doc/documentation.tex
+% will be included in ThornGuides built with the Cactus make system.
+% The scripts employed by the make system automatically include
+% pages about variables, parameters and scheduling parsed from the
+% relevent thorn CCL files.
+%
+% This template contains guidelines which help to assure that your
+% documentation will be correctly added to ThornGuides. More
+% information is available in the Cactus UsersGuide.
+%
+% Guidelines:
+% - Do not change anything before the line
+% % START CACTUS THORNGUIDE",
+% except for filling in the title, author, date etc. fields.
+% - Each of these fields should only be on ONE line.
+% - Author names should be sparated with a \\ or a comma
+% - You can define your own macros are OK, but they must appear after
+% the START CACTUS THORNGUIDE line, and do not redefine standard
+% latex commands.
+% - To avoid name clashes with other thorns, 'labels', 'citations',
+% 'references', and 'image' names should conform to the following
+% convention:
+% ARRANGEMENT_THORN_LABEL
+% For example, an image wave.eps in the arrangement CactusWave and
+% thorn WaveToyC should be renamed to CactusWave_WaveToyC_wave.eps
+% - Graphics should only be included using the graphix package.
+% More specifically, with the "includegraphics" command. Do
+% not specify any graphic file extensions in your .tex file. This
+% will allow us (later) to create a PDF version of the ThornGuide
+% via pdflatex. |
+% - References should be included with the latex "bibitem" command.
+% - use \begin{abstract}...\end{abstract} instead of \abstract{...}
+% - For the benefit of our Perl scripts, and for future extensions,
+% please use simple latex.
+%
+% *======================================================================*
+%
+% Example of including a graphic image:
+% \begin{figure}[ht]
+% \begin{center}
+% \includegraphics[width=6cm]{MyArrangement_MyThorn_MyFigure}
+% \end{center}
+% \caption{Illustration of this and that}
+% \label{MyArrangement_MyThorn_MyLabel}
+% \end{figure}
+%
+% Example of using a label:
+% \label{MyArrangement_MyThorn_MyLabel}
+%
+% Example of a citation:
+% \cite{MyArrangement_MyThorn_Author99}
+%
+% Example of including a reference
+% \bibitem{MyArrangement_MyThorn_Author99}
+% {J. Author, {\em The Title of the Book, Journal, or periodical}, 1 (1999),
+% 1--16. {\tt http://www.nowhere.com/}}
+%
+% *======================================================================*
+
+% If you are using CVS use this line to give version information
+% $Header: /home/eschnett/C/carpet/Carpet/Carpet/doc/documentation.tex,v 1.10 2003/07/21 18:48:08 schnetter Exp $
+
+\documentclass{article}
+
+% Use the Cactus ThornGuide style file
+% (Automatically used from Cactus distribution, if you have a
+% thorn without the Cactus Flesh download this from the Cactus
+% homepage at www.cactuscode.org)
+\usepackage{../../../doc/latex/cactus}
+\usepackage{hyperref}
+
+\begin{document}
+
+% The author of the documentation
+\author{Erik Schnetter \textless schnetter@uni-tuebingen.de\textgreater}
+
+% The title of the document (not necessarily the name of the Thorn)
+\title{Carpet}
+
+% the date your document was last changed, if your document is in CVS,
+% please use:
+\date{$ $Date: 2003/07/21 18:48:08 $ $}
+
+\maketitle
+
+% Do not delete next line
+% START CACTUS THORNGUIDE
+
+% Add all definitions used in this documentation here
+% \def\mydef etc
+
+% Add an abstract for this thorn's documentation
+\begin{abstract}
+This text describes the Carpet arrangement. Carpet is a mesh
+refinement driver for Cactus that can replace PUGH, the standard
+unigrid driver. Carpet supports multiple refinement levels and
+multiple grid patches. Carpet can run in parallel, but not yet very
+efficiently so. Carpet does not yet support multiple grid
+hierarchies, i.e.\ shadow hierarchies or automatic convergence tests.
+\end{abstract}
+
+
+
+\section{Overview}
+
+\subsection{Fixed Mesh Refinement, aka Box-in-Box}
+
+Fixed Mesh Refinement (FMR), also known as box-in-box, is a way to
+increase the local resolution in unigrid applications, while retaining
+the basic unigrid character of an application. A small number (maybe
+two or three) of grids with varying resolution overlay each other,
+where the coarsest grid has the largest extent. This allows the
+application to benefit from the higher resolution of the smaller grids
+while keeping the outer boundary far out at the same time. The main
+advantage of FMR are that it needs far less resources than globally
+increasing the resolution.
+
+\subsection{Carpet}
+
+Carpet is the name of an FMR driver, i.e.\ the back end that handles
+storage allocation for the grid functions, parallelism, I/O, and the
+various inter-grid operations. Carpet was developed in early summer
+of 2000 by Erik Schnetter \cite{Carpet__erik-schnetter}, then a
+research scholar in the Department for Astronomy and Astrophysics
+\cite{Carpet__astro-psu-edu} of Penn State University
+\cite{Carpet__psu-edu}. In spring 2001, Carpet was coupled to Cactus
+as a drop-in enhancement for the standard unigrid Cactus driver PUGH.
+
+\subsection{Cactus}
+
+From the main Cactus web pages \cite{Carpet__cactuscode-org}:
+\begin{quote}
+Cactus is an open source problem solving environment designed for
+scientests and engineers. Its modular structure easily enables
+parallel computation across different architectures and collaborative
+code development between different groups. Cactus originated in the
+academic research community, where it was developed and used over many
+years by a large international collaboration of physicists and
+computational scientists.
+\end{quote}
+
+
+
+\section{Introduction}
+
+\subsection{Fixed Mesh Refinement}
+
+A standard way of solving partial differential equations are finite
+differences on a regular grid. This is also called \emph{unigrid}.
+Such an application discretises its problem space onto a single,
+rectangular grid which has everywhere the same grid spacing. This
+grid might be broken up into several parts for parallelisation
+purposes, but parallelisation should be transparent to the physics
+part of the application.
+
+Increasing the resolution in a unigrid application is somewhat
+expensive. For example, increasing the resolution by a factor of two
+requires a factor of eight more storage in three dimensions. Given a
+constant Courant factor, the calculation time will even go up by a
+factor of sixteen. This behaviour makes it easy to find problems that
+cannot be solved on contemporary supercomputers, no matter how big and
+fast those computers are.
+
+Apart from physical insight, which often has to be used to decrease
+the problem size until it fits the current hardware, there are also
+numerical and algorithmic methods to decrease the resource
+requirements of the application. Most applications need the high
+resolution only in a part of the simulation domain. Discretisation
+methods that don't require a uniform resolution, such as finite
+elements, can implement non-uniform resolutions very naturally. One
+problem with finite elements is that many physicists today are not
+familiar with finite elements, or shy away from their perceived
+complexity, or are not willing to adapt existing finite difference
+code.
+
+Fixed Mesh Refinement (FMR) is a poor man's way of implementing a
+non-uniform resolution into a unigrid application with minimal changes
+to its structure. Instead of only one grid, there are several grids
+or grid patches with different resolutions. The coarsest grid usually
+encloses the whole simulation domain. Successively finer grids
+overlay the coarse grid at those locations where a higher resolutions
+is needed. The coarser grids provide boundary conditions to the finer
+grid through interpolation.
+
+Instead of updating only one grid, the application has to update all
+grids. The usual approach is to first take a step on the coarsest
+grid, and then recursively take several smaller steps on the finer
+grids. The Courant criterion requires that the step sizes on the
+finer grids be smaller than on the coarse grid. The boundary values
+for the finer grids are found through interpolation in space and time
+from the coarser grid. In the end, the information on the finer grids
+is injected into the coarse grids.
+
+Strictly speaking there is no need for a coarse grid on the regions
+covered by the finer grids. But as stated above, the resources
+required for treating the overlapping region on the coarse grid are
+only minimal compared to treating the finer grids. And because a
+coarse grid with a hole often creates complications, this obvious
+optimisation is often left out.
+
+\subsection{Carpet}
+
+Carpet is a C++ library that provides infrastructure to describe
+regions of varying resolution in a convenient and efficient way.
+Carpet contains routines to manage grid hierarchies, containing the
+relationships between the components of the grid on the different
+refinement and convergence levels. Carpet has a notion of simulation
+time and grid spacing, which are necessary for interpolation, and
+contains efficient interpolators.
+
+Carpet can run on several processors in parallel using MPI for
+communication. Each grid can be broken down into several components,
+and every component has a home processor. Carpet also contains
+operators to move certain regions to a different processor, or to
+synchronise all components of a grid.
+
+Carpet is also an arrangement of thorns for Cactus, implementing a
+driver and associated I/O routines for both ASCII and binary I/O. It
+should be possible to substitute Carpet for the standard Cactus driver
+PUGH without changes to the application thorns and thus use Carpet as
+a unigrid driver. Making use of the FMR capabilities of Carpet
+usually requires some rearranging of the application, comparable in
+general to the changes necessary for a uniprocessor application to run
+on multiple processors.
+
+The driver section of Carpet contains the logic to manage storage for
+the grid functions, to traverse the grid hierarchy for all scheduled
+routines, and to automatically apply the necessary inter-grid
+operators for prolongation (interpolation of the fine grid boundaries)
+and restriction (injecting the fine grid information back into the
+coarse grid).
+
+The ASCII I/O routines use the quasi-standard gnuplot
+\cite{Carpet__gnuplot-info} format. The binary I/O routines use the
+FlexIO library \cite{Carpet__FlexIO} written by John Shalf. It allows
+efficient and platform independent I/O. The FlexIO format is based on
+HDF \cite{Carpet__HDF} and also supported by several visualisation
+packages.
+
+Carpet is copyrighted by Erik Schnetter, and is available under the
+GPL licence from a CVS \cite{Carpet__CVS} repository.
+
+\subsection{WaveToy}
+
+Cactus comes with a sample application called \emph{WaveToy}, which
+solves the scalar wave equation with various initial data and boundary
+conditions. An an example, I have extended WaveToy so that is uses
+Carpet's FMR capabilities. WaveToy serves both as a test case for
+Carpet, and as example of how to convert an application to using FMR.
+
+The equation solved by WaveToy is the well known scalar wave equation,
+discretised using the Leapfrog method with three time levels, yielding
+second order accuracy in space and time. A typical set of initial
+data are a plane wave, and a typical boundary condition is
+periodicity. Those allow long term simulations as well as easy and
+meaningful comparisons to the analytic solution.
+
+
+
+\section{Compiling Cactus With Carpet}
+
+Carpet has been written in C++, using templates and the STL (Standard
+Template Library). Both templates and the STL make writing and
+debugging code a lot easier. Without templates, I would have had to
+put much effort into making Carpet support all of Cactus' data types.
+Without the STL, I would have had to spend quite some time
+implementing basic containers such as lists or sets. I still had to
+implement a custom vector type, because STL's vector type is optimised
+for large vectors only, and I needed threedimensional vectors of
+integers.
+
+The inner loops of Carpet are the inter-grid operators, that is the
+routines that copy, restrict, and prolongate between grids. Due to
+Cactus it was rather easy to write these in \textsc{Fortran 77}, which
+makes them both fast and portable.
+
+Carpet is an arrangement in Cactus. It can theoretically be compiled
+without any external library, if you omit the binary I/O support which
+requires the FlexIO library. FlexIO is already part of Cactus in the
+thorn CactusExternal/FlexIO. I suggest that you enable support for
+the HDF format in the FlexIO library, although this is not necessary.
+For that, you have to install the HDF5 libraries first.
+
+\subsection{Hurdle 1: STL}
+
+Some operating systems do not have a compliant STL (Standard Template
+Library) installed. If not, then you are in trouble. Carpet does
+make use of the STL, and there is no way around that.
+
+\subsection{Hurdle 2: Templates}
+
+Some compilers contain switches to instantiate some or all templates
+automatically. This usually does not work when files are put into
+libraries, which is what Cactus does. The scheme that I found working
+on all machines is to instantiate most templates by hand, and have the
+compiler instantiate the missing templates for every object file.
+This is the default for gcc. On SGIs, you have to pass the options
+\texttt{-no\_auto\_include -ptused} to the C++ compiler.
+
+The C++ standard specifies a limit when using templates as template
+parameters. Carpet's use of the GNU STL exceeds this limit. Gcc
+requires the option \texttt{-ftemplate-depth-30} to enable this.
+
+\subsection{WaveToy}
+
+Unfortunately, PUGH and Carpet cannot yet be both compiled into a
+single application. (This will be fixed soon.) That means that you
+will have separate executables for unigrid and for mesh refinement
+applications.
+
+Configuring Carpet is not quite trivial, because Cactus provides
+currently no way to autodetect the settings for Carpet. Hence you
+will have to set the settings manually. I propose that you start with
+on of the pre-made options files in the directory
+\texttt{Carpet/Carpet/options}. Try e.g.\ \texttt{carpet-harpo-sgi}
+for an SGI, or \texttt{carpet-lilypond} for Linux with gcc, or
+\texttt{carpet-lilypond-ic} for Linux with the Intel compilers. Once
+you have a working options file for your machine, send it to me, so
+that I can include it.
+
+As for the thorn list: Carpet has its own ASCII output thorn, which
+outputs more information than CactusBase/IOASCII. The thorn list that
+I use is
+
+\begin{verbatim}
+CactusBase/Boundary # boundary (grid) [ ] { }
+CactusBase/CartGrid3D # grid ( ) [ ] {driver}
+#CactusBase/IOASCII # IOASCII (IO,Hyperslab) [ ] {IO}
+CactusBase/IOBasic # IOBasic (IO) [ ] {IO}
+CactusBase/IOUtil # IO ( ) [ ] { }
+CactusBase/LocalInterp # LocalInterp ( ) [ ] { }
+CactusBase/Time # time ( ) [ ] { }
+CactusConnect/HTTPD # HTTPD (Socket) [ ] {Cactus}
+CactusConnect/HTTPDExtra # http_utils (httpd,IO) [ ] { }
+CactusConnect/Socket # Socket ( ) [ ] { }
+CactusExternal/FlexIO # FlexIO ( ) [ ] { }
+CactusExternal/jpeg6b # jpeg6b ( ) [ ] { }
+CactusIO/IOJpeg # IOJpeg (IO,Hyperslab,jpeg6b) [ ] {IO}
+CactusUtils/NaNChecker # NaNChecker ( ) [ ] { }
+CactusWave/IDScalarWave # idscalarwave (wavetoy,grid) [ ] {grid}
+CactusWave/IDScalarWaveC # idscalarwave (wavetoy,grid) [ ] {grid}
+CactusWave/IDScalarWaveCXX # idscalarwave (wavetoy,grid) [ ] {grid}
+#CactusWave/IDScalarWaveElliptic # idscalarwaveelliptic (grid,wavetoy,ellbase) [ ] {idscalarwave}
+CactusWave/WaveBinarySource # binarysource (wavetoy,grid,idscalarwave) [ ] { }
+CactusWave/WaveToyC # wavetoy (Grid,Boundary) [ ] { }
+CactusWave/WaveToyCXX # wavetoy (Grid,Boundary) [ ] { }
+CactusWave/WaveToyF77 # wavetoy (Grid,Boundary) [ ] { }
+#CactusWave/WaveToyF90 # wavetoy (Grid,Boundary) [ ] { }
+#CactusWave/WaveToyFreeF90 # wavetoy (Grid,Boundary) [ ] { }
+Carpet/Carpet # driver (CarpetLib) [ ] {Cactus,IO}
+Carpet/CarpetIOASCII # IOASCII (CarpetLib,driver,Hyperslab) [ ] {IO}
+Carpet/CarpetIOFlexIO # IOFlexIO (CarpetLib,driver,Hyperslab,FlexIO) [ ] {IO}
+#Carpet/CarpetIOHDF5 # IOHDF5 (CarpetLib,driver,Hyperslab) [ ] {IO}
+#Carpet/CarpetIOSer # IOSer (CarpetLib,driver,Hyperslab) [ ] {IO}
+Carpet/CarpetLib # CarpetLib ( ) [ ] { }
+Carpet/CarpetReduce # reduce (CarpetLib,driver) [ ] { }
+Carpet/CarpetRegrid # CarpetRegrid (CarpetLib,driver) [ ] { }
+Carpet/CarpetSlab # Hyperslab (CarpetLib,driver) [ ] { }
+\end{verbatim}
+
+The thorns prefixed with \texttt{\#} are disabled. IOASCII conflicts
+with CarpetIOASCII. I disabled IDScalarWaveElliptic because there is
+no elliptic solver for mesh refinement, and I disabled WaveToyF90 and
+WaveToyFreeF90 because gcc does not yet contain a Fortran 90 compiler.
+CarpetIOHDF5 is not yet finished, and CarpetIOSer needs the Ser
+library which is not publically available.
+
+The CactusConnect, CactusIO, and CactusUtils thorns are not necessary,
+but are nice to have around. You can safely omit these.
+
+
+
+\section{Running The Example Applications}
+
+Although Carpet works fine with the standard WaveToy thorns, all the
+example parameter files in the CactusWave arrangement use PUGH, and
+can therefore not be directly used.
+
+The coordinate thorn CactusBase/CartGrid3D does not provide periodic
+boundary conditions. These are normally provided by the driver PUGH.
+However, Carpet does not contain any boundary conditions. If you want
+to apply periodic boundaries, you will therefore have to use the
+AlphaThorns/Cart3d coordinate thorn instead, which does provide
+periodicity. Unfortunately, AlphaThorns/Cart3d is incompatible with
+CactusBase/CartGrid3D. There is a version of WaveToy in the Carpet
+arrangement that has been adapted to AlphaThorns/Cart3d. I suggest
+that you use this version of WaveToy instead of CactusWave to run test
+problems, because periodicity makes for nice testing setups.
+
+You can find quite a few example parameter files in the directory
+\texttt{Carpet/WaveToyF77/par}. I especially recommend the
+\texttt{wavetoyf77\_periodic\_*} set, which comes in two sizes
+(\texttt{coarse} and \texttt{fine}, corresponding to a small and a
+large simulation domain) and three different refinement hierarchies
+(with one, two, and three level altogether, respectively). This set
+thus forms a convergence test, which you can run and test yourself.
+The set \texttt{wavetoyf77\_rad\_full\_*} uses radiative instead of
+periodic boundaries and should also be nice to look at. The file
+\texttt{wavetoyf77\_rad\_automatic.par} is an attempt at adaptive mesh
+refinement, which may or may not work, depending on the current status
+of Carpet.
+
+Second order convergence requires second order interpolation in time,
+which requires that at least three time levels are present.
+
+
+
+\section{Fold Your Own FMR Application}
+
+There are three steps to take from a simple unigrid uniprocessor toy
+application to a full-blown FMR multiprocessor production application.
+Those steps are almost independent, and I would like to explain them
+and their implications in some detail below.
+
+\subsection{Multiple Processors}
+
+The probably best known of these is the step from using one to using
+several processors, also known as parallelisation. Because many
+people are already familiar with this step, I will describe it first.
+
+In a uniprocessor application, it is possible to access every grid
+point in arbitrary manners. In order to allow multiple processors to
+run efficiently in parallel, the grid is broken down into several
+rectangular components, and each processor is assigned one of these
+components.
+
+The components will usually overlap by a few grid points, so as to
+allow the processors to e.g.\ calculate spatial derivatives (which
+require neighbouring grid points) without having to communicate for
+every grid point. From time to time it is then necessary to
+synchronise the overlapping region, which is the only time at which
+communication happens. This allows the application to run almost
+unchanged, i.e.\ without invoking communication itself. The
+synchronisation routine is provided by the driver and not by the
+application.
+
+Of course a serial applicate usually will have to be changed to
+support multiple processors. In order to do so, all the operations
+that the application performs have to be classified into one of two
+categories:
+
+One category contains the so-called \emph{local} operations. These
+are operations that are applied to each and every grid point
+individually, and that do not depend on any other grid point except
+nearby neighbours. Each local operation will thus involve a loop over
+all grid points, and in order to run on multiple processors, after
+each such loop the synchronisation routine has to be called. An
+example of a local operation would be calculating a spatial
+derivative.
+
+The other category contains so-called \emph{global} operations. These
+operations do not depend on individual grid points, and thus do not
+involve loops over grid points. The result of a global operation is
+the same on all processors; therefore global operations don't involve
+communication and don't require synchronisation. An example of a
+global operation would be to check how many time steps have been
+taken, and decide whether the simulation should be terminated.
+
+Typically most operations can be classified or rewritten to be either
+local or global. But often there are operations that fit neither
+category, and these parts of an application are hardest to
+parallelise. Applying the boundary conditions, to give another
+example, might seem at first to be neither local nor global. But in a
+slight (yet completely correct) stretch of the term "applied to all
+grid points", boundary conditions can be classified as local; they are
+a local operation that just does nothing to most grid points.
+
+To give one more example, calculating an error norm does not fit these
+categories. It is neither local nor global. It is not local because
+the results involved all grid points (and not only nearby neighbours),
+and it is not global because it does involve the grid points. All
+operations that do not fit the two category require typically special
+handling, and often require hand-coded communication in the
+application. Luckily calculating various norms is such a common case
+that there are special routines for that already present, called
+\emph{reduction operators}.
+
+\subsection{Multiple Resolution Levels}
+
+There are several reasons why an application might want to incorporate
+more than one grid, overlapping and each with a different resolution.
+
+The most commonly known reason is probably a convergence test, where
+the very same problem is treated in different resolutions.
+Differences in the result are then likely caused by insufficient
+resolution on the coarser (or on all) grids. For a convergence test,
+the grids are completely independent, and it does not matter whether
+the simulation runs on all grids simultaneously or sequentially. In
+order to treat the grid sequentially, the application does not have to
+be changed at all.
+
+The reason of interest here is of course FMR. For FMR, the order in
+which the grids are treated is fixed. As described above, there is
+first a time step on the coarse grid, and then recursively several
+smaller steps on the finer grids. This order does require certain
+changes in the application. The sequence of operations that form a
+single time step have to be identified and isolated. (Which is to say
+that there has to be a routine that calculates a time step, that is, a
+complete time step, and nothing else.) It is then the task of the FMR
+driver to call this routine for the correct grids in the correct
+order.
+
+Other reasons for multiple resolution levels are e.g.\ multigrid
+algorithms for elliptic equations, which I do not want to mention
+here, or shadow hierarchies to determine truncation errors, which I
+also want to skip here. Shadow hierarchies are very similar to the
+convergence levels described above.
+
+Apart from this order in which the operations are performed on the
+grids, there is one more complication for FMR. The boundary values of
+the finer grids have to be calculated from the coarser grids through
+interpolation. An because the time steps on the finer grids are
+smaller, there is not always a corresponding value on the coarser
+grids available. This makes it necessary to interpolate in time
+between time steps on the coarser grids. The alternative would be to
+take smaller steps on the coarser grids, and this would be very
+expensive.
+
+These interpolations in time make it necessary that the driver knows
+which grid function contains values corresponding to what time. The
+usual way to achieve this is to have several time levels per grid
+function; three time levels allow for a second order interpolation in
+time. Only grid functions with enough time levels can be
+interpolated, i.e.\ boundary conditions can be calculated only for
+those.
+
+Fortunately time levels are rather widespread in applications, so they
+are no new concept to introduce. Unfortunately they are often abused,
+so that values corresponding to the wrong time are stored in a time
+level, usually with the excuse of saving storage. This will in
+general not work with FMR, because the driver then cannot interpolate
+in time, leading to incorrect values on the boundaries of the finer
+grids.
+
+\subsection{Multiple Grid Components}
+
+Sometimes it is convenient to have a simulation domain that is not a
+rectangle. It might instead be an L-shaped simulation domain, or a
+domain that consists of two disconnected rectangular regions. This
+issue becomes more important with FMR, because there it is often
+convenient to have several disconnected refined regions. As long as
+there are enough processors available, each processor can be assigned
+a region or a part thereof, and no new concept need be introduced.
+If, however, there are fewer processors than regions, then a new
+problem arises.
+
+A common case for that problem might be a simulation containing just
+two refined regions, and running on a single processor. The refined
+grid the consists of two component. The problem then is that the two
+components cannot be treated sequentially: Imagine the time evolution
+routine working on (say) the first component. It will at some time
+call the synchronisation routine. At that time there are no values
+from the second component available, because the second component has
+not been treated yet. Therefore the synchronisation routine cannot
+complete. That means in turn that the time evolution routine cannot
+complete working on the first component, leading to a deadlock. Work
+on neither component can be completed before work on the other
+component.
+
+The solution is to break up the time evolution routine into several
+smaller routines, each consisting of a single either local or global
+operation. (``Local'' and ``global'' have here the exact same
+meanings that were defined above for parallelisation.) A local
+operation works, by definition, on individual grid points. Hence the
+local routines have to be called once for every grid component. A
+global operation, by definition, does not depend on individual grid
+points. Hence it has to be called only once per processor, and not
+once per component. That means that the driver has to be told the
+category individual routine is in.
+
+\subsection{Example}
+
+Let me finish this section with an detailed example. Suppose you want
+to solve the equation
+\begin{eqnarray}
+ \frac{d}{dt} u & = & f(u) \quad ,
+\end{eqnarray}
+integrating using the midpoint rule, i.e.\ the simplemost second-order
+time integration scheme. Given values at the previous time $u^{n-1}$,
+one first calculates a first order solution using an Euler step,
+leading to the intermediate result
+\begin{eqnarray}
+ v^n & = & u^{n-1} + dt\; f(u^{n-1}) \quad .
+\end{eqnarray}
+The second and final step is then calculated via
+\begin{eqnarray}
+ u^n & = & u^{n-1} + dt\; f(\frac{1}{2} [u^{n-1} + v^n]) \quad .
+\end{eqnarray}
+
+The corresponding pseudo code would look like
+\begin{enumerate}
+\item
+Calculate Euler step, storing the result into $u^n$
+\item
+Apply boundary conditions to $u^n$
+\item
+Synchronise $u^n$
+\item
+Calculate average of $u^{n-1}$ and $u^n$, storing the result into
+$v^n$
+\item
+Calculate second step, storing the result again into $u^n$
+\item
+Apply boundary conditions again to $u^n$
+\item
+Synchronise again $u^n$
+\end{enumerate}
+
+The above algorithm looks a bit different from a naive implementation
+of the midpoint rule. One difference is that both the first and the
+second step store their result into $u^n$. This is necessary because
+it would be inconvenient to apply boundary conditions to the
+intermediate value $v^n$. Remember, in order to apply boundary
+conditions on the finer grids, there have to be several time levels
+present. With the above scheme, only $u$ needs several time levels.
+$v$ is used only as a temporary (and could conceivably be completely
+eliminated).
+
+Note also that the first step goes all the way from time level $n-1$
+to time level $n$. The midpoint rule can be rewritten (in fact, is
+usually written) so that the first step is only a half step, leading
+to the time level $n - \frac{1}{2}$. This is not possible for FMR,
+because interpolating to the time $n - \frac{1}{2}$ is not possible,
+and thus there could be no boundary conditions applied after the first
+step.
+
+The second thing to note is that the application of the boundary
+condition and the synchronisation have been separated rather
+artificially. Normally synchronisation would be considered part of
+the boundary condition. In this case, however, the applying the
+boundary condition is a local operation, whereas synchronisation
+counts as global operation. (It is not obvious that synchronisation
+should be global, but as the synchronisation routine is a part of
+Carpet, it was up to me to decide this.) As explained above, local
+and global operations have to be separated.
+
+Separating the evolution steps and the boundary condition routines is,
+on the other hand, just a notational convenience. There could well be
+a single routine implementing both.
+
+For Cactus, the order in which to call the individual parts of the
+time evolution routines is described in the schedule routines, i.e.\
+in the files called \texttt{schedule.ccl}. By default a routine is
+assumed to be local; global routines have to be tagged with
+\texttt{OPTIONS: GLOBAL}.
+
+The tag \texttt{SYNC: groupname} indicates that the group
+\texttt{groupname} should be synchronised after the scheduled routine
+has been called for all grid components. This obviously makes sense
+only for local routines. Using the \texttt{SYNC:} tag is preferred
+over calling the synchronisation routine \texttt{CCTK\_SyncGroup}
+directly.
+
+The example thorn WaveToy in Carpet's arrangement is a bit simpler
+than what is described here, because it uses the Leapfrog scheme which
+consists of only a single step. I would suggest looking at WaveToy as
+an initial FMR example.
+
+The thorn SpaceToy is implemented very close to the way described
+here. It evolves two variables phi and psi, but it is also coupled to
+the thorn HydroToy. This coupling introduces some additional
+complications. The thorn HydroToy, on the other hand uses a
+predictor-corrector scheme, which is also a two step scheme and thus
+more complex that WaveToy. All the coupling between SpaceToy and
+HydroToy is contained in SpaceToy. I would thus suggest looking at
+HydroToy first.
+
+I assume that converting an application to FMR is straightforward
+after handling the time levels has been straightened out.
+
+
+
+\section{Further documentation}
+
+The individual thorns in the Carpet arrangement might contain further
+documentation, which is also available in the thorn guide.
+Additionally, there is a document \texttt{internals.tex} in the
+arrangement's doc directory, and a document
+\texttt{threelev\_initdata.tex} in thorn \texttt{Carpet}'s doc
+directory.
+
+
+\section{Frequently Asked Questions}
+\label{sec:faq}
+
+Here are a few of the more frequently asked questions with some
+answers.
+\begin{enumerate}
+\item \textbf{If I run without any refined grids, why don't I get the
+ same results as with PUGH?}
+
+ There are two possible reasons. The most common is that the you are
+ not comparing exactly the same output. It used to be the case that
+ norms would disagree (this is no longer the case). If it is the
+ ASCII output that disagress, then you should note that the default
+ output format for CarpetIOASCII gives more digits than
+ CactusBase/IOASCII. If you want to get ``identical'' results for
+ this output, try setting \texttt{IOASCII::out\_format = ".14f"}).
+
+ The second reason is subtle differences are bugs in the
+ implementation. Good luck finding these...
+\item \textbf{I switch on a refined grid. Why do I not see it output?
+ Why is the output strange?}
+
+\begin{figure}[htbp]
+ \begin{center}
+ \includegraphics[scale=0.5]{Grid1.eps}
+ \caption{How the grids are indexed in Carpet. This is an
+ artificial three level example using C-style numbering (0
+ origin). Note that the numbering is with respect to the finest
+ grid.}
+ \label{fig:Grid1}
+ \end{center}
+\end{figure}
+ As soon as you switch on refinement the way the grids are numbered
+ by index changes. The numbering is done with respect to the
+ \textit{finest} grid but covers the entire domain. An example of how
+ the numbering works is given in figure~\ref{fig:Grid1}. It is
+ important to note that this also applies to the numbering in
+ time. So with the grid structure of figure~\ref{fig:Grid1} output
+ for the coarsest grid only occurs on iterations $0,4,8,\dots$, for
+ the medium grid only on iterations $0,2,4,\dots$, and for the finest
+ grid on iterations $0,1,2,\dots$. Note that here the finest grid is
+ not the finest \textit{existing} grid, but the finest
+ \textit{possible} grid. This is controlled by the
+ \texttt{Carpet::max\_refinement\_levels} parameter.
+
+ So, there are plenty of reasons why the output might be strange:
+ \begin{itemize}
+ \item You are requesting output on iterations when not all grids are
+ output. For example, requesting output every $5^{\text{}th}$
+ iteration with the above grid structure would only output the
+ coarse grid every 20 iterations.
+ \item You are requesting output along an index that does not
+ intersect with any grid points. For example, the line defined by
+ $j = 6$ in the example above corresponds to the center of the box,
+ but does not intersect the coarse grid at all!
+ \item Requesting output along a line defined by a coordinate value
+ will give you the index closest to it. This may not agree on the
+ different refinement levels. In the example above the coordinate
+ value $y=5.1$ is closest to $j=5$ on the fine grid, $j=6$ on the
+ medium grid, and $j=4$ on the coarse grid. All the different lines
+ will be output but you should not expect points that appear to
+ overlap in the output to agree as they're actually not at the same
+ point.
+ \item CarpetRegrid (which sets up the refined boxes) knows nothing
+ about symmetries. So if you have a simulation in, for example,
+ octant mode with $x,y,z\in[0,10]$ and you leave all the parameters
+ to be the defaults, the following will happen:
+ \begin{itemize}
+ \item CarpetRegrid creates a refined box at the center of the
+ \textit{index space}. This might cover something like
+ $x,y,z\in[3,7]$.
+ \item When the IO thorn requests the output lines and planes it
+ does know the symmetries, so tries to put the lines and planes
+ as close to the origin $x=y=z=0$ as possible.
+ \item When output occurs the lines and planes don't intersect the
+ fine grid and so you get no output.
+ \end{itemize}
+ \end{itemize}
+
+ Morals: Comparing 1D output on different refinement levels can be
+ very frustrating. 2D output is usually much more informative. Using
+ symmetry conditions with Carpet is tricky.
+
+\item {\bf I want to run with periodic boundaries. Why aren't things
+ working correctly?}
+
+ You thought symmetry boundaries were bad? Periodic boundaries are
+ even worse.
+
+ Firstly, Carpet does not itself implement periodic boundaries. The
+ thorn {\tt TAT/Periodic} is ``more or less'' driver independent and
+ does. This should be used to implement the actual boundary
+ conditions. You should not need to change your code - just activate
+ the thorn with the appropriate parameters.
+
+ Secondly, periodic boundaries do {\bf not} work the same way as
+ symmetry boundaries. This is because you cannot specify a point in
+ coordinate space where the boundary actually lies; it really lies in
+ the index space. The following example will hopefully help.
+
+ Take a 1D slice through the grid. There are 7 points with 2 boundary
+ (ghost) zones (0,2 and 10,12), so only 3 points are actually being
+ evolved (4, 6, 8). Periodic boundaries means that the boundary points
+ are identified with certain evolved points. For example, point 2 is
+ to the left of the first evolved point and so must be identified
+ with the \textit{last} evolved point (8). The identifications are
+ shown in figure~\ref{fig:Periodic1}.
+ \begin{figure}[htbp]
+ \begin{center}
+ \includegraphics[scale=0.5]{Periodic1.eps}
+ \caption{Periodic grids identify boundary points and interior
+ points. The interior points are given by circles and the
+ boundary points by squares. The identifications are shown by the
+ arrows.}
+ \label{fig:Periodic1}
+ \end{center}
+ \end{figure}
+
+ We then want to place a refined region across the entire width of
+ the domain but also have the correct periodic boundaries. The
+ crucial point is to ensure that points that are identified on the
+ coarse grid are identified in the same way on the fine grid. For
+ example, point 2 must still be identified with point 8. Therefore
+ point 2 must remain a boundary point and point 8 an interior
+ point. Point 4 must also be identified with point 10. There are
+ therefore 2 possibilities:
+ \begin{itemize}
+ \item Point 3 is the first interior point on the refined grid and
+ point 8 the last. Therefore the point to the ``left'' of point 3,
+ point 2, is still identified with point 8.
+ \item Point 4 is the first interior point on the refined grid and
+ point 9 the last. This possibility is illustrated in
+ figure~\ref{fig:Periodic2}.
+ \end{itemize}
+ \begin{figure}[htbp]
+ \begin{center}
+ \includegraphics[scale=0.5]{Periodic2.eps}
+ \caption{A periodic refined grid. The boundary zones are blue
+ plus signs, the interior blue crosses. Note that the interior
+ points on the refined grid extend \textit{outside} the
+ interior on the base grid. However, equivalent points on both
+ grids coincide.}
+ \label{fig:Periodic2}
+ \end{center}
+ \end{figure}
+
+ So to specify the particular refined grid shown in
+ figure~\ref{fig:Periodic2} you would specify a lower bound of 2, an
+ upper bound of 11, and that both boundaries are outer boundaries. An
+ example for a $44 \times 7 \times 7$ grid where the ``centre half''
+ of the grid in the $x$ direction is refined and the refined region
+ covers the entirety of the $y$ and $z$ directions, you could use
+\begin{verbatim}
+carpet::max_refinement_levels = 2
+carpetregrid::refinement_levels = 2
+carpetregrid::refined_regions = "manual-gridpoint-list"
+carpetregrid::gridpoints = "[ [ ([22,2,2]:[62,11,11]:[1,1,1]) ] ]"
+carpetregrid::outerbounds = "[ [ [[0,0],[1,1],[1,1]] ] ]"
+\end{verbatim}
+
+\end{enumerate}
+
+%% \bibliographystyle{amsalpha} % initials + year
+%% \bibliography{carpet}
+
+\begin{thebibliography}{{Pen}}
+
+\bibitem[AA]{Carpet__astro-psu-edu}
+{Department for} Astronomy and Astrophysics,
+ \emph{{http://www.astro.psu.edu/}}.
+
+\bibitem[{Cac}]{Carpet__cactuscode-org}
+{Cactus web pages}, \emph{{http://www.cactuscode.org/}}.
+
+\bibitem[CVS]{Carpet__CVS}
+CVS, \emph{{http://www.cvshome.org/}}.
+
+\bibitem[gnu]{Carpet__gnuplot-info}
+gnuplot, \emph{{http://www.gnuplot.info/}}.
+
+\bibitem[HDF]{Carpet__HDF}
+HDF, \emph{{http://hdf.ncsa.uiuc.edu/}}.
+
+\bibitem[{Pen}]{Carpet__psu-edu}
+{Penn State University}, \emph{{http://www.psu.edu/}}.
+
+\bibitem[Sch]{Carpet__erik-schnetter}
+Erik Schnetter, \emph{{\textless
+ schnetter@uni-tuebingen.de\textgreater}}.
+
+\bibitem[Sha]{Carpet__FlexIO}
+John Shalf, \emph{{FlexIO} library:
+ {http://zeus.ncsa.uiuc.edu/\textasciitilde jshalf/FlexIO/}}.
+
+\bibitem[TAT]{Carpet__tat-physik-uni-tuebingen-de}
+Theoretische Astrophysik T\"ubingen,
+ \emph{{http://www.tat.physik.uni-tuebingen.de/}}.
+
+\end{thebibliography}
+
+% Do not delete next line
+% END CACTUS THORNGUIDE
+
+\end{document}
diff --git a/Carpet/doc/first-steps.tex b/Carpet/doc/first-steps.tex
new file mode 100644
index 000000000..cfa0e7577
--- /dev/null
+++ b/Carpet/doc/first-steps.tex
@@ -0,0 +1,583 @@
+% $Header: /home/eschnett/C/carpet/Carpet/Carpet/doc/first-steps.tex,v 1.3 2004/08/05 14:43:52 schnetter Exp $
+
+\documentclass[11pt]{article}
+
+\usepackage{epsfig,amsmath,amssymb,amsfonts,mathrsfs}
+\RequirePackage{amssymb}
+\RequirePackage{amsfonts}
+\pagestyle{headings}
+
+%\bibliographystyle{chicagoa}
+%\bibliography{uli.bib}
+
+\setlength{\textwidth}{17cm}
+\setlength{\oddsidemargin}{-0.4cm}
+\setlength{\topmargin}{0.0cm}
+\setlength{\textheight}{22cm}
+%\setlength{\parindent}{0.5cm}
+
+\numberwithin{equation}{section}
+
+\renewcommand{\textfraction}{0}
+\renewcommand{\topfraction}{1}
+%\renewcommand{\bottomfraction}{1}
+\renewcommand{\floatpagefraction}{1}
+
+\begin{document}
+
+
+\title{A user's perspective on getting started with Carpet}
+\author{Ulrich Sperhake, Erik Schnetter}
+\date{$ $Date: 2004/08/05 14:43:52 $ $}
+\maketitle
+
+
+
+%=============================================================================
+\section{Introduction}
+
+These notes provide information on how to install and use the package
+Carpet as seen from a user's point of view. Carpet is a set of Thorns
+that provide fixed and to some extent adapted mesh refinement in
+the Cactus environment. As Cactus is a necessary requirement for
+using Carpet, these notes will inevitably contain some information about
+Cactus as well.
+
+The reader should regard these notes as a first draft and the information
+represents the author's personal experiences rather than an exhaustive
+recipe on getting Carpet to work on an arbitrary given platform. In this sense I
+am hopeful that users as well as developers will continue to add to this
+document to make it more useful in the future.
+
+Useful starting points for retrieving more detailed information on
+various issues are the project's web pages\\
+
+\hspace{1cm}{\tt http://www.cactuscode.org}\\
+
+\hspace{1cm}{\tt http://www.carpetcode.org}\\
+
+
+%=============================================================================
+\section{Downloading the necessary packages}
+
+One first needs to download the Cactus version 4.0.13 (or
+alternatively for the more daring the development version).
+A more detailed description about how this is done can be found on the
+Cactus web page
+%
+\begin{center}
+ {\tt http://www.cactuscode.org}
+\end{center}
+%
+Here we will summarize the required steps for downloading the complete
+Cactus-4.0.13 package. Change to a suitable directory on your system
+and log onto the Cactus cvs server via\\
+
+\hspace{1cm}{\tt cvs -d :pserver:cvs\_anon@cvs.cactuscode.org:/cactus login} \\
+
+which will prompt you for a password which is {\tt anon}.
+For the development version you will need to choose
+the directory {\tt /cactusdevcvs} instead.
+Next check out the Cactus
+flesh which will create a directory {\tt Cactus} under your current location\\
+
+\hspace{1cm}{\tt cvs -d :pserver:cvs\_anon@cvs.cactuscode.org:/cactus checkout Cactus} \\
+
+The rest of the cactus checkout is best done with the scripts that are shipped
+as part of Cactus. Change to that directory\\
+
+\hspace{1cm}{\tt cd Cactus} \\
+
+and enter the command\\
+
+\hspace{1cm}{\tt make checkout} \\
+
+That will give you various options to choose those parts of cactus you
+want to ckeckout. The default option {\em arrangements} is quite
+satisfactory for this purpose, so just hit return. You will then be
+given a list of (at the time of writing) 13 Cactus arrangements. Getting
+them all is a good idea, so choose once more the default option by pressing
+return. Depending on your internet connection this may take a while.
+Once all is downloaded you want to quit the script. This is not the
+default option, so type {\tt q} and hit return. \\
+
+In order to run the WaveToy example that comes with CarpetExtra
+(see below) you will need to check out
+Erik Schnetter's package TAT. First switch the directory to\\
+
+\hspace{1cm}{\tt cd arrangements} \\
+
+then checkout\\
+
+\hspace{1cm}{\tt cvs -d :pserver:cvs\_anon@cvs.cactuscode.org:/arrangements checkout TAT} \\
+
+Again this may take a little time. Finally you will have to check out the
+{\tt Carpet} package. As of mid April 2004 Carpet consists of 4 arrangements.
+{\tt Carpet} contains all the necessary thorns you will need to run Carpet
+in the first place. The latest cutting edge thorns currently under
+development are located in {\tt CarpetDev}. Do not be too surprised, though,
+if you find some the tools in there not to be fully functional.
+Packages not required to run {\tt Carpet}, but probably useful for
+various purposes, such as scalar wave examples,
+are located in {\tt CarpetExtra}.
+% Detailed instructions can be found on the
+% web page
+%
+% \begin{center}
+% {\tt http://www.carpetcode.org}\\
+% \end{center}
+%
+Remain in the {\tt arrangements} directory for this purpose and log
+into the carpet cvs-server\\
+
+\hspace{1cm}{\tt cvs -d :pserver:cvs\_anon@cvs.carpetcode.org:/home/cvs/carpet login}\\
+
+the password being once more {\tt anon}. Next checkout Carpet by typing\\
+
+\hspace{1cm}{\tt cvs -d :pserver:cvs\_anon@cvs.carpetcode.org:/home/cvs/carpet checkout Carpet}
+
+\hspace{1cm}{\tt cvs -d :pserver:cvs\_anon@cvs.carpetcode.org:/home/cvs/carpet checkout CarpetExtra}
+
+\hspace{1cm}{\tt cvs -d :pserver:cvs\_anon@cvs.carpetcode.org:/home/cvs/carpet checkout CarpetDev}\\
+
+
+
+%=============================================================================
+\section{Documentation}
+
+Documentation about Cactus, Carpet and their separate thorns comes
+in different forms. Most importantly you generate the UsersGuide and
+ReferenceManual for Cactus by going into the {\tt Cactus} directory
+and typing
+
+\hspace{1cm}{\tt make UsersGuide}\\
+
+\hspace{1cm}{\tt make ReferenceManual}\\
+
+\hspace{1cm}{\tt make ArrangementDoc}\\
+
+\hspace{1cm}{\tt make ThornDoc}\\
+
+(four separate commands). They will be created in postscript format under
+the directory\\
+
+\hspace{1cm}{\tt doc}\\
+
+relative to your current position, i.e.\,the {\rm Cactus} directory.
+In addition each thorn may contain a subdirectory {\tt doc} where
+the author (or users) may store additional documentation, typically
+in the form of a file {\tt documentation.tex}.
+
+
+%=============================================================================
+\section{Compilers}
+
+Before we indulge in using Cactus/Carpet, we have to address issues
+concerning the system you are working on. We begin with the compilers
+although we will not be able to deal with the subject in an exhaustive
+fashion.
+Basically these notes list our experiences with local machines
+(i.e. at Penn State) and may or may not be valid for your environment.
+Users are encouraged to add their experiences to this list.
+
+At Penn State we largely work with the Intel compilers and the success
+of compilations has been found to depend sensitively on which version of the
+Intel compilers we are using. We will discuss some error messages
+encountered in the process of compiler testing below.
+
+Free download (at least for Linux) of
+the Intel compiler (Fortran and C++) for
+non-commercial private or academic use is available from the web page
+%
+\begin{center}
+ {\tt http://downloadfinder.intel.com/scripts-df/support\_intel.asp}
+\end{center}
+%
+(click on Software Development, check for the compilers on your system
+and follow their instructions).
+
+In case you haven't got root access,
+you may need to install the compiler locally or you will have to ask your
+sys-admin. Additional difficulties may arise in case you have no
+root access, i.e. install locally, while your sys-admin keeps
+some older version installed. In order to make sure that no conflict
+arises thereof (e.g. by linking against old versions of the library)
+the environment variable {\tt LD\_LIBRARY\_PATH} must point to your local
+new version and not to the old version in {\tt /usr/local} or wherever.
+You will probably end up with error messages such as
+{\tt undefined symbols ...} otherwise. We decided to use the Intel compiler
+for both Fortran and C++ code. This was mainly a result of the current
+version of g++ not having the complete stl libraries that are made use of
+extensively in Carpet. \\
+An important aspect of the Intel compilers is that they come in various
+different versions. Even the same version number (say 7.1) comes in many
+different releases. You can check this by typing \\
+
+\hspace{1cm}{\tt ifc -V}\\
+
+and likewise for {\tt icc}. Note in particular the date of build given in the
+form of {\em 20030307Z}. This corresponds to the March 2003 build of
+version 7.1 and caused difficulties for me. I encountered an error message
+like\\
+
+\hspace{1cm}{\tt /home/terminator/sperhake/src/2004\_02\_16\_cactus-FMR/configs/test01/build/CarpetLib/}
+
+\hspace{1cm}{\tt data.cc(173): error: no instance of overloaded function "dist::datatype"}
+
+\hspace{1cm}{\tt matches the argument list}\\
+
+This can be rectified by switching to a newer release, at least the
+September 2003 build of version 7.1 (I'd recommend doing that for both
+the Fortran and the C++ compiler).
+
+Some Cactus-Carpet users have reported problems, such as segmentation faults,
+by using the most recent versions of the Intel compilers, namely the March
+2004 release of version 7.1 and the latest version 8.0. So far we have
+been using the former of these without encountering any difficulties,
+but you should probably stick to the December or September 2003 version
+of 7.1 if you can.
+
+On my Gentoo Linux laptop, on the other hand, I experienced trouble with
+the September 2003 version of 7.1.
+I received error messages like\\
+
+\hspace{1cm}{\tt struct stat stat\_bbox ...}
+
+\hspace{1cm}{\tt Incomplete components in structure not allowed}\\
+
+at compilation (I have forgotten the exact wording, but you'll recognize it).
+I managed to work around this by using the Intel Fortran
+and C++ compilers version 8.0 (build October 2003).
+As I have not done extensive code development on this laptop, though,
+I cannot really comment on the potential issues concerning the 8.0 version
+mentioned above. \\
+%In case you decide to use version 8.0, however, note that it does not
+%accept the {\tt rpath} option as specified in the {\tt LDFLAGS} entry
+%of the configuration file (see below). This option, for
+%what little I know, tells the linker about the paths for shared libraries.
+%The error message obtained with the {\tt rpath} option was something like\\
+
+%\hspace{1cm}{\tt undefined reference to CCTK\_FullVersion, CCTK\_TimeInfo}\\
+
+Trouble may also arise from preprocessing
+in case you are using RedHat 7.3 (possibly also with other versions).
+This is essentially related to the
+treatment of white space in Fortran files.
+Should you encounter rather stupid error messages which clearly indicate
+that proper lines of Fortran have been corrupted by introducing white space
+(e.g. line breaks) at preprocessing, you should check your cpp and possibly
+download another (probably older) version.
+Details about this can be found on
+%
+\begin{center}
+ {\tt http://www.cactuscode.org/Documentation/Architectures/Linux.html}
+\end{center}
+%
+which also gives a link to the preprocessor of the older RedHat 6.2
+distribution. I downloaded that older version and it solved the preprocessing
+problems I encountered prior to that. \\
+
+
+%=============================================================================
+\section{Libraries}
+\label{sec: libraries}
+
+As much as the compiler issue is strongly dependent on your platform,
+the extent to which you will have to install new libraries will depend on
+what your system administrator has already done for you. Again these notes
+cannot be exhaustive and rather focus on our experience. Feel free,
+as before, to add to our list.
+
+%=============================================================================
+\subsection{HDF library}
+
+The HDF5 library is required for handling in/output in
+a particular binary data
+format. The use of these libraries in Cactus/Carpet is entirely optional,
+but in the end I found it easier to install the libraries than to
+convince my system that I do not want to use them. They should be useful
+in the long run anyway, so I recommend their installation unless
+they are already part of your system.
+
+Let us start with the hdf5 libraries. The binary version can be obtained from
+%
+\begin{center}
+ {\tt ftp://ftp.ncsa.uiuc.edu/HDF/HDF5/hdf5-1.6.1/bin}
+\end{center}
+%
+As before I prefer compiling the source which you can get from
+%
+\begin{center}
+ {\tt ftp://ftp.ncsa.uiuc.edu/HDF/HDF5/hdf5-1.6.1/src}
+\end{center}
+%
+Again the instructions in the {\tt INSTALL} file are straightforward. I
+included the C++ interface by setting the options\\
+
+\hspace{1cm}{\tt ./configure --enable-cxx}\\
+
+and used the variables {\tt CPPFLAGS} and {\tt LDFLAGS} to ensure that the
+szip libraries were found (see {\tt INSTALL} file). The Fortran interface
+did not work for me, so I did not enable that. In future versions of
+this document this issued may be readdressed. Finally you may need to point
+the environment variable {\tt LD\_LIBRARY\_PATH} in your {\tt .bashrc}
+or {\tt .cshrc} to the directory containing the hdf5 library.\\
+
+
+%=============================================================================
+\subsection{Parallelization}
+
+This subsection is relevant only if you plan to do multi processor runs
+(which you are rather likely to do, though, since it is a key feature of
+Cactus/Carpet). There are various packages that take care of
+parallelization, such as {\tt MPICH} or {\tt lam} and your machine
+will probably come equipped with one of these.
+
+I have only had the need to install a message passing interface
+{\tt (MPI)} on my laptop. It's a single processor laptop but
+you can emulate multi-processor runs none the less. Furthermore
+it appears to me that Carpet expects {\tt MPI} at least in the
+form of a header file {\tt mpi.h}, so you'd better install it.
+I chose the {\tt lam} package for this
+purpose, so that is the only experience I have to report.
+
+Installation of this package was straightforward on my Gentoo
+Linux laptop by typing\\
+
+\hspace{1cm}{\tt emerge lam-mpi}\\
+
+Depending on your Linux flavor installation may be done differently,
+for example using {\tt rpm}. {\tt lam} is started by typing\\
+
+\hspace{1cm}{\tt lamboot}\\
+
+and then executables can be started via\\
+
+\hspace{1cm}{\tt mpirun -np} $<$n$>$ $<$executable$>$\\
+
+where $<$n$>$ is the number of processors and $<$executable$>$
+the binary file (with full path) you want to run.
+
+
+%=============================================================================
+\section{Creating a configuration}
+
+%=============================================================================
+\subsection{The configuration file}
+
+Eventually we can start writing a configuration file for a Cactus-Carpet
+project. In this configuration file the paths to various files, such as
+libraries and compilers need to be specified. Naturally these paths will
+differ from machine to machine. In this subsection I will assume
+the installation path
+
+\hspace{1cm}{\tt /usr/local/}$<$name$>$\\
+
+for most libraries,
+where $<$name$>$ is the name of the library, e.g. {\tt hdf4} or {\tt szip}.
+I further assume that each of these directories contains subdirectories
+{\tt lib} and {\tt include} which contain the libraries and header files.
+Similarly I presume that all compilers/preprocessors are installed in the
+directory\\
+
+\hspace{1cm}{\tt /usr/local/for\_carpet/bin}\\
+
+This is, of course, not where they reside on your machine
+(nor on mine), but it'll be sufficient for this document and
+you will merely have to replace each of these paths with the correct one
+on your system.
+
+We are now in the position to create the configuration file, say
+{\tt mycode\_carpet.cfg} (you can store that file wherever you think
+convenient). We will focus on the most important entries
+in this file only. Please refer to the Cactus documentation
+for a more detailed description. First we specify information
+about the compilers\\
+
+\hspace{1cm}{\tt F90 \hspace{1cm} /usr/local/for\_carpet/bin/ifc}
+
+\hspace{1cm}{\tt F77 \hspace{1cm} /usr/local/for\_carpet/bin/ifc}
+
+\hspace{1cm}{\tt CC \hspace{1.2cm} /usr/local/for\_carpet/bin/icc}
+
+\hspace{1cm}{\tt CXX \hspace{1cm} /usr/local/for\_carpet/bin/icc}
+
+\hspace{1cm}{\tt CPP \hspace{1cm} /usr/local/for\_carpet/bin/cpp}
+
+\hspace{1cm}{\tt FPP \hspace{1cm} /usr/local/for\_carpet/bin/cpp}\\
+
+
+(the exact amount of white space between the variables {\tt F90, F77,...} and
+their entries should not matter and you may even put in an $=$ sign).
+Note that you do not need to specify the
+full path if your environment variable {\tt PATH} points to the correct
+versions of the compilers/preprocessors already.\\
+Next we need to specify information about the message passing interface.
+In my case that was {\tt lam}, so the next entries in my
+file {\tt mycode\_carpet.cfg} are\\
+
+\hspace{1cm}{\tt MPI \hspace{2.5cm} LAM}
+
+\hspace{1cm}{\tt LAM\_INC\_DIR \hspace{1cm} /usr/include}
+
+\hspace{1cm}{\tt LAM\_LIB\_DIR \hspace{1cm} /usr/lib}\\
+
+In case you are using a different {\tt MPI} package refer to the Cactus
+users guide to find the correct entry for {\tt MPI}. Make sure that you
+specify the correct paths for the corresponding header files
+and libraries (ask your sys-admin if necessary).\\
+Next we specify the libraries to be included in the compilation. For the
+7.1 version of the Intel compilers in combination with {\tt lam}
+we found the following to work fine\\
+
+\hspace{1cm}{\tt LIBS \hspace{2cm} crypt lapack blas g2c z BINDF90 CEPCF90
+F90 IEPCF90 PEPCF90}
+
+\hspace{4.2cm}{\tt POSF90 cprts cxa guide imf intrins irc ircmt ompstub svml}
+
+\hspace{4.2cm}{\tt unwind X11 ieeeio df m mpi lam pmpi}\\
+
+(all in one line). It goes without saying that all these libraries must
+be installed on your machine. Most of them probably are and the
+installation of some that may not is described in more detail above
+in Sec.\,\ref{sec: libraries}. \\
+The paths to some of these libraries may not be known automatically by the
+linker and needs to be specified separately. This is done with the variable
+{\tt LIBDIRS} which I had to set to\\
+
+\hspace{1cm}{\tt LIBDIRS \hspace{1.4cm} /usr/local/intel/compiler70/ia32/lib}
+
+\hspace{4.2cm}{\tt /usr/X11R6/lib /usr/local/IEEEIO/lib /usr/local/hdf4/lib}
+
+\hspace{4.2cm}{\tt /usr/lib/gcc-lib/i386-redhat-linux/egcs-2.91.66}\\
+
+(again on all in one line). As before you will have to adjust this line to
+your demands.\\
+Finally I set
+
+%\hspace{1cm}{\tt LDFLAGS \hspace{1.4cm} -Wl,-rpath,/usr/nrlocal/petsc-2.1.0/lib/libO/linux}\\
+
+\hspace{1cm}{\tt PTHREADS \hspace{1.2cm} yes}\\
+
+though I am not sure what this is exactly doing.
+%As has been mentioned above the first of these lines is valid for
+%version 7.1 of the Intel compiler and should be omitted if you are using
+%version 8.0.
+
+
+
+%=============================================================================
+\subsection{make-config}
+
+In order to create a configuration change into the {\tt Cactus}
+directory and type\\
+
+\hspace{1cm}{\tt make} $<$name$>${\tt-config options=}$<$config-file$>$\\
+
+where you can choose an arbitrary $<$name$>$ for your configuration and
+$<$config-file$>$ is the file (with full path) created in the previous
+subsection.
+
+%=============================================================================
+\subsection{Creating a thornlist}
+
+Next you will need to generate a thornlist, i.e.\,a list of all those
+thorns you want to compile. This is done in the {\tt Cactus} directory
+by typing\\
+
+\hspace{1cm}{\tt make} $<$name$>${\tt-thornlist}\\
+
+where $<$name$>$ must be the same as in setting up the configuration.
+This command
+will search all arrangements for all thorns and eventually prompt you
+whether you want to modify the list. As all thorns are activated by default
+you do want to modify the list and type {\em yes} and hit return. This
+will open an editor session where you can unselect thorns by putting a
+hash '\#' at the beginning of the line. Unselect all thorns in this
+way except for the following\\
+
+\hspace{1cm}{\tt CactusBase/Boundary}
+
+\hspace{1cm}{\tt CactusBase/CartGrid3D}
+
+\hspace{1cm}{\tt CactusBase/CoordBase}
+
+\hspace{1cm}{\tt CactusBase/IOBasic}
+
+\hspace{1cm}{\tt CactusBase/IOUtil}
+
+\hspace{1cm}{\tt CactusBase/LocalInterp}
+
+\hspace{1cm}{\tt CactusBase/SymBase}
+
+\hspace{1cm}{\tt CactusBase/Time}
+
+\hspace{1cm}{\tt Carpet/Carpet}
+
+\hspace{1cm}{\tt Carpet/CarpetIOASCII}
+
+\hspace{1cm}{\tt Carpet/CarpetIOHDF5}
+
+\hspace{1cm}{\tt Carpet/CarpetInterp}
+
+\hspace{1cm}{\tt Carpet/CarpetLib}
+
+\hspace{1cm}{\tt Carpet/CarpetReduce}
+
+\hspace{1cm}{\tt Carpet/CarpetRegrid}
+
+\hspace{1cm}{\tt Carpet/CarpetSlab}
+
+\hspace{1cm}{\tt CarpetExtra/IDScalarWave}
+
+\hspace{1cm}{\tt CarpetExtra/WaveToyF77}\\
+
+Before you compile, you need to apply one modification to the file\\
+
+{\tt arrangements/CarpetExtra/WaveToyF77/configuration.ccl}\\
+
+namely remove the entry {\tt Cart3d} from the list of {\tt REQUIRED}
+thorns. This thorn is actually required but, for some reason unknown
+to me, must not be mentioned here. It gave an error message complaining
+that there is no thorn {\tt Cart3d}. Having applied this modification
+you can start compiling by typing\\
+
+\hspace{1cm}{\tt make} $<$name$>$
+
+There is no guarantee, but at least you have a chance of compiling through
+without error messages (do not be intimidated by the odd warning, though).
+In case you still cannot compile, please add your wisdom to this document to
+help future users.
+
+
+%=============================================================================
+\subsection{Running the first application: WaveToyF77}
+
+If you've gotten this far, you should be able to run your first
+simulation with mesh refinement. Change to some convenient directory
+for this purpose and copy over from relative to the main {\tt Cactus}
+directory the parameter file\\
+
+\hspace{1cm}{\tt arrangements/CarpetExtra/WaveToyF77/par/wavetoyf77\_rad\_full\_rl2.par}\\
+
+You will need to adjust this parameter file a little to get it running (I am
+not aware of a WaveToy-parameter file that does not require such minor
+modification). First add to the first line beginning with {\tt ActiveThorns}
+the thorns {\tt Slab CoordBase SymBase} (that is within the quotes).
+Finally you should be able to run this example by typing something like\\
+
+\hspace{1cm}{\tt mpirun -np 1} $\tilde{\,\,}${\tt /Cactus/exe/cactus-}$<$name$>$ {\tt wavetoyf77\_rad\_full\_rl2.par}\\
+
+where $<$name$>$ is again the name of the configuration above. In case you do
+not have your main {\tt Cactus} directory under your home directory you will
+need to adjust that part in the command.\\
+By running this command you should obtain a directory
+{\tt wavetoyf77\_rad\_full\_rl2} with the resulting data in ascii format.
+You can check for example the file\\
+
+\hspace{1cm}{\tt wavetoyf77\_rad\_full\_rl2/phi.x.asc}\\
+
+(relative to the directory where you ran the code)
+which lists the data on the separate refinement levels.
+
+\end{document}
diff --git a/Carpet/doc/internals.tex b/Carpet/doc/internals.tex
new file mode 100644
index 000000000..3acbaa9b1
--- /dev/null
+++ b/Carpet/doc/internals.tex
@@ -0,0 +1,777 @@
+% $Header: /home/eschnett/C/carpet/Carpet/Carpet/doc/internals.tex,v 1.4 2003/05/03 13:29:23 schnetter Exp $
+
+\documentclass{article}
+
+\usepackage{amsfonts}
+\usepackage{amssymb}
+\usepackage[english] {babel}
+\usepackage{exscale}
+\usepackage[final]{graphicx}
+\usepackage[backref,draft=false]{hyperref}
+%\usepackage{concrete}
+\usepackage{mathpple}
+%\usepackage{pslatex}
+
+\newcommand{\todo}[1]{\rule{1em}{1ex}~{\small [{#1}]}}
+
+\sloppypar
+
+\begin{document}
+
+\title{Carpet under the hood}
+\author{Erik Schnetter \textless schnetter@uni-tuebingen.de\textgreater}
+\date{$ $Date: 2003/05/03 13:29:23 $ $}
+\maketitle
+
+\begin{abstract}
+ This document describes the internal workings of the Carpet
+ arrangement. Its intended readership are people who extend Carpet,
+ or who use Carpet more thant the average user. This document is
+ supposed to be read in conjuction with and guiding through the
+ source code.
+\end{abstract}
+
+\tableofcontents
+
+\section{Overview}
+
+ The Carpet driver, which lives in the Carpet arrangement, is
+ divided into several parts. The thorn \texttt{Carpet} is the main
+ driver piece; it provides all the routines and structures that
+ Cactus expects from it. The thorn \texttt{CarpetLib} is the
+ workhorse that does all the bookkeeping and data shuffling. Those
+ two alone form a valid Cactus driver; the other thorns provide
+ additional functionality. The thorns \texttt{CarpetInterp},
+ \texttt{CarpetReduce}, and \texttt{CarpetSlab} provide the
+ corresponding interpolation, reduction, and slabbing interfaces.
+ The thorns \texttt{CarpetIOASCII} and \texttt{CarpetIOFlexIO}
+ provide I/O methods. Finally, thorn \texttt{CarpetRegrid} provides
+ a user interface to select where and what to refine. (The actual
+ refinement is handled in \texttt{CarpetLib}.)
+
+
+
+\section{Terminology}
+
+ Carpet is called ``Carpet'' because a carpet consists of many
+ individual patches.
+
+ Carpet is a mesh refinement driver. It knows about a hierarchy of
+ \emph{refinement levels}, where each level is decomposed into a set
+ of cuboid \emph{grid patches}. For historic reasons it also has a
+ notion of \emph{multigrid levels}, but those are currently unused.
+ They might conceivably be reactivated to form multigrid stacks to
+ solve elliptic equations. The grid patch is the smallest unit of
+ grid points that Carpet deals with. Carpet parallelises by
+ assigning sets of grid patches to processors.
+
+ A multi-patch run is a run where more than one grid patch (of the
+ same refinement level) is assigned to a single processor. This is
+ a situation that can occur even without refinement. This is also a
+ situation that cannot occur with PUGH, so that most thorns cannot
+ handle this situation. In multi-patch runs one has to distinguish
+ between \emph{local mode}, where one has access to a single grid
+ patch, and \emph{global mode}, where one cannot access individual
+ grid patches, but can instead perfom global operations such as
+ synchronisation, interpolation, or reduction. This part of Cactus
+ is currently (2003-04-30) undergoing changes.
+
+ Carpet uses vertex-centered refinement. That is, each coarse grid
+ point coincides with a fine grid point. To \emph{regrid} means to
+ select a new set of grid patches for each refinement level. To
+ \emph{recompose} the grid hierarchy means to move data around.
+ Regridding is only about bookkeeping, while recomposing is about
+ data munging.
+
+ Each grid patch can be divided in up to four zones: the interior,
+ the outer boundary, and the ghost zone, and the refinement
+ boundary. The interior is where the actual compuations go on. The
+ outer boundary is where the users' outer boundary condition is
+ applied; from Carpet's point of view, these two are the same. (The
+ only difference is that Carpet sets \texttt{cctk\_bbox}
+ correspondingly.) The ghost zones are boundaries to other grid
+ patches on the same refinement level (that might live on a
+ different processor). The refinement boundary is the boundary of
+ the refined region in a level, and it is filled by prolongation
+ (interpolation) from the next coarser level. Both the ghost zones
+ and the prolongation boundary are filled by \emph{synchronising}.
+
+ Grid patches that are on the same refinement level never overlap
+ except with their ghost zones. Conversly, all ghost zones must
+ overlap with a non-ghost zone of another grid patch of the same
+ level. All refinement boundaries must overlap with a grid patch on
+ the next coarser level. (This is also called \emph{proper
+ nesting}.)
+
+ Except for exceptions, Carpet numbers grid point indices and time
+ levels with integers. It counts always in terms of the finest
+ grid, so that coarser grids have \emph{strides} that are powers of
+ the refinement factor. This has the advantage that different
+ refinement levels can use the same global numbering scheme.
+
+ The grid patches are described by a \emph{bounding box}
+ (abbreviated bbox, see \texttt{CarpetLib/src/bbox.*}.). This is a
+ triplet of \emph{vectors} (see \texttt{CarpetLib/src/vect.*}),
+ where each triplet specifies \emph{lower bound}, \emph{upper
+ bound}, and \emph{stride}, much as is conventional in Fortran.
+ Triplets are enclosed in round parentheses $(\cdot:\cdot:\cdot)$,
+ and vectors are enclosed in square brackets $[\cdot,\cdot,\cdots]$.
+ A typical grid patch might have a bounding box which is denoted by
+ $([0,0,0]:[20,20,20]:[2,2,2])$. This is to be read as
+ $(\textrm{lower}:\textrm{upper}:\textrm{stride})$, meaning that the
+ grid patch has one corner grid point at $[0,0,0]$, the diagonally
+ opposite corner grid point at $[20,20,20]$, and the grid points are
+ spaced two ``fine grid spacings'' apart. This grid patch contains
+ $11 \times 11 \times 11$ grid points. Empty bboxes have an upper
+ bound that is strictly lower than the lower bound. The files
+ \texttt{CarpetLib/src/vect.*} contains many useful routines to deal
+ with short vectors, and the files \texttt{CarpetLib/src/bbox.*}
+ contain routines deal with an algebra of bboxes. The files
+ \texttt{CarpetLib/src/bboxset.*} contain routines that handle sets
+ of bboxes.
+
+
+
+\section{The driver}
+
+ The driver consists of the two thorns \texttt{Carpet} and
+ \texttt{CarpetLib}. \texttt{Carpet} is the front end to
+ Cactus, while \texttt{CarpetLib} is the back end to the
+ machine. \texttt{Carpet} specifies the grid shape, decides when to
+ allocate and deallocate storage, cycles through thes schedule bins,
+ and passes all internal information in the \texttt{cGH} structure
+ to the thorns.
+
+
+
+\subsection{Specifying the grid extent}
+
+ \texttt{Carpet} defines the usual parameters necessary to specify
+ the extent of the grid. Everything that has to do with coordinates
+ and symmetries is handled elsewhere, and the driver does not know
+ about that.
+
+ The \texttt{global\_*} parameters specify the global extent of the
+ coarsest grid. Not all of this grid needs to be covered by grid
+ patches. It is conceivable to have an L-shaped simulation domain
+ without any refinement. This situation can be described to Carpet
+ by specifying a global shape that is the convex hull of the domain,
+ and then using two cuboid grid patchs to fill in the shape of
+ the~L.
+
+ The \texttt{ghost\_*} parameters specify the number of ghost zones.
+ The \texttt{periodic*} parameters are unused; they are only there
+ because some thorns look at these parameters. Carpet itself does
+ not supply periodic boundary conditions; they have to be handled by
+ another thorn. The size of the prolongation boundary is the same
+ as the number of ghost zones.
+
+ The parameter \texttt{max\_refinement\_levels} specifies the
+ maximum number of levels that can be present in a run, including
+ the base level. This parameter, together with
+ \texttt{refinement\_factor}, define the grid point numbering
+ scheme, which (see above) depends on the finest possible grid.
+ However, none of the finer levels will be activated automatically.
+ The \texttt{multigrid\_*} parameters are unused.
+
+ The parameter \texttt{base\_extents} specifies the shapes of the
+ grid patches that are present on the coarsest grid. This can be
+ used to set up e.g.\ an L-shaped domain. The parameter
+ \texttt{base\_outerbounds} specifies which of the grid patches'
+ boundaries are to be treated as outer boundaries, i.e.\ for which
+ of those \texttt{cctk\_bbox} should be set to 1.
+
+ Carpet currently ignores \texttt{enable\_all\_storage} and always
+ enables all storage. This is because it is not yet clear how
+ individual grid function can be allocated and deallocated while
+ still keeping enough data for the boundary prolongation.
+
+ Checksumming and poisoning are means to find thorns that alter grid
+ variables that should not be altered, or that fail to fill in grid
+ variables that they should fill in.
+
+ None of the above specifies anything about refined grids. Refined
+ grid are created and destroyed at run time, possibly guided by the
+ thorn \texttt{CarpetRegrid} which provides a nice user interface.
+
+
+
+\subsection{The timeline}
+
+ It is \texttt{Carpet}'s task to walk through the schedule bins and
+ call all user routines. Only some fairly fundamental
+ initialisation happens in the flesh before Carpet takes control.
+ The overall picture of what happens when is:
+\begin{enumerate}
+\item
+ Startup (see file \texttt{Carpet/src/CarpetStartup.cc}). This is
+ the only scheduled routine; everything else happens by overloading
+ and registering. This routine does nothing but registering and
+ overloading.
+\item
+ SetupGH (see file \texttt{Carpet/src/SetupGH.cc}). This routine
+ does the bulk of initialising Carpet. It sets up the internal
+ structures for all grid variables.
+\item
+ Initialise (see file \texttt{Carpet/src/Initialise.cc}). This
+ routine walks the initialisation part of the scheduling bins.
+\item
+ Evolve (see file \texttt{Carpet/src/Evolve.cc}). This routine
+ walks the evolution part of the scheduling bins. It also contains
+ the main evolution loop.
+\item
+ Shutdown (see file \texttt{Carpet/src/Shutdown.cc}). This routine
+ walks the shutdown part of the scheduling bins. Normally, nothing
+ interesting happens here.
+\end{enumerate}
+ These stages are explained in the following sections.
+
+
+
+\subsubsection{Initialisation}
+
+ (See file \texttt{Carpet/src/Initialise.cc}.) In this stage Carpet
+ initialises the simulation. This includes setting up the grids,
+ calling routines to register symmetries and boundary conditions, as
+ well as calculating the actual initial data on several refinement
+ levels. It traverses the scheduling bins in the following order:
+\begin{enumerate}
+\itemsep 0pt
+\item
+ Set \texttt{cctk\_iteration} to zero
+\item
+ Set \texttt{cctk\_time} to the initial time
+\item
+ PARAMCHECK
+\item
+ Loop over refinement levels, starting from coarsest:
+\item \quad
+ BASEGRID
+\item \quad
+ INITIAL
+\item \quad
+ POSTINITIAL
+\item \quad
+ POSTSTEP
+\item \quad
+ Regrid (possibly creating new levels)
+\item
+ End loop over refinement levels
+\item
+ Restrict from finer to coarser grids
+\item
+ If desired, perform Scott Hawley's initialisation scheme for three
+ timelevels
+\item
+ Loop over refinement levels, starting from coarsest:
+\item \quad
+ RECOVER\_VARIABLES
+\item \quad
+ CPINITIAL
+\item \quad
+ ANALYSIS
+\item \quad
+ OutputGH
+\item
+ End loop over refinement levels
+\end{enumerate}
+
+ In the beginning, only the coarsest level exists. The first loop
+ starts by initialising this level. At the end of this loop, more
+ levels are created if desired. This makes it possible to make this
+ decision depend on an automatic refinement criterion.
+
+
+
+\subsubsection{Evolution}
+
+ (See file \texttt{Carpet/src/Evolve.cc}.) In this stage Carpet
+ performs the main time evolution loop. This is further complicated
+ by the fact that finer grids need to take more and smaller time
+ steps than coarser grids. In Carpet's time step counting scheme,
+ which is based on the finest grid time steps, this means that the
+ coarser grids are skipped in the remaining time steps. Thus the
+ elegant recursive scheme is flattened out. The scheduling bins in
+ the main time evolution loop are traversed in the following order:
+\begin{enumerate}
+\itemsep 0pt
+\item
+ Advance \texttt{cctk\_iteration}
+\item
+ Loop over refinement levels, starting from coarsest:
+\item \quad
+ If the current level needs to be treated at this iteration:
+\item \quad \quad
+ Calculate current \texttt{cctk\_time}
+\item \quad \quad
+ Cycle time levels
+\item \quad \quad
+ PRESTEP
+\item \quad \quad
+ EVOL
+\item \quad \quad
+ POSTSTEP
+\item \quad \quad
+ Regrid
+\item
+ End loop over refinement levels
+\item
+ Restrict from finer to coarser grids
+\item
+ Loop over refinement levels, starting from coarsest:
+\item \quad
+ If the current level needs to be treated at this iteration:
+\item \quad \quad
+ CHECKPOINT
+\item \quad \quad
+ ANALYSIS
+\item \quad \quad
+ OutputGH
+\item
+ End loop over refinement levels
+\end{enumerate}
+
+ The condition whether a refinement level needs to be treated at the
+ current iteration is different for the two loops. In the first
+ loop, the coarse grids need to be advanced before the finer grids,
+ so the condition is $iter \,\mathrm{mod}\, stride = 1$. Here
+ $iter$ is the current iteration, and $stride$ the stride of the
+ current refinement level, i.e.\ the factor by which the finest grid
+ is finer than the current grid. In the second loop above, the
+ coarse grids need to be treated after the finer grids, so that the
+ condition reads $iter \,\mathrm{mod}\, stride = stride$.
+
+
+
+\subsection{Calling scheduled routines}
+
+ (See file \texttt{Carpet/src/CallFunction.cc}.) The process by
+ which the scheduling bins are traversed is different from the
+ process which actually calls the routines within the scheduling
+ bins. The former has to do with mesh refinement, making sure that
+ the coarse and fine grids are evolved in the right order. The
+ latter has to do with treating multiple patches, i.e.\ with local
+ mode and global mode operations, as mentioned above.
+
+ In the function \texttt{CallFunction}, all the arguments that are
+ passed to the scheduled routines have to be set up. Additionally,
+ the \texttt{cGH} structure has to be filled in. Some fields in the
+ \texttt{cGH} structure are always kept up-to-date during the
+ refinement level loops, such as the time step size and the grid
+ spacing. The file \texttt{Carpet/src/helper.cc} contains helper
+ routines that allow easy looping over refinement levels and over
+ grid patches. (Grid patches are also called \emph{compoments} in
+ Carpet. The expression component seems to be confusing, so that I
+ switched to using \emph{patch} instead. Some source code still
+ reflects the old conventsion.)
+
+ The function \texttt{CallFunction} first distinguishes between
+ global mode functions and local mode functions.
+\begin{description}
+\item[Global mode functions]
+ are called once (on each processor). They are passed all the
+ global data, such as \texttt{cctk\_gsh} and
+ \texttt{cctk\_delta\_space}, but none of the local data, such as
+ \texttt{cctk\_lsh} or \texttt{cctk\_bbox}. Grid functions are not
+ accessible, and they are passed as null pointers. However, grid
+ scalars and grid arrays are accessible. There is an untested
+ gateway to directly call local mode functions from global mode
+ functions.
+\item[Local mode functions]
+ might be called several times (on each processor), once for each
+ grid patch that is assigned to this processor. They receive the
+ global data as well as data for a single grid patch. It is illegal
+ to perform global operations, such as synchronisation,
+ interpolation, or reduction, in local mode.
+\end{description}
+
+ The distinction between global and local mode is only important for
+ multi-patch runs. For single-patch runs, the distinction does not
+ exist.
+
+ Multi-patch runs are only necessary when there are more grid
+ patches on a refinement level than there are processors. This is
+ normally not the case for fixed mesh refinement. Things are
+ different for adaptive mesh refinement, which can create many
+ refined regions.
+
+
+
+\subsection{Grid arrays and grid scalars}
+
+ Grid scalars are implemented as zero-dimensional grid arrays with
+ \texttt{DISTRIB=CONSTANT}.
+
+ Grid arrays are implemented as grid functions, where each grid
+ array group has their own refinement hierarchy that consists of a
+ single level only and is never changed at run time. Grid arrays
+ with less than 3 dimension are extended to have an extent of 1 (and
+ no ghost zones) in the remaining dimensions, so that all quantities
+ in Carpet have 3 dimensions\footnote{This is set by a compile-time
+ constant and could be changed to allow for grid functions and
+ arrays with more than 3 dimensions.}. \texttt{DISTRIB=CONSTANT} grid arrays
+ are implemented by internally enlarging the grid array in the $z$
+ direction, and then distributing this array onto the processors.
+
+
+
+\subsection{Flesh interfaces}
+
+ The flesh has many interfaces that need to be filled in by a
+ driver. These are in particular all the routines that are
+ overloaded in the SetupGH stage. Those overloaded routines as well
+ as other helper function are implemented in the following files:
+\begin{description}
+\itemsep 0pt
+\item[\texttt{Carpet/src/Checksum.cc}]
+ catching illegal changes to grid variables
+\item[\texttt{Carpet/src/Comm.cc}]
+ synchronisation, prolongation
+\item[\texttt{Carpet/src/Cycle.cc}]
+ time level handling
+\item[\texttt{Carpet/src/Poison.cc}]
+ catching uninitialised grid variables
+\item[\texttt{Carpet/src/Restrict.cc}]
+ restriction from finer to coarser grids
+\item[\texttt{Carpet/src/Storage.cc}]
+ enabling and disabling storage
+\item[\texttt{Carpet/src/helpers.cc}]
+ small low-level helper routines
+\item[\texttt{Carpet/src/variables.cc}]
+ the global variables that keep Carpet's current state (this is used
+ instead of a GH extension --- should probably be changed some time)
+\end{description}
+
+ Most of these files are fairly self-contained, and they mostly
+ marshal the actual work to \texttt{CarpetLib}.
+
+
+
+\subsection{Interfaces to other thorns}
+
+ Some other thorns, mostly from the Carpet arrangement, do need to
+ access internal data of Carpet. Carpet keeps its internal state in
+ global variables which are declared in
+ \texttt{Carpet/src/carpet\_public.hh} and defined in
+ \texttt{Carpet/src/variables.cc}. Entities that can be accessed
+ from C are declared in \texttt{Carpet/src/carpet\_public.h}; some
+ of these would be quite useful if they were provided by the flesh.
+
+
+
+\subsection{Missing parts}
+
+ \texttt{Carpet} does not handle staggered grids. \texttt{Carpet}
+ does not provide cell-centered refinement. \texttt{Carpet} always
+ enables all storage. \texttt{Carpet} does not run efficiently in
+ parallel.
+
+
+
+\section{The workhorse}
+
+ While \texttt{Carpet} provides the necessary interfaces to the
+ flesh, the grunt work is actually done by \texttt{CarpetLib}. This
+ thorn grew from an earlier mesh refinement of mine (Erik Schnetter)
+ library that was independent of Cactus. It has in the mean time
+ been thoroughly changed, and it does not make sense any more to use
+ it independent of Cactus. \texttt{CarpetLib} contains of three
+ major parts: a set of generic useful helpers, the grid hierarchy
+ and data handling, and interpolation operators. Especially the
+ latter could probably be separated out. While \texttt{CarpetLib}
+ is written in C++, the interpolators are written in
+ \textsc{Fortran77}.
+
+
+
+\subsection{The helpers}
+
+ The helpers correspond closely to Carpet's terminology. A class
+ \texttt{vect<T,D>} provides small \texttt{D}-dimensional vectors of
+ the type \texttt{T}, with all the operators that one has learned to
+ enjoy from Haskell and Fortran 90. A \texttt{vect} corresponds to
+ a grid point location. The class \texttt{bbox<T,D>} provides
+ \texttt{D}-dimensional bounding boxes using type \texttt{T} as
+ indices. A \texttt{bbox} defines the location and shape of a grid
+ patch. Finally, \texttt{bboxset<T,D>} is a collection of \texttt{bbox}es.
+ \texttt{bboxsets} are a useful extension of the algebra of bboxes, as it
+ closes the \texttt{bbox} algebra under the union operation.
+
+ The files \texttt{CarpetLib/src/defs.*} defines useful small
+ helpers and instantiates the STL templates.
+ \texttt{CarpetLib/src/dist.*} provides some routines around MPI.
+ Carpet is closely coupled to MPI and does not work without it.
+
+ (Instead of inserting switches into Carpet to make it work without
+ MPI, it would make more sense to use a dummy version of MPI. PETSc
+ does contain such a dummy version. It is also easily possible to
+ use a free MPI version such as MPICH and use that to run on a
+ single processor. However, I cannot see any real need for making
+ Carpet work without MPI.)
+
+
+
+\subsection{The grid hierarchy}
+
+ The grid hierarchy is described by a set of classes. Except for
+ the actual data, all structures and all information is replicated
+ on all processors.
+\begin{description}
+\item[\texttt{gh}]
+ is a grid hierarchy. It describes, for each refinement level, the
+ location of the grid patches. This \texttt{gh} does not contain
+ ghost zones or prolongation boundaries. There exists only one
+ common \texttt{gh} for all grid functions.
+\item[\texttt{dh}]
+ is a data hierarchy. It extends the notion of a \texttt{gh} by
+ ghost zones and prolongation boundaries. The \texttt{dh} does most
+ of the bookkeeping work, deciding which grid patches interact with
+ what other grid patches through synchronisation, prolongation,
+ restriction, and boundary prolongation. Unexpected situations are
+ often caught in one of \texttt{dh}'s many self checks. As all grid
+ functions have the same number of ghost zones, there exists also
+ only one \texttt{dh} for all grid functions.
+\item[\texttt{th}]
+ is a time hierarchy. It extends the notion of a \texttt{gh} by
+ multiple time levels. There exists one \texttt{th} per grid
+ function group. This is a small class that keeps track of the
+ current time on the different refinement levels. (Note that
+ different refinement levels usually live at different times.)
+\item[\texttt{gf}]
+ is a grid function of a certain variable type. There is one
+ instance of \texttt{gf} for every grid function, whether it has
+ storage or not. Each \texttt{gf} is associated with a \texttt{dh}
+ and a \texttt{th} and holds the storage for all levels and all
+ patches. It provides interfaces to access and modify these data,
+ either directly or through interpolation operators. \texttt{gf}
+ also handles the data movement during a regridding operation.
+\item[\texttt{ggf}]
+ is an abstract superclass of \texttt{gf} which is independent of
+ the variable type. This is necessary in C++ to prevent egregious
+ code duplication due to class templates. Most of the routines in
+ \texttt{gf} are actually declared in \texttt{ggf}, and they either
+ are virtual functions themselves, or they call virtual functions
+ that are declared in \texttt{gf}.
+\item[\texttt{data}]
+ is a container for a grid patch of a certain variable type. This is
+ a glorified multi-dimensional array that knows how to move between
+ processors. \texttt{data} is not only used to store the grid
+ patches that make up a \texttt{gf}, it is also used to move parts
+ of patches around, e.g.\ for synchronisation or prolongation.
+\item[\texttt{gdata}]
+ is an abstract superclass of \texttt{data} for much the same
+ reasons as for \texttt{ggf}. All information that is independent
+ of the variable type is kept in \texttt{gdata}.
+\end{description}
+
+
+
+\subsection{The interpolators}
+
+ There are three kinds of ``interpolators'': for prolongation, for
+ restricting, and for copying. The latter is only a glorified
+ hyperslabber that moves parts of grid patches between grid patches.
+
+ The interpolators used for restriction and prolongation are
+ different from those used for the generic interpolation interface
+ in Cactus. The reason is that interpolation is expensive, and
+ hence the interpolation operators used for restriction and
+ prolongation have to be streamlined and optimised. As one knows
+ the location of the sampling points for the interpolation, one can
+ calculate the coefficients in advance, saving much time compared to
+ calling a generic interpolation interface.
+
+
+
+\subsubsection{Restriction}
+
+ Restriction operators move data from finer to coarser grids. They
+ are typically called after both the coarse and the fine grid have
+ been advanced to the same time, and they overwrite parts of the
+ coarse grid with information from the fine grid, coupling the
+ coarse grid evolution to the fine grid evolution. In principle,
+ there could be restriction operators with different orders of
+ accuracy. Currently only a single restriction operator is
+ implemented that uses sampling.
+
+ The interface of the restriction operator (see file
+ \texttt{CarpetLib/src/restrict\_3d\_real8.F77}) is
+\begin{verbatim}
+subroutine restrict_3d_real8
+ (src, srciext, srcjext, srckext,
+ dst, dstiext, dstjext, dstkext,
+ srcbbox, dstbbox, regbbox)
+
+ integer srciext, srcjext, srckext
+ CCTK_REAL8 src(srciext,srcjext,srckext)
+ integer dstiext, dstjext, dstkext
+ CCTK_REAL8 dst(dstiext,dstjext,dstkext)
+ integer srcbbox(3,3), dstbbox(3,3), regbbox(3,3)
+\end{verbatim}
+ This interpolator assumes that space has three dimensions. The
+ arrays \texttt{src} and \texttt{dst} contain the source (fine) and
+ destination (coarse) grid patches, stored in Fortran order, as is
+ customary in Cactus. The arrays \texttt{src} and \texttt{dst} have
+ the shapes \texttt{(srciext,srcjext,srckext)} and
+ \texttt{(dstiext,dstjext,dstkext)}, respectively --- this
+ corresponds to the \texttt{cctk\_lsh} field in the \texttt{cGH}
+ structure.
+
+ The three bboxes describe the location and shape of the two arrays
+ and of the region that should be prolongated in the global grid
+ point index system. That is, while the two arrays \texttt{src} and
+ \texttt{dst} are stored as dense arrays, they correspond to grid
+ patches which in general have non-unit strides in the global index
+ system. As prolongation is an operation that is performed between
+ overlapping grids, the prolongation region is the same for both the
+ coarse and the fine grids.
+
+ A few constraints must hold for these data. For example, the
+ shapes of the arrays must be the same as the shapes defined by the
+ bounding boxes; the strides in the bounding boxes must differ by
+ the refinement factor; the bounding boxes must overlap, and the
+ region's bounding box must be contained in the arrays bounding
+ boxes, etc. Checking these constraints makes up about three
+ quarters of the restriction routine.
+
+ The bboxes themselves are here represented as Fortran arrays.
+ Their meaning is
+\begin{description}
+\itemsep 0pt
+\item[\texttt{bbox(:,1)}]
+ lower boundary (inclusive)
+\item[\texttt{bbox(:,2)}]
+ upper boundary (inclusive)
+\item[\texttt{bbox(:,3)}]
+ stride
+\end{description}
+
+
+
+\subsubsection{Prolongation}
+
+ There are many prolongation operators implemented. They differ in
+ the order of their interpolation in space (first and third, or
+ linear and cubic interpolation) and in time (first and second, or
+ linear and quadratic). The higher the order of interpolation, the
+ larger is the stencil, i.e.\ the more ghost zones and time levels
+ are necessary, and the more expensive the operation becomes.
+
+ The prolongation operators live in the files
+ \texttt{CarpetLib/src/prolongate\_3d\_real8*.F77}, and the file
+ names indicate their orders: \texttt{$n$tl} stands for $n$ time
+ levels, and \texttt{o$n$} stands for an order $n$ interpolation in
+ space (which uses a stencil that is $n+1$ grid points wide).
+
+ Apart from taking more than one \texttt{src} array argument when
+ using more than one time level, the interface to the prolongation
+ operator is equivalent to that of the restriction operator
+ described above.
+
+
+
+\section{Regridding, how and where and when}
+
+ The thorn \texttt{Carpet} provides a routine
+ \texttt{RegisterRegridRoutine} where one can register a regridding
+ routine. Such a regridding routine does not have to actually
+ regrid anything, it only has to return the new desired grid
+ hierarchy, i.e.\ basically a description of a \texttt{gh}.
+
+ Thorn \texttt{CarpetRegrid} provides a user interface to the
+ regridding routines in Carpet. All it does is take a regridding
+ specification from the user and translate that into a \texttt{gh}.
+ As usual, the parts where the computer has to listen to what a
+ human being intends are the most complicated.
+
+ As humans are usually more adept at getting used to computers than
+ the other way around, it is useful and probably necessary to get
+ acquainted with how Carpet thinks in order to make it do what is
+ intended.
+
+ Carpet does not deal with real-valued coordinates. Carpet deals
+ with integer grid point locations only, and it counts grid points
+ in terms of the finest possible grid (not the finest currently
+ existing grid). The finest possible grid is defined by the maximum
+ number of refinement levels set in \texttt{Carpet}. Changing this
+ parameter will change the meaning of many other values in parameter
+ files, such as e.g.\ iteration numbers (termination, output). The
+ only parameter that is specified in terms of the coarsest grid is
+ the shape of the coarsest grid in the \texttt{global\_*} parameters
+ of \texttt{Carpet}. I therefore suggest to set
+ \texttt{max\_refinement\_levels} to some large number (e.g.\ 10),
+ and then not changing it while experimenting with other parameter
+ settings.
+
+ Carpet also does not know about symmetries. When specifying the
+ location of a fine grid in terms of grid points, it is the
+ responsibility of the user to place the fine grid correctly. For
+ that one has to take ghost zones and symmetry zones into account.
+
+ It is also possible to specify the fine grid locations in terms of
+ real-valued coordinates. In this case, \texttt{CarpetRegrid}
+ translates these into integer grid points. A good translation is
+ quite complicated, because it has to take many user expectations
+ into account, such as the location of the origin, staggering with
+ respect to the origin, symmetry boundary conditions, the number of
+ ghost zones etc. The current translation is naive and leads to
+ unexpected results in many cases. A routine that does most of the
+ time what the user expects while being easy to understand is
+ probably important for the ease of use of Carpet, but it might be
+ some time until it is written.
+
+ \texttt{CarpetRegrid} contains also a routine that measures the
+ error, as provided in a grid function, and the automatically
+ decides where to refine. This is called AMR (adaptive mesh
+ refinement) if it works efficiently.
+
+ Much of \texttt{CarpetRegrid} is just slabbed together in an
+ attempt to find out what people need and expect. The thorn is a
+ mess, and a complete rewrite might be a good idea, once one knows
+ what exactly the rewritten thorn should do.
+
+
+
+\section{Random ramblings}
+
+ Carpet uses the STL, because the STL provides very useful container
+ classes such as vectors, sets, and lists. Writing these abstract
+ datatypes oneself does not make sense in these times. It makes
+ much more sense to politick computer administrators to upgrade
+ their software.
+
+ The STL and \texttt{CarpetLib}'s classes need to be instantiated
+ explicitly. Several compilers have several ``automatic'' schemes
+ that handle all template issues ``just fine''. Except they don't.
+ One wants to select the following: No automatic inclusion of
+ \texttt{.cc} files, no automatic template instantiation at link
+ time. Instead, most templates are instantiated explicitly by
+ \texttt{CarpetLib}. It is also necessary to specify to instantiate
+ used templates automatically. The explicit instantiations of
+ \texttt{CarpetLib}'s classes live in the \texttt{.cc} files
+ corresponding to the \texttt{.hh} file that define the templates.
+ The STL templates are instantiated in the file
+ \texttt{CarpetLib/src/defs.cc}.
+
+ Carpet makes extensive use of the \texttt{assert()} macro in C.
+ This is a quick and easy way to ensure that a certain condition
+ holds. Assert statements abort the code if the condition does not
+ hold. Although I try to provide useful error messages to the user,
+ many unexpected cases are only caught deep inside Carpet and
+ manifest themselves as assertion failures. If you report an
+ assertion failure, it is vitally important to remember
+ theaccompanying file name and line number. It would also be useful
+ to extract from the core file a stack backtrace and the values of
+ the local variables of the current stack frame.
+
+ Using symmetry boundary conditions such as octant mode is currently
+ still awkward in Carpet. There are several reasons for this:
+ \texttt{CarpetRegrid} does not know about symmetries, and hence
+ doesn't take them into account when choosing refinement regions.
+ The symmetry conditions on the finer grid might be different from
+ the conditions on the coarser grids, and the symmetry thorns cannot
+ cope with this, so this situation must be avoided: one cannot use
+ \texttt{avoid\_origin=yes}, because the finer grids all have
+ \texttt{avoid\_origin=no} due to the vertex-centred refinement.
+
+\end{document}
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-17 06:52:57 +0000