summaryrefslogtreecommitdiff
path: root/doc/UsersGuide/RunningCactus.tex
blob: 4edb7685a0d30374b115386869dd5c8f16291d4f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
% /*@@
%   @file      RunningCactus.tex
%   @date      27 Jan 1999
%   @author    Tom Goodale, Gabrielle Allen, Gerd Lanferman
%   @desc 
%   How to run Cactus part of the Cactus User's Guide
%   @enddesc %   @version $Header$      
% @@*/
\begin{cactuspart}{1}{Installation and Running}{$RCSfile$}{$Revision$}
\renewcommand{\thepage}{\Alph{part}\arabic{page}}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Installation} 
\label{cha:in}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Required software}
\label{sec:reqo}

In general, Cactus {\em requires} the following set of software to function
in single processor mode. Please refer to the architecture section
\ref{sec:suar} for architecture specific items.
\begin{Lentry}
\item[{\tt Perl5.0}] Perl is used extensively during the Cactus
  thorn configuration phase. Perl is available for nearly all
  operating systems known to man and can be obtained at
  {\tt http://www.perl.org}
\item[{\tt GNU make}] The make
  process works with the GNU make utility (referred to as {\bf gmake} 
  henceforth). While other make utilities may also work, this is not 
  guaranteed. Gmake can be obtained from your favorite GNU site or
  from {\tt www.gnu.org}
\item[{\tt C/C++}] C and C++ compiler. For example, the GNU compilers. These
 are available for most supported platforms.  Platform specific compilers 
 should also work. 
\item[{\tt CPP}] C Pre-processor. For example, the GNU CPP.  These are
normally provided on most platforms, and many C compilers have an option
to just run as a preprocessor.
\item[{\tt CVS}] The {\em ``Concurrent Versioning System''} is not needed
  to run/compile Cactus, but you are strongly encourage to install
  this software to take advantage of the update procedures. It can be
  downloaded from your favorite GNU site.  Tar files of each release are
  also available.
\end{Lentry}

\noindent
To use Cactus, with the default driver ({\tt CactusPUGH/PUGH}) on multiple
processors you also need:
\begin{Lentry}
\item[{\tt MPI}] the {\it Message Passing Interface (MPI)} 
which provides inter-processor communication. 
Supercomputering sites often supply a native {\tt MPI} implementation with
which Cactus is very likely to be compatible. Otherwise there are
various freely available ones available, e.g. the {\tt MPICH}
version of {\tt MPI} is available for various architectures and operating
systems at {\tt http://www-unix.mcs.anl.gov/mpi/}. 
\end{Lentry}

\noindent
If you are using any thorns containing routines 
written in {\tt FORTRAN} you also need
\begin{Lentry}
\item[{\tt F90/F77}] For routines written in F77, either an F90 or an F77
 compiler can be used. For routines written in F90 a F90 compiler is
 obviously
 required. There is a very limited set of free F90 compilers available
 for the different architectures.
\end{Lentry}

\noindent
While not required for compiling or running Cactus, for thorn development
it is useful to install
\begin{Lentry}
\item[{\tt ctags/etags}] The program Tags enables you browse through the calling structure
  of a program by help of function call database. Navigating the flesh and
  arrangements becomes very easy. Emacs and vi both support this method. See
  \ref{sec:usta} for a short guide to ``tags''.
\end{Lentry}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Supported architectures}
\label{sec:suar}

Cactus runs on many machines, under a large number of Unix operating
systems. Here we list all the machines we have compiled and verified
Cactus on, including some architecture specific notes. 
\begin{Lentry} 
\item[{\bf SGI Origin 2000} running Irix]
\item[{\bf SGI} 32 or 64 bit running Irix]
\item[{\bf Cray T3E}] 
\item[{\bf Dec Alpha}]  Dec operating system and Linux. Single processor
  mode and {\tt MPI} supported. The Decs need to have the GNU {\tt C/C++} 
  compilers installed 
\item[{\bf Intel Linux (ia32, ia64, ppc)}] There is a
  free Linux F90 compiler available from  {\tt http://www.psrv.com}
  -- the only free we know of. Single processor mode and {\tt MPICH}, {\tt LAM}
  supported.
\item[{\bf Windows NT}] Compiles with Cygwin version B20, Digital Fortran Compiler and Microsoft Visual C++ compiler. Single processor mode and {\tt WMPI}, 
{\tt HPVM} supported.
\item[{\bf SP2}]
\end{Lentry}

The following machines are only partially supported
\begin{Lentry}
\item[{\bf HP Exemplar}] 
\item[{\bf Hitachi SR8000-F1}]
\item[{\bf NEC SX-5}]
\item[{\bf Sun Solaris}]  
\end{Lentry}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Checkout procedure}
\label{sec:chpr}

Cactus is distributed, extended, and maintained using the free CVS
software. CVS  allows many
people to work on a large software project together without getting
into a tangle. For the beginner, we summarize the basics in appendix
\ref{sec:uscv}, please refer to this section for a short description of
the CVS syntax.

The space required for an installation depends on the arrangements and
thorns used. The flesh on its own requires less than 5 MB.

\begin{Lentry}
\item[{\bf Login}] Prior to any CVS operation, you need to log into the Cactus
  repository. For an anonymous checkout, type:\\
  {\tt
    cvs -d :pserver:cvs\_anon@cvs.cactuscode.org:/cactus login
    }
  You will be prompted for a password which is {\bf anon}. 
\item[{\bf Checkout}] To obtain a fresh copy of Cactus, move to a directory
  which does not contain a previously checked out version, and type
  {\t
    cvs -z9 -d :pserver:cvs\_anon@cvs.cactuscode.org:/cactus checkout Cactus
    }
  The CVS checkout procedure will create a directory called {\bf
  Cactus} and install the code inside this directory.  From now on we
  will reference all directory names relative to {\bf Cactus}. 

\noindent
  If you want to compile Cactus with thorns, you now need to checkout 
  separately the required arrangements (or {\it arrangements} 
   into the {\bf arrangements} directory. To see  the 
  available Cactus arrangements and thorns type
  {\t 
    cvs -z9 -d :pserver:cvs\_anon@cvs.cactuscode.org:/cactus checkout -s
  }
  To check out a arrangement or thorn type go to the arrangements directory,  {\t cd arrangements},
  and  for a arrangement type
{\t 
  cvs -z9 checkout <arrangement\_name>
  }
        or for just one thorn
{\t 
cvs -z9 checkout <arrangement\_name/thorn\_name>
}
 
To simplify this procedure you may use {\t gmake checkout} in the Cactus
home directory which provides menus to pick arrangements and thorns from.
  
  
\item[{\bf Update}] to update an existing Cactus checkout (to patch in
  possible changes, etc.), do the following {\em within} the {\tt Cactus} directory.
  {\t
    cvs -z9 update
    }
  The update process will operate recursively downwards from your current position
  within the Cactus tree. To update only on certain directories, change
  into these directories and issue the update command.
\item{\bf CVS status}: to obtain a status report on the ``age'' of your
  Cactus or arrangement routines (from your current directory position
  downward), type
  {\t
    cvs status
    }
\item[{\bf non-anonymous CVS}] if you have an account at the central
  repository ({\tt cvs.cactuscode.org}) and would like to perform 
  any of the operation above
  {\em non-anonymously}, replace {\tt cvs\_anon} by your login name
  and provide the appropriate password during the CVS login
  process. Depending on your permissions, you may then make commits to Cactus
  or its arrangements.
\item[{\bf Commits}] you need to perform a personalized login and have
  proper permissions to commit code to the repository. 
\end{Lentry}
For more CVS commands on how to add files, etc. please refer to appendix 
\ref{sec:uscv}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Directory structure}
\label{sec:dist}

A fresh checkout creates a directory {\tt Cactus} with the
following subdirectories:

\begin{Lentry}

\item[{\tt CVS}] the CVS book-keeping directory, present in every subdirectory.

\item[{\tt doc}] Cactus documentation

\item[{\tt lib}] contains libraries.

\item[{\tt src}] contains the source code for Cactus

\item [{\tt arrangements}] contains the Cactus arrangements. The arrangements
  (the actual ``physics'') are not supplied by checking out just Cactus. 
  If the arrangements you want to use are standard Cactus arrangements, or
  reside on our CVS repository ({\tt cvs.cactuscode.org}), 
  they can be checked out in similar way to the Flesh. 
\end{Lentry}

When Cactus is first compiled it creates a new directory {\tt
Cactus/configs}, which will contain all the source code, object files and
libraries created during the build process.  Disk space may be a problem 
on supercomputers where home directories are small. 

A workaround is to first create a
configs directory on scratch space, say {\tt scratch/cactus\_configs/} (where
{\tt scratch/} is your scratch directory), and then either
\begin{itemize}
\item{} set the environment variable {\tt CONFIGS\_DIR} to point to 
this directory
\end{itemize}
or
\begin{itemize}
\item{}  soft link this directory ({\tt ln -s
scratch/cactus\_configs Cactus/configs/}) to the Cactus directory, if your
file-system supports soft-links.
\end{itemize}

Configurations are descibed in detail in section \ref{sec:coaco}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Getting help}
\label{sec:gehe}

For tracking problem reports and bugs we use GNATS, which is a bugtracking 
system published under the GNU license. We have set up a web interface at 
{\tt http://www.cactuscode.org} which allows easy submission and browsing 
of problem reports.    

A description of the GNATS categories which we use is provided in the appendix 
\ref{sec:usgn}.

% OK, there is NO emacs at the moment, because the GNATS setup is really stupid
% and sendpr handles like c.... besides the fact, that the user has to go 
% through a make-process which installs stuff somewhere on his HD. gerd.
% BUT, we could distribute our own, either copy cvsbug, or write a perl
% version.  Tom
% \begin{itemize}
% \item {\tt A web interface}
% \item {\tt SendPR}
% {FIXME: Mention the emacs thing here too...}
% \end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Compilation} 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Cactus can be built in different configurations from the same copy of
the source files, and these different configurations coexist in the
{\tt Cactus/configs} directory. Here are several instances in which
 this can be useful:

\begin{enumerate}
\item{}Different configurations can be for {\em different
architectures}. You can keep executables for multiple architecures
based on a single copy of source code, shared on a common file
system.
\item{} You can compare different {\em compiler options, debug-modes}.
  You might want to compile different communication protocols
  (e.g. {\tt MPI} or {\tt GLOBUS}) or leave them out all together.
\item{} You can have different configurations for {\em different thorn
    collections} compiled into your executable.
\end{enumerate}

Once a configuration has been created, by {\tt gmake <config>} as described
in detail in the next section, a single call to {\tt gmake <config>}
will compile the code.  The first time generate a compile 
{\tt ThornList},  and gives you the chance to edit it before continuing.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Creating a configuration}
\label{sec:coaco}

At its simplest, this is done by {\tt gmake <config>}.  This generates
a configuration with the name {\tt config}, doing its best to automatically
determine the default compilers and compilation flags suitable for the current 
architecture.

There are a number of additional command line arguments which may be supplied 
to override some parts of the procedure.

\subsection{Configuration options}

There are two ways to pass options to the configuration process from the
gmake command line.  
\begin{enumerate}
\item{} Add the options to a configuration file and use,

{\tt gmake <config>-config  options=<filename>} 

All available configuration options may be set in the file {\tt filename}, 
any which are not set will take a default value. The options file
should contain lines of the form:

{\tt <option> [=] ...}

The equals sign is optional.

\item{} Pass the options individually  on the command line,

{\tt gmake <configuration name>-config  <option name>=<chosen value>, ...}
Not all configuration options can be set on the command line. Those than can
be set are indicated in the table below.

\end{enumerate}

Note that if a configuration file is used, and options are also passed
on the command line, the configuration file will currently override the command line
options, although this behaviour may change.

It is important to note that these methods cannot be used to, for example add
options to the default values for {\tt CFLAGS}. Setting {\tt CFLAGS} in the
configuration file, or the command line will overwrite completely the 
default values.

\subsection{Available options}

There is a plethora of available options.

\begin{itemize}
\item {\tt Compiler and tool specification}

These are used to specify which compilers and other tools to use. Entries followed
by * may be specified on the command line.

\begin{Lentry}
\item [{\tt CC}]
* The C compiler.
\item [{\tt CXX}]
The C++ compiler
\item [{\tt F90}]
* The Fortran 90 compiler
\item [{\tt F77}]
* The FORTRAN 77 compiler.
\item [{\tt CPP}]
  The preprocessor used to generate dependencies and to preprocess Fortran code
\item [{\tt LD}]
* The linker.
\item [{\tt AR}]
The archiver used for generating libraries.
\item [{\tt RANLIB}]
The archive indexer to use.
\item [{\tt MKDIR}]
The program to use to create a directory.
\item [{\tt SHELL}]
The shell to use.
\item [{\tt PERL}]
The name of the perl executable

\end{Lentry}

\item {\tt Compilation and tool flags}

Flags which are passed to the compilers and the tools.

\begin{Lentry}
\item [{\tt CFLAGS}]
Flags for the C compiler

\item [{\tt CXXFLAGS}]
Flags for the C++ compiler

\item [{\tt F90FLAGS}]
* Flags for the Fortran 90 compiler

\item [{\tt F77FLAGS}]
* Flags for the FORTRAN 77 compiler

\item [{\tt MKDIRFLAGS}]
  Flags for MKDIR so that no error is given if the directory exists
\item [{\tt LDFLAGS}]
* Flags for the linker

\item [{\tt ARLAGS}]
Flags for the archiver

\item [{\tt DEBUG}]
* Specifies what type of debug mode should be used, 
the default is no debugging.
Current options are {\tt yes}, {\tt no}, or {\tt memory}. The option 
{\tt yes} switches on all debugging features, whereas {\tt memory} just 
employs memory tracing (\ref{sec:metr}).


\item [{\tt OPTIMISE}]
* Specifies what type of optimisation should be used. The only option
currently available is {\tt no}. The default is to use optimisation.

\item [{\tt C\_OPTIMISE\_FLAGS}]
Optimisation flags for the C compiler, their use depends on the type of
optimisation being used.

\item [{\tt CXX\_OPTIMISE\_FLAGS}]
Optimisation flags for the C++ compiler, their use depends on the type of
optimisation being used.

\item [{\tt F90\_OPTIMISE\_FLAGS}]
Optimisation flags for the FORTRAN 90 compiler, their use depends on the 
type of optimisation being used.

\item [{\tt F77\_OPTIMISE\_FLAGS}]
Optimisation flags for the FORTRAN 77 compiler, their use depends on the 
type of optimisation being used.

\item [{\tt C\_WARN\_FLAGS}]
Warning flags for the C compiler, their use depends on the type of
warnings used during compilation (\ref{sec:gmopfobuco}).

\item [{\tt CXX\_WARN\_FLAGS}]
Warning flags for the C++ compiler, their use depends on the type of
warnings used during compilation (\ref{sec:gmopfobuco}).

\item [{\tt F90\_WARN\_FLAGS}]
Warning flags for the FORTRAN 90 compiler, their use depends on the type of
warnings used during compilation (\ref{sec:gmopfobuco}).

\item [{\tt F77\_WARN\_FLAGS}]
Warning flags for the Fortran 77 compiler, their use depends on the type of
warnings used during compilation (\ref{sec:gmopfobuco}).

\end{Lentry}

\item {\tt Library flags}

Used to specify auxiliary libraries and directories to find them in.

\begin{Lentry}

\item [{\tt LIBS}]
The additional libraries.

\item [{\tt LIBDIRS}]
Any other library directories.

\end{Lentry}

\item {\tt Extra include directories}

\begin{Lentry}
\item [{\tt SYS\_INC\_DIRS}]
Used to specify any additional directories for system include files
\end{Lentry}


\item {\tt Precision options}

Used to specify the precision of the default real and integer data types,
specified as the number of bytes the data takes up.  Note that not all
values will be valid on all architectures.

\begin{Lentry}

\item [{\tt REAL\_PRECISION}]
* Allowed values are 16, 8, 4 .

\item [{\tt INTEGER\_PRECISION}]
* Allowed values are 8, 4 and 2 .

\end{Lentry}

\item {\tt Executable name}

\begin{Lentry}
\item [{\tt EXE\_DIR}] The directory in which to place the executable
\item [{\tt EXE}]
The name of the executable.
\end{Lentry}

\item{\tt Extra packages}

Compiling with extra packages is described fully in 
Section \ref{sec:cowiexpa},
which should be consulted for the full range of configuration options.

\begin{Lentry}
\item [{\tt MPI} *] The {\tt MPI} package to use, if required. Supported values are
        {\tt CUSTOM}, {\tt NATIVE}, {\tt MPICH} or {\tt LAM}

\item [{\tt HDF5}]
Supported values are YES

\end{Lentry}

\end{itemize}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Compiling with extra packages}
\label{sec:cowiexpa}


\subsubsection{MPI: Message Passing Interface}

{\tt MPI} (the {\it Message Passing Interface}) can provides inter-processor 
communication. It can either be implemented natively on a machine
(this is usual on most supercomputers), or through a standard package
such as {\tt MPICH}, {\tt LAM}, {WMPI}, or {PACX}.  

To compile with MPI, the configure option is
\newline
\newline
{\tt MPI = <MPI\_TYPE>}
\newline
\newline
where {\tt <MPI\_TYPE>} can take the values (entries followed by * 
may be specified on the configuration command line):

\begin{Lentry}

\item[{\tt CUSTOM}] For a custom {\tt MPI} configuation set the variables
  \begin{Lentry}
  \item [{\tt MPI\_LIBS} *] libraries
  \item [{\tt MPI\_LIB\_DIRS} *] library directories
  \item [{\tt MPI\_INC\_DIRS} *] include file directories
  \end{Lentry}

\item[{\tt NATIVE}] Use the native {\tt MPI} for this machine, as indicated in
        the {\tt known-architectures} directory 
	({\tt lib/make/known-architectures})

\item[{\tt MPICH}] 
Use MPICH ({\tt http://www-unix.mcs.anl.gov/mpi/mpich}). This is controlled
by the options
  \begin{Lentry}
  \item [{\tt MPICH\_ARCH} *] machine architecture
  \item [{\tt MPICH\_DIR} *] directory in which {\tt MPICH} is installed.
        If this option is not defined it will be searched for.
  \item [{\tt MPICH\_DEVICE} *] the device used by {\tt MPICH}. If not 
        defined, the configuration process will search for this in a 
        few defined places.
        Supported devices are currently {\tt ch\_p4}, {\tt ch\_shmem}, 
	{\tt globus} and {\tt myrinet}.
	For versions of MPICH prior to 1.2.0 the devices are searched for
 	in this order, for 1.2.0 you may need to specify {\tt MPICH\_DEVICE},
	depending on the installation.
  \end{Lentry}

If {\tt MPICH\_DEVICE} is chosen to be {\tt globus}, ({\tt www.globus.org}), 
an additional variable must be set
  \begin{Lentry}
  \item[{\tt GLOBUS\_LIB\_DIR} *] directory in which Globus libraries are installed
  \end{Lentry}

\item[{\tt LAM}]
Use {\tt LAM} (Local Area Multicomputer, {\tt http://www.mpi.nd.edu/lam/}). This is 
controlled by the variables 
  \begin{Lentry}
  \item[{\tt LAM\_DIR} *] directory in which {\tt LAM} is installed. This 
     will be searched for, in a few provided places,  if not given.
  \end{Lentry}
if the {\tt LAM} installation splits libraries and include files into different
directories, instead of setting {\tt LAM\_DIR} set the two variables
  \begin{Lentry}
  \item[{\tt LAM\_LIB\_DIR} *] directory in which {\tt LAM} libraries are installed. 
  \item[{\tt LAM\_INC\_DIR} *] directory in which {\tt LAM} include files are installed. 
  \end{Lentry}


\item[{\tt WMPI}] 
Use WMPI (Win32 Message Passing Interface, {\tt http://dsg.dei.uc.pt/w32mpi/intro.html}). This is controlled by the variable
  \begin{Lentry}
  \item[{\tt WMPI\_DIR} *] directory in which {\tt WMPI} is installed.
  \end{Lentry}

\item[{\tt PACX}] 
Use the PACX Metacomputing package (PArallel Computer eXtension,\\
{\tt http://www.hlrs.de/structure/organisation/par/projects/pacx-mpi/}). This is controlled by the variables
  \begin{Lentry}
  \item[{\tt PACX\_DIR} *] directory in which {\tt PACX} is installed.
        If this option is not defined it will be searched for.
  \item[{\tt PACX\_MPI} *] the MPI package {\tt PACX} uses for node-local
        communication. This can be any of the above MPI packages.
  \end{Lentry}

\end{Lentry}

Note that the searches for libraries etc. mentioned above use the 
locations given in the files in {\tt lib/make/extras/MPI}.


\subsubsection{HDF5: Hierarchical Data Format version 5}

To compile with HDF5 ({\tt http://hdf.ncsa.uiuc.edu/whatishdf5.html}),
the configure options are
\newline
\newline
{\tt HDF5 = YES [HDF5\_DIR = <dir>] [LIBZ\_DIR = <dir>]}
\newline
\newline
If HDF5\_DIR is not given the configuration process will search for an
installed HDF5 package in some standard places (defined in
{\tt lib/make/extras/HDF5}). If the found HDF5 library was compiled with
libz.a, the configuration process also searches for that library and adds it 
to the linker flags. You may also point directly to an installation of libz.a
by setting LIBZ\_DIR.\\






\subsection{File layout after a configuration has been created }

The configuration process sets up various subdirectories and files in the 
{\tt configs} directory to contain the configuration specific files, these
are placed in a directory with the name of the configuration.

\begin{Lentry}

\item [{\tt config-data}] contains the files created by the configure
script:

The most important ones are

\begin{Lentry}

\item [{\tt make.config.defn}] 
contains compilers and compilation flags for a configuration.  

\item [{\tt make.extra.defn}]
contains details about extra packages used in the configuration.

\item [{\tt cctk\_Config.h}]
The main configuration header file, containing architecture specific
definitions.

\item [{\tt cctk\_Archdefs.h}]
An architecture specific header file containing things which cannot be
automatically detected, and have thus been hand-coded for this architecture.
\end{Lentry}

These are the first files which should be checked or modified to suite any
peculiarities of this configuration.

In addition the following files may be informative:

\begin{Lentry}
\item [{\tt fortran\_name.pl}] 
A perl script used to determine how the Fortran compiler names subroutines.  
This is used to make some C routines callable from Fortran, and Fortran 
routines callable from C.

\item [{\tt make.config.deps}]
Initially empty.  Can be edited to add extra architecture specific dependencies
needed to generate the executable.

\item [{\tt make.config.rule}] 
Make rules for generating object files from source files.

\end{Lentry}

Finally, autoconf generates the following files.

\begin{Lentry}

\item [{\tt config.log}]
A log of the autoconf process.

\item [{\tt config.status}]
A script which may be used to regenerate the configuration.

\item [{\tt config.cache}]
An internal file used by autoconf.

\end{Lentry}

\item [{\tt lib}] 
An empty directory which will contain the libraries created for each thorn.

\item [{\tt build}] 
An empty directory which will contain the object files generated for this 
configuration, and preprocessed source files.

\item [{\tt config-info}] 
An file containing information about the configuration.

\end{Lentry}


\section{Building and Administering a configuration}
\label{sec:buanadaco}

Once you have created a new configuration, the command
\\ \\ 
{\tt gmake <configuration name>}
\\ \\
will build an executable, prompting you along the way for the 
thorns which should be included. There is a range of  gmake 
targets and options which are detailed in the following sections.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{gmake targets for building and adminstering configurations}
\label{sec:gmtafobuanadco}

A target for {\tt gmake} can be naively thought of as an argument
that tells it which of several things listed in the {\tt Makefile} it
is to do. The command {\tt gmake help} lists all gmake targets.

\begin{Lentry}

\item [{\tt gmake <config>}] 
	builds a configuration. If the configuation doesn't exist
        it will create it.

\item [{\tt gmake <config>-clean}] removes all object and dependency files from
  a configuration. 

\item [{\tt gmake <config>-cleandeps}] removes all dependency files from
  a configuration. 

\item [{\tt gmake <config>-cleanobjs}] removes all object files from
  a configuration. 

\item [{\tt gmake <config>-config}] creates a new configuration or reconfigures an old one.

\item [{\tt gmake <config>-delete}] deletes a configuration ({\tt rm -r configs/<config>}).

\item [{\tt gmake <config>-editthorns}] edits the ThornList

\item [{\tt gmake <config>-realclean}] removes from a configuration
  all object and dependency files, 
  as well as files generated from the CST (that is, only the files
  generated by configure and the ThornList files remain).

\item [{\tt gmake <config>-rebuild}] rebuilds a configuration (reruns the CST -- the perl scripts which parse the thorn configuration files).

\item [{\tt gmake <config>-testsuite}] runs the test programs associated with
 each thorn in the configuration. See section \ref{sec:te} for information about the 
 testsuite mechanism.

\item [{\tt gmake <config>-thornlist}] regenerates the ThornList for a configuration.

\item [{\tt gmake <config>-examples}] copies all the example parameter files relevant for this configuration to the directory {\tt examples} in the Cactus home directory. If a file of the same name is already there, it will not overwrite it.

\end{Lentry}



\subsection{Compiling in thorns}
\label{sec:cointh}

Cactus will try to compile all thorns listed in 
{\tt configs/<config>/ThornList}.
The {\tt ThornList} file is simply a list of the form
{\t <arrangement>/<thorn>}.  All text after a \# sign
on a line is treated as a comment and ignored.
The first time that you compile a configuration, 
you will be shown a list of all the thorns in your arrangement
directory, and asked if you with to edit them. You can regenerate
this list at anytime by typing

\begin{verbatim}
gmake <config>-thornlist
\end{verbatim}

or you can edit it using

\begin{verbatim}
gmake <config>-editthorns
\end{verbatim}

Instead of using the editor to specify the thorns you want to
  have compiled, you can {\em edit} the {\em ThornList} outside
  the make process. It is located in {\tt configs/<config>/ThornList},
  where {\tt <config>} refers to the name of your configuration.
  The directory, {\tt ./configs}, exists {\em
    after} the very first  make phase for the first configuration.

\subsection{Notes and Caveats}
\begin{itemize}
\item{} If during the build you see the error ``{\tt missing
    separator}'' you are probably not using GNU make. 
\item{} {\em The EDITOR environment variable}. You may not be aware of
  this, but this thing very often exists and may be set  by default to
  something scary like {\tt vi}. If you don't know how to use {\tt vi}
  or wish to
  use your favorite editor instead, reset this environment variable.
  (To exit {\tt vi} type {\tt <ESC> :q!})
\end{itemize}

\subsection{gmake options for building configurations}
\label{sec:gmopfobuco}

An {\it option} for gmake can be thought of as an argument which tells
it how it should make a {\tt target}. Note that the final result is always
the same.

\begin{Lentry}

\item [{\tt gmake <target> THORNLIST=<file> [THORNLIST\_DIR=<dir>]}] uses the file {\tt dir/file} as the ThornList for the configuration. The directory defaults to the current directory
\item [{\tt gmake <target> SILENT=no}] print the commands that gmake is executing
\item [{\tt gmake <target> WARN=yes}] show compiler warnings during compilation.
\item [{\tt gmake <target> FJOBS=<number>}] compile in parallel, across files within each thorn
\item [{\tt gmake <target> TJOBS=<number>}] compile in parallel, across thorns 

\end{Lentry}

Note that with more modern versions of gmake, it is sufficient to pass the normal
 {\tt -j <number>} flag to gmake to get parallel compilation. 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Other gmake targets}

\begin{Lentry}

\item [{\tt gmake help}] lists all make options

\item [{\tt gmake checkout}] allows you to easily checkout Cactus arrangements and thorns.

\item [{\tt gmake cvsdiff}] differences between checked out version of Cactus and that in the CVS repositories.

\item [{\tt gmake cvsstatus}] status of checked out version of Cactus, reporting which files have been modified or need updating.

\item [{\tt gmake default}] creates a new configuration with a default name.

\item [{\tt gmake distclean}] delete your {\tt configs} directory and hence all your configurations.

\item [{\tt gmake doc}] places a postscript version of the Users Guide documentation in your Cactus home directory.

\item [{\tt gmake downsize}] removes non essential files as documents
  and testsuites to allow for minimal installation size.

\item [{\tt gmake newthorn}] creates a new thorn, prompting for the necessary 
  information and creating template files.

\item [{\tt gmake TAGS}] creates an Emacs style TAGS file. See section
  \ref{sec:usta} for using TAGS within Cactus.

\item [{\tt gmake tags}] creates a {\tt vi} style tags file. See section
  \ref{sec:usta} for using TAGS within Cactus.

\item [{\tt gmake UsersGuide}] runs LaTeX to produce a copy of the Users' Guide.
\item [{\tt gmake MaintGuide}] runs Latex to produce a copy of the Mainainers' Guide.

\end{Lentry}


\section{Testing} 
\label{sec:te}

Some thorns come with a testsuite, consisting of example parameter files
and the output files generated by running these. To run the testsuites
for the thorns you have compiled use
\newline

{\tt gmake <configuration>-testsuite}
\newline

These testsuites serve the dual purpose of

\begin{Lentry}
\item [Regression testing]
i.e. making sure that changes to the thorn or the flesh don't affect the
output from a known parameter file.
\item [Portability testing]
i.e. checking that the results are independent of the architecture - this
is also of use when trying to get Cactus to work on a new architecture.
\end{Lentry}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\chapter{Running Cactus}

This chapter covers all aspects for running your Cactus executable.
These include: command line options; parameter file syntax; understanding
screen output and environment variables.

\section{Command Line Options}
\label{sec:coliop}

The cactus executable accepts numerous command line arguments:
\newline

{\tt
\begin{tabular}{|l|l|}
\hline
Short Version & Long Version \\
\hline
 -O & -describe-all-parameters \\
\hline
 -o <param> & -describe-parameter <param> \\
\hline
 -T & -list-thorns\\
\hline
 -t <arrangement/thorn>& -test-thorn-compiled <arrangement/thorn>\\
\hline
 -h,-? & -help\\
\hline
 -v & -version \\
\hline
 -x [<nprocs>] & -test-parameters [<nprocs>] \\
\hline
 -W <level> & -warning-level <level> \\
\hline
 -E <level> & -error-level <level> \\
\hline
 -r & -redirect-stdout \\
\hline
\end{tabular}
}
\newline

\begin{Lentry}
\item [{\tt -O} or {\tt -describe-all-parameters}]
Produces a full list of all parameters from all thorns which were compiled,
along with descriptions and allowed values.  This can take an optional extra
parameter {\tt v}  (i.e. {\tt -Ov} to give verbose information about
all parameters).
\item [{\tt -o <param>} or {\tt -describe-parameter <param>}] 
Produces the description and allowed values for a given parameter - takes one
argument.
\item [{\tt -T} or {\tt -list-thorns}] 
Produces a list of all the thorns which were compiled in.
\item [{\tt -t <arrangement or thorn>} or {\tt -test-thorn-compiled <arrangement or thorn>} ] 
Checks if a given thorn was compiled in - takes one argument.
\item [{\tt -h}, {\tt -?} or {\tt -help}]
Produces a help message.
\item [{\tt -v} or {\tt -version}] 
Produces version information of the code.
\item [{\tt -x <nprocs>} or {\tt -test-parameters <nprocs>}] 
Runs the code far enough to check the consistency of the parameters.  If
given a numeric argument it will attempt to simulate being on that number 
of processors. [To be implemented]
\item [{\tt -W <level>} or {\tt -waring-level <level>}]
Sets the warning level of the code.  All warning messages are given a level--
the lower the level the greater the severity.  This parameter controls the
level of messages to be seen.  The default is a warning level of 1, with 
0 indicating the only those messages which are (by default) fatal should 
be seen.
\item [{\tt -E <level} or {\tt -error-level <level>}]
This works in concert with {\tt -W} - it controls which warning level is
treated as a fatal error.  This cannot be set to a higher value than 
{\tt -W}. The default value is zero.
\item [{\tt -r} or {\tt -redirect-stout}]
This redirects the standard output of each processor to a file.  By default
the output from processors other than processor 0 is discarded.
\end{Lentry}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Parameter File Syntax}

The parameter file is used to control the behaviour of the code at runtime.
It is of a text file with lines which are either comments, denoted
by a `\#', or parameter statements. A parameter statement consists 
of one or more parameter names, followed by
and `=', followed by the value(s) for this (these) parameter(s).

The name of a parameter consists of:

\begin{Lentry}
\item [{\tt Global parameters}]
Just name of the parameter itself.
\item [{\tt Restricted parameters}]
The name of the {\em implementation} which defined the parameter, two colons,
and the name of the parameter --- e.g. {\tt driver::global\_nx}.
\item [{\tt Private parameters}]
The name of the {\em thorn} which defined the parameter, two colons,
and the name of the parameter --- e.g. {\tt wavetoyF77::amplitude}.
\end{Lentry}

In addition there is a parameter {\tt ActiveThorns} which tells the code
which {\em thorns} are to be activated.  Only parameters from active thorns can
be set  (and only those routines {\it scheduled} by active thorns are run).  
By default all thorns are inactive.  {\bf This should be the first
parameter in your parameter file.}
\newline

Note that you can obtain lists of the parameters associated with
each thorn using the command line options {\tt -o} and {\tt -O}
(Section~\ref{sec:coliop}).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\chapter{Getting and looking at output}


\section{Screen output}

As your Cactus executable runs, standard output and standard error
are usually written to the screen. Standard output provides you
with information about the run, and standard error reports warnings
and errors from the flesh and thorns.

As the program runs, the normal output provides the following information

\begin{Lentry}

\item [Active thorns]
	A report is made as each of the thorns in the {\tt ActiveThorns} parameters from the parameter file is attempted to be activated. This report 
shows whether the activation was successful, and if successful gives the 
thorns implementation. For example

{\tt
\begin{verbatim}
Activating thorn idscalarwave...Success -> active implementation idscalarwave
\end{verbatim}
}

\item [Failed parameters] 
 	If any of the parameters in the parameter file does not belong to any of the active thorns, or if the parameter value is not in the allowed range, an
error is registered. For example, if the parameter is not recognised

{\tt
\begin{verbatim}
Unknown parameter time::ddtfac�
\end{verbatim}
}
or if the parameter value is not in the allowed range

{\tt
\begin{verbatim}
Unable to set keyword CartGrid3D::type - ByMouth not in any active rangey
\end{verbatim}
}

\item [Scheduling information]
	A complete list of all scheduled routines is given, in the 
order that they will be executed. For example

{\tt
\begin{verbatim}
----------------------------------------------------------------------
  Startup routines
    Cactus: Register banner for Cactus
    CartGrid3D: Register GH Extension for GridSymmetry
    CartGrid3D: Register coordinates for the Cartesian grid
    IOASCII: Startup routine
    IOBasic: Startup routine
    IOUtil: IOUtil startup routine
    PUGH: Startup routine.
    WaveToyC: Register banner

  Parameter checking routines
    CartGrid3D: Check coordinates for CartGrid3D
    IDScalarWave: Check parameters

  Initialisation
    CartGrid3D: Set up spatial 3D Cartesian coordinates on the GH
    PUGH: Report on PUGH set up
    Time: Set timestep based on speed one Courant condition
    WaveToyC: Schedule symmetries
    IDScalarWave: Initial data for 3D wave equation

  do loop over timesteps
    WaveToyC: Evolution of 3D wave equation
    t = t+dt
    if (analysis)
    endif
  enddo
----------------------------------------------------------------------
\end{verbatim}
}

\item [Thorn banners]
	Any banners registered from the thorns are displayed

\end{Lentry}


\section{Output}
Output methods in Cactus are all provided by thorns. Any number
of output methods can be used for each run. The behaviour of 
the output thorns in the standard arrangements are described in
those thorns documentation. In general, these thorns decide
what to output by parsing a string parameter containing the 
names of those grid variables, or groups of variables, for 
which output is required. The names should be fully qualified
with the {\tt implementation} and {\tt group} or {\tt variable} names. 
There is usually a parameter for each method to denote how often, in evolution
iterations, this output should be performed.  There is also usually
a parameter to define the directory in which the output should be
placed, defaulting to the directory from which the executable is 
run.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Checkpointing}

Checkpointing is defined as writing all data from a run to a file,
so that the run can be restarted from reading all the data from the
file. Checkpointing methods in Cactus are provided by thorns.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\end{cactuspart}