Git stuff:

Replace shell commands in Makefile with explicit perl scripts.

Handle symbolic links to arrangements or thorns explicitly, so that
newer versions of git work fine.


git-svn-id: http://svn.cactuscode.org/arrangements/CactusUtils/Formaline/trunk@181 83718e91-0e4f-0410-abf4-91180603181f

11 files changed, 500 insertions, 161 deletions

diff --git a/src/make.code.deps b/src/make.code.deps
index 7eb710f..606383d 100644
--- a/src/make.code.deps
+++ b/src/make.code.deps
@@ -6,20 +6,13 @@
 # file make.configuration.deps because only here the location of the
 # source directory is known.
 
-TARBALL_DIR = $(SCRATCH_BUILD)
+FORMALINE_BIN_DIR = $(SCRATCH_BUILD)/formaline-bin
 
-$(CCTK_TARGET): $(TARBALL_DIR)/gethostname.pl $(TARBALL_DIR)/makeblob.pl $(TARBALL_DIR)/makemetablob.pl $(SCRATCH_BUILD)/git-lock.pl
+FORMALINE_TAR_FILES = gethostname.pl makeblob.pl makemetablob.pl
+FORMALINE_GIT_FILES = git-add-thorn.pl git-commit-everything.pl git-gc-repo.pl git-init-master-repo.pl git-init-repo.pl git-lock.pl git-push-everything.pl git-rm-thorn.pl
 
+$(CCTK_TARGET): $(FORMALINE_TAR_FILES:%=$(FORMALINE_BIN_DIR)/%) $(FORMALINE_GIT_FILES:%=$(FORMALINE_BIN_DIR)/%)
 
-
-$(TARBALL_DIR)/gethostname.pl: $(SRCDIR)/util/gethostname.pl
-	cd $(TARBALL_DIR) && cp $^ $@
-
-$(TARBALL_DIR)/makeblob.pl: $(SRCDIR)/util/makeblob.pl
-	cd $(TARBALL_DIR) && cp $^ $@
-
-$(TARBALL_DIR)/makemetablob.pl: $(SRCDIR)/util/makemetablob.pl
-	cd $(TARBALL_DIR) && cp $^ $@
-
-$(SCRATCH_BUILD)/git-lock.pl: $(SRCDIR)/util/git-lock.pl
-	cd $(SCRATCH_BUILD) && cp $^ $@
+$(FORMALINE_BIN_DIR)/%.pl: $(SRCDIR)/util/%.pl
+	mkdir -p $(FORMALINE_BIN_DIR)
+	cp $^ $@
diff --git a/src/make.configuration.deps b/src/make.configuration.deps
index 100513a..726960e 100644
--- a/src/make.configuration.deps
+++ b/src/make.configuration.deps
@@ -20,6 +20,7 @@ TR_C := $(shell tr -C 'a' 'b' < /dev/null > /dev/null 2> /dev/null && echo 'tr -
 # into an executable
 
 TARBALL_DIR = $(SCRATCH_BUILD)
+FORMALINE_BIN_DIR = $(SCRATCH_BUILD)/formaline-bin
 
 
 
@@ -62,7 +63,7 @@ $(FRM-LIB): $(FRM-OBJS)
 .PHONY: $(TOP)/BUILD-ID
 $(TOP)/BUILD-ID: $(CCTK_LIBDIR)/$(LIBNAME_PREFIX)$(CCTK_LIBNAME_PREFIX)Formaline$(LIBNAME_SUFFIX)
 	config=`echo "$(EXE:cactus_%=%)" | $(TR_C) -d '[:alnum:]+-._]'` && \
-	hostname=`$(TARBALL_DIR)/gethostname.pl` && \
+	hostname=`$(FORMALINE_BIN_DIR)/gethostname.pl` && \
 	user="$$USER" && \
 	dirname=`echo "$(subst /,-,$(CCTK_HOME:/%=%))" | $(TR_C) -d '[:alnum:]+-._]'` && \
 	timestamp=`date -u +%Y.%m.%d-%H.%M.%S` && \
@@ -95,7 +96,7 @@ $(TARBALL_DIR)/cactus-meta-source.o: $(TARBALL_DIR)/cactus-meta-source.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 $(TARBALL_DIR)/cactus-meta-source.c: $(TARBALL_DIR)/cactus-flesh-source.c $(patsubst %,$(TARBALL_DIR)/cactus-thorn-source-%.c,$(notdir $(THORNS)))
-	$(TARBALL_DIR)/makemetablob.pl Cactus $(notdir $(THORNS)) > $@
+	$(FORMALINE_BIN_DIR)/makemetablob.pl Cactus $(notdir $(THORNS)) > $@
 
 .PRECIOUS: $(TARBALL_DIR)/cactus-meta-source.c $(TARBALL_DIR)/cactus-meta-source.o
 
@@ -115,7 +116,7 @@ $(TARBALL_DIR)/cactus-flesh-source.o: $(TARBALL_DIR)/cactus-flesh-source.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 $(TARBALL_DIR)/cactus-flesh-source.c: $(TARBALL_DIR)/cactus-flesh-source.tar.gz $(CCTK_HOME)/arrangements/AEIThorns/Formaline/src/util/VERSION
-	$(TARBALL_DIR)/makeblob.pl '' Cactus < $< > $@
+	$(FORMALINE_BIN_DIR)/makeblob.pl '' Cactus < $< > $@
 
 $(TARBALL_DIR)/cactus-flesh-source.tar.gz: $(TARBALL_DIR)/cactus-flesh-source.files
 	cd $(CCTK_HOME) && \
@@ -133,6 +134,8 @@ $(TARBALL_DIR)/cactus-flesh-source.files: $(CCTK_LIBDIR)/$(LIBNAME_PREFIX)$(CCTK
 		configs/$(notdir $(TOP))/config-data \
 		configs/$(notdir $(TOP))/config-info \
 		-name 'CVS' -prune -o \
+		-name '*.log' -prune -o \
+		-name '*.log.gz' -prune -o \
 		-name '.?*' -prune -o \
 		-name '*~' -prune -o \
 		! -type d \
@@ -163,7 +166,7 @@ $(TARBALL_DIR)/cactus-thorn-source-%.o: $(TARBALL_DIR)/cactus-thorn-source-%.c
 	$(CC) $(CFLAGS) -c -o $@ $^
 
 $(TARBALL_DIR)/cactus-thorn-source-%.c: $(TARBALL_DIR)/cactus-thorn-source-%.tar.gz $(CCTK_HOME)/arrangements/AEIThorns/Formaline/src/util/VERSION
-	$(TARBALL_DIR)/makeblob.pl $(patsubst %/,%,$(dir $(filter %/$(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%),$(THORNS)))) $(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%) < $< > $@
+	$(FORMALINE_BIN_DIR)/makeblob.pl $(patsubst %/,%,$(dir $(filter %/$(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%),$(THORNS)))) $(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%) < $< > $@
 
 $(TARBALL_DIR)/cactus-thorn-source-%.tar.gz: $(TARBALL_DIR)/cactus-thorn-source-%.files
 	cd $(CCTK_HOME) && \
@@ -175,10 +178,13 @@ $(TARBALL_DIR)/cactus-thorn-source-%.tar.gz: $(TARBALL_DIR)/cactus-thorn-source-
 $(TARBALL_DIR)/cactus-thorn-source-%.files: $(CCTK_LIBDIR)/$(LIBNAME_PREFIX)$(CCTK_LIBNAME_PREFIX)%$(LIBNAME_SUFFIX) $(TARBALL_DIR)/cactus-thorn-source-%.ccldeps
 	cd $(CCTK_HOME) && \
 	find arrangements/$(filter %/$*,$(THORNS))/. \
+		-name 'data' -prune -o \
 		-name 'doc' -prune -o \
 		-name 'par' -prune -o \
 		-name 'test' -prune -o \
 		-name 'CVS' -prune -o \
+		-name '*.log' -prune -o \
+		-name '*.log.gz' -prune -o \
 		-name '.?*' -prune -o \
 		-name '*~' -prune -o \
 		! -type d \
@@ -260,14 +266,10 @@ endif
 
 
 
-# Suppress all output from git commands
-GIT-SILENT = > /dev/null 2>&1
-
 # Does git exist, or should we do nothing instead?
-GIT-CMD := $(shell env PATH=$$HOME/git/bin:$$PATH which git)
+GIT-CMD := $(shell env PATH=$(SCRATCH_BUILD)/git-1.6.6.1/bin:$(SCRATCH_BUILD)/git-1.6.0.6/bin:$$HOME/git/bin:$$PATH which git)
 HAVE-GIT := $(shell '$(GIT-CMD)' --version > /dev/null 2>&1 && echo 'true' || echo 'false')
-GIT1 := $(shell $(HAVE-GIT) && echo '$(SCRATCH_BUILD)/git-lock.pl $(GIT-CMD)' || { echo 'WARNING: git command not found' >&2 && echo 'true'; })
-GIT := $(shell $(HAVE-GIT) && echo '$(SCRATCH_BUILD)/git-lock.pl $(GIT-CMD) $(GIT-SILENT)' || { echo 'WARNING: git command not found' >&2 && echo 'true'; })
+GIT := $(shell $(HAVE-GIT) && echo '$(FORMALINE_BIN_DIR)/git-lock.pl $(GIT-CMD)' || { echo 'Formaline: WARNING: git command not found' >&2 && echo 'true'; })
 
 
 
@@ -282,6 +284,7 @@ git-store-source: git-commit-everything git-push-everything
 	echo "Formaline:  If the build process seems to hang, some other command"
 	echo "Formaline:  is still running, e.g. the final link stage.)"
 
+GIT-BIN         = $(FORMALINE_BIN_DIR)
 GIT-ROOT        = $(CCTK_HOME)
 GIT-REPO        = $(TOP)/configjar.git
 GIT-MASTER-REPO = $(CCTK_HOME)/cactusjar.git
@@ -291,161 +294,50 @@ GIT-THORNS      = $(notdir $(THORNS))
 BUILD-ID-FILE   = $(TOP)/BUILD-ID
 CONFIG-ID-FILE  = $(TOP)/CONFIG-ID
 
-define GIT-INIT-MASTER-REPO
-	{ \
-	if [ ! -e $(GIT-MASTER-REPO)/.git/HEAD ]; then \
-		echo "Formaline: Creating master git repository..." && \
-		mkdir -p $(GIT-MASTER-REPO) && \
-		cd $(GIT-MASTER-REPO) && \
-		$(GIT) init-db; \
-	fi && \
-	if [ ! -e $(GIT-MASTER-REPO)/README ]; then \
-		{ \
-		echo "This directory $$(pwd)" && \
-		echo "is not empty -- it contains a git repository with the Cactus source" && \
-		echo "trees of all previous builds, starting on $$(date)." && \
-		echo "" && \
-		echo "You can use the command \"git branch\" to list all configurations that" && \
-		echo "are stored in this repository.  The history of each branch is the" && \
-		echo "sequence in which the configuration was built.  The most recent build" && \
-		echo "is stored in the branch head, as usual.  In order to check out a" && \
-		echo "certain branch into a directory <name>, issue the following commands:" && \
-		echo "        cd <somewhere_else>" && \
-		echo "        mkdir <name>" && \
-		echo "        cd <name>" && \
-		echo "        git init" && \
-		echo "        git pull $$(pwd) <branch>" && \
-		echo "" && \
-		echo "You can also use the command \"git tag -l\" to list all builds that are" && \
-		echo "stored in this repository.  This keeps the source tree for each build" && \
-		echo "directly accessible.  In order to check out a certain tag into a" && \
-		echo "directory <name>, issue the following commands:" && \
-		echo "        cd <somewhere_else>" && \
-		echo "        git clone -o <name> $$(pwd)" && \
-		echo "        git checkout <tag>"; \
-		} > $(GIT-MASTER-REPO)/README; \
-	fi; \
-	} || echo "WARNING: Error while handling git repository"
-endef
-
-define GIT-INIT-REPO
-	{ \
-	if [ ! -e $(GIT-REPO)/.git/HEAD ]; then \
-		echo "Formaline: Creating git repository..." && \
-		mkdir -p $(GIT-REPO) && \
-		cd $(GIT-REPO) && \
-		$(GIT) init-db; \
-	fi; \
-	} || echo "WARNING: Error while handling git repository"
-endef
-
-# Call git-gc if the repository has grown in size by more than a
-# factor of 10.  This macro needs to be called in the directory where
-# the .git subdirectory lives.
-define GIT-COMPACT-REPO
-	{ \
-	if [ ! -e .oldreposize ]; then \
-		echo 0 > .oldreposize; \
-	fi && \
-	oldreposize=$$(cat .oldreposize) && \
-	maxreposize=$$((oldreposize * 10)) && \
-	reposize=$$(du -s .git | awk '{ print $$1; }') && \
-	if [ $$reposize -gt $$maxreposize ]; then \
-		$(GIT) gc && \
-		reposize=$$(du -s .git | awk '{ print $$1; }') && \
-		echo $$reposize > .oldreposize; \
-	fi; \
-	} || echo "WARNING: Error while handling git repository"
-endef
-
 .PHONY: git-push-everything
 git-push-everything: git-commit-everything $(BUILD-ID-FILE)
-	$(GIT-INIT-MASTER-REPO)
-	{ \
-	echo "Formaline: Pushing source tree to master git repository..." && \
-	cd $(GIT-ROOT) && \
-	export GIT_DIR=$(GIT-REPO)/.git && \
-	build_id=$$(cat $(BUILD-ID-FILE)) && \
-	$(GIT) push -f $(GIT-MASTER-REPO) "$$build_id" && \
-	config_id=$$(cat $(CONFIG-ID-FILE)) && \
-	$(GIT) push -f $(GIT-MASTER-REPO) "$$config_id"; \
-	} || echo "WARNING: Error while handling git repository"
-	true || { \
-	cd $(GIT-ROOT) && \
-	export GIT_DIR=$(GIT-MASTER-REPO)/.git && \
-	echo "Formaline: Optimising master git repository..." && \
-	$(GIT) gc; \
-	} || echo "WARNING: Error while handling git repository"
-	{ \
-	cd $(GIT-ROOT) && \
-	export GIT_DIR=$(GIT-MASTER-REPO)/.git && \
-	echo "Formaline: Optimising master git repository..." && \
-	cd $(GIT-MASTER-REPO) && \
-	$(GIT-COMPACT-REPO); \
-	} || echo "WARNING: Error while handling git repository"
 	{ \
+	'$(GIT-BIN)/git-init-master-repo.pl' '$(GIT)' '$(GIT-MASTER-REPO)' && \
 	if [ -e "$(CCTK_HOME)/cactus.config" ]; then \
-		source "$(CCTK_HOME)/cactus.config"; \
+		source "$(CCTK_HOME)/cactus.config" && \
+		export CACTUS_CENTRAL_GIT_REPO; \
 	fi && \
-	if [ -n "$$CACTUS_CENTRAL_GIT_REPO" ]; then \
-		cd $(GIT-ROOT) && \
-		export GIT_DIR=$(GIT-MASTER-REPO)/.git && \
-		echo "Formaline: Pushing to central repository $$CACTUS_CENTRAL_GIT_REPO..." && \
-		config_id=$$(cat $(CONFIG-ID-FILE)) && \
-		{ $(GIT1) tag -l && echo "$$config_id"; } | \
-		xargs $(GIT) push -f "$$CACTUS_CENTRAL_GIT_REPO"; \
-	fi; \
-	} || echo "WARNING: Error while handling git repository"
-
-# Try to use the previous commit as parent, if possible
+	'$(GIT-BIN)/git-push-everything.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-MASTER-REPO)'; \
+	} || echo "Formaline: WARNING: Error while pushing to master repository"
+
 .PHONY: git-commit-everything
 git-commit-everything: $(SCRATCH_BUILD)/cactus-flesh-source.git-tag $(GIT-THORNS:%=$(SCRATCH_BUILD)/cactus-thorn-%.git-tag) $(BUILD-ID-FILE)
 	{ \
-	echo "Formaline: Committing source tree to git repository..." && \
-	cd $(GIT-ROOT) && \
-	export GIT_DIR=$(GIT-REPO)/.git && \
-	build_id="$$(cat $(BUILD-ID-FILE))" && \
-	{ $(GIT) commit -m "$$build_id" || true; } && \
-	$(GIT) tag -f "$$build_id" && \
-	echo "Formaline: Created git tag $$build_id" && \
-	config_id="$$(cat $(CONFIG-ID-FILE))" && \
-	$(GIT) branch -f "$$config_id" && \
-	echo "Formaline: Updated git branch $$config_id" && \
-	echo "Formaline: Optimising git repository (slow only the first time)..." && \
-	true || $(GIT) gc; \
-	cd $(GIT-REPO) && \
-	$(GIT-COMPACT-REPO); \
-	} || echo "WARNING: Error while handling git repository"
+	'$(GIT-BIN)/git-commit-everything.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' "$$(cat $(BUILD-ID-FILE))" "$$(cat $(CONFIG-ID-FILE))"; \
+	} || echo "Formaline: WARNING: Error while committing to repository"
 
 .PRECIOUS: $(SCRATCH_BUILD)/cactus-flesh-source.git-tag
 $(SCRATCH_BUILD)/cactus-flesh-source.git-tag: $(TARBALL_DIR)/cactus-flesh-source.files
-	$(GIT-INIT-REPO)
 	{ \
+	'$(GIT-BIN)/git-init-repo.pl' '$(GIT)' '$(GIT-REPO)' && \
 	echo "Formaline: Adding flesh to git repository..." && \
-	cd $(GIT-ROOT) && \
-	export GIT_DIR=$(GIT-REPO)/.git && \
-	{ $(GIT) rm --cached -r CONTRIBUTORS COPYRIGHT Makefile lib src configs > /dev/null 2>&1 || true; } && \
-	xargs ls < $(TARBALL_DIR)/cactus-flesh-source.files \
-		> $(TARBALL_DIR)/cactus-flesh-source.files2 2> /dev/null; \
-	xargs $(GIT) add < $(TARBALL_DIR)/cactus-flesh-source.files2 && \
-	rm $(TARBALL_DIR)/cactus-flesh-source.files2 && \
+	'$(GIT-BIN)/git-rm-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' 'cactus' CONTRIBUTORS COPYRIGHT Makefile lib src configs && \
+	: xargs ls < $(TARBALL_DIR)/cactus-flesh-source.files \
+		> $(TARBALL_DIR)/cactus-flesh-source.files2 2> /dev/null && \
+	xargs '$(GIT-BIN)/git-add-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' 'cactus' \
+		< $(TARBALL_DIR)/cactus-flesh-source.files && \
+	: rm $(TARBALL_DIR)/cactus-flesh-source.files2 && \
 	: > $@; \
-	} || echo "WARNING: Error while handling git repository"
+	} && echo "Formaline: WARNING: Error while adding flesh to git repository"
 
 .PRECIOUS: $(SCRATCH_BUILD)/cactus-thorn-%.git-tag
 $(SCRATCH_BUILD)/cactus-thorn-%.git-tag: $(TARBALL_DIR)/cactus-thorn-source-%.files
-	$(GIT-INIT-REPO)
 	{ \
+	'$(GIT-BIN)/git-init-repo.pl' '$(GIT)' '$(GIT-REPO)' && \
 	echo "Formaline: Adding thorn $* to git repository..." && \
-	cd $(GIT-ROOT) && \
-	export GIT_DIR=$(GIT-REPO)/.git && \
-	{ $(GIT) rm --cached -r arrangements/*/$* > /dev/null 2>&1 || true; } &&\
-	xargs ls < $(TARBALL_DIR)/cactus-thorn-source-$*.files \
-		> $(TARBALL_DIR)/cactus-thorn-source-$*.files2 2> /dev/null; \
-	xargs $(GIT) add < $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \
-	rm $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \
+	'$(GIT-BIN)/git-rm-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' '$*' "'"'arrangements/*/$*'"'" && \
+	: xargs ls < $(TARBALL_DIR)/cactus-thorn-source-$*.files \
+		> $(TARBALL_DIR)/cactus-thorn-source-$*.files2 2> /dev/null && \
+	xargs '$(GIT-BIN)/git-add-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' '$*' \
+		< $(TARBALL_DIR)/cactus-flesh-source.files && \
+	: rm $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \
 	: > $@; \
-	} || echo "WARNING: Error while handling git repository"
+	} || echo "Formaline: WARNING: Error while adding thorn $* to git repository"
 
 endif
 
@@ -468,6 +360,21 @@ endif
 
 
 # TODO:
+# 8. Create a separate git repository for every thorn, which would
+#    make git operations more parallel
+# 9. Do not copy all branches/tags to the central repo all the time
+# 10. Add a "git reset --mixed" in the beginning to ensure the
+#     repository is in a consistent state.  Add "git rm -r --cached ."
+#     as well?
+# 11. Execute a "rm scratch/*.git-tag" if there are problems, and try
+#     again
+# 12. Use subdirectories in scratch directory
+# 13. Calculate and output md5sum for source files and the source tree
+#     to see more easily whether and which files changed
+# 14. cd into thorn directory, and use lstat/readlink to follow
+#     symbolic links, to make thing work with recent versions of git
+
+
 
 # DONE:
 # 1. Remember git-archive to get a tarball
diff --git a/src/util/git-add-thorn.pl b/src/util/git-add-thorn.pl
new file mode 100755
index 0000000..9c83a33
--- /dev/null
+++ b/src/util/git-add-thorn.pl
@@ -0,0 +1,59 @@
+#! /usr/bin/perl -w
+
+# Add a thorn to the repository
+
+# 2010-01-29 Erik Schnetter <schnetter@cct.lsu.edu>
+
+use strict;
+use File::Path;
+
+
+
+$#ARGV >= 3 or die;
+my ($git_cmd, $git_repo, $git_root, $thorn, @files) = @ARGV;
+
+my $scratch = $ENV{'SCRATCH_BUILD'};
+defined $scratch or die;
+
+my $silent = $ENV{'SILENT'};
+$silent = 'yes' if ! defined $silent;
+$silent = $silent !~ /^no$/i;
+my $silencer = $silent ? '> /dev/null 2>&1' : '';
+
+
+
+# Ensure that the repository exists
+die unless -e "$git_repo/.git";
+$ENV{'GIT_DIR'} = "$git_repo/.git";
+
+
+
+# This does not work, because arrangements or thorns may be symbolic
+# links, and git refuses to follow them.  Instad, we copy the whole
+# thorn (using hard links), and then add the copy.
+
+# system "cd $git_root && $git_cmd add @files" or die;
+
+my $srcdir = $git_root;
+my $dstdir = "$scratch/tmp-$thorn";
+
+rmtree $dstdir;                 # ignore errors
+
+for my $file (@files) {
+    my $dir = $file;
+    if ($dir =~ m+/+) {
+        $dir =~ s+/[^/]*$++;
+    } else {
+        $dir = '.';
+    }
+    mkpath "$dstdir/$dir";      # ignore errors
+    link "$srcdir/$file", "$dstdir/$file" or die;
+}
+
+print "Executing: cd $dstdir && $git_cmd add @files\n" unless $silent;
+system "cd $dstdir && $git_cmd add @files $silencer";
+if ($?) {
+    die "Could not add thorn $thorn to git repository";
+}
+
+rmtree $dstdir or die;
diff --git a/src/util/git-commit-everything.pl b/src/util/git-commit-everything.pl
new file mode 100755
index 0000000..d972218
--- /dev/null
+++ b/src/util/git-commit-everything.pl
@@ -0,0 +1,58 @@
+#! /usr/bin/perl -w
+
+# Commit everything to the repository
+
+# 2010-01-29 Erik Schnetter <schnetter@cct.lsu.edu>
+
+use strict;
+
+
+
+$#ARGV == 4 or die;
+my ($git_cmd, $git_repo, $git_root, $build_id, $config_id) = @ARGV;
+
+# Path where the git-*.pl commands are installed
+my $bindir = $ENV{'SCRATCH_BUILD'} . '/formaline-bin';
+
+my $silent = $ENV{'SILENT'};
+$silent = 'yes' if ! defined $silent;
+$silent = $silent !~ /^no$/i;
+my $silencer = $silent ? '> /dev/null 2>&1' : '';
+
+
+
+# Ensure that the repository exists
+die unless -e "$git_repo/.git";
+$ENV{'GIT_DIR'} = "$git_repo/.git";
+
+
+
+print "Formaline: Committing source tree to git repository...\n";
+
+# Try to use the previous commit as parent, if possible
+print "Executing: $git_cmd commit -m $build_id\n" unless $silent;
+system "$git_cmd commit -m $build_id $silencer";
+# Ignore errors
+#if ($?) {
+#    die "Could not commit";
+#}
+
+print "Executing: $git_cmd tag $build_id\n" unless $silent;
+system "$git_cmd tag $build_id $silencer";
+if ($?) {
+    die "Could not tag";
+}
+print "Formaline: Created git tag $build_id\n";
+
+print "Executing: $git_cmd branch -f $config_id\n" unless $silent;
+system "$git_cmd branch -f $config_id $silencer";
+if ($?) {
+    die "Could not update branch";
+}
+print "Formaline: Updated git branch $config_id\n";
+
+print "Executing: $bindir/git-gc-repo.pl '$git_cmd' $git_repo\n" unless $silent;
+system "$bindir/git-gc-repo.pl '$git_cmd' $git_repo $silencer";
+if ($?) {
+    die "Could not collect garbage";
+}
diff --git a/src/util/git-gc-repo.pl b/src/util/git-gc-repo.pl
new file mode 100755
index 0000000..21bb03f
--- /dev/null
+++ b/src/util/git-gc-repo.pl
@@ -0,0 +1,65 @@
+#! /usr/bin/perl -w
+
+# Collect garbage in the repository if it grown in size by more than a
+# factor of 10.
+
+# 2010-01-29 Erik Schnetter <schnetter@cct.lsu.edu>
+
+use strict;
+
+
+
+$#ARGV == 1 or die;
+my ($git_cmd, $git_repo) = @ARGV;
+
+my $silent = $ENV{'SILENT'};
+$silent = 'yes' if ! defined $silent;
+$silent = $silent !~ /^no$/i;
+my $silencer = $silent ? '> /dev/null 2>&1' : '';
+
+
+
+# Ensure that the repository exists
+die unless -e "$git_repo/.git";
+$ENV{'GIT_DIR'} = "$git_repo/.git";
+
+
+
+my $git_dir = "$git_repo/.git";
+my $sizefile = "$git_repo/.oldreposize";
+
+# Determine current repository size
+my $reposize = `du -s $git_dir` or die;
+$reposize = (split ' ', $reposize)[0];
+
+# Read old repository size
+my $oldreposize;
+if (open FILE, '<', $sizefile) {
+    $oldreposize = <FILE>;
+    close FILE;
+}
+if (! defined $oldreposize) {
+    $oldreposize = 0;
+}
+
+my $maxreposize = 10 * $oldreposize;
+if ($reposize > $maxreposize) {
+    
+    print "Formaline: Optimising git repository (slow only the first time)...\n";
+    
+    # Collect garbage
+    print "Executing: $git_cmd gc\n" unless $silent;
+    system "$git_cmd gc $silencer";
+    if ($?) {
+        die "Could not compact repository";
+    }
+    
+    # Determine new repository size
+    my $newreposize = `du -s $git_dir` or die;
+    $newreposize = (split ' ', $newreposize)[0];
+    
+    # Write new repository size
+    open (FILE, "> $sizefile") or die;
+    print FILE "$newreposize\n" or die;
+    close FILE or die;
+}
diff --git a/src/util/git-init-master-repo.pl b/src/util/git-init-master-repo.pl
new file mode 100755
index 0000000..ef5d5e5
--- /dev/null
+++ b/src/util/git-init-master-repo.pl
@@ -0,0 +1,71 @@
+#! /usr/bin/perl -w
+
+# Initialise the master repository
+
+# 2010-01-29 Erik Schnetter <schnetter@cct.lsu.edu>
+
+use strict;
+
+
+
+$#ARGV == 1 or die;
+my ($git_cmd, $git_master_repo) = @ARGV;
+
+my $silent = $ENV{'SILENT'};
+$silent = 'yes' if ! defined $silent;
+$silent = $silent !~ /^no$/i;
+my $silencer = $silent ? '> /dev/null 2>&1' : '';
+
+
+
+# If the repository exists already, do nothing
+if (-e "$git_master_repo/.git") {
+    exit;
+}
+
+
+
+print "Formaline: Creating git master repository...\n";
+
+# Create the directory for the repository
+mkdir $git_master_repo;
+$ENV{'GIT_DIR'} = $git_master_repo;
+
+# Create the repository
+print "Executing: $git_cmd init-db\n" unless $silent;
+system "$git_cmd init-db $silencer";
+if ($?) {
+    die "Formaline: WARNING: Error while initialising master git repository";
+}
+
+
+
+# Add a README
+open README, "> $git_master_repo/README" or die;
+my $today = `date`;
+print README "\
+This directory $git_master_repo
+
+is not empty -- it contains a git repository with the Cactus source
+trees of all previous builds, starting on $today.
+
+You can use the command \"git branch\" to list all configurations that
+are stored in this repository.  The history of each branch is the
+sequence in which the configurations were built.  The most recent
+build is stored in the branch head, as usual.  In order to check out a
+certain branch into a directory <name>, issue the following commands:
+        cd <somewhere_else>
+        mkdir <name>
+        cd <name>
+        git init
+        git pull $git_master_repo <branch>
+
+You can also use the command \"git tag -l\" to list all builds that
+are stored in this repository.  This keeps the source tree for each
+build directly accessible.  In order to check out a certain tag into a
+directory <name>, issue the following commands:
+        cd <somewhere_else>
+        git clone -o <name> $git_master_repo
+        git checkout <tag>"
+    or die;
+close README or die;
diff --git a/src/util/git-init-repo.pl b/src/util/git-init-repo.pl
new file mode 100755
index 0000000..07a6f43
--- /dev/null
+++ b/src/util/git-init-repo.pl
@@ -0,0 +1,71 @@
+#! /usr/bin/perl -w
+
+# Initialise a repository
+
+# 2010-01-29 Erik Schnetter <schnetter@cct.lsu.edu>
+
+use strict;
+
+
+
+$#ARGV == 1 or die;
+my ($git_cmd, $git_repo) = @ARGV;
+
+my $silent = $ENV{'SILENT'};
+$silent = 'yes' if ! defined $silent;
+$silent = $silent !~ /^no$/i;
+my $silencer = $silent ? '> /dev/null 2>&1' : '';
+
+
+
+# If the repository exists already, do nothing
+if (-e "$git_repo/.git") {
+    exit;
+}
+
+
+
+print "Formaline: Creating git repository...\n";
+
+# Create the directory for the repository
+mkdir $git_repo;
+$ENV{'GIT_DIR'} = "$git_repo/.git";
+
+# Create the repository
+print "Executing: $git_cmd init-db\n" unless $silent;
+system "$git_cmd init-db $silencer";
+if ($?) {
+    die "Formaline: WARNING: Error while initialising git repository";
+}
+
+
+
+# Add a README
+open README, "> $git_repo/README" or die;
+my $today = `date`;
+print README "\
+This directory $git_repo
+
+is not empty -- it contains a git repository with the Cactus source
+trees of all previous builds, starting on $today.
+
+You can use the command \"git branch\" to list all configurations that
+are stored in this repository.  The history of each branch is the
+sequence in which the configuration was built.  The most recent
+build is stored in the branch head, as usual.  In order to check out a
+certain branch into a directory <name>, issue the following commands:
+        cd <somewhere_else>
+        mkdir <name>
+        cd <name>
+        git init
+        git pull $git_repo <branch>
+
+You can also use the command \"git tag -l\" to list all builds that
+are stored in this repository.  This keeps the source tree for each
+build directly accessible.  In order to check out a certain tag into a
+directory <name>, issue the following commands:
+        cd <somewhere_else>
+        git clone -o <name> $git_repo
+        git checkout <tag>"
+    or die;
+close README or die;
diff --git a/src/util/git-lock.pl b/src/util/git-lock.pl
index 0eb7922..c05a31a 100755
--- a/src/util/git-lock.pl
+++ b/src/util/git-lock.pl
@@ -29,7 +29,7 @@ my $lockdir = "$git_dir/GITLOCK";
 
 
 
-my $waittime = 1;
+my $waittime = 0.01;
 my $maxwaittime = 10;
 while (! (mkdir $lockdir)) {
     # Wait some time
@@ -38,6 +38,7 @@ while (! (mkdir $lockdir)) {
     system "sleep $waittime";
     # Back off exponentially
     $waittime *= 2;
+    $waittime = 1 if $waittime>1 && $waittime<2;
     $waittime = $maxwaittime if $waittime > $maxwaittime;
 }
 
diff --git a/src/util/git-push-everything.pl b/src/util/git-push-everything.pl
new file mode 100755
index 0000000..faebdf7
--- /dev/null
+++ b/src/util/git-push-everything.pl
@@ -0,0 +1,74 @@
+#! /usr/bin/perl -w
+
+# Push everything to the master repository
+
+# 2010-01-29 Erik Schnetter <schnetter@cct.lsu.edu>
+
+use strict;
+
+
+
+$#ARGV == 2 or die;
+my ($git_cmd, $git_repo, $git_master_repo) = @ARGV;
+
+# Central Cactus repository
+my $git_central_repo = $ENV{'CACTUS_CENTRAL_GIT_REPO'};
+
+# Path where the git-*.pl commands are installed
+my $bindir = $ENV{'SCRATCH_BUILD'} . '/formaline-bin';
+
+my $silent = $ENV{'SILENT'};
+$silent = 'yes' if ! defined $silent;
+$silent = $silent !~ /^no$/i;
+my $silencer = $silent ? '> /dev/null 2>&1' : '';
+
+
+
+# Ensure that the repositories exist
+die unless -e "$git_repo/.git";
+$ENV{'GIT_DIR'} = "$git_repo/.git";
+die unless -e "$git_master_repo/.git";
+
+
+
+print "Formaline: Pushing source tree to master git repository...\n";
+
+print "Executing: $git_cmd push -v -f --all $git_master_repo\n" unless $silent;
+system "$git_cmd push -v -f --all $git_master_repo $silencer";
+if ($?) {
+    die "Could not push branches";
+}
+
+print "Executing: $git_cmd push -v -f --tags $git_master_repo\n" unless $silent;
+system "$git_cmd push -v -f --tags $git_master_repo $silencer";
+if ($?) {
+    die "Could not push tags";
+}
+
+print "Executing: $bindir/git-gc-repo.pl '$git_cmd' $git_master_repo\n"
+    unless $silent;
+system "$bindir/git-gc-repo.pl '$git_cmd' $git_master_repo $silencer";
+if ($?) {
+    die "Could not collect garbage";
+}
+
+
+
+if (defined $git_central_repo) {
+    $ENV{'GIT_DIR'} = "$git_master_repo/.git";
+    
+    print "Formaline: Pushing to central repository $git_central_repo...\n";
+    
+    print "Executing: $git_cmd push -v -f --all $git_central_repo\n"
+        unless $silent;
+    system "$git_cmd push -v -f --all $git_central_repo $silencer";
+    if ($?) {
+        die "Could not push branches";
+    }
+    
+    print "$git_cmd push -v -f --tags $git_central_repo\n" unless $silent;
+    system "$git_cmd push -v -f --tags $git_central_repo $silencer";
+    if ($?) {
+        die "Could not push tags";
+    }
+}
diff --git a/src/util/git-rm-thorn.pl b/src/util/git-rm-thorn.pl
new file mode 100755
index 0000000..9d5342b
--- /dev/null
+++ b/src/util/git-rm-thorn.pl
@@ -0,0 +1,40 @@
+#! /usr/bin/perl -w
+
+# Remote a Cactus thorn from the repository index, if and wherever it
+# exists
+
+# 2010-01-29 Erik Schnetter <schnetter@cct.lsu.edu>
+
+use strict;
+
+
+
+$#ARGV >= 3 or die;
+my ($git_cmd, $git_repo, $git_root, $thorn, @files) = @ARGV;
+
+my $silent = $ENV{'SILENT'};
+$silent = 'yes' if ! defined $silent;
+$silent = $silent !~ /^no$/i;
+my $silencer = $silent ? '> /dev/null 2>&1' : '';
+
+
+
+# Ensure that the repository exists
+die unless -e "$git_repo/.git";
+$ENV{'GIT_DIR'} = "$git_repo/.git";
+
+
+
+# Remove the files one by one because we want to ignore errors, but
+# git aborts after the first error
+for my $file (@files) {
+    
+    print "Executing: $git_cmd rm --cached -r $file 2> /dev/null\n"
+        unless $silent;
+    system "$git_cmd rm --cached -r $file > /dev/null 2>&1";
+    # Ignore errors
+    #if ($?) {
+    #    die "Could not remove thorn $thorn from git repository";
+    #}
+    
+}
diff --git a/src/util/makeblob.pl b/src/util/makeblob.pl
index 8eaed08..14fd49c 100755
--- a/src/util/makeblob.pl
+++ b/src/util/makeblob.pl
@@ -3,7 +3,7 @@
 use strict;
 
 my $items_per_line = 16;
-my $items_per_file = 1024 * 1024;
+my $items_per_file = 128 * 1024;
 
 $#ARGV == 1 or die;