From 1f053de3b2531ccad7f4e96e18892435705996c2 Mon Sep 17 00:00:00 2001 From: schnetter Date: Sat, 6 Feb 2010 17:24:06 +0000 Subject: Git stuff: Replace shell commands in Makefile with explicit perl scripts. Handle symbolic links to arrangements or thorns explicitly, so that newer versions of git work fine. git-svn-id: http://svn.cactuscode.org/arrangements/CactusUtils/Formaline/trunk@181 83718e91-0e4f-0410-abf4-91180603181f --- src/make.code.deps | 21 ++-- src/make.configuration.deps | 197 ++++++++++---------------------------- src/util/git-add-thorn.pl | 59 ++++++++++++ src/util/git-commit-everything.pl | 58 +++++++++++ src/util/git-gc-repo.pl | 65 +++++++++++++ src/util/git-init-master-repo.pl | 71 ++++++++++++++ src/util/git-init-repo.pl | 71 ++++++++++++++ src/util/git-lock.pl | 3 +- src/util/git-push-everything.pl | 74 ++++++++++++++ src/util/git-rm-thorn.pl | 40 ++++++++ src/util/makeblob.pl | 2 +- 11 files changed, 500 insertions(+), 161 deletions(-) create mode 100755 src/util/git-add-thorn.pl create mode 100755 src/util/git-commit-everything.pl create mode 100755 src/util/git-gc-repo.pl create mode 100755 src/util/git-init-master-repo.pl create mode 100755 src/util/git-init-repo.pl create mode 100755 src/util/git-push-everything.pl create mode 100755 src/util/git-rm-thorn.pl diff --git a/src/make.code.deps b/src/make.code.deps index 7eb710f..606383d 100644 --- a/src/make.code.deps +++ b/src/make.code.deps @@ -6,20 +6,13 @@ # file make.configuration.deps because only here the location of the # source directory is known. -TARBALL_DIR = $(SCRATCH_BUILD) +FORMALINE_BIN_DIR = $(SCRATCH_BUILD)/formaline-bin -$(CCTK_TARGET): $(TARBALL_DIR)/gethostname.pl $(TARBALL_DIR)/makeblob.pl $(TARBALL_DIR)/makemetablob.pl $(SCRATCH_BUILD)/git-lock.pl +FORMALINE_TAR_FILES = gethostname.pl makeblob.pl makemetablob.pl +FORMALINE_GIT_FILES = git-add-thorn.pl git-commit-everything.pl git-gc-repo.pl git-init-master-repo.pl git-init-repo.pl git-lock.pl git-push-everything.pl git-rm-thorn.pl +$(CCTK_TARGET): $(FORMALINE_TAR_FILES:%=$(FORMALINE_BIN_DIR)/%) $(FORMALINE_GIT_FILES:%=$(FORMALINE_BIN_DIR)/%) - -$(TARBALL_DIR)/gethostname.pl: $(SRCDIR)/util/gethostname.pl - cd $(TARBALL_DIR) && cp $^ $@ - -$(TARBALL_DIR)/makeblob.pl: $(SRCDIR)/util/makeblob.pl - cd $(TARBALL_DIR) && cp $^ $@ - -$(TARBALL_DIR)/makemetablob.pl: $(SRCDIR)/util/makemetablob.pl - cd $(TARBALL_DIR) && cp $^ $@ - -$(SCRATCH_BUILD)/git-lock.pl: $(SRCDIR)/util/git-lock.pl - cd $(SCRATCH_BUILD) && cp $^ $@ +$(FORMALINE_BIN_DIR)/%.pl: $(SRCDIR)/util/%.pl + mkdir -p $(FORMALINE_BIN_DIR) + cp $^ $@ diff --git a/src/make.configuration.deps b/src/make.configuration.deps index 100513a..726960e 100644 --- a/src/make.configuration.deps +++ b/src/make.configuration.deps @@ -20,6 +20,7 @@ TR_C := $(shell tr -C 'a' 'b' < /dev/null > /dev/null 2> /dev/null && echo 'tr - # into an executable TARBALL_DIR = $(SCRATCH_BUILD) +FORMALINE_BIN_DIR = $(SCRATCH_BUILD)/formaline-bin @@ -62,7 +63,7 @@ $(FRM-LIB): $(FRM-OBJS) .PHONY: $(TOP)/BUILD-ID $(TOP)/BUILD-ID: $(CCTK_LIBDIR)/$(LIBNAME_PREFIX)$(CCTK_LIBNAME_PREFIX)Formaline$(LIBNAME_SUFFIX) config=`echo "$(EXE:cactus_%=%)" | $(TR_C) -d '[:alnum:]+-._]'` && \ - hostname=`$(TARBALL_DIR)/gethostname.pl` && \ + hostname=`$(FORMALINE_BIN_DIR)/gethostname.pl` && \ user="$$USER" && \ dirname=`echo "$(subst /,-,$(CCTK_HOME:/%=%))" | $(TR_C) -d '[:alnum:]+-._]'` && \ timestamp=`date -u +%Y.%m.%d-%H.%M.%S` && \ @@ -95,7 +96,7 @@ $(TARBALL_DIR)/cactus-meta-source.o: $(TARBALL_DIR)/cactus-meta-source.c $(CC) $(CFLAGS) -c -o $@ $^ $(TARBALL_DIR)/cactus-meta-source.c: $(TARBALL_DIR)/cactus-flesh-source.c $(patsubst %,$(TARBALL_DIR)/cactus-thorn-source-%.c,$(notdir $(THORNS))) - $(TARBALL_DIR)/makemetablob.pl Cactus $(notdir $(THORNS)) > $@ + $(FORMALINE_BIN_DIR)/makemetablob.pl Cactus $(notdir $(THORNS)) > $@ .PRECIOUS: $(TARBALL_DIR)/cactus-meta-source.c $(TARBALL_DIR)/cactus-meta-source.o @@ -115,7 +116,7 @@ $(TARBALL_DIR)/cactus-flesh-source.o: $(TARBALL_DIR)/cactus-flesh-source.c $(CC) $(CFLAGS) -c -o $@ $^ $(TARBALL_DIR)/cactus-flesh-source.c: $(TARBALL_DIR)/cactus-flesh-source.tar.gz $(CCTK_HOME)/arrangements/AEIThorns/Formaline/src/util/VERSION - $(TARBALL_DIR)/makeblob.pl '' Cactus < $< > $@ + $(FORMALINE_BIN_DIR)/makeblob.pl '' Cactus < $< > $@ $(TARBALL_DIR)/cactus-flesh-source.tar.gz: $(TARBALL_DIR)/cactus-flesh-source.files cd $(CCTK_HOME) && \ @@ -133,6 +134,8 @@ $(TARBALL_DIR)/cactus-flesh-source.files: $(CCTK_LIBDIR)/$(LIBNAME_PREFIX)$(CCTK configs/$(notdir $(TOP))/config-data \ configs/$(notdir $(TOP))/config-info \ -name 'CVS' -prune -o \ + -name '*.log' -prune -o \ + -name '*.log.gz' -prune -o \ -name '.?*' -prune -o \ -name '*~' -prune -o \ ! -type d \ @@ -163,7 +166,7 @@ $(TARBALL_DIR)/cactus-thorn-source-%.o: $(TARBALL_DIR)/cactus-thorn-source-%.c $(CC) $(CFLAGS) -c -o $@ $^ $(TARBALL_DIR)/cactus-thorn-source-%.c: $(TARBALL_DIR)/cactus-thorn-source-%.tar.gz $(CCTK_HOME)/arrangements/AEIThorns/Formaline/src/util/VERSION - $(TARBALL_DIR)/makeblob.pl $(patsubst %/,%,$(dir $(filter %/$(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%),$(THORNS)))) $(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%) < $< > $@ + $(FORMALINE_BIN_DIR)/makeblob.pl $(patsubst %/,%,$(dir $(filter %/$(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%),$(THORNS)))) $(@:$(TARBALL_DIR)/cactus-thorn-source-%.c=%) < $< > $@ $(TARBALL_DIR)/cactus-thorn-source-%.tar.gz: $(TARBALL_DIR)/cactus-thorn-source-%.files cd $(CCTK_HOME) && \ @@ -175,10 +178,13 @@ $(TARBALL_DIR)/cactus-thorn-source-%.tar.gz: $(TARBALL_DIR)/cactus-thorn-source- $(TARBALL_DIR)/cactus-thorn-source-%.files: $(CCTK_LIBDIR)/$(LIBNAME_PREFIX)$(CCTK_LIBNAME_PREFIX)%$(LIBNAME_SUFFIX) $(TARBALL_DIR)/cactus-thorn-source-%.ccldeps cd $(CCTK_HOME) && \ find arrangements/$(filter %/$*,$(THORNS))/. \ + -name 'data' -prune -o \ -name 'doc' -prune -o \ -name 'par' -prune -o \ -name 'test' -prune -o \ -name 'CVS' -prune -o \ + -name '*.log' -prune -o \ + -name '*.log.gz' -prune -o \ -name '.?*' -prune -o \ -name '*~' -prune -o \ ! -type d \ @@ -260,14 +266,10 @@ endif -# Suppress all output from git commands -GIT-SILENT = > /dev/null 2>&1 - # Does git exist, or should we do nothing instead? -GIT-CMD := $(shell env PATH=$$HOME/git/bin:$$PATH which git) +GIT-CMD := $(shell env PATH=$(SCRATCH_BUILD)/git-1.6.6.1/bin:$(SCRATCH_BUILD)/git-1.6.0.6/bin:$$HOME/git/bin:$$PATH which git) HAVE-GIT := $(shell '$(GIT-CMD)' --version > /dev/null 2>&1 && echo 'true' || echo 'false') -GIT1 := $(shell $(HAVE-GIT) && echo '$(SCRATCH_BUILD)/git-lock.pl $(GIT-CMD)' || { echo 'WARNING: git command not found' >&2 && echo 'true'; }) -GIT := $(shell $(HAVE-GIT) && echo '$(SCRATCH_BUILD)/git-lock.pl $(GIT-CMD) $(GIT-SILENT)' || { echo 'WARNING: git command not found' >&2 && echo 'true'; }) +GIT := $(shell $(HAVE-GIT) && echo '$(FORMALINE_BIN_DIR)/git-lock.pl $(GIT-CMD)' || { echo 'Formaline: WARNING: git command not found' >&2 && echo 'true'; }) @@ -282,6 +284,7 @@ git-store-source: git-commit-everything git-push-everything echo "Formaline: If the build process seems to hang, some other command" echo "Formaline: is still running, e.g. the final link stage.)" +GIT-BIN = $(FORMALINE_BIN_DIR) GIT-ROOT = $(CCTK_HOME) GIT-REPO = $(TOP)/configjar.git GIT-MASTER-REPO = $(CCTK_HOME)/cactusjar.git @@ -291,161 +294,50 @@ GIT-THORNS = $(notdir $(THORNS)) BUILD-ID-FILE = $(TOP)/BUILD-ID CONFIG-ID-FILE = $(TOP)/CONFIG-ID -define GIT-INIT-MASTER-REPO - { \ - if [ ! -e $(GIT-MASTER-REPO)/.git/HEAD ]; then \ - echo "Formaline: Creating master git repository..." && \ - mkdir -p $(GIT-MASTER-REPO) && \ - cd $(GIT-MASTER-REPO) && \ - $(GIT) init-db; \ - fi && \ - if [ ! -e $(GIT-MASTER-REPO)/README ]; then \ - { \ - echo "This directory $$(pwd)" && \ - echo "is not empty -- it contains a git repository with the Cactus source" && \ - echo "trees of all previous builds, starting on $$(date)." && \ - echo "" && \ - echo "You can use the command \"git branch\" to list all configurations that" && \ - echo "are stored in this repository. The history of each branch is the" && \ - echo "sequence in which the configuration was built. The most recent build" && \ - echo "is stored in the branch head, as usual. In order to check out a" && \ - echo "certain branch into a directory , issue the following commands:" && \ - echo " cd " && \ - echo " mkdir " && \ - echo " cd " && \ - echo " git init" && \ - echo " git pull $$(pwd) " && \ - echo "" && \ - echo "You can also use the command \"git tag -l\" to list all builds that are" && \ - echo "stored in this repository. This keeps the source tree for each build" && \ - echo "directly accessible. In order to check out a certain tag into a" && \ - echo "directory , issue the following commands:" && \ - echo " cd " && \ - echo " git clone -o $$(pwd)" && \ - echo " git checkout "; \ - } > $(GIT-MASTER-REPO)/README; \ - fi; \ - } || echo "WARNING: Error while handling git repository" -endef - -define GIT-INIT-REPO - { \ - if [ ! -e $(GIT-REPO)/.git/HEAD ]; then \ - echo "Formaline: Creating git repository..." && \ - mkdir -p $(GIT-REPO) && \ - cd $(GIT-REPO) && \ - $(GIT) init-db; \ - fi; \ - } || echo "WARNING: Error while handling git repository" -endef - -# Call git-gc if the repository has grown in size by more than a -# factor of 10. This macro needs to be called in the directory where -# the .git subdirectory lives. -define GIT-COMPACT-REPO - { \ - if [ ! -e .oldreposize ]; then \ - echo 0 > .oldreposize; \ - fi && \ - oldreposize=$$(cat .oldreposize) && \ - maxreposize=$$((oldreposize * 10)) && \ - reposize=$$(du -s .git | awk '{ print $$1; }') && \ - if [ $$reposize -gt $$maxreposize ]; then \ - $(GIT) gc && \ - reposize=$$(du -s .git | awk '{ print $$1; }') && \ - echo $$reposize > .oldreposize; \ - fi; \ - } || echo "WARNING: Error while handling git repository" -endef - .PHONY: git-push-everything git-push-everything: git-commit-everything $(BUILD-ID-FILE) - $(GIT-INIT-MASTER-REPO) - { \ - echo "Formaline: Pushing source tree to master git repository..." && \ - cd $(GIT-ROOT) && \ - export GIT_DIR=$(GIT-REPO)/.git && \ - build_id=$$(cat $(BUILD-ID-FILE)) && \ - $(GIT) push -f $(GIT-MASTER-REPO) "$$build_id" && \ - config_id=$$(cat $(CONFIG-ID-FILE)) && \ - $(GIT) push -f $(GIT-MASTER-REPO) "$$config_id"; \ - } || echo "WARNING: Error while handling git repository" - true || { \ - cd $(GIT-ROOT) && \ - export GIT_DIR=$(GIT-MASTER-REPO)/.git && \ - echo "Formaline: Optimising master git repository..." && \ - $(GIT) gc; \ - } || echo "WARNING: Error while handling git repository" - { \ - cd $(GIT-ROOT) && \ - export GIT_DIR=$(GIT-MASTER-REPO)/.git && \ - echo "Formaline: Optimising master git repository..." && \ - cd $(GIT-MASTER-REPO) && \ - $(GIT-COMPACT-REPO); \ - } || echo "WARNING: Error while handling git repository" { \ + '$(GIT-BIN)/git-init-master-repo.pl' '$(GIT)' '$(GIT-MASTER-REPO)' && \ if [ -e "$(CCTK_HOME)/cactus.config" ]; then \ - source "$(CCTK_HOME)/cactus.config"; \ + source "$(CCTK_HOME)/cactus.config" && \ + export CACTUS_CENTRAL_GIT_REPO; \ fi && \ - if [ -n "$$CACTUS_CENTRAL_GIT_REPO" ]; then \ - cd $(GIT-ROOT) && \ - export GIT_DIR=$(GIT-MASTER-REPO)/.git && \ - echo "Formaline: Pushing to central repository $$CACTUS_CENTRAL_GIT_REPO..." && \ - config_id=$$(cat $(CONFIG-ID-FILE)) && \ - { $(GIT1) tag -l && echo "$$config_id"; } | \ - xargs $(GIT) push -f "$$CACTUS_CENTRAL_GIT_REPO"; \ - fi; \ - } || echo "WARNING: Error while handling git repository" - -# Try to use the previous commit as parent, if possible + '$(GIT-BIN)/git-push-everything.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-MASTER-REPO)'; \ + } || echo "Formaline: WARNING: Error while pushing to master repository" + .PHONY: git-commit-everything git-commit-everything: $(SCRATCH_BUILD)/cactus-flesh-source.git-tag $(GIT-THORNS:%=$(SCRATCH_BUILD)/cactus-thorn-%.git-tag) $(BUILD-ID-FILE) { \ - echo "Formaline: Committing source tree to git repository..." && \ - cd $(GIT-ROOT) && \ - export GIT_DIR=$(GIT-REPO)/.git && \ - build_id="$$(cat $(BUILD-ID-FILE))" && \ - { $(GIT) commit -m "$$build_id" || true; } && \ - $(GIT) tag -f "$$build_id" && \ - echo "Formaline: Created git tag $$build_id" && \ - config_id="$$(cat $(CONFIG-ID-FILE))" && \ - $(GIT) branch -f "$$config_id" && \ - echo "Formaline: Updated git branch $$config_id" && \ - echo "Formaline: Optimising git repository (slow only the first time)..." && \ - true || $(GIT) gc; \ - cd $(GIT-REPO) && \ - $(GIT-COMPACT-REPO); \ - } || echo "WARNING: Error while handling git repository" + '$(GIT-BIN)/git-commit-everything.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' "$$(cat $(BUILD-ID-FILE))" "$$(cat $(CONFIG-ID-FILE))"; \ + } || echo "Formaline: WARNING: Error while committing to repository" .PRECIOUS: $(SCRATCH_BUILD)/cactus-flesh-source.git-tag $(SCRATCH_BUILD)/cactus-flesh-source.git-tag: $(TARBALL_DIR)/cactus-flesh-source.files - $(GIT-INIT-REPO) { \ + '$(GIT-BIN)/git-init-repo.pl' '$(GIT)' '$(GIT-REPO)' && \ echo "Formaline: Adding flesh to git repository..." && \ - cd $(GIT-ROOT) && \ - export GIT_DIR=$(GIT-REPO)/.git && \ - { $(GIT) rm --cached -r CONTRIBUTORS COPYRIGHT Makefile lib src configs > /dev/null 2>&1 || true; } && \ - xargs ls < $(TARBALL_DIR)/cactus-flesh-source.files \ - > $(TARBALL_DIR)/cactus-flesh-source.files2 2> /dev/null; \ - xargs $(GIT) add < $(TARBALL_DIR)/cactus-flesh-source.files2 && \ - rm $(TARBALL_DIR)/cactus-flesh-source.files2 && \ + '$(GIT-BIN)/git-rm-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' 'cactus' CONTRIBUTORS COPYRIGHT Makefile lib src configs && \ + : xargs ls < $(TARBALL_DIR)/cactus-flesh-source.files \ + > $(TARBALL_DIR)/cactus-flesh-source.files2 2> /dev/null && \ + xargs '$(GIT-BIN)/git-add-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' 'cactus' \ + < $(TARBALL_DIR)/cactus-flesh-source.files && \ + : rm $(TARBALL_DIR)/cactus-flesh-source.files2 && \ : > $@; \ - } || echo "WARNING: Error while handling git repository" + } && echo "Formaline: WARNING: Error while adding flesh to git repository" .PRECIOUS: $(SCRATCH_BUILD)/cactus-thorn-%.git-tag $(SCRATCH_BUILD)/cactus-thorn-%.git-tag: $(TARBALL_DIR)/cactus-thorn-source-%.files - $(GIT-INIT-REPO) { \ + '$(GIT-BIN)/git-init-repo.pl' '$(GIT)' '$(GIT-REPO)' && \ echo "Formaline: Adding thorn $* to git repository..." && \ - cd $(GIT-ROOT) && \ - export GIT_DIR=$(GIT-REPO)/.git && \ - { $(GIT) rm --cached -r arrangements/*/$* > /dev/null 2>&1 || true; } &&\ - xargs ls < $(TARBALL_DIR)/cactus-thorn-source-$*.files \ - > $(TARBALL_DIR)/cactus-thorn-source-$*.files2 2> /dev/null; \ - xargs $(GIT) add < $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \ - rm $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \ + '$(GIT-BIN)/git-rm-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' '$*' "'"'arrangements/*/$*'"'" && \ + : xargs ls < $(TARBALL_DIR)/cactus-thorn-source-$*.files \ + > $(TARBALL_DIR)/cactus-thorn-source-$*.files2 2> /dev/null && \ + xargs '$(GIT-BIN)/git-add-thorn.pl' '$(GIT)' '$(GIT-REPO)' '$(GIT-ROOT)' '$*' \ + < $(TARBALL_DIR)/cactus-flesh-source.files && \ + : rm $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \ : > $@; \ - } || echo "WARNING: Error while handling git repository" + } || echo "Formaline: WARNING: Error while adding thorn $* to git repository" endif @@ -468,6 +360,21 @@ endif # TODO: +# 8. Create a separate git repository for every thorn, which would +# make git operations more parallel +# 9. Do not copy all branches/tags to the central repo all the time +# 10. Add a "git reset --mixed" in the beginning to ensure the +# repository is in a consistent state. Add "git rm -r --cached ." +# as well? +# 11. Execute a "rm scratch/*.git-tag" if there are problems, and try +# again +# 12. Use subdirectories in scratch directory +# 13. Calculate and output md5sum for source files and the source tree +# to see more easily whether and which files changed +# 14. cd into thorn directory, and use lstat/readlink to follow +# symbolic links, to make thing work with recent versions of git + + # DONE: # 1. Remember git-archive to get a tarball diff --git a/src/util/git-add-thorn.pl b/src/util/git-add-thorn.pl new file mode 100755 index 0000000..9c83a33 --- /dev/null +++ b/src/util/git-add-thorn.pl @@ -0,0 +1,59 @@ +#! /usr/bin/perl -w + +# Add a thorn to the repository + +# 2010-01-29 Erik Schnetter + +use strict; +use File::Path; + + + +$#ARGV >= 3 or die; +my ($git_cmd, $git_repo, $git_root, $thorn, @files) = @ARGV; + +my $scratch = $ENV{'SCRATCH_BUILD'}; +defined $scratch or die; + +my $silent = $ENV{'SILENT'}; +$silent = 'yes' if ! defined $silent; +$silent = $silent !~ /^no$/i; +my $silencer = $silent ? '> /dev/null 2>&1' : ''; + + + +# Ensure that the repository exists +die unless -e "$git_repo/.git"; +$ENV{'GIT_DIR'} = "$git_repo/.git"; + + + +# This does not work, because arrangements or thorns may be symbolic +# links, and git refuses to follow them. Instad, we copy the whole +# thorn (using hard links), and then add the copy. + +# system "cd $git_root && $git_cmd add @files" or die; + +my $srcdir = $git_root; +my $dstdir = "$scratch/tmp-$thorn"; + +rmtree $dstdir; # ignore errors + +for my $file (@files) { + my $dir = $file; + if ($dir =~ m+/+) { + $dir =~ s+/[^/]*$++; + } else { + $dir = '.'; + } + mkpath "$dstdir/$dir"; # ignore errors + link "$srcdir/$file", "$dstdir/$file" or die; +} + +print "Executing: cd $dstdir && $git_cmd add @files\n" unless $silent; +system "cd $dstdir && $git_cmd add @files $silencer"; +if ($?) { + die "Could not add thorn $thorn to git repository"; +} + +rmtree $dstdir or die; diff --git a/src/util/git-commit-everything.pl b/src/util/git-commit-everything.pl new file mode 100755 index 0000000..d972218 --- /dev/null +++ b/src/util/git-commit-everything.pl @@ -0,0 +1,58 @@ +#! /usr/bin/perl -w + +# Commit everything to the repository + +# 2010-01-29 Erik Schnetter + +use strict; + + + +$#ARGV == 4 or die; +my ($git_cmd, $git_repo, $git_root, $build_id, $config_id) = @ARGV; + +# Path where the git-*.pl commands are installed +my $bindir = $ENV{'SCRATCH_BUILD'} . '/formaline-bin'; + +my $silent = $ENV{'SILENT'}; +$silent = 'yes' if ! defined $silent; +$silent = $silent !~ /^no$/i; +my $silencer = $silent ? '> /dev/null 2>&1' : ''; + + + +# Ensure that the repository exists +die unless -e "$git_repo/.git"; +$ENV{'GIT_DIR'} = "$git_repo/.git"; + + + +print "Formaline: Committing source tree to git repository...\n"; + +# Try to use the previous commit as parent, if possible +print "Executing: $git_cmd commit -m $build_id\n" unless $silent; +system "$git_cmd commit -m $build_id $silencer"; +# Ignore errors +#if ($?) { +# die "Could not commit"; +#} + +print "Executing: $git_cmd tag $build_id\n" unless $silent; +system "$git_cmd tag $build_id $silencer"; +if ($?) { + die "Could not tag"; +} +print "Formaline: Created git tag $build_id\n"; + +print "Executing: $git_cmd branch -f $config_id\n" unless $silent; +system "$git_cmd branch -f $config_id $silencer"; +if ($?) { + die "Could not update branch"; +} +print "Formaline: Updated git branch $config_id\n"; + +print "Executing: $bindir/git-gc-repo.pl '$git_cmd' $git_repo\n" unless $silent; +system "$bindir/git-gc-repo.pl '$git_cmd' $git_repo $silencer"; +if ($?) { + die "Could not collect garbage"; +} diff --git a/src/util/git-gc-repo.pl b/src/util/git-gc-repo.pl new file mode 100755 index 0000000..21bb03f --- /dev/null +++ b/src/util/git-gc-repo.pl @@ -0,0 +1,65 @@ +#! /usr/bin/perl -w + +# Collect garbage in the repository if it grown in size by more than a +# factor of 10. + +# 2010-01-29 Erik Schnetter + +use strict; + + + +$#ARGV == 1 or die; +my ($git_cmd, $git_repo) = @ARGV; + +my $silent = $ENV{'SILENT'}; +$silent = 'yes' if ! defined $silent; +$silent = $silent !~ /^no$/i; +my $silencer = $silent ? '> /dev/null 2>&1' : ''; + + + +# Ensure that the repository exists +die unless -e "$git_repo/.git"; +$ENV{'GIT_DIR'} = "$git_repo/.git"; + + + +my $git_dir = "$git_repo/.git"; +my $sizefile = "$git_repo/.oldreposize"; + +# Determine current repository size +my $reposize = `du -s $git_dir` or die; +$reposize = (split ' ', $reposize)[0]; + +# Read old repository size +my $oldreposize; +if (open FILE, '<', $sizefile) { + $oldreposize = ; + close FILE; +} +if (! defined $oldreposize) { + $oldreposize = 0; +} + +my $maxreposize = 10 * $oldreposize; +if ($reposize > $maxreposize) { + + print "Formaline: Optimising git repository (slow only the first time)...\n"; + + # Collect garbage + print "Executing: $git_cmd gc\n" unless $silent; + system "$git_cmd gc $silencer"; + if ($?) { + die "Could not compact repository"; + } + + # Determine new repository size + my $newreposize = `du -s $git_dir` or die; + $newreposize = (split ' ', $newreposize)[0]; + + # Write new repository size + open (FILE, "> $sizefile") or die; + print FILE "$newreposize\n" or die; + close FILE or die; +} diff --git a/src/util/git-init-master-repo.pl b/src/util/git-init-master-repo.pl new file mode 100755 index 0000000..ef5d5e5 --- /dev/null +++ b/src/util/git-init-master-repo.pl @@ -0,0 +1,71 @@ +#! /usr/bin/perl -w + +# Initialise the master repository + +# 2010-01-29 Erik Schnetter + +use strict; + + + +$#ARGV == 1 or die; +my ($git_cmd, $git_master_repo) = @ARGV; + +my $silent = $ENV{'SILENT'}; +$silent = 'yes' if ! defined $silent; +$silent = $silent !~ /^no$/i; +my $silencer = $silent ? '> /dev/null 2>&1' : ''; + + + +# If the repository exists already, do nothing +if (-e "$git_master_repo/.git") { + exit; +} + + + +print "Formaline: Creating git master repository...\n"; + +# Create the directory for the repository +mkdir $git_master_repo; +$ENV{'GIT_DIR'} = $git_master_repo; + +# Create the repository +print "Executing: $git_cmd init-db\n" unless $silent; +system "$git_cmd init-db $silencer"; +if ($?) { + die "Formaline: WARNING: Error while initialising master git repository"; +} + + + +# Add a README +open README, "> $git_master_repo/README" or die; +my $today = `date`; +print README "\ +This directory $git_master_repo + +is not empty -- it contains a git repository with the Cactus source +trees of all previous builds, starting on $today. + +You can use the command \"git branch\" to list all configurations that +are stored in this repository. The history of each branch is the +sequence in which the configurations were built. The most recent +build is stored in the branch head, as usual. In order to check out a +certain branch into a directory , issue the following commands: + cd + mkdir + cd + git init + git pull $git_master_repo + +You can also use the command \"git tag -l\" to list all builds that +are stored in this repository. This keeps the source tree for each +build directly accessible. In order to check out a certain tag into a +directory , issue the following commands: + cd + git clone -o $git_master_repo + git checkout " + or die; +close README or die; diff --git a/src/util/git-init-repo.pl b/src/util/git-init-repo.pl new file mode 100755 index 0000000..07a6f43 --- /dev/null +++ b/src/util/git-init-repo.pl @@ -0,0 +1,71 @@ +#! /usr/bin/perl -w + +# Initialise a repository + +# 2010-01-29 Erik Schnetter + +use strict; + + + +$#ARGV == 1 or die; +my ($git_cmd, $git_repo) = @ARGV; + +my $silent = $ENV{'SILENT'}; +$silent = 'yes' if ! defined $silent; +$silent = $silent !~ /^no$/i; +my $silencer = $silent ? '> /dev/null 2>&1' : ''; + + + +# If the repository exists already, do nothing +if (-e "$git_repo/.git") { + exit; +} + + + +print "Formaline: Creating git repository...\n"; + +# Create the directory for the repository +mkdir $git_repo; +$ENV{'GIT_DIR'} = "$git_repo/.git"; + +# Create the repository +print "Executing: $git_cmd init-db\n" unless $silent; +system "$git_cmd init-db $silencer"; +if ($?) { + die "Formaline: WARNING: Error while initialising git repository"; +} + + + +# Add a README +open README, "> $git_repo/README" or die; +my $today = `date`; +print README "\ +This directory $git_repo + +is not empty -- it contains a git repository with the Cactus source +trees of all previous builds, starting on $today. + +You can use the command \"git branch\" to list all configurations that +are stored in this repository. The history of each branch is the +sequence in which the configuration was built. The most recent +build is stored in the branch head, as usual. In order to check out a +certain branch into a directory , issue the following commands: + cd + mkdir + cd + git init + git pull $git_repo + +You can also use the command \"git tag -l\" to list all builds that +are stored in this repository. This keeps the source tree for each +build directly accessible. In order to check out a certain tag into a +directory , issue the following commands: + cd + git clone -o $git_repo + git checkout " + or die; +close README or die; diff --git a/src/util/git-lock.pl b/src/util/git-lock.pl index 0eb7922..c05a31a 100755 --- a/src/util/git-lock.pl +++ b/src/util/git-lock.pl @@ -29,7 +29,7 @@ my $lockdir = "$git_dir/GITLOCK"; -my $waittime = 1; +my $waittime = 0.01; my $maxwaittime = 10; while (! (mkdir $lockdir)) { # Wait some time @@ -38,6 +38,7 @@ while (! (mkdir $lockdir)) { system "sleep $waittime"; # Back off exponentially $waittime *= 2; + $waittime = 1 if $waittime>1 && $waittime<2; $waittime = $maxwaittime if $waittime > $maxwaittime; } diff --git a/src/util/git-push-everything.pl b/src/util/git-push-everything.pl new file mode 100755 index 0000000..faebdf7 --- /dev/null +++ b/src/util/git-push-everything.pl @@ -0,0 +1,74 @@ +#! /usr/bin/perl -w + +# Push everything to the master repository + +# 2010-01-29 Erik Schnetter + +use strict; + + + +$#ARGV == 2 or die; +my ($git_cmd, $git_repo, $git_master_repo) = @ARGV; + +# Central Cactus repository +my $git_central_repo = $ENV{'CACTUS_CENTRAL_GIT_REPO'}; + +# Path where the git-*.pl commands are installed +my $bindir = $ENV{'SCRATCH_BUILD'} . '/formaline-bin'; + +my $silent = $ENV{'SILENT'}; +$silent = 'yes' if ! defined $silent; +$silent = $silent !~ /^no$/i; +my $silencer = $silent ? '> /dev/null 2>&1' : ''; + + + +# Ensure that the repositories exist +die unless -e "$git_repo/.git"; +$ENV{'GIT_DIR'} = "$git_repo/.git"; +die unless -e "$git_master_repo/.git"; + + + +print "Formaline: Pushing source tree to master git repository...\n"; + +print "Executing: $git_cmd push -v -f --all $git_master_repo\n" unless $silent; +system "$git_cmd push -v -f --all $git_master_repo $silencer"; +if ($?) { + die "Could not push branches"; +} + +print "Executing: $git_cmd push -v -f --tags $git_master_repo\n" unless $silent; +system "$git_cmd push -v -f --tags $git_master_repo $silencer"; +if ($?) { + die "Could not push tags"; +} + +print "Executing: $bindir/git-gc-repo.pl '$git_cmd' $git_master_repo\n" + unless $silent; +system "$bindir/git-gc-repo.pl '$git_cmd' $git_master_repo $silencer"; +if ($?) { + die "Could not collect garbage"; +} + + + +if (defined $git_central_repo) { + $ENV{'GIT_DIR'} = "$git_master_repo/.git"; + + print "Formaline: Pushing to central repository $git_central_repo...\n"; + + print "Executing: $git_cmd push -v -f --all $git_central_repo\n" + unless $silent; + system "$git_cmd push -v -f --all $git_central_repo $silencer"; + if ($?) { + die "Could not push branches"; + } + + print "$git_cmd push -v -f --tags $git_central_repo\n" unless $silent; + system "$git_cmd push -v -f --tags $git_central_repo $silencer"; + if ($?) { + die "Could not push tags"; + } +} diff --git a/src/util/git-rm-thorn.pl b/src/util/git-rm-thorn.pl new file mode 100755 index 0000000..9d5342b --- /dev/null +++ b/src/util/git-rm-thorn.pl @@ -0,0 +1,40 @@ +#! /usr/bin/perl -w + +# Remote a Cactus thorn from the repository index, if and wherever it +# exists + +# 2010-01-29 Erik Schnetter + +use strict; + + + +$#ARGV >= 3 or die; +my ($git_cmd, $git_repo, $git_root, $thorn, @files) = @ARGV; + +my $silent = $ENV{'SILENT'}; +$silent = 'yes' if ! defined $silent; +$silent = $silent !~ /^no$/i; +my $silencer = $silent ? '> /dev/null 2>&1' : ''; + + + +# Ensure that the repository exists +die unless -e "$git_repo/.git"; +$ENV{'GIT_DIR'} = "$git_repo/.git"; + + + +# Remove the files one by one because we want to ignore errors, but +# git aborts after the first error +for my $file (@files) { + + print "Executing: $git_cmd rm --cached -r $file 2> /dev/null\n" + unless $silent; + system "$git_cmd rm --cached -r $file > /dev/null 2>&1"; + # Ignore errors + #if ($?) { + # die "Could not remove thorn $thorn from git repository"; + #} + +} diff --git a/src/util/makeblob.pl b/src/util/makeblob.pl index 8eaed08..14fd49c 100755 --- a/src/util/makeblob.pl +++ b/src/util/makeblob.pl @@ -3,7 +3,7 @@ use strict; my $items_per_line = 16; -my $items_per_file = 1024 * 1024; +my $items_per_file = 128 * 1024; $#ARGV == 1 or die; -- cgit v1.2.3