From 9cbf586eb3a7bb7a57f23045cce2f4331a40f526 Mon Sep 17 00:00:00 2001 From: schnetter Date: Mon, 10 Mar 2008 18:12:39 +0000 Subject: Change git repository structure. Introduce one repository per configuration, so that many fewer git-add and git-remove are necessary, speeding up things considerably. Move the main git repository into a subdirectory of the main Cactus directory. Lock the git repository before calling git, so that parallel make works. git-svn-id: http://svn.cactuscode.org/arrangements/CactusUtils/Formaline/trunk@171 83718e91-0e4f-0410-abf4-91180603181f --- src/make.code.deps | 6 +- src/make.configuration.deps | 198 +++++++++++++++++++++++++++----------------- src/util/git-commit.pl | 177 --------------------------------------- src/util/git-lock.pl | 58 +++++++++++++ 4 files changed, 183 insertions(+), 256 deletions(-) delete mode 100755 src/util/git-commit.pl create mode 100755 src/util/git-lock.pl diff --git a/src/make.code.deps b/src/make.code.deps index 7f48ec8..7eb710f 100644 --- a/src/make.code.deps +++ b/src/make.code.deps @@ -8,7 +8,7 @@ TARBALL_DIR = $(SCRATCH_BUILD) -$(CCTK_TARGET): $(TARBALL_DIR)/gethostname.pl $(TARBALL_DIR)/makeblob.pl $(TARBALL_DIR)/makemetablob.pl $(TARBALL_DIR)/git-commit.pl +$(CCTK_TARGET): $(TARBALL_DIR)/gethostname.pl $(TARBALL_DIR)/makeblob.pl $(TARBALL_DIR)/makemetablob.pl $(SCRATCH_BUILD)/git-lock.pl @@ -21,5 +21,5 @@ $(TARBALL_DIR)/makeblob.pl: $(SRCDIR)/util/makeblob.pl $(TARBALL_DIR)/makemetablob.pl: $(SRCDIR)/util/makemetablob.pl cd $(TARBALL_DIR) && cp $^ $@ -$(TARBALL_DIR)/git-commit.pl: $(SRCDIR)/util/git-commit.pl - cd $(TARBALL_DIR) && cp $^ $@ +$(SCRATCH_BUILD)/git-lock.pl: $(SRCDIR)/util/git-lock.pl + cd $(SCRATCH_BUILD) && cp $^ $@ diff --git a/src/make.configuration.deps b/src/make.configuration.deps index d55d6a8..f1f5c66 100644 --- a/src/make.configuration.deps +++ b/src/make.configuration.deps @@ -57,6 +57,7 @@ $(FRM-LIB): $(FRM-OBJS) # (force a new ID to be created every time) # (do this after the thorn's library has been built, so that the # script gethostname.pl has been copied to the scratch directory) +# (generates also $(TOP)/CONFIG-ID) .PRECIOUS: $(TOP)/BUILD-ID $(TOP)/BUILD-ID: $(CCTK_LIBDIR)/$(LIBNAME_PREFIX)$(CCTK_LIBNAME_PREFIX)Formaline$(LIBNAME_SUFFIX) config=`echo "$(EXE:cactus_%=%)" | $(TR_C) -d '[:alnum:]+-._]'` && \ @@ -77,13 +78,11 @@ $(TARBALL_DIR)/build-id.o: $(TARBALL_DIR)/build-id.c $(CC) $(CFLAGS) -c -o $@ $^ .PRECIOUS: $(TARBALL_DIR)/build-id.c -$(TARBALL_DIR)/build-id.c: $(TOP)/BUILD-ID $(TOP)/GIT-COMMIT-ID +$(TARBALL_DIR)/build-id.c: $(TOP)/BUILD-ID { \ echo '/* This is an auto-generated file -- do not edit */' && \ build_id="$$(cat $(TOP)/BUILD-ID)" && \ echo 'char const * const build_id = "'$$build_id'";' && \ - git_commit_id="$$(cat $(TOP)/GIT-COMMIT-ID || echo 'NO-GIT-ID')" && \ - echo 'char const * const source_id = "git-'$$git_commit_id'";' && \ config_id="$$(cat $(TOP)/CONFIG-ID || echo 'NO-CONFIG-ID')" && \ echo 'char const * const config_id = "'$$config_id'";'; \ } > $@ @@ -249,101 +248,148 @@ endif -# Does git exist, or should we do nothing instead? -GIT-REPO = $(CCTK_HOME) -GIT-DIR = $(CCTK_HOME)/.git -HAVE-GIT := $(shell env PATH=$$PATH:$$HOME/git/bin git --version > /dev/null 2>&1 && echo 'true' || echo 'false') -GIT := $(shell $(HAVE-GIT) && echo 'env PATH=$$PATH:$$HOME/git/bin git' || { echo 'WARNING: git command not found' >&2 && echo 'true'; }) - +# Suppress all output from git commands +#GIT-SILENT = > /dev/null 2>&1 +# Does git exist, or should we do nothing instead? +GIT-CMD := $(shell env PATH=$$PATH:$$HOME/git/bin which git) +HAVE-GIT := $(shell '$(GIT-CMD)' --version > /dev/null 2>&1 && echo 'true' || echo 'false') +GIT := $(shell $(HAVE-GIT) && echo '$(SCRATCH_BUILD)/git-lock.pl $(GIT-CMD) $(GIT-SILENT)' || { echo 'WARNING: git command not found' >&2 && echo 'true'; }) -$(EXEDIR)$(DIRSEP)$(EXE): git-commit -.PHONY: git-commit -git-commit: $(TOP)/GIT-COMMIT-ID ifeq ($(HAVE-GIT),true) -.PRECIOUS: $(TOP)/GIT-COMMIT-ID -$(TOP)/GIT-COMMIT-ID: $(TOP)/BUILD-ID $(TARBALL_DIR)/cactus-flesh-source.files $(patsubst %,$(TARBALL_DIR)/cactus-thorn-source-%.files,$(notdir $(THORNS))) - $(TARBALL_DIR)/git-commit.pl --git='$(GIT)' --git-repo='$(GIT-REPO)' \ - --CCTK_HOME='$(CCTK_HOME)' --TOP='$(TOP)' \ - --TARBALL_DIR='$(TARBALL_DIR)' \ - $(notdir $(THORNS)) - -unused-$(TOP)/GIT-COMMIT-ID: $(TOP)/BUILD-ID $(TARBALL_DIR)/cactus-flesh-source.files $(patsubst %,$(TARBALL_DIR)/cactus-thorn-source-%.files,$(notdir $(THORNS))) - cd $(GIT-REPO) && \ - true "Ensure that the git repository exists" && \ - if ! test -e $(GIT-DIR) -a -e $(GIT-DIR)/HEAD; then \ - mkdir -p $(GIT-DIR) && \ - echo "Formaline: Creating new git repository..." && \ - $(GIT) init && \ - true "Add a root commit to please some tools" && \ - : >> README && \ - $(GIT) add README && \ - $(GIT) commit -m 'README'; \ +main: git-store-source + +.PHONY: git-store-source +git-store-source: git-commit-everything git-push-everything + echo "Formaline: Done." + +GIT-ROOT = $(CCTK_HOME) +GIT-REPO = $(TOP)/configjar.git +GIT-MASTER-REPO = $(CCTK_HOME)/cactusjar.git + +GIT-THORNS = $(notdir $(THORNS)) + +BUILD-ID-FILE = $(TOP)/BUILD-ID +CONFIG-ID-FILE = $(TOP)/CONFIG-ID + +define GIT-INIT-MASTER-REPO + if [ ! -e $(GIT-MASTER-REPO)/.git/HEAD ]; then \ + echo "Formaline: Creating master git repository..." && \ + mkdir -p $(GIT-MASTER-REPO) && \ + cd $(GIT-MASTER-REPO) && \ + $(GIT) init-db; \ + fi +endef + +define GIT-INIT-REPO + if [ ! -e $(GIT-REPO)/.git/HEAD ]; then \ + echo "Formaline: Creating git repository..." && \ + mkdir -p $(GIT-REPO) && \ + cd $(GIT-REPO) && \ + $(GIT) init-db; \ + fi +endef + +.PHONY: git-push-everything +git-push-everything: git-commit-everything $(BUILD-ID-FILE) + $(GIT-INIT-MASTER-REPO) + echo "Formaline: Pushing source tree to master git repository..." && \ + cd $(GIT-ROOT) && \ + export GIT_DIR=$(GIT-REPO)/.git && \ + build_id=$$(cat $(BUILD-ID-FILE)) && \ + $(GIT) push -f $(GIT-MASTER-REPO) "$$build_id" && \ + config_id=$$(cat $(CONFIG-ID-FILE)) && \ + $(GIT) push -f $(GIT-MASTER-REPO) "$$config_id" + cd $(GIT-ROOT) && \ + export GIT_DIR=$(GIT-MASTER-REPO)/.git && \ + echo "Formaline: Optimising master git repository..." && \ + $(GIT) gc + if [ -e "$(CCTK_HOME)/cactus.config" ]; then \ + source "$(CCTK_HOME)/cactus.config"; \ fi && \ - true "Remove all staged files" && \ - echo "Formaline: Preparing git repository..." && \ - $(GIT) rm --cached -r . > /dev/null 2>&1; \ + if [ -n "$$CACTUS_CENTRAL_GIT_REPO" ]; then \ + cd $(GIT-ROOT) && \ + export GIT_DIR=$(GIT-MASTER-REPO)/.git && \ + echo "Formaline: Pushing to central repository $$CACTUS_CENTRAL_GIT_REPO..." && \ + config_id=$$(cat $(CONFIG-ID-FILE)) && \ + { $(GIT) tag -l && echo "$$config_id"; } | \ + xargs $(GIT) push -f "$$CACTUS_CENTRAL_GIT_REPO"; \ + fi + +# Try to use the previous commit as parent, if possible +.PHONY: git-commit-everything +git-commit-everything: $(SCRATCH_BUILD)/cactus-flesh-source.git-tag $(GIT-THORNS:%=$(SCRATCH_BUILD)/cactus-thorn-%.git-tag) $(BUILD-ID-FILE) + echo "Formaline: Committing source tree to git repository..." && \ + cd $(GIT-ROOT) && \ + export GIT_DIR=$(GIT-REPO)/.git && \ + build_id="$$(cat $(BUILD-ID-FILE))" && \ + { $(GIT) commit -m "$$build_id" || true; } && \ + $(GIT) tag -f "$$build_id" && \ + echo "Formaline: Created git tag $$build_id" && \ + config_id="$$(cat $(CONFIG-ID-FILE))" && \ + $(GIT) branch -f "$$config_id" && \ + echo "Formaline: Updated git branch $$config_id" && \ + echo "Formaline: Optimising git repository (slow only the first time)..." && \ + $(GIT) gc + +.PRECIOUS: $(SCRATCH_BUILD)/cactus-flesh-source.git-tag +$(SCRATCH_BUILD)/cactus-flesh-source.git-tag: $(TARBALL_DIR)/cactus-flesh-source.files + $(GIT-INIT-REPO) echo "Formaline: Adding flesh to git repository..." && \ + cd $(GIT-ROOT) && \ + export GIT_DIR=$(GIT-REPO)/.git && \ + { $(GIT) rm --cached -r CONTRIBUTORS COPYRIGHT Makefile lib src configs > /dev/null 2>&1 || true; } && \ xargs ls < $(TARBALL_DIR)/cactus-flesh-source.files \ > $(TARBALL_DIR)/cactus-flesh-source.files2 2> /dev/null; \ xargs $(GIT) add < $(TARBALL_DIR)/cactus-flesh-source.files2 && \ rm $(TARBALL_DIR)/cactus-flesh-source.files2 && \ - for thorn in $(notdir $(THORNS)); do \ - echo "Formaline: Adding thorn $$thorn to git repository..." && \ - xargs ls < $(TARBALL_DIR)/cactus-thorn-source-$$thorn.files \ - > $(TARBALL_DIR)/cactus-thorn-source-$$thorn.files2 \ - 2> /dev/null; \ - xargs $(GIT) add \ - < $(TARBALL_DIR)/cactus-thorn-source-$$thorn.files2 && \ - rm $(TARBALL_DIR)/cactus-thorn-source-$$thorn.files2; \ - done && \ - true "Write source tree and create a commit" && \ - echo "Formaline: Committing source tree to git repository..." && \ - $(GIT) add configs/$(notdir $(TOP))/BUILD-ID && \ - $(GIT) write-tree > $(TARBALL_DIR)/GIT-TREE-ID && \ - true "Try to use the previous commit as parent, if possible" && \ - if test -e $@ -a -n "$$(cat $@)"; then \ - $(GIT) commit-tree $$(cat $(TARBALL_DIR)/GIT-TREE-ID) -p "$$(cat $@)" < $(TOP)/BUILD-ID > $@.new || \ - $(GIT) commit-tree $$(cat $(TARBALL_DIR)/GIT-TREE-ID) < $(TOP)/BUILD-ID > $@.new; \ - else \ - $(GIT) commit-tree $$(cat $(TARBALL_DIR)/GIT-TREE-ID) < $(TOP)/BUILD-ID > $@.new; \ - fi && \ - mv $@.new $@ && \ - true "Remember the commit id" && \ - mkdir -p $(CCTK_HOME)/GIT-COMMIT-IDS && \ - cp $@ $(CCTK_HOME)/GIT-COMMIT-IDS/$$(cat $@) && \ - echo "Formaline: Git commit id is $$(cat $@)" && \ - true "Update branch" && \ - config_id=$$(cat $(TOP)/CONFIG-ID) && \ - $(GIT) branch -f $$config_id $$(cat $@) && \ - echo "Formaline: Cleaning up git repository..." && \ - $(GIT) rm --cached -r . > /dev/null 2>&1; \ - true "Call git-gc for good measure" && \ - echo "Formaline: Optimising git repository (slow only the first time)..." && \ - $(GIT) gc > /dev/null 2>&1 && \ - echo "Formaline: Done."; \ + : > $@ + +.PRECIOUS: $(SCRATCH_BUILD)/cactus-thorn-%.git-tag +$(SCRATCH_BUILD)/cactus-thorn-%.git-tag: $(TARBALL_DIR)/cactus-thorn-source-%.files + $(GIT-INIT-REPO) + echo "Formaline: Adding thorn $* to git repository..." && \ + cd $(GIT-ROOT) && \ + export GIT_DIR=$(GIT-REPO)/.git && \ + { $(GIT) rm --cached -r arrangements/*/$* > /dev/null 2>&1 || true; } &&\ + xargs ls < $(TARBALL_DIR)/cactus-thorn-source-$*.files \ + > $(TARBALL_DIR)/cactus-thorn-source-$*.files2 2> /dev/null; \ + xargs $(GIT) add < $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \ + rm $(TARBALL_DIR)/cactus-thorn-source-$*.files2 && \ + : > $@ -else +endif -.PRECIOUS: $(TOP)/GIT-COMMIT-ID -$(TOP)/GIT-COMMIT-ID: $(TOP)/BUILD-ID $(TARBALL_DIR)/cactus-flesh-source.files $(patsubst %,$(TARBALL_DIR)/cactus-thorn-source-%.files,$(notdir $(THORNS))) - echo 'NO-GIT-ID' > $(TOP)/GIT-COMMIT-ID -endif + +# NOTE: + +# Define environment variable CACTUS_CENTRAL_GIT_REPO where the source trees +# are automatically pushed + +# git-archive --prefix=Cactus/ wavetoy | gzip > wavetoy.tar.gz +# git-archive --prefix=Cactus/ wavetoy | { cd /somewhere && tar xf -; } + +# git init +# git pull + +# git clone +# git checkout # TODO: -# 1. Remember git-archive to get a tarball -# 2. Don't create a version history; instead, leave all commits independent # 3. Push the commits automatically to a central (write-only?) repository + +# DONE: +# 1. Remember git-archive to get a tarball # 4. What if the Cactus directory is alread the root of a git repository? # 5. Place the .git repository into the config subdirectory # 6. Push it automatically into a subdirectory of the main Cactus directory # 7. Figure out how to handle conflicting branch names -# git-archive --prefix=Cactus/ wavetoy | gzip > wavetoy.tar.gz -# git-archive --prefix=Cactus/ wavetoy | { cd /somewhere && tar xf -; } +# WONT: +# 2. Don't create a version history; instead, leave all commits independent diff --git a/src/util/git-commit.pl b/src/util/git-commit.pl deleted file mode 100755 index d28e023..0000000 --- a/src/util/git-commit.pl +++ /dev/null @@ -1,177 +0,0 @@ -#! /usr/bin/perl -w - -# Commit the current source tree to a git repository - -# 2008-03-02 Erik Schnetter - -use strict; -#use Fcntl ':flock'; -use Getopt::Long; - -# $(GIT-REPO) -# $(CCTK-HOME) -# $(CONFIG-DIR) -# $(SCRATCH-DIR) - -my %options; -GetOptions - (\%options, - 'git=s', # git command - 'git-repo=s', # git repository - 'CCTK_HOME=s', # main Cactus directory - 'TOP=s', # main directory of the configuration - 'TARBALL_DIR=s', # Formaline's directory - ) or die "Could not parse command line options"; - -my $git = $options{'git'}; -my $git_repo = $options{'git-repo'}; -my $cctk_home = $options{'CCTK_HOME'}; -my $config_dir = $options{'TOP'}; -my $tarball_dir = $options{'TARBALL_DIR'}; - -defined $git or die; -defined $git_repo or die; -defined $cctk_home or die; -defined $config_dir or die; -defined $tarball_dir or die; - -chdir $git_repo or die; - -# Use relative path names -$config_dir =~ m+/configs/+; -$config_dir = "configs/$'"; - -my $config_id_file = "$config_dir/CONFIG-ID"; -my $build_id_file = "$config_dir/BUILD-ID"; -my $commit_id_file = "$config_dir/GIT-COMMIT-ID"; - -my @thorns = @ARGV; - - - -# Obtain exclusive access to the repository - -# We cannot use flock since the file system may be remote -#open LOCKFILE, "$cctk_home/Makefile"; -#flock LOCKFILE, LOCK_EX; - -my $gitlock = "$cctk_home/GITLOCK"; -my $waittime = 1; -my $maxwaittime = 30; -while (! (mkdir $gitlock)) { - # Check whether the locking process still exists - my $pid = `cat $gitlock/PID`; - chomp $pid; - if ($pid ne '') { - system "ps $pid > /dev/null 2>&1"; - if ($?) { - # Break the lock - print "Git repository seems free -- breaking the lock\n"; - unlink "$gitlock/PID" or die; - rmdir "$gitlock" or die; - $waittime = 1; - next; - } - } - # Wait some time - my $unit = $waittime==1 ? "second" : "seconds"; - print "Git repository is busy; waiting $waittime $unit...\n"; - system "sleep $waittime"; - # Back off exponentially - $waittime *= 2; - $waittime = $maxwaittime if $waittime > $maxwaittime; -} -# Ensure that the lock exists -die unless -d $gitlock; -# Add our PID -open PID, '>', "$gitlock/PID.tmp" or die; -print PID "$$\n"; -close PID; -rename "$gitlock/PID.tmp", "$gitlock/PID" or die; -# Check whether it is really our PID -my $pid = `cat $gitlock/PID`; -chomp $pid; -die unless $pid == $$; - - - -# Ensure that the git repository exists -if (! -e "$git_repo/.git/HEAD") { - print "Formaline: Creating new git repository...\n"; - system "$git init"; $? and die; - # Add a root commit to please some tools - system ": >> README"; $? and die; - system "$git add README"; $? and die; - system "$git commit -m 'README'"; $? and die; -} - -# Remove all staged files -print "Formaline: Preparing git repository...\n"; -system "$git rm --cached -r . > /dev/null 2>&1"; # may fail - -# Add flesh files to repository -print "Formaline: Adding flesh to git repository...\n"; -system "xargs ls < $tarball_dir/cactus-flesh-source.files > $tarball_dir/cactus-flesh-source.files.tmp 2> /dev/null"; $? and die; -system "xargs $git add < $tarball_dir/cactus-flesh-source.files.tmp"; $? and die; -unlink "$tarball_dir/cactus-flesh-source.files.tmp" or die; - -# Add thorn files to repository -foreach my $thorn (@thorns) { - print "Formaline: Adding thorn $thorn to git repository...\n"; - system "xargs ls < $tarball_dir/cactus-thorn-source-$thorn.files > $tarball_dir/cactus-thorn-source-$thorn.files.tmp 2> /dev/null"; $? and die; - system "xargs $git add < $tarball_dir/cactus-thorn-source-$thorn.files.tmp"; $? and die; - unlink "$tarball_dir/cactus-thorn-source-$thorn.files.tmp" or die; -} - -# Write source tree and create a commit -print "Formaline: Committing source tree to git repository...\n"; -system "$git add $config_id_file $build_id_file"; $? and die; -my $tree_id_file = "$tarball_dir/GIT-TREE-ID"; -system "$git write-tree > $tree_id_file"; $? and die; -my $tree_id = `cat $tree_id_file`; -chomp $tree_id; - -# Try to use the previous commit as parent, if possible -my $old_commit_id; -if (-e $commit_id_file) { - $old_commit_id = `cat $commit_id_file`; - chomp $old_commit_id; -} - -# (Use the build id as commit message) -my $did_commit = 0; -if (defined $old_commit_id && $old_commit_id ne '') { - system "$git commit-tree $tree_id -p $old_commit_id < $build_id_file > $commit_id_file.new"; - $did_commit = $? == 0; -} -if (! $did_commit) { - system "$git commit-tree $tree_id < $build_id_file > $commit_id_file.new"; $? and die; -} -rename "$commit_id_file.new", "$commit_id_file" or die; -my $commit_id = `cat $commit_id_file`; -chomp $commit_id; -print "Formaline: Git commit id is $commit_id\n"; - -# Remember the commit id -mkdir "$cctk_home/GIT-COMMIT-IDS"; -system "cp $commit_id_file $cctk_home/GIT-COMMIT-IDS/$commit_id"; $? and die; - -# Update branch -my $config_id = `cat $config_id_file`; -chomp $config_id; -die unless defined $config_id && $config_id ne ''; -system "$git branch -f $config_id $commit_id"; $? and die; - -print "Formaline: Cleaning up git repository...\n"; -system "$git rm --cached -r . > /dev/null 2>&1"; # may fail - -# Call git-gc for good measure -print "Formaline: Optimising git repository (slow only the first time)...\n"; -system "$git gc > /dev/null 2>&1"; $? and die; - - - -# Release the lock -system "rm -rf $gitlock"; - -print "Formaline: Done.\n"; diff --git a/src/util/git-lock.pl b/src/util/git-lock.pl new file mode 100755 index 0000000..0eb7922 --- /dev/null +++ b/src/util/git-lock.pl @@ -0,0 +1,58 @@ +#! /usr/bin/perl -w + +# Obtain a lock for a git repository + +# 2008-03-06 Erik Schnetter + +use strict; +use Cwd; +#use Fcntl ':flock'; +use sigtrap qw(die normal-signals); + + + +# Obtain exclusive access to the repository + +# We cannot use flock since the file system may be remote +#open LOCKFILE, "$cctk_home/Makefile"; +#flock LOCKFILE, LOCK_EX; + +my $git_dir = $ENV{'GIT_DIR'}; +$git_dir = getcwd unless defined $git_dir; +$git_dir =~ s+/.git/*$+/+; + +# We are very conservative as to what characters we allow. Other +# characters can be added, but be careful. +$git_dir =~ m|^[[:alnum:]+-._/]+$| or die; + +my $lockdir = "$git_dir/GITLOCK"; + + + +my $waittime = 1; +my $maxwaittime = 10; +while (! (mkdir $lockdir)) { + # Wait some time + my $unit = $waittime==1 ? "second" : "seconds"; + print "Git repository is busy; waiting $waittime $unit...\n"; + system "sleep $waittime"; + # Back off exponentially + $waittime *= 2; + $waittime = $maxwaittime if $waittime > $maxwaittime; +} + + + +# Execute the command +system @ARGV; +$? != -1 or die; +exit $? >> 8; + + + +# Release the lock +END { + my $retval = $?; + rmdir $lockdir; + $? = $retval; +} -- cgit v1.2.3