github · rubiojr · Jun 21, 2016 · Jun 21, 2016 · Jun 21, 2016 · Jun 21, 2016
diff --git a/bin/ghe-backup b/bin/ghe-backup
@@ -190,8 +190,10 @@ if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
         failures="$failures hookshot"
 
         echo "Backing up custom Git hooks ..."
-        ghe-backup-userdata git-hooks ||
-        failures="$failures git-hooks"
+        ghe-backup-userdata git-hooks/environments/tarballs ||
+        failures="$failures git-hooks-environments"
+        ghe-backup-userdata git-hooks/repos ||
+        failures="$failures git-hooks-repos"
     fi
 fi
 

diff --git a/bin/ghe-restore b/bin/ghe-restore
@@ -274,7 +274,9 @@ elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
   ghe-restore-userdata hookshot "$GHE_HOSTNAME" 1>&3
 
   echo "Restoring custom Git hooks ..."
-  ghe-restore-userdata git-hooks "$GHE_HOSTNAME" 1>&3
+  ghe-restore-userdata git-hooks/environments/tarballs "$GHE_HOSTNAME" 1>&3
+  ghe-restore-userdata git-hooks/repos "$GHE_HOSTNAME" 1>&3
+  ghe-restore-git-hooks-extract "$GHE_HOSTNAME" 1>&3
 fi
 
 if $cluster; then

diff --git a/share/github-backup-utils/ghe-backup-git-hooks-cluster b/share/github-backup-utils/ghe-backup-git-hooks-cluster
@@ -58,12 +58,6 @@ cleanup() {
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
 
-# If we have a previous increment, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-if [ -d "$backup_current" ]; then
-    link_dest="--link-dest=../../current/git-hooks"
-fi
-
 # Transfer Git hooks data from a GitHub instance to the current snapshot
 # directory, using a previous snapshot to avoid transferring files that have
 # already been transferred. A set of rsync filter rules are provided on stdin
@@ -72,16 +66,42 @@ rsync_git_hooks_data () {
     port=$(ssh_port_part "$1")
     host=$(ssh_host_part "$1")
 
-    shift
-    ghe-rsync -a \
+    subpath=$2
+    shift 2
+
+    # If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
+    # --link-dest support. This also decreases physical space usage considerably.
+    if [ -d "$backup_current/$subpath" ] && [ "$(ls -A $backup_current/$subpath)" ]; then
+
+      subdir="git-hooks/$subpath"
+      link_path=".."
+      while true; do
+        if [ $(dirname $subdir) = "." ]; then
+          break
+        fi
+
+        if [ $(dirname $subdir) = "/" ]; then
+          break
+        fi
+
+        link_path="../$link_path"
+        subdir=$(dirname $subdir)
+      done
+
+      local link_dest="--link-dest=../${link_path}/current/git-hooks/$subpath"
+    fi
+
+    # Ensure target directory exists, is needed with subdirectories
+    mkdir -p "$backup_dir/$subpath"
+
+    ghe-rsync -av \
         -e "ssh -q $opts -p $port -F $config_file -l $user" $link_dest \
         --rsync-path='sudo -u git rsync' \
-        "$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/" \
-        "$backup_dir" 1>&3
+        "$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/$subpath/" \
+        "$backup_dir/$subpath" 1>&3
 }
 
-for hostname in $hostnames; do
-  ghe-ssh -F $config_file "$hostname:122" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks' ]" || exit 0
-  # We can stop after the first successful sync since hooks are the same everywhere
-  rsync_git_hooks_data $hostname:122 && break
-done
+hostname=$(echo $hostnames | awk '{ print $1; }')
+ghe-ssh -F $config_file "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks' ]" || exit 0
+rsync_git_hooks_data $hostname:122 environments/tarballs
+rsync_git_hooks_data $hostname:122 repos
diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata
@@ -23,14 +23,33 @@ ghe_remote_version_required "$host"
 
 # Verify that the user data directory exists. Bail out if not, which may be due
 # to an older version of GHE or no data has been added to this directory yet.
-ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0
+ghe-ssh "$host" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0
 
 # If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
 # --link-dest support. This also decreases physical space usage considerably.
 if [ -d "$GHE_DATA_DIR/current/$dirname" ] && [ "$(ls -A $GHE_DATA_DIR/current/$dirname)" ]; then
-    link_dest="--link-dest=../../current/$dirname"
+
+  subdir=$dirname
+  link_path=".."
+  while true; do
+    if [ $(dirname $subdir) = "." ]; then
+      break
+    fi
+
+    if [ $(dirname $subdir) = "/" ]; then
+      break
+    fi
+
+    link_path="../$link_path"
+    subdir=$(dirname $subdir)
+  done
+
+  link_dest="--link-dest=../${link_path}/current/$dirname"
 fi
 
+# Ensure target directory exists, is needed with subdirectories
+mkdir -p "$GHE_SNAPSHOT_DIR/$dirname"
+
 # Transfer all data from the user data directory using rsync.
 ghe-rsync -avz \
     -e "ghe-ssh -p $(ssh_port_part "$host")" \

diff --git a/share/github-backup-utils/ghe-restore-alambic-cluster-ng b/share/github-backup-utils/ghe-restore-alambic-cluster-ng
@@ -23,7 +23,7 @@ GHE_HOSTNAME="$1"
 : ${GHE_RESTORE_SNAPSHOT:=current}
 
 # Find the objets to restore
-storage_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find storage -mindepth 4 -maxdepth 4 -type f -printf '%p %s\n' | cut -d / -f2-)
+storage_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find storage -mindepth 4 -maxdepth 4 -type f -exec wc -c {} \;)
 
 # No need to restore anything, early exit
 if [ -z "$storage_paths" ]; then
@@ -80,8 +80,8 @@ trap 'cleanup' EXIT
 # ...
 OLDIFS=$IFS; IFS=$'\n'
 for path in $storage_paths; do
-  oid=$(echo $path | cut -d ' ' -f 1 | awk -F/ '{print $(NF)}')
-  size=$(echo $path | cut -d ' ' -f 2)
+  oid=$(echo $path | awk -F/ '{print $(NF)}')
+  size=$(echo $path | awk '{print $1}')
   echo $oid $size
 done > $tmp_list
 IFS=$OLDIFS

diff --git a/share/github-backup-utils/ghe-restore-git-hooks-cluster b/share/github-backup-utils/ghe-restore-git-hooks-cluster
@@ -58,13 +58,31 @@ cleanup() {
 }
 trap 'cleanup' INT TERM EXIT
 
-if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks" ]; then
+if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs" ]; then
+  tarballs=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs && find . -type f)
+  hostname=$(echo $hostnames | awk '{ print $1; }')
+
+  if [ -n "$hostname" ]; then
+    ghe-rsync -avH --delete \
+    -e "ssh -q $opts -p $port -F $config_file -l $user" \
+    --rsync-path="sudo -u git rsync" \
+    "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs/" \
+    "$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" 1>&3
+
+    for tarball in $tarballs; do
+      env_id=$(echo $tarball | cut -d '/' -f 2)
+      ssh -q $opts -p $port -F $config_file -l $user $hostname "/bin/bash -c 'export PATH=\$PATH:/usr/local/share/enterprise && ghe-hook-env-update $env_id $GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs/$tarball'" 1>&3 2>&3
+    done
+  fi
+fi
+
+if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/repos" ]; then
   for hostname in $hostnames; do
-    ghe-rsync -aH --delete \
+    ghe-rsync -avH --delete \
     -e "ssh -q $opts -p $port -F $config_file -l $user" \
     --rsync-path="sudo -u git rsync" \
-    "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/" \
-    "$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/" &
+    "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/repos/" \
+    "$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" 1>&3 &
   done
 
   for pid in $(jobs -p); do

diff --git a/share/github-backup-utils/ghe-restore-git-hooks-extract b/share/github-backup-utils/ghe-restore-git-hooks-extract
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-restore-git-hooks-extract <host>
+#/ Restore custom Git hooks environments from tarballs
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-restore
+set -e
+
+# Bring in the backup configuration
+. $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+# Grab host arg
+GHE_HOSTNAME="$1"
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$GHE_HOSTNAME"
+
+# The snapshot to restore should be set by the ghe-restore command but this lets
+# us run this script directly.
+: ${GHE_RESTORE_SNAPSHOT:=current}
+
+if [ ! -d $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs ]; then
+  echo "Warning: No pre-receive hook environments. Skipping ..."
+  exit 0
+fi
+
+tarballs=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs && find . -type f)
+
+for tarball in $tarballs; do
+  env_id=$(echo $tarball | cut -d '/' -f 2)
+  ghe-ssh "$GHE_HOSTNAME" "/bin/bash -c 'export PATH=\$PATH:/usr/local/share/enterprise && ghe-hook-env-update $env_id $GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs/$tarball'"
+done
diff --git a/share/github-backup-utils/ghe-restore-userdata b/share/github-backup-utils/ghe-restore-userdata
@@ -35,7 +35,7 @@ ghe_remote_version_required "$GHE_HOSTNAME"
 if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/$dirname" ]; then
   ghe-rsync -avz --delete \
       -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \
-      --rsync-path="sudo -u git rsync" \
+      --rsync-path="sudo -u git mkdir -p $GHE_REMOTE_DATA_USER_DIR/$dirname && sudo -u git rsync" \
       "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/$dirname/" \
       "$(ssh_host_part "$GHE_HOSTNAME"):$GHE_REMOTE_DATA_USER_DIR/$dirname" 1>&3
 fi

diff --git a/test/bin/ghe-hook-env-update b/test/bin/ghe-hook-env-update
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Emulates a remote GitHub import command. Each of the ghe-import-* utilities
+# that are run on the remote side have corresponding commands in this directory
+# that symlink to this file. The command just gobbles up stdin and writes a
+# simple success message.
+env_id=$1
+tarball=$2
+
+path=$(dirname $tarball)/../../
+mkdir -p $path/$1
+cd $path/$i
+tar -zxf $tarball
diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh
@@ -17,21 +17,30 @@ touch alice/index.html bob/index.html
 mkdir -p "$GHE_REMOTE_DATA_USER_DIR/common"
 echo "fake password hash data" > "$GHE_REMOTE_DATA_USER_DIR/common/manage-password"
 
-# Create some fake hookshot data in the remote data directory
 if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
+    # Create some fake data in the remote data directory
     mkdir -p "$GHE_REMOTE_DATA_USER_DIR/hookshot"
     cd "$GHE_REMOTE_DATA_USER_DIR/hookshot"
     mkdir -p repository-123 repository-456
     touch repository-123/test.bpack repository-456/test.bpack
 
-    mkdir -p "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
-    cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks"
-    mkdir -p repository-123 repository-456
-    touch repository-123/script.sh repository-456/foo.sh
-fi
+    # Create some fake hooks in the remote data directory
+    mkdir -p "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs"
+    mkdir -p "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos"
 
-# Create some fake alambic data in the remote data directory
-if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
+    cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments"
+    mkdir -p 123/abcdef 456/fed314
+    touch 123/abcdef/script.sh 456/fed314/foo.sh
+
+    cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs"
+    mkdir -p 123/abcdef 456/fed314
+    touch 123/abcdef/script.tar.gz 456/fed314/foo.tar.gz
+
+    cd "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos"
+    mkdir -p 321 654
+    touch 321/script.sh 654/foo.sh
+
+    # Create some fake alambic data in the remote data directory
     mkdir -p "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-assets/0000"
     touch "$GHE_REMOTE_DATA_USER_DIR/alambic_assets/github-enterprise-assets/0000/test.png"
 
@@ -127,8 +136,14 @@ begin_test "ghe-backup first snapshot"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
 
-        # verify all git hooks data was transferred
-        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
+        # verify all git hooks tarballs were transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" "$GHE_DATA_DIR/current/git-hooks/environments/tarballs"
+
+        # verify the extracted environments were not transferred
+        ! diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments" "$GHE_DATA_DIR/current/git-hooks/environments"
+
+        # verify the extracted repositories were transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" "$GHE_DATA_DIR/current/git-hooks/repos"
 
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
@@ -204,8 +219,14 @@ begin_test "ghe-backup subsequent snapshot"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
 
-        # verify all git hooks data was transferred
-        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
+        # verify all git hooks tarballs were transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" "$GHE_DATA_DIR/current/git-hooks/environments/tarballs"
+
+        # verify the extracted environments were not transferred
+        ! diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments" "$GHE_DATA_DIR/current/git-hooks/environments"
+
+        # verify the extracted repositories were transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" "$GHE_DATA_DIR/current/git-hooks/repos"
 
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
@@ -282,8 +303,14 @@ begin_test "ghe-backup with relative data dir path"
         # verify all hookshot user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/hookshot" "$GHE_DATA_DIR/current/hookshot"
 
-        # verify all git hooks data was transferred
-        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks" "$GHE_DATA_DIR/current/git-hooks"
+        # verify all git hooks tarballs were transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" "$GHE_DATA_DIR/current/git-hooks/environments/tarballs"
+
+        # verify the extracted environments were not transferred
+        ! diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments" "$GHE_DATA_DIR/current/git-hooks/environments"
+
+        # verify the extracted repositories were transferred
+        diff -ru "$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" "$GHE_DATA_DIR/current/git-hooks/repos"
 
         # verify all alambic assets user data was transferred
         diff -ru "$GHE_REMOTE_DATA_USER_DIR/alambic_assets" "$GHE_DATA_DIR/current/alambic_assets"
@@ -375,8 +402,8 @@ begin_test "ghe-backup empty git-hooks directory"
 (
   set -e
 
-  rm -rf $GHE_REMOTE_DATA_USER_DIR/git-hooks/repository-*
-  rm -rf $GHE_DATA_DIR/current/git-hooks/repository-*
+  rm -rf $GHE_REMOTE_DATA_USER_DIR/git-hooks/*
+  rm -rf $GHE_DATA_DIR/current/git-hooks/*
   ghe-backup
 
   # Check that the "--link-dest arg does not exist" message hasn't occurred.