github · rubiojr · May 31, 2016 · Mar 14, 2016 · May 31, 2016
diff --git a/bin/ghe-restore b/bin/ghe-restore
@@ -216,7 +216,12 @@ if $cluster; then
   ghe-restore-repositories-dgit "$GHE_HOSTNAME" 1>&3
 
   echo "Restoring Gists into cluster ..."
-  ghe-restore-repositories-gist "$GHE_HOSTNAME" 1>&3
+  if ghe-ssh "$GHE_HOSTNAME" test -f /data/github/current/script/gist-cluster-restore-routes; then
+    ghe_verbose "* Using ghe-restore-repositories-gist-ng to restore"
+    ghe-restore-repositories-gist-ng "$GHE_HOSTNAME" 1>&3
+  else
+    ghe-restore-repositories-gist "$GHE_HOSTNAME" 1>&3
+  fi
 else
   # Remove temporary 2.2 storage migration directory if it exists
   echo "if [ -d /data/user/repositories-nw-backup ]; then sudo rm -rf /data/user/repositories-nw-backup; fi" |

diff --git a/share/github-backup-utils/ghe-restore-repositories-gist-ng b/share/github-backup-utils/ghe-restore-repositories-gist-ng
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-restore-repositories-gist-ng <host>
+#/ Restore repositories fron an rsync snapshot of all Git repository data to a GitHub cluster.
+#/
+#/ Note: This script typically isn't called directly. It's invoked by the
+#/ ghe-restore command when restoring into a cluster.
+set -e
+
+# Bring in the backup configuration
+. $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
+
+# Show usage and bail with no arguments
+[ -z "$*" ] && print_usage
+
+# Grab host arg
+GHE_HOSTNAME="$1"
+
+# The snapshot to restore should be set by the ghe-restore command but this lets
+# us run this script directly.
+: ${GHE_RESTORE_SNAPSHOT:=current}
+
+# Find the gists to restore
+gist_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/  && find repositories -mindepth 6 -maxdepth 7 -name \*.git | grep gist | cut -d / -f2-)
+
+# No need to restore anything, early exit
+if [ -z "$gist_paths" ]; then
+  echo "Warning: Gist backup missing. Skipping ..."
+  exit 0
+fi
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$GHE_HOSTNAME"
+
+# Generate SSH config for forwarding
+# Split host:port into parts
+port=$(ssh_port_part "$GHE_HOSTNAME")
+host=$(ssh_host_part "$GHE_HOSTNAME")
+
+# Add user / -l option
+user="${host%@*}"
+[ "$user" = "$host" ] && user="admin"
+
+tempdir=$(mktemp -d)
+ssh_config_file=$tempdir/ssh_config
+opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
+tmp_list=$tempdir/tmp_list
+to_restore=$tempdir/to_restore
+
+hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | cut -d ' ' -f 2)
+for hostname in $hostnames; do
+  echo "
+Host $hostname
+  ServerAliveInterval 60
+  ProxyCommand ssh -q $GHE_EXTRA_SSH_OPTS -p $port $user@$host nc.openbsd %h %p" >> $ssh_config_file
+done
+
+cleanup() {
+  rm -rf $tempdir
+}
+trap cleanup EXIT
+
+# Find the routes (servers) for each gist available locally
+OLDIFS=$IFS; IFS=$'\n'
+for path in $gist_paths; do
+   echo $path
+done > $tmp_list
+IFS=$OLDIFS
+
+cat $tmp_list | ghe-ssh "$GHE_HOSTNAME" github-env ./bin/gist-cluster-restore-routes \
+ | while read route; do
+  servers=$(echo $route | cut -d ' ' -f2-)
+  for server in $servers; do
+    gist=$(echo $route | cut -d ' ' -f1)
+    ghe_verbose "Adding $gist to $tempdir/$server.rsync"
+    echo "$gist" >> $tempdir/$server.rsync
+ done
+
+ gist_id=$(basename $(echo $route | cut -d ' ' -f 1) .git)
+ ghe_verbose "Route: $gist_id /data/repositories/$gist $servers"
+ echo "$gist_id /data/repositories/$gist $servers" >> $to_restore
+done
+
+# rsync all the gist repositories
+for route in $tempdir/*.rsync; do
+  ghe-rsync -aHR --delete \
+    -e "ssh -q $opts -p $port -F $ssh_config_file -l $user" \
+    --rsync-path="sudo -u git rsync" \
+    --files-from=$route \
+    "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/./" \
+    "$(basename $route .rsync):$GHE_REMOTE_DATA_USER_DIR/repositories/"
+done
+
+cat $to_restore | ghe-ssh "$GHE_HOSTNAME" github-env ./bin/gist-cluster-restore-finalize