diff --git a/.travis.yml b/.travis.yml index cdb3b944f..308e30ca9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ matrix: - brew install shellcheck - brew install jq - brew install coreutils + - brew install pigz script: make test - os: linux dist: trusty @@ -26,4 +27,5 @@ matrix: - fakeroot - jq - coreutils + - pigz script: debuild -uc -us diff --git a/bin/ghe-backup b/bin/ghe-backup index 3abe96749..1acf095c1 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -81,6 +81,25 @@ if [ "$(ls -il dest1/testfile | awk '{ print $1 }')" != "$(ls -il dest2/testfile fi rm -rf src dest1 dest2 +# if we should use gitbackups to backup repositories +should_use_gitbackups_for_repositories(){ + ghe-ssh "$GHE_HOSTNAME" ghe-config --true "app.github.gitbackups" +} + +# check that the appliance supports using gitbackups for repositories +can_use_gitbackups_for_repositories(){ + ghe-ssh "$GHE_HOSTNAME" test -e /data/github/current/bin/ghe-backup-repositories +} + +# Exit early if the appliance is missing script to backup repositories using gitbackups +if should_use_gitbackups_for_repositories; then + if ! can_use_gitbackups_for_repositories; then + echo "Error: Configuration setting 'app.github.gitbackups' is enabled but this version of GHES cannot use gitbackups to back up repositories via 'ghe-backup'." + echo "Disable configuration setting 'app.github.gitbackups' and re-run 'ghe-backup' to use rsync." + exit 1 + fi +fi + # To prevent multiple backup runs happening at the same time, we create a # in-progress file with the timestamp and pid of the backup process, # giving us a form of locking. @@ -174,10 +193,23 @@ ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar || failures="$failures ssh-host-keys" bm_end "ghe-export-ssh-host-keys" +# if we are going to take a binary backup +is_binary_backup(){ + ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" +} + echo "Backing up MySQL database ..." bm_start "ghe-export-mysql" -echo 'set -o pipefail; ghe-export-mysql | gzip' | +export_command="ghe-export-mysql" +if ! is_binary_backup; then + # binary backup is already compressed + export_command+=" | pigz" +fi +echo "set -o pipefail; $export_command" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz || failures="$failures mysql" +if is_binary_backup; then + echo "NO_ADDITIONAL_COMPRESSION" > mysql-binary-backup-sentinel +fi bm_end "ghe-export-mysql" echo "Backing up Redis database ..." diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 9f1ed3c8a..bbcf3ab9a 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -28,18 +28,67 @@ ghe_remote_version_required "$GHE_HOSTNAME" # The directory holding the snapshot to restore snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" +# Check if the backup is binary by looking up the sentinel file +is_binary_backup(){ + test -f $snapshot_dir/mysql-binary-backup-sentinel +} + +# if mysql.backup.binary feature flag is on +is_binary_backup_feature_on(){ + ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" +} + +if ghe-ssh "$GHE_HOSTNAME" test -f /etc/github/cluster ; then + ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master") + if [ -z $ghe_mysql_master ]; then + ghe_mysql_master=$GHE_HOSTNAME + else + port=$(ssh_port_part "$GHE_HOSTNAME") + ghe_mysql_master=$ghe_mysql_master${port:+:$port} + fi +else + ghe_mysql_master=$GHE_HOSTNAME +fi + +if is_binary_backup_feature_on; then + # Feature "mysql.backup.binary" is on, which means new backup scripts are available + if is_binary_backup; then + # Check if the decompress needed by looking into the sentinel file + # In 2.19.5 we compress the binary backup twice + if [ "$(cat $snapshot_dir/mysql-binary-backup-sentinel)" = "NO_ADDITIONAL_COMPRESSION" ]; then + IMPORT_MYSQL=ghe-import-mysql-xtrabackup + GHE_RESTORE_HOST=$ghe_mysql_master + else + IMPORT_MYSQL="unpigz | ghe-import-mysql-xtrabackup" + GHE_RESTORE_HOST=$ghe_mysql_master + fi + else + IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump" + GHE_RESTORE_HOST=$GHE_HOSTNAME + fi +else + # We do not allow to restore binary backup without "mysql.backup.binary" set + if is_binary_backup; then + echo "To restore from a binary backup, you have to set ghe-config \"mysql.backup.binary\" to true" >&2 + exit 2 + else + # legacy mode + IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump" + GHE_RESTORE_HOST=$GHE_HOSTNAME + fi +fi + cleanup() { - ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz" + ghe-ssh "$GHE_RESTORE_HOST" -- "sudo rm -rf $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz" } trap 'cleanup' INT TERM EXIT -ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 +ghe-ssh "$GHE_RESTORE_HOST" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 # Transfer MySQL data from the snapshot to the GitHub instance. -cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" +cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_RESTORE_HOST" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" # Import the database -echo "gunzip -cd $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | ghe-import-mysql" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 - +echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_RESTORE_HOST" -- /bin/bash 1>&3 bm_end "$(basename $0)" diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 17bdb70fa..6ef6ef5ac 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -2.19.2 +2.19.3 diff --git a/test/bin/ghe-import-mysql-mysqldump b/test/bin/ghe-import-mysql-mysqldump new file mode 120000 index 000000000..bc329368a --- /dev/null +++ b/test/bin/ghe-import-mysql-mysqldump @@ -0,0 +1 @@ +ghe-fake-import-command \ No newline at end of file diff --git a/test/bin/ghe-import-mysql-xtrabackup b/test/bin/ghe-import-mysql-xtrabackup new file mode 120000 index 000000000..bc329368a --- /dev/null +++ b/test/bin/ghe-import-mysql-xtrabackup @@ -0,0 +1 @@ +ghe-fake-import-command \ No newline at end of file