From 317957204c80b73b9f4d6e6f9a3b20faf889c7e6 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Wed, 5 Feb 2020 12:53:25 -0800 Subject: [PATCH 01/17] Use pigz instead of gzip in backup-util --- bin/ghe-backup | 2 +- share/github-backup-utils/ghe-restore-mysql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index f67b4dc71..822301096 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -200,7 +200,7 @@ is_binary_backup(){ echo "Backing up MySQL database ..." bm_start "ghe-export-mysql" -echo 'set -o pipefail; ghe-export-mysql | gzip' | +echo 'set -o pipefail; ghe-export-mysql | pigz' | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz || failures="$failures mysql" if is_binary_backup; then touch mysql-binary-backup-sentinel diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 4c8cec9e4..7db8c6d05 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -67,6 +67,6 @@ ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" # Import the database -echo "gunzip -cd $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 +echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | unpigz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 bm_end "$(basename $0)" From a6e6d7d61b1a5c59dcf2caa05098e85b0d5f231c Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 6 Feb 2020 08:30:52 -0800 Subject: [PATCH 02/17] Use unpigz -cd option --- share/github-backup-utils/ghe-restore-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 7db8c6d05..8336ea61c 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -67,6 +67,6 @@ ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" # Import the database -echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | unpigz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 +unpigz -cd "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 bm_end "$(basename $0)" From 16465cdf82e62aeeb9b389befb2e5c44aac17561 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 6 Feb 2020 10:18:02 -0800 Subject: [PATCH 03/17] Fix issue with last commit --- share/github-backup-utils/ghe-restore-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 8336ea61c..3cefb77ff 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -67,6 +67,6 @@ ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" # Import the database -unpigz -cd "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 +echo "unpigz -cd $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 bm_end "$(basename $0)" From 05a63423e3bf2525fa2c503faf230302fd7099e1 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 6 Feb 2020 13:07:28 -0800 Subject: [PATCH 04/17] Not to compress for binary backup --- bin/ghe-backup | 9 +++++++-- share/github-backup-utils/ghe-restore-mysql | 14 ++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 822301096..abcb9b5d1 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -200,10 +200,15 @@ is_binary_backup(){ echo "Backing up MySQL database ..." bm_start "ghe-export-mysql" -echo 'set -o pipefail; ghe-export-mysql | pigz' | +export_command="ghe-export-mysql" +if ! is_binary_backup; then + # binary backup is already compressed + export_command+=" | pigz" +fi +echo "set -o pipefail; ${export_command}" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz || failures="$failures mysql" if is_binary_backup; then - touch mysql-binary-backup-sentinel + echo "NO_ADDITIONAL_COMPRESSION" > mysql-binary-backup-sentinel fi bm_end "ghe-export-mysql" diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 3cefb77ff..0abe00da3 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -46,9 +46,15 @@ is_binary_backup_feature_on(){ if is_binary_backup_feature_on; then # Feature "mysql.backup.binary" is on, which means new backup scripts are available if is_binary_backup; then - IMPORT_MYSQL=ghe-import-mysql-xtrabackup + # Check if the decompress needed by looking into the sentinel file + # In 2.19.5 we compress the binary backup twice + if [ $(cat $snapshot_dir/mysql-binary-backup-sentinel) = "NO_ADDITIONAL_COMPRESSION" ]; then + IMPORT_MYSQL=ghe-import-mysql-xtrabackup + else + IMPORT_MYSQL="unpigz | ghe-import-mysql-xtrabackup" + fi else - IMPORT_MYSQL=ghe-import-mysql-mysqldump + IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump" fi else # We do not allow to restore binary backup without "mysql.backup.binary" set @@ -57,7 +63,7 @@ else exit 2 else # legacy mode - IMPORT_MYSQL=ghe-import-mysql + IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump" fi fi @@ -67,6 +73,6 @@ ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" # Import the database -echo "unpigz -cd $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 +echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 bm_end "$(basename $0)" From 41ad92ded8be2ccdf7d03b3f1cc4fd1694a73126 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 7 Feb 2020 06:50:44 -0800 Subject: [PATCH 05/17] Make sure xtrabackup restore will be imported on mysql master --- share/github-backup-utils/ghe-restore-mysql | 23 ++++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 0abe00da3..f1b2c85d7 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -28,11 +28,6 @@ ghe_remote_version_required "$GHE_HOSTNAME" # The directory holding the snapshot to restore snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" -cleanup() { - ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz" -} -trap 'cleanup' INT TERM EXIT - # Check if the backup is binary by looking up the sentinel file is_binary_backup(){ test -f $snapshot_dir/mysql-binary-backup-sentinel @@ -43,6 +38,9 @@ is_binary_backup_feature_on(){ ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" } +port=$(ssh_port_part "$GHE_HOSTNAME") +ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master"):$port + if is_binary_backup_feature_on; then # Feature "mysql.backup.binary" is on, which means new backup scripts are available if is_binary_backup; then @@ -50,11 +48,14 @@ if is_binary_backup_feature_on; then # In 2.19.5 we compress the binary backup twice if [ $(cat $snapshot_dir/mysql-binary-backup-sentinel) = "NO_ADDITIONAL_COMPRESSION" ]; then IMPORT_MYSQL=ghe-import-mysql-xtrabackup + GHE_RESTORE_HOST=$ghe_mysql_master else IMPORT_MYSQL="unpigz | ghe-import-mysql-xtrabackup" + GHE_RESTORE_HOST=$ghe_mysql_master fi else IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump" + GHE_RESTORE_HOST=$GHE_HOSTNAME fi else # We do not allow to restore binary backup without "mysql.backup.binary" set @@ -64,15 +65,21 @@ else else # legacy mode IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump" + GHE_RESTORE_HOST=$GHE_HOSTNAME fi fi -ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 +cleanup() { + ghe-ssh "$GHE_RESTORE_HOST" -- "sudo rm -rf $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz" +} +trap 'cleanup' INT TERM EXIT + +ghe-ssh "$GHE_RESTORE_HOST" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3 # Transfer MySQL data from the snapshot to the GitHub instance. -cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" +cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_RESTORE_HOST" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" # Import the database -echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3 +echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_RESTORE_HOST" -- /bin/bash 1>&3 bm_end "$(basename $0)" From 74f23c42ea40cf82cd60ea25968feb12645d75e1 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 7 Feb 2020 07:16:41 -0800 Subject: [PATCH 06/17] ignore port if not set in hostname --- share/github-backup-utils/ghe-restore-mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index f1b2c85d7..2f7cd3a39 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -39,7 +39,7 @@ is_binary_backup_feature_on(){ } port=$(ssh_port_part "$GHE_HOSTNAME") -ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master"):$port +ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master")${port:+:$port} if is_binary_backup_feature_on; then # Feature "mysql.backup.binary" is on, which means new backup scripts are available From d645bf46c3c3a554af2cbf6646816fa67c28ff5e Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 7 Feb 2020 08:13:58 -0800 Subject: [PATCH 07/17] Check if sql master exists --- share/github-backup-utils/ghe-restore-mysql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 2f7cd3a39..3f2250160 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -39,7 +39,12 @@ is_binary_backup_feature_on(){ } port=$(ssh_port_part "$GHE_HOSTNAME") -ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master")${port:+:$port} +ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master") +if [ -z $ghe_mysql_master ]; then + ghe_mysql_master=$GHE_HOSTNAME +else + ghe_mysql_master=$ghe_mysql_master${port:+:$port} +fi if is_binary_backup_feature_on; then # Feature "mysql.backup.binary" is on, which means new backup scripts are available From f29f0c1fbc80e492f78d02148b2680afe57ce071 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 7 Feb 2020 10:33:54 -0800 Subject: [PATCH 08/17] check /etc/github/cluster as well --- share/github-backup-utils/ghe-restore-mysql | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 3f2250160..bbcf3ab9a 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -38,12 +38,16 @@ is_binary_backup_feature_on(){ ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" } -port=$(ssh_port_part "$GHE_HOSTNAME") -ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master") -if [ -z $ghe_mysql_master ]; then - ghe_mysql_master=$GHE_HOSTNAME +if ghe-ssh "$GHE_HOSTNAME" test -f /etc/github/cluster ; then + ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master") + if [ -z $ghe_mysql_master ]; then + ghe_mysql_master=$GHE_HOSTNAME + else + port=$(ssh_port_part "$GHE_HOSTNAME") + ghe_mysql_master=$ghe_mysql_master${port:+:$port} + fi else - ghe_mysql_master=$ghe_mysql_master${port:+:$port} + ghe_mysql_master=$GHE_HOSTNAME fi if is_binary_backup_feature_on; then @@ -51,7 +55,7 @@ if is_binary_backup_feature_on; then if is_binary_backup; then # Check if the decompress needed by looking into the sentinel file # In 2.19.5 we compress the binary backup twice - if [ $(cat $snapshot_dir/mysql-binary-backup-sentinel) = "NO_ADDITIONAL_COMPRESSION" ]; then + if [ "$(cat $snapshot_dir/mysql-binary-backup-sentinel)" = "NO_ADDITIONAL_COMPRESSION" ]; then IMPORT_MYSQL=ghe-import-mysql-xtrabackup GHE_RESTORE_HOST=$ghe_mysql_master else From f6ffabf37a02ff8d7f21bae6831381a1b40bc22b Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 11:06:38 -0800 Subject: [PATCH 09/17] add pigz to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e8b540837..36e27a0c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,4 +24,5 @@ matrix: - moreutils - fakeroot - jq + - pigz script: debuild -uc -us From e20f1baa1f284ddd2c3e13278ee00bd4c581da55 Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 13:13:19 -0800 Subject: [PATCH 10/17] check git version --- test/test-ghe-backup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 9fe8eb9bb..9c338aeac 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -54,6 +54,8 @@ begin_test "ghe-backup logs the benchmark" # wait a second for snapshot timestamp sleep 1 + echo "GIT VERSION" + echo $(git --version) export BM_TIMESTAMP=foo From 7a18097986bccf6def40736d12987a2beeded013 Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 13:33:33 -0800 Subject: [PATCH 11/17] try with git --- .travis.yml | 1 + test/test-ghe-backup.sh | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36e27a0c6..a2e1ab83e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,4 +25,5 @@ matrix: - fakeroot - jq - pigz + - git script: debuild -uc -us diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 9c338aeac..9fe8eb9bb 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -54,8 +54,6 @@ begin_test "ghe-backup logs the benchmark" # wait a second for snapshot timestamp sleep 1 - echo "GIT VERSION" - echo $(git --version) export BM_TIMESTAMP=foo From 86b0698acab7d2c7c8de8b306321109c229255d6 Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 13:50:58 -0800 Subject: [PATCH 12/17] git for mac --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a2e1ab83e..78ce5be59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ matrix: - brew install moreutils - brew install shellcheck - brew install jq + - brew install git script: make test - os: linux dist: trusty From aa72af40994605dd23141bc3cc88f6f0da8403ec Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 13:58:46 -0800 Subject: [PATCH 13/17] remove brew git --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 78ce5be59..a2e1ab83e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ matrix: - brew install moreutils - brew install shellcheck - brew install jq - - brew install git script: make test - os: linux dist: trusty From ddd329341483d1391b70cbd87975c343f0b62439 Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 14:12:57 -0800 Subject: [PATCH 14/17] change to true --- bin/ghe-backup | 4 ++-- share/github-backup-utils/ghe-backup-audit-log | 2 +- share/github-backup-utils/ghe-restore-mysql | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 93ea8d23c..ee60abcdf 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -83,7 +83,7 @@ rm -rf src dest1 dest2 # if we should use gitbackups to backup repositories should_use_gitbackups_for_repositories(){ - ghe-ssh "$GHE_HOSTNAME" ghe-config --true "app.github.gitbackups" + ghe-ssh "$GHE_HOSTNAME" ghe-config "app.github.gitbackups" true } # check that the appliance supports using gitbackups for repositories @@ -195,7 +195,7 @@ bm_end "ghe-export-ssh-host-keys" # if we are going to take a binary backup is_binary_backup(){ - ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" + ghe-ssh "$GHE_HOSTNAME" ghe-config "mysql.backup.binary" true } echo "Backing up MySQL database ..." diff --git a/share/github-backup-utils/ghe-backup-audit-log b/share/github-backup-utils/ghe-backup-audit-log index 6030b2c5b..c59274d1b 100755 --- a/share/github-backup-utils/ghe-backup-audit-log +++ b/share/github-backup-utils/ghe-backup-audit-log @@ -35,7 +35,7 @@ audit_entries_deleted(){ } is_binary_backup(){ - ghe-ssh "$host" ghe-config --true "mysql.backup.binary" + ghe-ssh "$host" ghe-config "mysql.backup.binary" true } backup_mysql(){ diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index bbcf3ab9a..c65db3ad1 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -35,7 +35,7 @@ is_binary_backup(){ # if mysql.backup.binary feature flag is on is_binary_backup_feature_on(){ - ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" + ghe-ssh "$GHE_HOSTNAME" ghe-config "mysql.backup.binary" true } if ghe-ssh "$GHE_HOSTNAME" test -f /etc/github/cluster ; then From b7dd1bbf4da8e5194bf6132c5f71be35289803c7 Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 14:43:52 -0800 Subject: [PATCH 15/17] Revert "change to true" This reverts commit ddd329341483d1391b70cbd87975c343f0b62439. --- bin/ghe-backup | 4 ++-- share/github-backup-utils/ghe-backup-audit-log | 2 +- share/github-backup-utils/ghe-restore-mysql | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index ee60abcdf..93ea8d23c 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -83,7 +83,7 @@ rm -rf src dest1 dest2 # if we should use gitbackups to backup repositories should_use_gitbackups_for_repositories(){ - ghe-ssh "$GHE_HOSTNAME" ghe-config "app.github.gitbackups" true + ghe-ssh "$GHE_HOSTNAME" ghe-config --true "app.github.gitbackups" } # check that the appliance supports using gitbackups for repositories @@ -195,7 +195,7 @@ bm_end "ghe-export-ssh-host-keys" # if we are going to take a binary backup is_binary_backup(){ - ghe-ssh "$GHE_HOSTNAME" ghe-config "mysql.backup.binary" true + ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" } echo "Backing up MySQL database ..." diff --git a/share/github-backup-utils/ghe-backup-audit-log b/share/github-backup-utils/ghe-backup-audit-log index c59274d1b..6030b2c5b 100755 --- a/share/github-backup-utils/ghe-backup-audit-log +++ b/share/github-backup-utils/ghe-backup-audit-log @@ -35,7 +35,7 @@ audit_entries_deleted(){ } is_binary_backup(){ - ghe-ssh "$host" ghe-config "mysql.backup.binary" true + ghe-ssh "$host" ghe-config --true "mysql.backup.binary" } backup_mysql(){ diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index c65db3ad1..bbcf3ab9a 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -35,7 +35,7 @@ is_binary_backup(){ # if mysql.backup.binary feature flag is on is_binary_backup_feature_on(){ - ghe-ssh "$GHE_HOSTNAME" ghe-config "mysql.backup.binary" true + ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" } if ghe-ssh "$GHE_HOSTNAME" test -f /etc/github/cluster ; then From 178da66f3cbefbb16b0909cc3ed8310f47bfb0d1 Mon Sep 17 00:00:00 2001 From: Courtney Oka Date: Fri, 7 Feb 2020 15:31:42 -0800 Subject: [PATCH 16/17] add pigz to macos --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a2e1ab83e..ef07336ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ matrix: - brew install moreutils - brew install shellcheck - brew install jq + - brew install pigz script: make test - os: linux dist: trusty @@ -25,5 +26,4 @@ matrix: - fakeroot - jq - pigz - - git script: debuild -uc -us From c96c6d23eac47dfe35c6de908119c01fb48c41ff Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Mon, 10 Feb 2020 08:55:42 -0800 Subject: [PATCH 17/17] Adopt comments --- bin/ghe-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 93ea8d23c..2f94cb408 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -205,7 +205,7 @@ if ! is_binary_backup; then # binary backup is already compressed export_command+=" | pigz" fi -echo "set -o pipefail; ${export_command}" | +echo "set -o pipefail; $export_command" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz || failures="$failures mysql" if is_binary_backup; then echo "NO_ADDITIONAL_COMPRESSION" > mysql-binary-backup-sentinel