From d4d4e313485cb2331deaffcee88fe4edf8f154d2 Mon Sep 17 00:00:00 2001 From: boxofyellow <54955040+boxofyellow@users.noreply.github.com> Date: Thu, 15 Jun 2023 20:12:18 +0000 Subject: [PATCH 01/68] Merge pull request #348 from github/users/boxofyellow/2023-06/mssql-backup_0 Allow the caller to handle setting failures --- share/github-backup-utils/ghe-backup-mssql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql index ead33f6bc..b5a3203fb 100755 --- a/share/github-backup-utils/ghe-backup-mssql +++ b/share/github-backup-utils/ghe-backup-mssql @@ -30,11 +30,11 @@ export_tool_available() { } ghe_ssh_mssql() { - ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@" + ghe-ssh "${opts[@]}" "${ssh_config_file_opt[@]}" "$GHE_MSSQL_PRIMARY_HOST" "$@" } cleanup() { - rm -rf $tempdir + rm -rf "$tempdir" } trap 'cleanup' EXIT INT @@ -55,8 +55,8 @@ isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)" # get server hostnames under cluster and HA if [ "$GHE_BACKUP_STRATEGY" = "cluster" ] || [ "$isHA" = "true" ] ; then ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" + ssh_config_file_opt=("-F" "$ssh_config_file") + opts=("-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no" "-o" "PasswordAuthentication=no") ghe-ssh-config "$GHE_HOSTNAME" "$GHE_MSSQL_PRIMARY_HOST" > "$ssh_config_file" fi @@ -69,10 +69,10 @@ add_minute() { # Expect date string in the format of yyyymmddTHHMMSS # Here parse date differently depending on GNU Linux vs BSD MacOS if date -v -1d > /dev/null 2>&1; then - echo "$(date -v +$2M -ujf'%Y%m%dT%H%M%S' $1 +%Y%m%dT%H%M%S)" + date -v +"$2"M -ujf'%Y%m%dT%H%M%S' "$1" +%Y%m%dT%H%M%S else dt=$1 - echo "$(date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes")" + date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes" fi } @@ -337,7 +337,7 @@ if [ -n "$backup_type" ]; then fi bm_start "$(basename "$0")" - ghe_ssh_mssql -- "$backup_command" || failures="$failures mssql" + ghe_ssh_mssql -- "$backup_command" bm_end "$(basename "$0")" # Configure the backup cadence on the appliance, which is used for diagnostics From ee5bad46d1f260a6a4ffd1495f1ec730f39828b0 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Fri, 16 Jun 2023 19:10:34 +0000 Subject: [PATCH 02/68] Merge pull request #280 from github/progress-indicator Progress Indicator --- bin/ghe-backup | 13 +++- bin/ghe-backup-progress | 56 ++++++++++++++ bin/ghe-restore | 77 +++++++++++++++++-- share/github-backup-utils/bm.sh | 3 + share/github-backup-utils/ghe-backup-config | 71 ++++++++++++++++- share/github-backup-utils/ghe-restore-redis | 26 +++++++ .../github-backup-utils/ghe-restore-ssh-keys | 28 +++++++ share/github-backup-utils/track-progress | 13 ++++ 8 files changed, 278 insertions(+), 9 deletions(-) create mode 100755 bin/ghe-backup-progress create mode 100755 share/github-backup-utils/ghe-restore-redis create mode 100755 share/github-backup-utils/ghe-restore-ssh-keys create mode 100755 share/github-backup-utils/track-progress diff --git a/bin/ghe-backup b/bin/ghe-backup index ad8147f82..98f831a51 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -37,11 +37,21 @@ while true; do esac done + export CALLING_SCRIPT="ghe-backup" + # Bring in the backup configuration # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" +# Setup progress tracking +init-progress +export PROGRESS_TYPE="Backup" +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +export PROGRESS=0 # Used to track progress of backup +echo "$PROGRESS" > /tmp/backup-utils-progress +export PROGRESS_TOTAL=18 # Maximum number of steps in backup + # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -121,7 +131,7 @@ ghe_restore_check # Check to see if there is a running backup if [ -h ../in-progress ]; then - log_error "Error: detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." 1>&2 + log_error "Detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." >&2 exit 1 fi @@ -306,6 +316,7 @@ else steps="${failures// /, }" ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. " + ghe_backup_finished exit 1 fi diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress new file mode 100755 index 000000000..2f4b267fb --- /dev/null +++ b/bin/ghe-backup-progress @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +#/ Usage: ghe-backup-progress [--once] +#/ Tracks the completed steps of a backup or restore operation. +#/ +#/ By default the progress is printed every continuously or until a key is pressed. +#/ Use the --once option to print the current progress once and exit. +#/ +#/ Options: +#/ --once Don't loop, just print the current progress once. +# +set -e + +while true; do + case "$1" in + -o|--once) + ONCE=1 + shift + ;; + -h|--help) + export GHE_SHOW_HELP=true + shift + ;; + -*) + echo "Unknown option: $1" >&2 + exit 1 + ;; + *) + break + ;; + esac +done + +check_for_progress_file() { + if [ ! -f /tmp/backup-utils-progress-info ]; then + echo "No progress file found. Has a backup or restore been started?" + exit 1 + fi +} + +if [ -n "$ONCE" ]; then + check_for_progress_file + cat /tmp/backup-utils-progress-info +else + check_for_progress_file + clear + cat /tmp/backup-utils-progress-info + while true; do + if read -r -t 1 -n 1; then + clear + exit ; + else + clear + cat /tmp/backup-utils-progress-info + fi + done +fi diff --git a/bin/ghe-restore b/bin/ghe-restore index c0aae6ed9..6a5b69ea0 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -86,6 +86,8 @@ while true; do esac done + + start_cron () { log_info "Starting cron ..." if $CLUSTER; then @@ -164,6 +166,10 @@ cleanup_cluster_nodes() { # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" + + + + # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -265,6 +271,48 @@ if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then prompt_for_confirmation "Please confirm this before continuing." fi +# Calculate the actual amounts of steps in the restore process +# taking into account the options passed to the script and the appliance configuration +# calculate restore steps +OPTIONAL_STEPS=0 +# Cluster restores add an additional step +if $CLUSTER ; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring UUID +if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring Actions +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring minio +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring Elasticsearch +if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring audit log +if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then + if [[ "$GHE_RESTORE_SKIP_AUDIT_LOG" != "yes" ]]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) + fi +fi +# Replica cleanup +if ! $CLUSTER && $instance_configured; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Maximum restore steps +export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6)) + +init-progress +export PROGRESS_TYPE="Restore" +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +export PROGRESS=0 # Used to track progress of restore +echo "$PROGRESS" > /tmp/backup-utils-progress # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) @@ -343,10 +391,11 @@ log_warn "Warning: storing backup-utils version remotely failed." # Stop cron and timerd, as scheduled jobs may disrupt the restore process. log_info "Stopping cron and github-timerd ..." if $CLUSTER; then + bm_start "$(basename $0) - Stopping cron and github-timerd on cluster" if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then log_warn "Failed to stop cron on one or more nodes" 1>&3 fi - + bm_end "$(basename $0) - Stopping cron and github-timerd on cluster" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then @@ -358,11 +407,14 @@ if $CLUSTER; then log_warn "Failed to stop github-timerd on one or more nodes" 1>&3 fi fi + else + bm_start "$(basename $0) - Stopping cron and github-timerd" + echo "$(basename $0) - Stopping cron and github-timerd" if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then log_warn "Failed to stop cron" 1>&3 fi - + bm_end "$(basename $0) - Stopping cron and github-timerd" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then @@ -374,9 +426,11 @@ else log_warn "Failed to stop github-timerd" 1>&3 fi fi + fi CRON_RUNNING=false + # Restore settings and license if restoring to an unconfigured appliance or when # specified manually. if $RESTORE_SETTINGS; then @@ -394,11 +448,15 @@ fi # Restore UUID if present and not restoring to cluster. if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then log_info "Restoring UUID ..." + + bm_start "$(basename $0) - Restore UUID" ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" <"$GHE_RESTORE_SNAPSHOT_PATH/uuid" ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft" + bm_end "$(basename $0) - Restore UUID" fi + if is_external_database_snapshot; then appliance_strategy="external" backup_snapshot_strategy="external" @@ -448,7 +506,7 @@ fi cmd_title=$(log_info "Restoring Redis database ...") commands=(" echo \"$cmd_title\" -ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3") +ghe-restore-redis \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"") cmd_title=$(log_info "Restoring Git Repositories ...") commands+=(" @@ -470,7 +528,7 @@ fi cmd_title=$(log_info "Restoring SSH authorized keys ...") commands+=(" echo \"$cmd_title\" -ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-authorized-keys' < \"$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json\" 1>&3") +ghe-restore-ssh-keys \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"") cmd_title=$(log_info "Restoring storage data ...") commands+=(" @@ -517,14 +575,17 @@ if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then else log_info "Restoring data serially ..." 1>&3 for c in "${commands[@]}"; do + . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/bm.sh" eval "$c" done fi # Restart an already running memcached to reset the cache after restore -log_info "Restarting memcached ..." 1>&3 +log_info "Restarting memcached ..." 1>&3 +bm_start "$(basename $0) - Restarting memcached" echo "sudo restart -q memcached 2>/dev/null || true" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh +bm_end "$(basename $0) - Restarting memcached" # Prevent GitHub Connect jobs running before we've had a chance to reset # the configuration by setting the last run date to now. @@ -538,20 +599,24 @@ fi # config run to perform data migrations. if $CLUSTER; then log_info "Configuring cluster ..." + bm_start "$(basename $0) - configure cluster" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3 elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 fi ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3 + bm_end "configure_cluster" elif $instance_configured; then log_info "Configuring appliance ..." + bm_start "configure_appliance" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3 elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 fi ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3 + bm_end "$(basename $0) - configure appliance" fi # Clear GitHub Connect settings stored in the restored database. @@ -569,6 +634,7 @@ CRON_RUNNING=true # Clean up all stale replicas on configured instances. if ! $CLUSTER && $instance_configured; then log_info "Cleaning up replicas..." 1>&3 + bm_start "$(basename $0) - Cleanup replicas" restored_uuid=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid") other_nodes=$(echo " set -o pipefail; \ @@ -584,6 +650,7 @@ if ! $CLUSTER && $instance_configured; then echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3 done fi + bm_end "$(basename $0) - Cleanup replicas" fi # Update the remote status to "complete". This has to happen before importing diff --git a/share/github-backup-utils/bm.sh b/share/github-backup-utils/bm.sh index b714d144b..561bcfffb 100755 --- a/share/github-backup-utils/bm.sh +++ b/share/github-backup-utils/bm.sh @@ -49,6 +49,7 @@ bm_end() { local tend tstart total tend=$(date +%s) tstart=$(eval "echo \$$(bm_desc_to_varname "$@")_start") + total=$(($tend - $tstart)) echo "$1 took ${total}s" >> $BM_FILE_PATH @@ -57,4 +58,6 @@ bm_end() { if [ -n "$GHE_DEBUG" ]; then echo "Debug: $1 took ${total}s (bm_end)" fi + # track progress + progress "$1 took ${total}s" } diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 83533ee97..31e9aff0a 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -19,9 +19,68 @@ RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' -NC='\033[0m' # No Colo# Logging display and formatting functions +NC='\033[0m' # No Color + + +# Assume this script lives in share/github-backup-utils/ when setting the root +GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# Get the version from the version file. +BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")" + +# If a version check was requested, show the current version and exit +if [ -n "$GHE_SHOW_VERSION" ]; then + echo "GitHub backup-utils v$BACKUP_UTILS_VERSION" + exit 0 +fi + +# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage +# shellcheck disable=SC2120 # the script name is always referenced +print_usage() { + grep '^#/' <"$0" | cut -c 4- + exit "${1:-1}" +} + +if [ -n "$GHE_SHOW_HELP" ]; then + print_usage +else + for a in "$@"; do + if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then + print_usage + fi + done +fi + +# Add the bin and share/github-backup-utils dirs to PATH +PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" +# shellcheck source=share/github-backup-utils/bm.sh +. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" +# Save off GHE_HOSTNAME from the environment since we want it to override the +# backup.config value when set. +GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" + +# Source in the backup config file from the copy specified in the environment +# first and then fall back to the backup-utils root, home directory and system. +config_found=false +for f in "$GHE_BACKUP_CONFIG" "$GHE_BACKUP_ROOT/backup.config" \ + "$HOME/.github-backup-utils/backup.config" "/etc/github-backup-utils/backup.config"; do + if [ -f "$f" ]; then + GHE_BACKUP_CONFIG="$f" + # shellcheck disable=SC1090 # This is a user-supplied value that can't be predicted + . "$GHE_BACKUP_CONFIG" + config_found=true + break + fi +done + +GHE_RESTORE_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-restore") +GHE_BACKUP_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-backup") + +export GHE_RESTORE_IN_PROGRESS +export GHE_BACKUP_IN_PROGRESS + +# Logging display and formatting functions -# Log a message to stdout log_level() { local level=$1 shift @@ -124,7 +183,8 @@ fi PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" # shellcheck source=share/github-backup-utils/bm.sh . "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" - +# shellcheck source=share/github-backup-utils/track-progress +. "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress" # Save off GHE_HOSTNAME from the environment since we want it to override the # backup.config value when set. GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" @@ -285,6 +345,7 @@ else exec 3>/dev/null fi + # Restore saved off hostname. [ -n "$GHE_HOSTNAME_PRESERVE" ] && GHE_HOSTNAME="$GHE_HOSTNAME_PRESERVE" @@ -646,6 +707,10 @@ restore-secret() { fi } +#initialize progress tracking by clearing out the temp files used to track +init-progress() { + rm -f /tmp/backup-utils-progress* +} diff --git a/share/github-backup-utils/ghe-restore-redis b/share/github-backup-utils/ghe-restore-redis new file mode 100755 index 000000000..1bdfe7642 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-redis @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-redis +#/ Restore redis files from an rsync snapshot. +#/ +#/ Note: This script typically isn't called directly. It's invoked by the +#/ ghe-restore command. +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" + +# Show usage and bail with no arguments +[[ -z ${*} ]] && print_usage + +# Grab host arg +GHE_HOSTNAME="${1}" + +# Grab snapshot path arg +GHE_RESTORE_SNAPSHOT_PATH="${2}" + +bm_start "$(basename "${0}")" + +ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-redis' < "$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb" 1>&3 + +bm_end "$(basename "${0}")" \ No newline at end of file diff --git a/share/github-backup-utils/ghe-restore-ssh-keys b/share/github-backup-utils/ghe-restore-ssh-keys new file mode 100755 index 000000000..216803d7a --- /dev/null +++ b/share/github-backup-utils/ghe-restore-ssh-keys @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-ssh-keys +#/ Restore ssh keys from an rsync snapshot. +#/ +#/ Note: This script typically isn't called directly. It's invoked by the +#/ ghe-restore command. +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" + +# Show usage and bail with no arguments +[[ -z ${*} ]] && print_usage + +bm_start "$(basename "${0}")" + +# Grab host arg +GHE_HOSTNAME="${1}" + +# Grab snapshot path arg +GHE_RESTORE_SNAPSHOT_PATH="${2}" + +bm_start "$(basename "${0}")" + +ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-authorized-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json" 1>&3 + +bm_end "$(basename "${0}")" \ No newline at end of file diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress new file mode 100755 index 000000000..3f67ca9e8 --- /dev/null +++ b/share/github-backup-utils/track-progress @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +#/ track-progress: track progress of backup or restore tasks +set -e + +# Current version is working solely with backups +progress(){ + + PROGRESS=$(cat /tmp/backup-utils-progress) + PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) + PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) + echo $((PROGRESS +1)) > /tmp/backup-utils-progress + echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info +} From 945691ca688a75c25dcf9c58a4ac5b0dbed49c94 Mon Sep 17 00:00:00 2001 From: Joseph Franks Date: Wed, 21 Jun 2023 20:11:46 +0000 Subject: [PATCH 03/68] Merge pull request #354 from github/adjust-release-script-to-support-patch-version Adjusting backup-utils Release Script To Support Patch Versions --- script/release | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/script/release b/script/release index a27194d4b..b7b61615e 100755 --- a/script/release +++ b/script/release @@ -31,7 +31,8 @@ GH_REPO = ENV['GH_REPO'] || 'backup-utils' GH_OWNER = ENV['GH_OWNER'] || 'github' GH_AUTHOR = ENV['GH_AUTHOR'] DEB_PKG_NAME = 'github-backup-utils' -GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' +GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly? +GH_STABLE_BRANCH = "" CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium @@ -137,7 +138,8 @@ def beautify_changes(changes) end def changelog - changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) + puts "building changelog by comparing origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH}" + changes = `git log --pretty=oneline origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) raise 'Building the changelog failed' if $CHILD_STATUS != 0 changes @@ -228,12 +230,12 @@ def push_release_branch(version) end def update_stable_branch - `git checkout --quiet stable` + `git checkout --quiet #{GH_STABLE_BRANCH}` unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`) - warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}" + warn "Merging #{GH_BASE_BRANCH} into #{GH_STABLE_BRANCH} failed:\n\n#{out}" end - unless (out = `git push --quiet origin stable`) - warn "Failed pushing the stable branch:\n\n#{out}" + unless (out = `git push --quiet origin #{GH_STABLE_BRANCH}`) + warn "Failed pushing the #{GH_STABLE_BRANCH} branch:\n\n#{out}" end end @@ -333,9 +335,38 @@ def clean_up(version) `git branch --quiet -D tmp-packaging >/dev/null 2>&1` end +def is_base_branch_valid?(branch) + if branch == "master" || branch.match(/^\d+\.\d+-main$/) + return true + else + return false + end +end + +def get_stable_branch_name(branch) + ## derive the proper stable branch. if the base branch is "master" the stable branch is just "stable" + ## if the base branch is a release branch, the stable branch will be "x.y-stable" + result = "" + if branch == "master" + result = "stable" + else + result = branch.gsub(/-main$/, "-stable") + end + + result +end + #### All the action starts #### if $PROGRAM_NAME == __FILE__ begin + ## validate base branch. this must either be "master" or a release branch which will match the pattern "x.y-main" + raise "The branch #{GH_BASE_BRANCH} is not valid for releasing backup-utils. branch name must be master or match x.y-main" if !is_base_branch_valid?(GH_BASE_BRANCH) + + GH_STABLE_BRANCH = get_stable_branch_name(GH_BASE_BRANCH) + + puts "base branch = " + GH_BASE_BRANCH + puts "stable branch = " + GH_STABLE_BRANCH + args = ARGV.dup dry_run = false skip_version_bump_check = false @@ -455,7 +486,7 @@ if $PROGRAM_NAME == __FILE__ puts 'Cleaning up...' clean_up version - puts 'Updating stable branch...' + puts "Updating #{GH_STABLE_BRANCH} branch..." update_stable_branch puts 'Released!' From b034dc0fdfd1fd7e6163cc962950abc2fde4c21b Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Mon, 26 Jun 2023 22:08:50 +0000 Subject: [PATCH 04/68] Merge pull request #376 from github/fix-host-check-avail-space Update "available_space" unit in ghe-host-check --- bin/ghe-host-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index d020d78a4..586739916 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -186,7 +186,7 @@ actions: $actions_disk_size MB mssql: $mssql_disk_size MB DATA_TRANSFER_SIZE - if [[ $available_space -lt $min_disk_req ]]; then + if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2 exit 1 fi From 49acc149450c76dab8a3d17d62b366df1148c67d Mon Sep 17 00:00:00 2001 From: Kylie Stradley <4666485+KyFaSt@users.noreply.github.com> Date: Tue, 27 Jun 2023 18:43:15 +0000 Subject: [PATCH 05/68] Merge pull request #349 from github/kyfast-always-restore-column-encryption-keys Always restore column encryption keys --- bin/ghe-restore | 6 ++ share/github-backup-utils/ghe-backup-settings | 12 +++- .../ghe-restore-column-encryption-keys | 42 +++++++++++++ .../github-backup-utils/ghe-restore-settings | 6 -- test/test-ghe-backup.sh | 62 ++++++++++++++++++- test/test-ghe-restore.sh | 62 ++++++++++++++++++- 6 files changed, 178 insertions(+), 12 deletions(-) create mode 100755 share/github-backup-utils/ghe-restore-column-encryption-keys diff --git a/bin/ghe-restore b/bin/ghe-restore index 6a5b69ea0..04fa861f9 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -437,6 +437,12 @@ if $RESTORE_SETTINGS; then ghe-restore-settings "$GHE_HOSTNAME" fi +# Always restore column encryption keys +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + log_info "Always restore encrypted column encryption keys on GHES verions 3.7.0+" +fi +ghe-restore-column-encryption-keys "$GHE_HOSTNAME" + # Make sure mysql and elasticsearch are prep'd and running before restoring. # These services will not have been started on appliances that have not been # configured yet. diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index c71085a0b..c12abcd92 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -78,8 +78,16 @@ backup-secret "management console password" "manage-password" "secrets.manage" backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" -backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" -backup-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" + +# backup encryption keying material for GHES 3.7.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" +fi + +# backup current encryption key for GHES 3.8.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + backup-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" +fi # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then diff --git a/share/github-backup-utils/ghe-restore-column-encryption-keys b/share/github-backup-utils/ghe-restore-column-encryption-keys new file mode 100755 index 000000000..d30cacd51 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-column-encryption-keys @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-column-encryption-keys +#/ Restore the column encryption keys from a snapshot to the given . +#/ This script will be run automatically by `ghe-restore +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +bm_start "$(basename $0)" + +# Grab host arg +GHE_HOSTNAME="$1" + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$GHE_HOSTNAME" + +# The snapshot to restore should be set by the ghe-restore command but this lets +# us run this script directly. +: ${GHE_RESTORE_SNAPSHOT:=current} + +# Path to snapshot dir we're restoring from +: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} + +# Restore encrypted column encryption keying material for GHES 3.7.0 onward +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + log_info "Restoring encrypted column encryption keying material" + restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" +fi + +# Restore encrypted column current encryption key for GHES 3.8.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + log_info "Restoring encrypted column current encryption key" + restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" +fi + + +bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-settings b/share/github-backup-utils/ghe-restore-settings index ad06f30a6..b489626c2 100755 --- a/share/github-backup-utils/ghe-restore-settings +++ b/share/github-backup-utils/ghe-restore-settings @@ -56,12 +56,6 @@ restore-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hm # Restore kredz.varz HMAC key if present. restore-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" -# Restore encrypted column encryption keying material if present -restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" - -# Restore encrypted column current encryption key if present -restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" - # Restore SAML keys if present. if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/saml-keys.tar" ]; then log_info "Restoring SAML keys ..." diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 135b26170..00837d50c 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -555,7 +555,18 @@ begin_test "ghe-backup takes backup of kredz-varz settings" ) end_test -begin_test "ghe-backup takes backup of encrypted column encryption keying material" +begin_test "ghe-backup does not take backup of encrypted column encryption keying material for versions below 3.7.0" +( + GHE_REMOTE_VERSION=2.1.10 ghe-backup -v | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + + GHE_REMOTE_VERSION=3.6.1 ghe-backup -v | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + +) +end_test + +begin_test "ghe-backup takes backup of encrypted column encryption keying material for versions 3.7.0+" ( set -e @@ -567,6 +578,24 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" done + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + ghe-backup required_files=( @@ -580,7 +609,18 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ) end_test -begin_test "ghe-backup takes backup of encrypted column current encryption key" +begin_test "ghe-backup does not take backup of encrypted column current encryption key for versions below 3.8.0" +( + GHE_REMOTE_VERSION=2.1.10 ghe-backup -v | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + + GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + +) +end_test + +begin_test "ghe-backup takes backup of encrypted column current encryption key for versions 3.8.0+" ( set -e @@ -592,6 +632,24 @@ begin_test "ghe-backup takes backup of encrypted column current encryption key" ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" done + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + ghe-backup required_files=( diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index c39c042b7..8028cca03 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -281,7 +281,18 @@ begin_test "ghe-restore with no pages backup" ) end_test -begin_test "ghe-restore with encrypted column encryption keying material" +begin_test "ghe-restore does not restore encrypted column encryption keying material for versions below 3.7.0" +( + GHE_REMOTE_VERSION=2.1.10 ghe-restore -v -f localhost | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + + GHE_REMOTE_VERSION=3.6.1 ghe-restore -v -f localhost | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + +) +end_test + +begin_test "ghe-restore with encrypted column encryption keying material for versions 3.7.0+" ( set -e rm -rf "$GHE_REMOTE_ROOT_DIR" @@ -295,6 +306,23 @@ begin_test "ghe-restore with encrypted column encryption keying material" echo "foo" > "$GHE_DATA_DIR/current/$file" done + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-keying-material" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + ghe-restore -v -f localhost required_secrets=( "secrets.github.encrypted-column-keying-material" @@ -306,7 +334,19 @@ begin_test "ghe-restore with encrypted column encryption keying material" ) end_test -begin_test "ghe-restore with encrypted column current encryption key" + +begin_test "ghe-restore does not encrypted column current encryption key for versions below 3.8.0" +( + GHE_REMOTE_VERSION=2.1.10 restore -v -f | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + + GHE_REMOTE_VERSION=3.7.0 restore -v -f | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + +) +end_test + +begin_test "ghe-restore with encrypted column current encryption key for versions 3.8.0+" ( set -e rm -rf "$GHE_REMOTE_ROOT_DIR" @@ -320,6 +360,24 @@ begin_test "ghe-restore with encrypted column current encryption key" echo "foo" > "$GHE_DATA_DIR/current/$file" done + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-current-encryption-key" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done + + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + ghe-restore -v -f localhost required_secrets=( "secrets.github.encrypted-column-current-encryption-key" From fdb0dbabddc91c78e236921533d7ec26069754de Mon Sep 17 00:00:00 2001 From: Joseph Franks Date: Thu, 29 Jun 2023 22:36:16 +0000 Subject: [PATCH 06/68] Bump version: 3.9.0 [ci skip] --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index ee3c73176..1fbe304ae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +github-backup-utils (3.9.0) UNRELEASED; urgency=medium + + * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 + * Cleanup SSH multiplexing on exit #363 + * Allow extra rsync options to override default options #370 + * Retry with the admin ssh port on network unreachable too. #377 + + -- Joseph Franks Thu, 29 Jun 2023 22:36:16 +0000 + github-backup-utils (3.9.0) UNRELEASED; urgency=medium * Set restore status on all cluster nodes #274 From f02815e43cde3d3b8e32524566a2d1c63d86e612 Mon Sep 17 00:00:00 2001 From: Robert Bolender Date: Mon, 10 Jul 2023 23:40:32 +0000 Subject: [PATCH 07/68] Merge pull request #351 from github/backup-restore-secret-scanning-encryption-keys Backup and restore secret scanning encrypted secrets encryption keys --- bin/ghe-restore | 6 ++ share/github-backup-utils/ghe-backup-settings | 5 ++ ...he-restore-secret-scanning-encryption-keys | 39 +++++++++++ .../github-backup-utils/ghe-restore-settings | 6 ++ test/test-ghe-backup.sh | 30 +++++++++ test/test-ghe-restore.sh | 65 +++++++++++++++++++ 6 files changed, 151 insertions(+) create mode 100755 share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys diff --git a/bin/ghe-restore b/bin/ghe-restore index 04fa861f9..24151afcf 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -443,6 +443,12 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then fi ghe-restore-column-encryption-keys "$GHE_HOSTNAME" +# Always restore secret scanning encryption keys +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+" + ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME" +fi + # Make sure mysql and elasticsearch are prep'd and running before restoring. # These services will not have been started on appliances that have not been # configured yet. diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index c12abcd92..4d08c7134 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -89,6 +89,11 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then backup-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" fi +backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" +backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" +backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" +backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" diff --git a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys new file mode 100755 index 000000000..aa225bc07 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-secret-scanning-encryption-keys +#/ Restore the secret scanning encryption keys from a snapshot to the given . +#/ This script will be run automatically by `ghe-restore` +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +bm_start "$(basename $0)" + +# Grab host arg +GHE_HOSTNAME="$1" + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$GHE_HOSTNAME" + +# The snapshot to restore should be set by the ghe-restore command but this lets +# us run this script directly. +: ${GHE_RESTORE_SNAPSHOT:=current} + +# Path to snapshot dir we're restoring from +: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} + +# Restore secret scanning encrypted secrets storage keys if present +log_info "Restoring secret scanning encrypted secrets storage keys" +restore-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" +restore-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + +# Restore secret scanning encrypted secrets transit keys if present +log_info "Restoring secret scanning encrypted secrets transit keys" +restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" +restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + +bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-settings b/share/github-backup-utils/ghe-restore-settings index b489626c2..ad06f30a6 100755 --- a/share/github-backup-utils/ghe-restore-settings +++ b/share/github-backup-utils/ghe-restore-settings @@ -56,6 +56,12 @@ restore-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hm # Restore kredz.varz HMAC key if present. restore-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" +# Restore encrypted column encryption keying material if present +restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" + +# Restore encrypted column current encryption key if present +restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" + # Restore SAML keys if present. if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/saml-keys.tar" ]; then log_info "Restoring SAML keys ..." diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 00837d50c..6aeb94342 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -663,6 +663,36 @@ begin_test "ghe-backup takes backup of encrypted column current encryption key f ) end_test +begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + ghe-backup + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done +) +end_test + begin_test "ghe-backup takes backup of Actions settings" ( set -e diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index 8028cca03..ef13b7697 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -389,6 +389,71 @@ begin_test "ghe-restore with encrypted column current encryption key for version ) end_test +begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions below 3.8.0" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.7.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.8.0 + done +) +end_test + + +begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions 3.8.0+" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # expecting this to have been restored successfully for versions 3.8.0+ + done +) +end_test + # Setup Actions data for the subsequent tests setup_actions_test_data "$GHE_DATA_DIR/1" From 86ec0b33d3b1fede0ce5990da1537ca3825a0b2f Mon Sep 17 00:00:00 2001 From: boxofyellow <54955040+boxofyellow@users.noreply.github.com> Date: Wed, 12 Jul 2023 15:42:36 +0000 Subject: [PATCH 08/68] Merge pull request #393 from github/users/boxofyellow/2023_07/Issue1054_0 Init array to Fix https://github.com/github/backup-utils/issues/1054 --- share/github-backup-utils/ghe-backup-mssql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql index b5a3203fb..37b09ca9d 100755 --- a/share/github-backup-utils/ghe-backup-mssql +++ b/share/github-backup-utils/ghe-backup-mssql @@ -47,8 +47,8 @@ if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then fi tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -ssh_config_file_opt= -opts= +ssh_config_file_opt=() +opts=() isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)" From ca339cf5184ca1e78ad094ced4705a99d7f56dd5 Mon Sep 17 00:00:00 2001 From: Joseph Franks Date: Thu, 13 Jul 2023 13:43:32 -0500 Subject: [PATCH 09/68] Update 3.9 version to be 3.9 and not 3.8 - seems the backward sync was missed Update 3.9 version to be 3.9 and not 3.8 - seems the backward sync was missed --- share/github-backup-utils/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 19811903a..a5c4c7633 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.8.0 +3.9.0 From 5727e0fb9a98fa714afaecc979fd67675f8e9b9b Mon Sep 17 00:00:00 2001 From: Joseph Franks Date: Thu, 13 Jul 2023 13:45:33 -0500 Subject: [PATCH 10/68] update minimum supported version to be 3.7 for backup utils 3.9 --- bin/ghe-host-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 586739916..e404d5447 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -131,7 +131,7 @@ fi # backup-utils 2.13 onwards limits support to the current and previous two releases # of GitHub Enterprise Server. -supported_minimum_version="3.6.0" +supported_minimum_version="3.7.0" if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then supported=1 From 71c04e08c4a908aed7c0dc1886f4745e0195f509 Mon Sep 17 00:00:00 2001 From: Ant Kaynak Date: Fri, 14 Jul 2023 17:42:53 +0000 Subject: [PATCH 11/68] Merge pull request #378 from github/fix-es-translog-setting Replace deprecated translog flush setting in ES --- share/github-backup-utils/ghe-backup-es-rsync | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync index 00946279a..ee61741b2 100755 --- a/share/github-backup-utils/ghe-backup-es-rsync +++ b/share/github-backup-utils/ghe-backup-es-rsync @@ -4,6 +4,7 @@ #/ #/ Note: This command typically isn't called directly. It's invoked by #/ ghe-backup when the rsync strategy is used. +# shellcheck disable=SC2086 set -e # Bring in the backup configuration @@ -54,7 +55,7 @@ log_rsync "END elasticsearch rsync" 1>&3 # Set up a trap to re-enable flushing on exit and remove temp file cleanup () { ghe_verbose "* Enabling ES index flushing ..." - echo '{"index":{"translog.disable_flush":false}}' | + echo '{"index":{"translog.flush_threshold_size":"512MB"}}' | ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null } trap 'cleanup' EXIT @@ -62,7 +63,7 @@ trap 'exit $?' INT # ^C always terminate # Disable ES flushing and force a flush right now ghe_verbose "* Disabling ES index flushing ..." -echo '{"index":{"translog.disable_flush":true}}' | +echo '{"index":{"translog.flush_threshold_size":"1PB"}}' | ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null From a195abd4de0454e65e634558520175d28f462296 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Wed, 19 Jul 2023 16:12:54 +0000 Subject: [PATCH 12/68] add msg in the logs for skipping snapshots (#417) --- bin/ghe-backup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/ghe-backup b/bin/ghe-backup index 98f831a51..e81bf11e3 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -303,6 +303,8 @@ if [ -z "$failures" ]; then ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current" ghe-prune-snapshots +else + log_info "Skipping pruning snapshots, since some backups failed..." fi END_TIME=$(date +%s) From cb3c557df6f4463dfe9ce1f1e0c442ddcd4b8b23 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Thu, 20 Jul 2023 13:39:15 +0000 Subject: [PATCH 13/68] Merge pull request #379 from github/fix-bm-start-step-desc Update bm_start desc to match bm_end in ghe-restore --- bin/ghe-restore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index 04fa861f9..69e53def9 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -612,10 +612,10 @@ if $CLUSTER; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 fi ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3 - bm_end "configure_cluster" + bm_end "$(basename $0) - configure cluster" elif $instance_configured; then log_info "Configuring appliance ..." - bm_start "configure_appliance" + bm_start "$(basename $0) - configure appliance" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3 elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then From 955d392c6639883c31d39212fd7618a03a06e326 Mon Sep 17 00:00:00 2001 From: Kylie Stradley <4666485+KyFaSt@users.noreply.github.com> Date: Mon, 24 Jul 2023 20:00:49 -0400 Subject: [PATCH 14/68] Manual backport of #443 - resolved conflict --- share/github-backup-utils/ghe-backup-settings | 6 +- test/test-ghe-backup.sh | 77 +++++++++++++++---- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index c12abcd92..63ac96264 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -79,14 +79,14 @@ backup-secret "password pepper" "password-pepper" "secrets.github.user-password- backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" -# backup encryption keying material for GHES 3.7.0 onwards +# backup encryption keying material and create backup value current encryption for GHES 3.7.0 onwards +# this is for forwards compatibility with GHES 3.8.0 onwards if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" fi -# backup current encryption key for GHES 3.8.0 onwards if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then - backup-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" + cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" fi # Backup argon secrets for multiuser from ghes version 3.8 onwards diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 00837d50c..e4215eeec 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -566,7 +566,7 @@ begin_test "ghe-backup does not take backup of encrypted column encryption keyin ) end_test -begin_test "ghe-backup takes backup of encrypted column encryption keying material for versions 3.7.0+" +begin_test "ghe-backup takes backup of encrypted column encryption keying material and create encrypted column current encryption key for versions 3.7.0+" ( set -e @@ -609,27 +609,61 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ) end_test -begin_test "ghe-backup does not take backup of encrypted column current encryption key for versions below 3.8.0" +begin_test "ghe-backup takes backup of encrypted column encryption keying material and encrypted column current encryption key for versions 3.8.0+" ( - GHE_REMOTE_VERSION=2.1.10 ghe-backup -v | grep -q "encrypted column current encryption key not set" && exit 1 - [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + set -e + + required_secrets=( + "secrets.github.encrypted-column-keying-material" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION - GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "encrypted column current encryption key not set" && exit 1 - [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + ghe-backup + + required_files=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done ) end_test -begin_test "ghe-backup takes backup of encrypted column current encryption key for versions 3.8.0+" +begin_test "ghe-backup takes backup of encrypted column encryption keying material and encrypted column current encryption key accounting for multiple encryption keying materials for versions 3.7.0+" ( set -e required_secrets=( - "secrets.github.encrypted-column-current-encryption-key" + "secrets.github.encrypted-column-keying-material" ) for secret in "${required_secrets[@]}"; do - ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + echo "ghe-config '$secret' 'foo;bar'" | + ghe-ssh "$GHE_HOSTNAME" -- /bin/bash done # GHES version 3.8.0 @@ -639,13 +673,22 @@ begin_test "ghe-backup takes backup of encrypted column current encryption key f ghe-backup required_files=( - "encrypted-column-current-encryption-key" + "encrypted-column-encryption-keying-material" ) for file in "${required_files[@]}"; do - [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] done + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + + # GHES version 3.9.0 GHE_REMOTE_VERSION=3.9.0 export GHE_REMOTE_VERSION @@ -653,11 +696,19 @@ begin_test "ghe-backup takes backup of encrypted column current encryption key f ghe-backup required_files=( - "encrypted-column-current-encryption-key" + "encrypted-column-encryption-keying-material" ) for file in "${required_files[@]}"; do - [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] + done + + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] done ) From 9a99fd40984177fa2a9a530ba8cc005b7d22d51e Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Wed, 26 Jul 2023 01:46:33 +0000 Subject: [PATCH 15/68] Bump version: 3.9.1 [ci skip] --- debian/changelog | 9 +++++++++ share/github-backup-utils/version | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 1fbe304ae..f3d432df3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +github-backup-utils (3.9.1) UNRELEASED; urgency=medium + + * Make it clear the settings need to be applied after restoring to an unconfigured instance #381 + * Remove check for git from ghe-ssh #393 + * Use remote's mktemp to create temp dir on remote host #395 + * Enabling concurrent `ghe-backup` tasks from separate backup hosts #441 + + -- Devin Dooley Wed, 26 Jul 2023 01:46:33 +0000 + github-backup-utils (3.9.0) UNRELEASED; urgency=medium * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index a5c4c7633..6bd10744a 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.9.0 +3.9.1 From 12197c5a5c152c816772136a8c2d79364bbb71a8 Mon Sep 17 00:00:00 2001 From: Kylie Stradley <4666485+KyFaSt@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:36:35 +0000 Subject: [PATCH 16/68] Merge pull request #453 from github/kyfast-create-current-encryption-key-3.7.0+ Create the encrypted column current encryption key backup on 3.7.0+ --- share/github-backup-utils/ghe-backup-settings | 3 - test/test-ghe-backup.sh | 66 +++++++------------ 2 files changed, 25 insertions(+), 44 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 63ac96264..3d7410041 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -83,9 +83,6 @@ backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-s # this is for forwards compatibility with GHES 3.8.0 onwards if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" -fi - -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" fi diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index e4215eeec..6fcad4731 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -555,17 +555,6 @@ begin_test "ghe-backup takes backup of kredz-varz settings" ) end_test -begin_test "ghe-backup does not take backup of encrypted column encryption keying material for versions below 3.7.0" -( - GHE_REMOTE_VERSION=2.1.10 ghe-backup -v | grep -q "encrypted column encryption keying material not set" && exit 1 - [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] - - GHE_REMOTE_VERSION=3.6.1 ghe-backup -v | grep -q "encrypted column encryption keying material not set" && exit 1 - [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] - -) -end_test - begin_test "ghe-backup takes backup of encrypted column encryption keying material and create encrypted column current encryption key for versions 3.7.0+" ( set -e @@ -586,6 +575,7 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi required_files=( "encrypted-column-encryption-keying-material" + "encrypted-column-current-encryption-key" ) for file in "${required_files[@]}"; do @@ -598,35 +588,6 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ghe-backup - required_files=( - "encrypted-column-encryption-keying-material" - ) - - for file in "${required_files[@]}"; do - [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] - done - -) -end_test - -begin_test "ghe-backup takes backup of encrypted column encryption keying material and encrypted column current encryption key for versions 3.8.0+" -( - set -e - - required_secrets=( - "secrets.github.encrypted-column-keying-material" - ) - - for secret in "${required_secrets[@]}"; do - ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" - done - - # GHES version 3.8.0 - GHE_REMOTE_VERSION=3.8.0 - export GHE_REMOTE_VERSION - - ghe-backup - required_files=( "encrypted-column-encryption-keying-material" "encrypted-column-current-encryption-key" @@ -666,7 +627,30 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ghe-ssh "$GHE_HOSTNAME" -- /bin/bash done - # GHES version 3.8.0 + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] + done + + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + + + # GHES version 3.8.0 GHE_REMOTE_VERSION=3.8.0 export GHE_REMOTE_VERSION From c7063698873beb3790a9bebf6cde1e402d26ef8a Mon Sep 17 00:00:00 2001 From: Solmaz Abbaspour <5240896+solmazabbaspour@users.noreply.github.com> Date: Thu, 27 Jul 2023 16:10:23 +0000 Subject: [PATCH 17/68] Merge pull request #383 from github/solmaz/docker-update Install bc on Docker --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 667739729..710cb9d52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ libssl-dev \ git \ jq \ + bc \ curl \ tar \ gzip \ @@ -54,8 +55,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ git \ openssh-client \ jq \ + bc \ moreutils \ - gawk \ + gawk \ ca-certificates \ xxhash \ && rm -rf /var/lib/apt/lists/* From 16422d989dcfd33f255fcf50653f6416ccccad10 Mon Sep 17 00:00:00 2001 From: Zheng Zeng <7733657+zheng022@users.noreply.github.com> Date: Wed, 26 Jul 2023 21:50:17 +0200 Subject: [PATCH 18/68] Merge pull request #408 from github/zheng/6791-indicator fix indicator when doing backup and restore --- bin/ghe-backup | 27 ++++++++++++++++++- bin/ghe-restore | 20 ++++++++------ share/github-backup-utils/ghe-backup-config | 6 +++++ share/github-backup-utils/ghe-backup-pages | 4 ++- .../ghe-backup-repositories | 4 ++- share/github-backup-utils/ghe-backup-storage | 3 +++ share/github-backup-utils/ghe-restore-pages | 5 ++++ .../ghe-restore-repositories | 5 ++++ .../ghe-restore-repositories-gist | 5 ++++ share/github-backup-utils/ghe-restore-storage | 5 ++++ share/github-backup-utils/track-progress | 3 ++- 11 files changed, 75 insertions(+), 12 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 98f831a51..4bd8d5bb3 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -46,12 +46,36 @@ export CALLING_SCRIPT="ghe-backup" # Setup progress tracking init-progress +export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total export PROGRESS_TYPE="Backup" echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type export PROGRESS=0 # Used to track progress of backup echo "$PROGRESS" > /tmp/backup-utils-progress -export PROGRESS_TOTAL=18 # Maximum number of steps in backup +OPTIONAL_STEPS=0 +# Backup actions+mssql +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) +fi + +# Backup fsck +if [ "$GHE_BACKUP_FSCK" = "yes" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup minio +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup pages +if [ "$GHE_BACKUP_PAGES" != "no" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -269,6 +293,7 @@ echo \"$cmd_title\" ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"") if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then + increment-progress-total-count 1 cmd_title=$(log_info "Backing up Elasticsearch indices ...") commands+=(" echo \"$cmd_title\" diff --git a/bin/ghe-restore b/bin/ghe-restore index 69e53def9..9fe17863d 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -275,17 +275,14 @@ fi # taking into account the options passed to the script and the appliance configuration # calculate restore steps OPTIONAL_STEPS=0 -# Cluster restores add an additional step -if $CLUSTER ; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi + # Restoring UUID if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) fi -# Restoring Actions +# Restoring Actions + MSSQL if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) fi # Restoring minio if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then @@ -305,10 +302,16 @@ fi if ! $CLUSTER && $instance_configured; then OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) fi -# Maximum restore steps -export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6)) +# Restoring settings + restore-chat-integration + restore-packages +if $RESTORE_SETTINGS; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3)) +fi + +# Minimum number of steps is 7 +export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) init-progress +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total export PROGRESS_TYPE="Restore" echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type export PROGRESS=0 # Used to track progress of restore @@ -484,6 +487,7 @@ if is_external_database_target_or_snapshot && $SKIP_MYSQL; then log_info "Skipping MySQL restore." else log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..." + increment-progress-total-count 2 ghe-restore-mysql "$GHE_HOSTNAME" 1>&3 fi diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 31e9aff0a..65e863b56 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -712,6 +712,12 @@ init-progress() { rm -f /tmp/backup-utils-progress* } +#increase total count of progress +increment-progress-total-count() { + ((PROGRESS_TOTAL += $1)) + echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +} + diff --git a/share/github-backup-utils/ghe-backup-pages b/share/github-backup-utils/ghe-backup-pages index 21c2ac465..b0c9f1fed 100755 --- a/share/github-backup-utils/ghe-backup-pages +++ b/share/github-backup-utils/ghe-backup-pages @@ -63,6 +63,7 @@ if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/page link_dest="--link-dest=../../current/pages" fi +count=0 for hostname in $hostnames; do bm_start "$(basename $0) - $hostname" echo 1>&3 @@ -82,6 +83,7 @@ for hostname in $hostnames; do "$GHE_SNAPSHOT_DIR/pages" 1>&3 log_rsync "END: pages rsync" 1>&3 bm_end "$(basename $0) - $hostname" + count=$((count + 1)) done - +increment-progress-total-count $count bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index a3ba533d7..e90d8c37a 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -144,6 +144,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "no routes found, skipping repositories backup ..." exit 0 +else + increment-progress-total-count 3 fi # Transfer repository data from a GitHub instance to the current snapshot @@ -377,7 +379,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." - + increment-progress-total-count 1 bm_end "$(basename $0) - Verifying Routes" fi diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage index 2f98a0541..9752de21d 100755 --- a/share/github-backup-utils/ghe-backup-storage +++ b/share/github-backup-utils/ghe-backup-storage @@ -113,6 +113,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "no routes found, skipping storage backup ..." exit 0 +else + increment-progress-total-count 2 fi # rsync all the storage objects @@ -149,6 +151,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." + increment-progress-total-count 1 bm_end "$(basename $0) - Verifying Routes" fi diff --git a/share/github-backup-utils/ghe-restore-pages b/share/github-backup-utils/ghe-restore-pages index 640aa5d62..0b104863e 100755 --- a/share/github-backup-utils/ghe-restore-pages +++ b/share/github-backup-utils/ghe-restore-pages @@ -29,6 +29,8 @@ pages_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find pages -mindepth 5 if [ -z "$pages_paths" ]; then log_warn "Warning: Pages backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -125,6 +127,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping pages restore ..." exit 0 +else + increment-progress-total-count 2 fi bm_start "$(basename $0) - Restoring pages" @@ -154,6 +158,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/dpages-cluster-restore-finalize" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/ghe-restore-repositories b/share/github-backup-utils/ghe-restore-repositories index d59864196..9a749bc74 100755 --- a/share/github-backup-utils/ghe-restore-repositories +++ b/share/github-backup-utils/ghe-restore-repositories @@ -30,6 +30,8 @@ network_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find repositories -mi if [ -z "$network_paths" ]; then log_warn "Warning: Repositories backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -142,6 +144,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping repositories restore ..." exit 0 +else + increment-progress-total-count 3 fi # rsync all the repository networks to the git server where they belong. @@ -190,6 +194,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/dgit-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index addb18514..7faae5260 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -29,6 +29,8 @@ gist_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find repositories -mind if [ -z "$gist_paths" ]; then log_warn "Warning: Gist backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -128,6 +130,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping gists restore ..." exit 0 +else + increment-progress-total-count 2 fi # rsync all the gist repositories @@ -157,6 +161,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/gist-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/ghe-restore-storage b/share/github-backup-utils/ghe-restore-storage index af6c24a23..8f0d73c48 100755 --- a/share/github-backup-utils/ghe-restore-storage +++ b/share/github-backup-utils/ghe-restore-storage @@ -33,6 +33,8 @@ storage_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find storage -mindept if [ -z "$storage_paths" ]; then log_warn "Warning: Storage backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -120,6 +122,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping storage restore ..." exit 0 +else + increment-progress-total-count 2 fi # rsync all the objects to the storage server where they belong. @@ -169,6 +173,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/storage-cluster-restore-finalize" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index 3f67ca9e8..bf50c42aa 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -6,8 +6,9 @@ set -e progress(){ PROGRESS=$(cat /tmp/backup-utils-progress) + PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress-total) PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) - echo $((PROGRESS +1)) > /tmp/backup-utils-progress + echo $((PROGRESS + 1)) > /tmp/backup-utils-progress echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info } From 424a0115eefc8411c078eb436203e92828942325 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Mon, 7 Aug 2023 16:01:03 -0400 Subject: [PATCH 19/68] Manual backport of #467 - resolved conflict --- backup.config-example | 4 ++++ bin/ghe-backup | 12 +++++++++--- bin/ghe-host-check | 9 ++++++++- docs/README.md | 2 +- docs/usage.md | 10 ++++++++-- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/backup.config-example b/backup.config-example index 4c78a103b..8aadc98e2 100644 --- a/backup.config-example +++ b/backup.config-example @@ -16,6 +16,10 @@ GHE_DATA_DIR="data" # be available for the past N days ... GHE_NUM_SNAPSHOTS=10 +# If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check +# disk space validation and software version checks on the backup-host will be disabled. +#GHE_SKIP_CHECKS=false + # The hostname of the GitHub appliance to restore. If you've set up a separate # GitHub appliance to act as a standby for recovery, specify its IP or hostname # here. The host to restore to may also be specified directly when running diff --git a/bin/ghe-backup b/bin/ghe-backup index 8ac5565b0..9482fc5b4 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -5,9 +5,11 @@ #/ the MySQL database, instance settings, GitHub Pages data, etc. #/ #/ OPTIONS: -#/ -v | --verbose Enable verbose output. -#/ -h | --help Show this message. -#/ --version Display version information. +#/ -v | --verbose Enable verbose output. +#/ -h | --help Show this message. +#/ --version Display version information. +#/ -i | --incremental Incremental backup +#/ --skip-checks Skip storage/sw version checks #/ set -e @@ -27,6 +29,10 @@ while true; do export GHE_VERBOSE=true shift ;; + --skip-checks) + export GHE_SKIP_CHECKS=true + shift + ;; -*) echo "Error: invalid argument: '$1'" 1>&2 exit 1 diff --git a/bin/ghe-host-check b/bin/ghe-host-check index e404d5447..6c22d2d15 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -144,7 +144,13 @@ if [ -z "$supported" ]; then exit 1 fi -if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then +if [[ "$CALLING_SCRIPT" == "ghe-backup" && "$GHE_SKIP_CHECKS" != "true" ]]; then + cat << SKIP_MSG 1>&2 +**You can disable the following storage & version checks by running ghe-backup with option "--skip-checks" +OR updating GHE_SKIP_CHECKS to 'true' in your backup.config file. + +SKIP_MSG + # Bring in the requirements file min_rsync="" min_openssh="" @@ -184,6 +190,7 @@ minio: $minio_disk_size MB mysql: $mysql_disk_size MB actions: $actions_disk_size MB mssql: $mssql_disk_size MB + DATA_TRANSFER_SIZE if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then diff --git a/docs/README.md b/docs/README.md index 153549d14..e84d313df 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ - **[GitHub Enterprise Server version requirements](requirements.md#github-enterprise-version-requirements)** - **[Getting started](getting-started.md)** - **[Using the backup and restore commands](usage.md)** -- **[Scheduling backups](scheduling-backups.md)** +- **[Scheduling backups & snapshot pruning](scheduling-backups.md)** - **[Backup snapshot file structure](backup-snapshot-file-structure.md)** - **[How does Backup Utilities differ from a High Availability replica?](faq.md)** - **[Docker](docker.md)** diff --git a/docs/usage.md b/docs/usage.md index b4db45e73..c603c5e36 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -73,13 +73,18 @@ appliance at IP "5.5.5.5": Restore of 5.5.5.5:122 from snapshot 20180326T020444 finished. To complete the restore process, please visit https://5.5.5.5/setup/settings to review and save the appliance configuration. -A different backup snapshot may be selected by passing the `-s` argument and the -datestamp-named directory from the backup location. +A different backup snapshot may be selected by passing the `-s` argument to `ghe-restore` and specifying the +datestamp-named directory from the backup location as the value. The `ghe-backup` and `ghe-restore` commands also have a verbose output mode (`-v`) that lists files as they're being transferred. It's often useful to enable when output is logged to a file. +Every time you execute `ghe-backup` we verify the storage and software setup of the host +you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are +met. You can disable this check using the `--skip-checks` argument or by +adding `GHE_SKIP_CHECKS=true` to your configuration file. + ### Restoring settings, TLS certificate, and license When restoring to a new GitHub Enterprise Server instance, settings, certificate, and @@ -107,3 +112,4 @@ GitHub Actions enabled, the following steps are required: Please refer to [GHES Documentation](https://docs.github.com/en/enterprise-server/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled) for more details. [1]: https://github.com/github/backup-utils/blob/master/docs/getting-started.md +[2]: requirements.md From cf3f87f7202ea3b753bfc379d576b27a08b17eda Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Mon, 7 Aug 2023 21:02:29 +0000 Subject: [PATCH 20/68] Manual backport of #467 - removed pruning/incremental comments for 3.9 --- bin/ghe-backup | 1 - docs/README.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 9482fc5b4..caff0490d 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -8,7 +8,6 @@ #/ -v | --verbose Enable verbose output. #/ -h | --help Show this message. #/ --version Display version information. -#/ -i | --incremental Incremental backup #/ --skip-checks Skip storage/sw version checks #/ diff --git a/docs/README.md b/docs/README.md index e84d313df..153549d14 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ - **[GitHub Enterprise Server version requirements](requirements.md#github-enterprise-version-requirements)** - **[Getting started](getting-started.md)** - **[Using the backup and restore commands](usage.md)** -- **[Scheduling backups & snapshot pruning](scheduling-backups.md)** +- **[Scheduling backups](scheduling-backups.md)** - **[Backup snapshot file structure](backup-snapshot-file-structure.md)** - **[How does Backup Utilities differ from a High Availability replica?](faq.md)** - **[Docker](docker.md)** From bf329758d55a88b750d6df4839dae13f9cc16172 Mon Sep 17 00:00:00 2001 From: Hubot Date: Fri, 11 Aug 2023 17:29:22 +0200 Subject: [PATCH 21/68] Remove duplicate definitions in ghe-backup-config (#480) (#485) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While investigating an unrelated issue, I noticed that some definitions in ghe-backup-config are present twice. The reason for this appears to be a faulty merge conflict resolution [1], as both parent commits [2, 3] only have a single copy of these definitions but the merge commit has duplicates. It seems that an unwieldy conflict came up while merging the progress-indicator with the master branch, as a result of which both the old and new locations of these definitions were accidentally kept, causing the duplication. This removes one copy of each definition to avoid confusion and potential future bugs. [1] 1eaf80941c84c21f148ea68783415a920d32ce15 [2] a8e7eca745a738c4c02cba4ce85f51bfb97bfe6b [3] 8aaccc3c8b6599d591239d22cb789572df05465a Co-authored-by: Patrick Lühne --- share/github-backup-utils/ghe-backup-config | 38 +-------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 65e863b56..dc5f5bae0 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -35,7 +35,7 @@ if [ -n "$GHE_SHOW_VERSION" ]; then fi # Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage -# shellcheck disable=SC2120 # the script name is always referenced +# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's print_usage() { grep '^#/' <"$0" | cut -c 4- exit "${1:-1}" @@ -51,10 +51,6 @@ else done fi -# Add the bin and share/github-backup-utils dirs to PATH -PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" -# shellcheck source=share/github-backup-utils/bm.sh -. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" # Save off GHE_HOSTNAME from the environment since we want it to override the # backup.config value when set. GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" @@ -150,44 +146,12 @@ log_ssh(){ log_level "ssh" "$1" } -# Assume this script lives in share/github-backup-utils/ when setting the root -GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -# Get the version from the version file. -BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")" - -# If a version check was requested, show the current version and exit -if [ -n "$GHE_SHOW_VERSION" ]; then - echo "GitHub backup-utils v$BACKUP_UTILS_VERSION" - exit 0 -fi - -# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage -# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's -print_usage() { - grep '^#/' <"$0" | cut -c 4- - exit "${1:-1}" -} - -if [ -n "$GHE_SHOW_HELP" ]; then - print_usage -else - for a in "$@"; do - if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then - print_usage - fi - done -fi - # Add the bin and share/github-backup-utils dirs to PATH PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" # shellcheck source=share/github-backup-utils/bm.sh . "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" # shellcheck source=share/github-backup-utils/track-progress . "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress" -# Save off GHE_HOSTNAME from the environment since we want it to override the -# backup.config value when set. -GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" # Source in the backup config file from the copy specified in the environment # first and then fall back to the backup-utils root, home directory and system. From 7e69fc36333ba5d5fa65eb75f75c45a80314ab6d Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Fri, 11 Aug 2023 17:08:47 +0000 Subject: [PATCH 22/68] Merge pull request #486 from github/fix-es-backup-failures Remove "exit on error" mode from helper script track-progress --- share/github-backup-utils/track-progress | 1 - test/test-shellcheck.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index bf50c42aa..d88b3704f 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,6 +1,5 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -set -e # Current version is working solely with backups progress(){ diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 92ac538cc..231d78edb 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -49,7 +49,7 @@ begin_test "shellopts: set -e set on all scripts" # Check all executable scripts checked into the repo, except bm.sh, ghe-backup-config, ghe-rsync and the dummy test scripts set +x cd $BASE_PATH - git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do + git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|track-progress|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do if head -n1 "$script" | grep -E -w "sh|bash" >/dev/null 2>&1; then grep -q "set -e" $script || echo $script >> $results || true fi From 5f6901aab2f86474aeab9594276cd8d9c4b35988 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Tue, 15 Aug 2023 12:13:10 -0700 Subject: [PATCH 23/68] Leave releases as draft --- script/release | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/script/release b/script/release index b7b61615e..891cd9f60 100755 --- a/script/release +++ b/script/release @@ -34,6 +34,9 @@ DEB_PKG_NAME = 'github-backup-utils' GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly? GH_STABLE_BRANCH = "" +# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI +PUBLISH = ENV['PUBLISH'] == 'true' || false + CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium <%- changes.each do |ch| -%> @@ -480,8 +483,10 @@ if $PROGRAM_NAME == __FILE__ attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] - puts 'Publishing release...' - publish_release res['id'] + if PUBLISH + puts 'Publishing release...' + publish_release res['id'] + end puts 'Cleaning up...' clean_up version @@ -489,6 +494,10 @@ if $PROGRAM_NAME == __FILE__ puts "Updating #{GH_STABLE_BRANCH} branch..." update_stable_branch + if !PUBLISH + puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.' + end + puts 'Released!' rescue RuntimeError => e $stderr.puts "Error: #{e}" From bdc647d734baba7f546e4932b4d6175615ecc181 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:43:03 +0000 Subject: [PATCH 24/68] Merge pull request #499 from github/shellcheck-stable Change shellcheck to use stable, not latest --- .github/workflows/main.yml | 6 +++--- test/test-shellcheck.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 26ce0090a..0b1ec8bc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,9 +18,9 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man - wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz" - tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz" - sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck + wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" + tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" + sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck if: matrix.os != 'macos-latest' - name: Install Dependencies (macOS) run: | diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 231d78edb..82c0fbb4f 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings" set -e # We manually install the latest Shellcheck on Linux builds as other options # are too old. - if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then - shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; } + if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then + shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; } fi if ! type shellcheck 1>/dev/null 2>&1; then From 7efdf40035086287dbce487be03a277ebfbe5ffe Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Wed, 16 Aug 2023 23:15:04 +0000 Subject: [PATCH 25/68] Bump version: 3.9.2 [ci skip] --- debian/changelog | 10 ++++++++++ share/github-backup-utils/version | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index f3d432df3..ae1c3c760 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +github-backup-utils (3.9.2) UNRELEASED; urgency=medium + + * Move check for git for ssh muxing into ghe-ssh #378 + * Optionally log verbose output to a file instead of stdout #391 + * Raise an error if the current symlink doesn't exist when attempting to restore it #404 + * Capture and display repository/gist warnings during cluster restore #423 + * Redirect ghe-export-audit-logs stderr output unless using verbose output #497 + + -- Devin Dooley Wed, 16 Aug 2023 23:15:04 +0000 + github-backup-utils (3.9.1) UNRELEASED; urgency=medium * Make it clear the settings need to be applied after restoring to an unconfigured instance #381 diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 6bd10744a..2009c7dfa 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.9.1 +3.9.2 From d466629818779893d13c062cbd1851f64aef61ec Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Tue, 15 Aug 2023 21:16:18 +0200 Subject: [PATCH 26/68] Manual backport of #488 - resolved conflict --- bin/ghe-host-check | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 6c22d2d15..e9ee842ba 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -177,21 +177,24 @@ SKIP_MSG mssql_disk_size=$(transfer_size mssql /tmp) min_disk_req=$((repos_disk_size + pages_disk_size + es_disk_size + stor_disk_size + minio_disk_size + mysql_disk_size + actions_disk_size + mssql_disk_size)) - echo "Available space: $((available_space / (1024 ** 2))) MB" 1>&2 - echo -e "Min Disk required for this backup is at least $min_disk_req MB\n" 1>&2 - -cat <&2 -### Data Transfer Sizes -repositories: $repos_disk_size MB -pages: $pages_disk_size MB -elasticsearch: $es_disk_size MB -storage: $stor_disk_size MB -minio: $minio_disk_size MB -mysql: $mysql_disk_size MB -actions: $actions_disk_size MB -mssql: $mssql_disk_size MB - -DATA_TRANSFER_SIZE + recommended_disk_req=$((min_disk_req * 5)) + echo " - Available space: $((available_space / (1024 ** 2))) MB" 1>&2 + echo " - Min Disk required for this backup is at least $min_disk_req MB" 1>&2 + echo " - Recommended Disk requirement is $recommended_disk_req MB" 1>&2 + echo "" 1>&2 + + printf '### Estimated Data Transfer Sizes + + - repositories: %d MB + - pages: %d MB + - elasticsearch: %d MB + - storage: %d MB + - minio: %d MB + - mysql: %d MB + - actions: %d MB + - mssql: %d MB +\n' \ + "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2 if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2 From a87a3e27cbc2f9f17ac20e4cf81a4f38e92057f3 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Thu, 17 Aug 2023 15:38:55 +0000 Subject: [PATCH 27/68] Adding host online check (#492) * adding host online check * fix test --------- Co-authored-by: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> --- bin/ghe-host-check | 12 +++++++++--- test/bin/ghe-cluster-host-check | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100755 test/bin/ghe-cluster-host-check diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 6c22d2d15..6a73f5526 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -91,13 +91,19 @@ if ghe-ssh "$host" -- \ CLUSTER=true fi -# ensure all nodes in the cluster are running the same version +# ensure all nodes in the cluster are online/reachable and running the same version if "$CLUSTER"; then + online_status=$(ghe-ssh "$host" ghe-cluster-host-check) + if [ "$online_status" != "Cluster is ready to configure." ]; then + echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 + exit 1 + fi + node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version) distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) if [ "$distinct_versions" -ne 1 ]; then - echo "$node_version_list" 1>&2 - echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&3 + echo "Version mismatch: $node_version_list" 1>&2 + echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 exit 1 fi fi diff --git a/test/bin/ghe-cluster-host-check b/test/bin/ghe-cluster-host-check new file mode 100755 index 000000000..3120d85de --- /dev/null +++ b/test/bin/ghe-cluster-host-check @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Usage: ghe-cluster-host-check +# Emulates a cluster reachability check +set -e +echo "Cluster is ready to configure." From df0d37e7e6bcd0525dabbd31c35030c865b569bc Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Thu, 17 Aug 2023 17:40:09 -0400 Subject: [PATCH 28/68] Host key fix (#513) --- bin/ghe-backup | 66 ++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index caff0490d..eb9059158 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -49,38 +49,7 @@ export CALLING_SCRIPT="ghe-backup" # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" -# Setup progress tracking -init-progress -export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total -export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type -export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress - -OPTIONAL_STEPS=0 -# Backup actions+mssql -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) -fi - -# Backup fsck -if [ "$GHE_BACKUP_FSCK" = "yes" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -# Backup minio -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi -# Backup pages -if [ "$GHE_BACKUP_PAGES" != "no" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -187,9 +156,44 @@ log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERS # Perform a host connection check and establish the remote appliance version. # The version is available in the GHE_REMOTE_VERSION variable and also written # to a version file in the snapshot directory itself. +# ghe_remote_version_required should be run before any other instances of ghe-ssh +# to ensure that there are no problems with host key verification. ghe_remote_version_required echo "$GHE_REMOTE_VERSION" > version +# Setup progress tracking +init-progress +export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +export PROGRESS_TYPE="Backup" +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +export PROGRESS=0 # Used to track progress of backup +echo "$PROGRESS" > /tmp/backup-utils-progress + +OPTIONAL_STEPS=0 +# Backup actions+mssql +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) +fi + +# Backup fsck +if [ "$GHE_BACKUP_FSCK" = "yes" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup minio +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup pages +if [ "$GHE_BACKUP_PAGES" != "no" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups." fi From ece63df72641bfe5826a883d4750c0d92bea8757 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 21 Aug 2023 12:56:06 -0700 Subject: [PATCH 29/68] Revert "Bump version: 3.9.2" --- debian/changelog | 10 ---------- share/github-backup-utils/version | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/debian/changelog b/debian/changelog index ae1c3c760..f3d432df3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,13 +1,3 @@ -github-backup-utils (3.9.2) UNRELEASED; urgency=medium - - * Move check for git for ssh muxing into ghe-ssh #378 - * Optionally log verbose output to a file instead of stdout #391 - * Raise an error if the current symlink doesn't exist when attempting to restore it #404 - * Capture and display repository/gist warnings during cluster restore #423 - * Redirect ghe-export-audit-logs stderr output unless using verbose output #497 - - -- Devin Dooley Wed, 16 Aug 2023 23:15:04 +0000 - github-backup-utils (3.9.1) UNRELEASED; urgency=medium * Make it clear the settings need to be applied after restoring to an unconfigured instance #381 diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 2009c7dfa..6bd10744a 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.9.2 +3.9.1 From 5304b3f2f60535671dc502869665b656af481a10 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 21 Aug 2023 20:51:10 +0000 Subject: [PATCH 30/68] Bump version: 3.9.2 [ci skip] --- debian/changelog | 5 +++++ share/github-backup-utils/version | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index f3d432df3..7f8db98b7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,8 @@ +github-backup-utils (3.9.2) UNRELEASED; urgency=medium + + + -- Devin Dooley Mon, 21 Aug 2023 20:51:10 +0000 + github-backup-utils (3.9.1) UNRELEASED; urgency=medium * Make it clear the settings need to be applied after restoring to an unconfigured instance #381 diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 6bd10744a..2009c7dfa 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.9.1 +3.9.2 From a58b12b0d7cdb64ad013cc9d08812d01fdc82c24 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Mon, 21 Aug 2023 12:03:51 -0400 Subject: [PATCH 31/68] Manual backport of #490 - resolved conflict in docs/requirements.md --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index a7dc45ebe..0d1921294 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), and [jq][11] v1.5 or newer. See below for an update on rsync. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](#april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. From d0b43195e3ebc127f88d9d5006e989d65353b548 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Wed, 23 Aug 2023 15:18:53 -0400 Subject: [PATCH 32/68] Added bc to host requirements (this was not backported previously) --- docs/requirements.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.md b/docs/requirements.md index 0d1921294..1320eb583 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -99,3 +99,4 @@ Due to how some components of Backup Utilities (e.g. MSSQL) take incremental bac [9]: https://joeyh.name/code/moreutils [10]: https://www.gnu.org/software/gawk [11]: https://stedolan.github.io/jq/ +[12]: https://www.gnu.org/software/bc/ From 0c35c61b54857792c18dedfa4bd46869e0974de0 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Thu, 24 Aug 2023 15:08:27 +0000 Subject: [PATCH 33/68] Merge pull request #528 from github/suppress-secret-scanning-encrypted-secrets-warning Only backup secret scanning secrets on GHES versions 3.8.0+ --- share/github-backup-utils/ghe-backup-settings | 11 +++--- test/test-ghe-backup.sh | 34 +++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 3a05feba7..aac02dde7 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -86,10 +86,13 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" fi -backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" -backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" -backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" -backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +# secret scanning encrypted secrets keys were added in GHES 3.8.0 +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" + backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +fi # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index f1b730d4e..7e9b86802 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -698,7 +698,7 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ) end_test -begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys" +begin_test "ghe-backup does not take backups of secret scanning encrypted secrets encryption keys on versions below 3.8.0" ( set -e @@ -713,7 +713,37 @@ begin_test "ghe-backup takes backup of secret scanning encrypted secrets encrypt ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" done - ghe-backup + GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "secret scanning encrypted secrets" && exit 1 + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "" ] + done +) +end_test + +begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys on versions 3.8.0+" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-backup required_files=( "secret-scanning-encrypted-secrets-current-storage-key" From 5aaf7797c2c0b75df5eba5bcaeac83109fa2c36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Thu, 31 Aug 2023 21:38:47 +0000 Subject: [PATCH 34/68] Merge pull request #527 from github/pluehne/find-parallel-in-more-locations Find parallel in more locations --- share/github-backup-utils/ghe-backup-config | 2 ++ test/testlib.sh | 2 ++ 2 files changed, 4 insertions(+) mode change 100644 => 100755 test/testlib.sh diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index dc5f5bae0..d832eea2f 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -240,8 +240,10 @@ ghe_parallel_check() { GHE_PARALLEL_COMMAND="parallel" local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do diff --git a/test/testlib.sh b/test/testlib.sh old mode 100644 new mode 100755 index 631049e30..c504eccff --- a/test/testlib.sh +++ b/test/testlib.sh @@ -588,8 +588,10 @@ setup_moreutils_parallel() { # We need moreutils parallel local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do From 32b37fe7e01544b4d3ef86d51eccec4566e289d9 Mon Sep 17 00:00:00 2001 From: Dax Amin Date: Thu, 31 Aug 2023 22:36:37 +0000 Subject: [PATCH 35/68] Merge pull request #534 from github/pluehne/allow-manually-triggering-testing-workflow Allow manually triggering testing workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0b1ec8bc1..3baf7e0a1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,6 @@ name: Test and build -on: [pull_request] +on: [pull_request, workflow_dispatch] jobs: From 76cac022eb174e06aad419969e348b5ae2b90e28 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Wed, 6 Sep 2023 22:07:17 +0000 Subject: [PATCH 36/68] Merge pull request #557 from github/hao/unit-test-fix Run backup-utils unit tests in serial --- .github/workflows/main.yml | 10 ---------- script/cibuild | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3baf7e0a1..ef3d887d7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,9 +7,6 @@ jobs: build: strategy: matrix: - # macos-latest references are kept here for historical purposes. removed macos-latest from the - #matrix as it is not a typical case for users and causes a lot of friction with other linux-based - # installs. Recommend developing on codespaces or using an ubuntu container. os: ['ubuntu-22.04', 'ubuntu-20.04'] fail-fast: false runs-on: ${{ matrix.os }} @@ -21,12 +18,6 @@ jobs: wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck - if: matrix.os != 'macos-latest' - - name: Install Dependencies (macOS) - run: | - brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget - brew install moreutils gawk - if: matrix.os == 'macos-latest' - name: Get Sources uses: actions/checkout@v3 - name: Test @@ -36,4 +27,3 @@ jobs: shell: bash - name: Build (Linux) run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc - if: matrix.os != 'macos-latest' diff --git a/script/cibuild b/script/cibuild index 7f57f444d..73dc9ae0a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -5,7 +5,7 @@ set -e # Enable verbose logging of ssh commands export GHE_VERBOSE_SSH=true -if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then +if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then exit 1 fi From 1b803ca766be7c65875f16f1446c2f718eae0cac Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Thu, 7 Sep 2023 13:47:19 +0000 Subject: [PATCH 37/68] Merge pull request #545 from github/fix-ghe-unsupported-version-test Fix GHE unsupported version check --- test/test-ghe-host-check.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 6bfa12a69..aeb41551d 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION) bu_major_minor="$bu_version_major.$bu_version_minor" releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json) - supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")') + latest_value=$(echo "$releases" | jq -r '.latest') + latest_major_version=$(echo $latest_value | cut -d "." -f 1-2) + # Replace "latest" with the derived major version in the releases string + releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g') + # Use the modified releases string as needed + supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo $releases | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) if [ -z "$supported" ] then From 6ecc5db46ac603a14ef799d30963fb82e2d9f16a Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Tue, 12 Sep 2023 09:06:16 +0000 Subject: [PATCH 38/68] add post backup cleanup and move progress to folder (#558) --- bin/ghe-backup | 8 ++++---- bin/ghe-backup-progress | 8 ++++---- bin/ghe-restore | 6 +++--- share/github-backup-utils/ghe-backup-config | 13 +++++++++++-- share/github-backup-utils/track-progress | 11 +++++------ test/test-ghe-backup.sh | 2 ++ test/testlib.sh | 6 ++++++ 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index eb9059158..2c020c54e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -164,11 +164,11 @@ echo "$GHE_REMOTE_VERSION" > version # Setup progress tracking init-progress export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress OPTIONAL_STEPS=0 # Backup actions+mssql @@ -192,7 +192,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then fi PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups." diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress index 2f4b267fb..7ab36e084 100755 --- a/bin/ghe-backup-progress +++ b/bin/ghe-backup-progress @@ -31,7 +31,7 @@ while true; do done check_for_progress_file() { - if [ ! -f /tmp/backup-utils-progress-info ]; then + if [ ! -f /tmp/backup-utils-progress/info ]; then echo "No progress file found. Has a backup or restore been started?" exit 1 fi @@ -39,18 +39,18 @@ check_for_progress_file() { if [ -n "$ONCE" ]; then check_for_progress_file - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info else check_for_progress_file clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info while true; do if read -r -t 1 -n 1; then clear exit ; else clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info fi done fi diff --git a/bin/ghe-restore b/bin/ghe-restore index 7c2d386ea..1929b5cdf 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -311,11 +311,11 @@ fi export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) init-progress -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Restore" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of restore -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index d832eea2f..8e7eac8fd 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -675,13 +675,22 @@ restore-secret() { #initialize progress tracking by clearing out the temp files used to track init-progress() { - rm -f /tmp/backup-utils-progress* + if [ -d /tmp/backup-utils-progress ]; then + rm -rf /tmp/backup-utils-progress/* + else + mkdir /tmp/backup-utils-progress + fi + touch /tmp/backup-utils-progress/total + touch /tmp/backup-utils-progress/type + touch /tmp/backup-utils-progress/progress + touch /tmp/backup-utils-progress/info + chmod -R 777 /tmp/backup-utils-progress } #increase total count of progress increment-progress-total-count() { ((PROGRESS_TOTAL += $1)) - echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total } diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index d88b3704f..a560ba540 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,13 +1,12 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -# Current version is working solely with backups progress(){ - PROGRESS=$(cat /tmp/backup-utils-progress) - PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress-total) - PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) + PROGRESS=$(cat /tmp/backup-utils-progress/progress) + PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total) + PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type) PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) - echo $((PROGRESS + 1)) > /tmp/backup-utils-progress - echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info + echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress + echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info } diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 7e9b86802..ed383b003 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -47,6 +47,8 @@ begin_test "ghe-backup subsequent snapshot" [ "$first_snapshot" != "$this_snapshot" ] verify_all_backedup_data + + verify_progress_cleanup_process ) end_test diff --git a/test/testlib.sh b/test/testlib.sh index c504eccff..56b22da69 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -486,6 +486,12 @@ verify_all_backedup_data() { verify_common_data } +# A unified method to make sure post backup, the cleanup process works +verify_progress_cleanup_process() { + set -e + sudo -u nobody rm -rf /tmp/backup-utils-progress/* +} + # A unified method to check everything restored when performing a full restore # during testing. verify_all_restored_data() { From caa5f77a9e12aa87ee5e08a393e1f9841c8194e8 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Wed, 13 Sep 2023 01:18:56 +0000 Subject: [PATCH 39/68] Merge pull request #569 from github/check-datadir-is-nfs Add NFS check for GHE_DATA_DIR --- bin/ghe-host-check | 7 +++++++ docs/requirements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index de554358b..8d845cd40 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "Checking host for sufficient space for a backup..." 1>&2 available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') diff --git a/docs/requirements.md b/docs/requirements.md index 7f501a02c..1288846ac 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -59,6 +59,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From 69155431b4d071ccbaed4d57c045f85305eac7f2 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Tue, 26 Sep 2023 22:09:54 +0000 Subject: [PATCH 40/68] fix mkdir issues or progress tracking (#612) Fixing https://github.com/github/ghes/issues/7409 --- share/github-backup-utils/ghe-backup-config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 8e7eac8fd..943d61e7c 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -677,14 +677,15 @@ restore-secret() { init-progress() { if [ -d /tmp/backup-utils-progress ]; then rm -rf /tmp/backup-utils-progress/* - else - mkdir /tmp/backup-utils-progress fi + + mkdir -p /tmp/backup-utils-progress + chmod -R 777 /tmp/backup-utils-progress + touch /tmp/backup-utils-progress/total touch /tmp/backup-utils-progress/type touch /tmp/backup-utils-progress/progress touch /tmp/backup-utils-progress/info - chmod -R 777 /tmp/backup-utils-progress } #increase total count of progress From 5735a6a8f19c19c1f5b518cc0520cd18c8613705 Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:21:35 -0500 Subject: [PATCH 41/68] Merge pull request #568 from github/543-workflow-build-release-part-2 Build and Release Workflow Part 2 --- .github/linters/.yaml-lint.yml | 53 +++++++++ .github/workflows/build-and-release.yml | 139 ++++++++++++++++++++++++ .github/workflows/lint.yml | 2 + .releaseignore | 2 + script/package-deb | 11 ++ script/package-tarball | 13 +++ 6 files changed, 220 insertions(+) create mode 100644 .github/linters/.yaml-lint.yml create mode 100644 .github/workflows/build-and-release.yml create mode 100644 .releaseignore diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..030c37f04 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,53 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable \ No newline at end of file diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 000000000..90b44086e --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,139 @@ +--- +name: Build and Release + +on: + workflow_dispatch: + inputs: + gh-token: + description: 'GitHub Token - used to create a commit in the backup-utils repo' + required: true + type: string + version: + description: 'Version - patch version of the release (e.g. x.y.z)' + required: true + type: string + draft: + description: 'Draft - true if the release should be a draft' + required: true + type: boolean + default: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils + ref: master + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: master + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "${{ github.actor }}" + commit_user_email: "ghes-releases-team@github.com" + commit_options: "--allow-empty" + skip_dirty_check: true + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ github.event.inputs.gh-token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ steps.empty-commit.outputs.commit_hash }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" + + + + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 44eec8716..12b4a3df2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +--- name: Lint Code Base on: @@ -21,3 +22,4 @@ jobs: env: VALIDATE_ALL_CODEBASE: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILTER_REGEX_EXCLUDE: .*release-notes/.* diff --git a/.releaseignore b/.releaseignore new file mode 100644 index 000000000..bb2f6b31c --- /dev/null +++ b/.releaseignore @@ -0,0 +1,2 @@ +ownership.yaml +.github \ No newline at end of file diff --git a/script/package-deb b/script/package-deb index dda90cbe3..15a698e9c 100755 --- a/script/package-deb +++ b/script/package-deb @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" @@ -22,6 +25,14 @@ mkdir -p dist/debuild distdir="$(pwd)/dist/debuild/$PKG_NAME" git clone -q . "$distdir" cd "$distdir" + +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +echo "Removing .releaseignore ..." +rm -f .releaseignore git checkout -q "$PKG_HEAD" debuild -uc -us 1>&2 diff --git a/script/package-tarball b/script/package-tarball index be653b77a..bf1510e8f 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -8,11 +8,24 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" PKG_NAME="${PKG_BASE}-${PKG_VERS}" +# Remove all files or directories listed in .releaseignore +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +# Remove the .releaseignore file itself +echo "Removing .releaseignore ..." +rm -f .releaseignore + # Run git-archive to generate tarball echo "Creating ${PKG_NAME}.tar.gz ..." mkdir -p dist From 712d4c404bb6246e0f30013d786fc4cf8b82792f Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 27 Sep 2023 08:53:21 -0400 Subject: [PATCH 42/68] Manual backport of #568 - resolved conflict From be27efa8b5c11424c20b907a30c7ea33648ccc1d Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 2 Oct 2023 11:37:42 -0700 Subject: [PATCH 43/68] Revert "Backport 569 for 3.9: Add NFS check for GHE_DATA_DIR" --- bin/ghe-host-check | 7 ------- docs/requirements.md | 2 -- 2 files changed, 9 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 8d845cd40..de554358b 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,13 +168,6 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" - #Check if GHE_DATA_DIR is NFS mounted - fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true - if [ "$fs_info" == "nfs" ]; then - echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 - echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 - fi - #Display dir requirements for repositories and mysql echo "Checking host for sufficient space for a backup..." 1>&2 available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') diff --git a/docs/requirements.md b/docs/requirements.md index 1288846ac..7f501a02c 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -59,8 +59,6 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. -Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. - ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From 7f2671839a190decdfd7ca9d729df5adae5da889 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:24:43 +0000 Subject: [PATCH 44/68] Merge pull request #569 from github/check-datadir-is-nfs Add NFS check for GHE_DATA_DIR --- bin/ghe-host-check | 7 +++++++ docs/requirements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index de554358b..8d845cd40 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "Checking host for sufficient space for a backup..." 1>&2 available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') diff --git a/docs/requirements.md b/docs/requirements.md index 7f501a02c..1288846ac 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -59,6 +59,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From c5d7d1aacf7608c316dfc99cedf487bbeafdf594 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 2 Oct 2023 20:36:46 +0000 Subject: [PATCH 45/68] Bump version: 3.9.3 [ci skip] --- debian/changelog | 8 ++++++++ share/github-backup-utils/version | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 7f8db98b7..47c44ef96 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +github-backup-utils (3.9.3) UNRELEASED; urgency=medium + + * Replace "sed -E" in ghe-host-check with a more portable solution #509 + * Fix `ghe-backup-repositories` performance for large instances #541 + * Turn off POSIX for ghe-backup-config #632 + + -- Devin Dooley Mon, 02 Oct 2023 20:36:46 +0000 + github-backup-utils (3.9.2) UNRELEASED; urgency=medium diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 2009c7dfa..820476af9 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.9.2 +3.9.3 From 55e9f2900e6abaeb8331828c19d6a825f0d41afd Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Mon, 9 Oct 2023 19:19:25 +0000 Subject: [PATCH 46/68] Merge pull request #579 from github/hao/integration-actions Add integration tests to backup-utils-private --- .github/actions/proxy-janky-build/action.yml | 33 ++++ .github/actions/proxy-janky-build/go.mod | 7 + .github/actions/proxy-janky-build/go.sum | 9 + .github/actions/proxy-janky-build/main.go | 180 +++++++++++++++++++ .github/workflows/integration-tests.yml | 41 +++++ 5 files changed, 270 insertions(+) create mode 100644 .github/actions/proxy-janky-build/action.yml create mode 100644 .github/actions/proxy-janky-build/go.mod create mode 100644 .github/actions/proxy-janky-build/go.sum create mode 100644 .github/actions/proxy-janky-build/main.go create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/actions/proxy-janky-build/action.yml b/.github/actions/proxy-janky-build/action.yml new file mode 100644 index 000000000..988c4a289 --- /dev/null +++ b/.github/actions/proxy-janky-build/action.yml @@ -0,0 +1,33 @@ +name: 'Trigger a CI Job on Janky' +description: 'Action to trigger and poll a Janky CI job' +inputs: + janky-token: + description: 'Token for making request to Janky' + required: true + job-name: + description: 'The name of the job to run' + required: true + branch-name: + description: 'The name of the branch to use' + required: true + force: + description: 'Force the job to run even if it is already passed' + required: false + envVars: + description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3' + required: false +runs: + using: 'composite' + steps: + - uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260 + with: + go-version: '1.21' + - run: | + go run main.go \ + -token ${{ inputs.janky-token }} \ + -job ${{ inputs.job-name }} \ + -branch ${{ inputs.branch-name }} \ + -force ${{ inputs.force }} \ + -envVars ${{ inputs.envVars }} + shell: bash + working-directory: .github/actions/proxy-janky-build diff --git a/.github/actions/proxy-janky-build/go.mod b/.github/actions/proxy-janky-build/go.mod new file mode 100644 index 000000000..b7560d7b9 --- /dev/null +++ b/.github/actions/proxy-janky-build/go.mod @@ -0,0 +1,7 @@ +module github.com/github/enterprise2/actions/proxy-janky-build + +go 1.21 + +require github.com/hashicorp/go-retryablehttp v0.7.2 + +require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/.github/actions/proxy-janky-build/go.sum b/.github/actions/proxy-janky-build/go.sum new file mode 100644 index 000000000..5c59c1d2e --- /dev/null +++ b/.github/actions/proxy-janky-build/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go new file mode 100644 index 000000000..e0342f121 --- /dev/null +++ b/.github/actions/proxy-janky-build/main.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +// Define our Janky Response Structs +type JankyBuildStruct struct { + Result string + Url string +} +type JankyStatusStruct struct { + Id string + Green bool + Completed bool + StartedAt string + CompletedAt string + Sha string + BuildableName string +} + +const ( + pollWaitTime = 10 * time.Second + jankyPollTimeout = 5 * time.Hour + jankyHttpRetryMax = 5 + jankyUrl = "https://janky.githubapp.com" +) + +func main() { + // Parse command-line arguments + job := flag.String("job", "", "Name of the Janky job") + token := flag.String("token", "", "Name of the Janky token") + branch := flag.String("branch", "", "Name of the Git branch") + force := flag.String("force", "false", "Force a build even if one is already passed") + envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") + flag.Parse() + + // Validate command-line arguments + if *job == "" || *token == "" || *branch == "" { + log.Fatal("job, token and branch flags must be specified") + } + + // Set up the token + request payload + authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token)) + type buildRequestObject struct { + BuildableName string `json:"buildable_name"` + BranchName string `json:"branch_name"` + Force string `json:"force"` + EnvVars map[string]string `json:"env_vars"` + } + + requestBody := buildRequestObject{ + BuildableName: *job, + BranchName: *branch, + Force: *force, + } + + // Parse the envVars flag into a map and add to the request payload + fmt.Println("Environment Variables:") + fmt.Println(*envVars) + if *envVars != "" { + envVarsMap := make(map[string]string) + for _, envVar := range strings.Split(*envVars, ",") { + envVarSplit := strings.Split(envVar, "=") + envVarsMap[envVarSplit[0]] = envVarSplit[1] + } + requestBody.EnvVars = envVarsMap + } + + payloadBytes, err := json.Marshal(requestBody) + if err != nil { + log.Fatal("Failed to marshal the JSON payload!\n" + err.Error()) + } + + // Send build request to Janky + buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes)) + if err != nil { + log.Fatal("Failed to create build request!\n" + err.Error()) + } + buildRequest.Header.Set("Content-Type", "application/json") + buildRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient := retryablehttp.NewClient() //nolint:all + retryClient.RetryMax = jankyHttpRetryMax + retryClient.Logger = nil // disable debug logging + client := retryClient.StandardClient() // uses *http.Client + resp, err := client.Do(buildRequest) + if err != nil { + log.Fatal("Failed to send build request!\n" + err.Error()) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatal("Error reading build response!\n" + err.Error()) + } + + // Check if the build was triggered successfully + if resp.StatusCode == 404 { + log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.") + } + if resp.StatusCode != 201 { + log.Fatal("Failed to trigger build! Got exception: " + string(body)) + } + + // Parse the build request response + var buildResponse JankyBuildStruct + json.Unmarshal(body, &buildResponse) + log.Println("Succesfully triggered janky!\n" + buildResponse.Result) + + // Parse the request response for the buildId + r, err := regexp.Compile("/[0-9]+/") + if err != nil { + log.Fatal("Failed to trigger build!\n" + err.Error()) + } + buildId := strings.Trim(r.FindString(buildResponse.Result), "/") + + // Setup our second HTTP client for reuse in during status polling + jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status" + statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil) + if err != nil { + log.Fatal("Failed to create status request!\n" + err.Error()) + } + statusRequest.Header.Set("Content-Type", "application/json") + statusRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient2 := retryablehttp.NewClient() //nolint:all + retryClient2.RetryMax = jankyHttpRetryMax + retryClient2.Logger = nil // disable debug logging + client2 := retryClient2.StandardClient() // uses *http.Client + + // Wait for a completed status from Janky or break the loop after a certain amount of time + timeout := time.NewTimer(jankyPollTimeout) + poll := time.NewTicker(pollWaitTime) + +jobLoop: + for { + select { + case <-timeout.C: + log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours") + case <-poll.C: + // Send build status request to Janky + statusResponse, err := client2.Do(statusRequest) + if err != nil { + log.Fatal("Failed to send status request!\n" + err.Error()) + } + defer statusResponse.Body.Close() + statusBody, err := io.ReadAll(statusResponse.Body) + if err != nil { + log.Fatal("Error reading status response!\n" + err.Error()) + } + + // Parse the status response for a green completed build + var jankyStatusResponse JankyStatusStruct + json.Unmarshal(statusBody, &jankyStatusResponse) + //fmt.Println("Janky Status Response:") + //fmt.Println(string(statusBody)) + if jankyStatusResponse.Completed && jankyStatusResponse.Green { + log.Println("Janky build Succeeded!") + break jobLoop + } + if jankyStatusResponse.Completed && !jankyStatusResponse.Green { + log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url) + } + + // wait for a bit and try again + log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]") + continue + } + } +} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..804c6d5c5 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,41 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'Branch that would be merged into' + required: true + source-branch: + description: 'Branch that would be merged' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.TARGET_BRANCH }}' + force : "true" + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }}" From 739a56be5b2a12e988f087d10365aadfddb1ece5 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Wed, 20 Sep 2023 19:52:34 -0600 Subject: [PATCH 47/68] Manual backport of #597 - resolved conflict --- .github/workflows/integration-tests.yml | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..8727233ea --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,42 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'Branch that would be merged into' + required: true + source-branch: + description: 'Branch that would be merged' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' + force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" From 6df936abba3b0ff7a0683823e8d9d6e8605f5ba6 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:18:14 +0000 Subject: [PATCH 48/68] Merge pull request #599 from github/hao/fix-issues-for-actions Fix issue with actions --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8727233ea..c7ccf30f0 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,8 +15,8 @@ on: # Get target and source branch from different variables depending on how it was triggered env: - TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' - SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' jobs: integration-tests: From 032dc441f0070c6625a42c4945f2bc3c8b799288 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Mon, 25 Sep 2023 16:04:55 -0600 Subject: [PATCH 49/68] Manual backport of #602 - resolved conflict --- .github/pull_request_template.md | 40 +++++++++++++++++++++++++ .github/workflows/integration-tests.yml | 27 ++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ade3e5c58 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,40 @@ + + + + +# PR Details + +## Description + +## Testing + + + +## Ownership + + +## Related Links + diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c7ccf30f0..f48de957e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,6 +37,31 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.SOURCE_BRANCH }}' - force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" + + # Cluster integration tests are optional based on label and PR titles + cluster-integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-cluster-binary-backup + - enterprise2-backup-utils-cluster-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + if: | + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.title, '[cluster]') || + contains(github.event.pull_request.labels.*.name, 'cluster') + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" From 7e50e000876d8d6f1e1a5227a0ce347219ce8fc2 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Sat, 14 Oct 2023 01:49:13 +0000 Subject: [PATCH 50/68] Merge pull request #613 from github/jianghao0718-patch-4 Fix description for target and source branches for integration tests CI --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f48de957e..efdd23ebe 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,10 +7,10 @@ on: workflow_dispatch: inputs: target-branch: - description: 'Branch that would be merged into' + description: 'enterprise2 branch to test against' required: true source-branch: - description: 'Branch that would be merged' + description: 'backup-utils-private topic branch to test' required: true # Get target and source branch from different variables depending on how it was triggered From 2869f0a10242e19209c0ad888db32a25bdf499a5 Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:30:09 +0000 Subject: [PATCH 51/68] Merge pull request #664 from github/enterprise-3.9-backport-662-timreimherr/actions-46-multi-repo-token-scope Backport 662 for 3.9: Implement App Token --- .github/workflows/build-and-release.yml | 76 ++++++++++++------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 90b44086e..c9b6b66e8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -4,10 +4,6 @@ name: Build and Release on: workflow_dispatch: inputs: - gh-token: - description: 'GitHub Token - used to create a commit in the backup-utils repo' - required: true - type: string version: description: 'Version - patch version of the release (e.g. x.y.z)' required: true @@ -21,22 +17,20 @@ on: jobs: build: runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" - name: Checkout backup-utils-private uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils-private + token: ${{ steps.app-token.outputs.token }} - name: Install dependencies run: | sudo apt-get update -y @@ -79,35 +73,40 @@ jobs: outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" + - name: Get major-feature from version + id: get-major-feature + run: | + echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + - name: Verify major-feature + run: | + echo "major_feature: $MAJOR_FEATURE" - name: Checkout backup-utils uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - ref: master - name: Create empty commit uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: master + branch: ${{ env.MAJOR_FEATURE }}-stable commit_message: "${{ github.event.inputs.version }} release" - commit_user_name: "${{ github.actor }}" - commit_user_email: "ghes-releases-team@github.com" + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" commit_options: "--allow-empty" + push_options: "--force" skip_dirty_check: true - - name: Checkout backup-utils + - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 @@ -120,20 +119,17 @@ jobs: - name: Create Release uses: ncipollo/release-action@v1 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} + owner: github repo: backup-utils name: | GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} + commit: ${{ env.MAJOR_FEATURE }}-stable bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true - artifactContentType: "raw" - - - - + artifactContentType: "raw" \ No newline at end of file From f80afe769be4181f1948f25c692e4bba2acc3fdd Mon Sep 17 00:00:00 2001 From: Hubot Date: Fri, 20 Oct 2023 14:49:20 -0400 Subject: [PATCH 52/68] Update debian/changelog and release-notes/3.9.4.md for 3.9.4 release --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 47c44ef96..e7e939593 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +github-backup-utils (3.9.4) UNRELEASED; urgency=medium + + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/652, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Fri, 20 Oct 2023 18:49:19 +0000 + github-backup-utils (3.9.3) UNRELEASED; urgency=medium * Replace "sed -E" in ghe-host-check with a more portable solution #509 From 7b72de2769ef25b667b4f1bb3720c19f42b6c6e2 Mon Sep 17 00:00:00 2001 From: Hubot Date: Fri, 20 Oct 2023 14:49:20 -0400 Subject: [PATCH 53/68] Create 3.9.4 release notes --- release-notes/3.9.4.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 release-notes/3.9.4.md diff --git a/release-notes/3.9.4.md b/release-notes/3.9.4.md new file mode 100644 index 000000000..241b0d274 --- /dev/null +++ b/release-notes/3.9.4.md @@ -0,0 +1,8 @@ +## Bug Fixes + +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/652, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} + +## Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + From 09de863fafbc317d9261ba1a7faa97399ac33f03 Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:10:55 +0000 Subject: [PATCH 54/68] Merge pull request #681 from github/enterprise-3.9-backport-679-timreimherr/build-and-release-edit Backport 679 for 3.9: Remove file rename step in build-and-release workflow --- .github/workflows/build-and-release.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c9b6b66e8..edf115738 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,14 +44,6 @@ jobs: - name: Package deb run: | ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - name: Upload deb artifact uses: actions/upload-artifact@v3 with: From 3fcf0116cc2ae2d49f053d97e18d822619f69486 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Mon, 23 Oct 2023 16:06:57 +0000 Subject: [PATCH 55/68] 3.9.4 release From fa3737c95fd2e774ce8240e037927d1936373ba6 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Wed, 1 Nov 2023 16:55:31 +0000 Subject: [PATCH 56/68] 3.9.4 release From 47fd4ab451489a79a41d9d4b4cce86ebf40bcf1a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 24 Oct 2023 15:33:03 +0000 Subject: [PATCH 57/68] remove comments --- debian/changelog | 4 ++-- release-notes/3.9.4.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index e7e939593..8e260e7ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ github-backup-utils (3.9.4) UNRELEASED; urgency=medium - * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/652, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. +* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. -- ghes-releases-team Fri, 20 Oct 2023 18:49:19 +0000 diff --git a/release-notes/3.9.4.md b/release-notes/3.9.4.md index 241b0d274..5d35007e3 100644 --- a/release-notes/3.9.4.md +++ b/release-notes/3.9.4.md @@ -1,8 +1,8 @@ ## Bug Fixes -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/652, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. ## Backups and Disaster Recovery -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. +* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. From af5cb7a7d422e5a095c2242bf71ed684650c1a25 Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:20 -0400 Subject: [PATCH 58/68] Update debian/changelog, version file, and release-notes/3.9.4.md for 3.9.4 release --- debian/changelog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index 8e260e7ff..7804a6aba 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +github-backup-utils (3.9.4) UNRELEASED; urgency=medium + + ### Bug Fixes + +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + +### Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Wed, 01 Nov 2023 14:28:19 +0000 + github-backup-utils (3.9.4) UNRELEASED; urgency=medium * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. From 5e792198b6ac6059bff096051e5670e79e6e12fd Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:21 -0400 Subject: [PATCH 59/68] Create 3.9.4 release notes --- release-notes/3.9.4.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-notes/3.9.4.md b/release-notes/3.9.4.md index 5d35007e3..edaf871e8 100644 --- a/release-notes/3.9.4.md +++ b/release-notes/3.9.4.md @@ -1,8 +1,8 @@ -## Bug Fixes +### Bug Fixes -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. -## Backups and Disaster Recovery +### Backups and Disaster Recovery -* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. From 526df0e371566e7ca475e30a82af0f324586a4a0 Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:21 -0400 Subject: [PATCH 60/68] Update version file to 3.9.4 --- share/github-backup-utils/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 820476af9..ed7555c71 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.9.3 +3.9.4 \ No newline at end of file From 3ccb44baec1e8eb40593f1ee27a8db19893fa082 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 1 Nov 2023 14:33:33 +0000 Subject: [PATCH 61/68] no need to update change log --- debian/changelog | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7804a6aba..8e260e7ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,15 +1,3 @@ -github-backup-utils (3.9.4) UNRELEASED; urgency=medium - - ### Bug Fixes - -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. - -### Backups and Disaster Recovery - -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. - - -- ghes-releases-team Wed, 01 Nov 2023 14:28:19 +0000 - github-backup-utils (3.9.4) UNRELEASED; urgency=medium * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. From 44922a3c74d6eae4112f76d9a1128f5f680b254b Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Mon, 23 Oct 2023 16:06:57 +0000 Subject: [PATCH 62/68] 3.9.4 release From 0f8e133e9c35276855cb128fee53b48a9d6cfb9b Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Wed, 1 Nov 2023 16:55:31 +0000 Subject: [PATCH 63/68] 3.9.4 release From cf608a32d837aab3c23c147c22db41f0a09898d5 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Wed, 17 Jan 2024 20:02:56 +0000 Subject: [PATCH 64/68] 3.9.5 release From 308a2b247ce981c9bde22741dac6e33e10b17833 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Fri, 19 Jan 2024 21:10:19 +0000 Subject: [PATCH 65/68] 3.9.5 release From 6379607e12666881af204b98ffb01e798c74fb00 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Thu, 21 Mar 2024 16:48:20 +0000 Subject: [PATCH 66/68] 3.9.6 release From f4fbffcd28123dbc86219687fb8e6508fcd856f4 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Fri, 19 Apr 2024 04:44:01 +0000 Subject: [PATCH 67/68] 3.9.7 release From 3958023984921ca0e902bdf66ec90f0579010f0b Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Fri, 2 Aug 2024 20:32:19 +0000 Subject: [PATCH 68/68] 3.9.8 release