diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..030c37f04 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,53 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable \ No newline at end of file diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 000000000..edf115738 --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,127 @@ +--- +name: Build and Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version - patch version of the release (e.g. x.y.z)' + required: true + type: string + draft: + description: 'Draft - true if the release should be a draft' + required: true + type: boolean + default: true + +jobs: + build: + runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" + - name: Get major-feature from version + id: get-major-feature + run: | + echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + - name: Verify major-feature + run: | + echo "major_feature: $MAJOR_FEATURE" + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: ${{ env.MAJOR_FEATURE }}-stable + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" + commit_options: "--allow-empty" + push_options: "--force" + skip_dirty_check: true + - name: Checkout backup-utils-private for release notes + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ steps.app-token.outputs.token }} + owner: github + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ env.MAJOR_FEATURE }}-stable + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9d97805de..12b4a3df2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +--- name: Lint Code Base on: @@ -17,7 +18,8 @@ jobs: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 - name: Lint Code Base - uses: github/super-linter@v4 + uses: github/super-linter@v5 env: VALIDATE_ALL_CODEBASE: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILTER_REGEX_EXCLUDE: .*release-notes/.* diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b4ceacdcd..430aff599 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,9 +18,9 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man - wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz" - tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz" - sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck + wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" + tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" + sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck if: matrix.os != 'macos-latest' - name: Install Dependencies (macOS) run: | diff --git a/.releaseignore b/.releaseignore new file mode 100644 index 000000000..bb2f6b31c --- /dev/null +++ b/.releaseignore @@ -0,0 +1,2 @@ +ownership.yaml +.github \ No newline at end of file diff --git a/backup.config-example b/backup.config-example index 8c98655ab..2ca0551fa 100644 --- a/backup.config-example +++ b/backup.config-example @@ -42,6 +42,11 @@ GHE_NUM_SNAPSHOTS=10 # #GHE_EXTRA_RSYNC_OPTS="" + +# If set to 'yes', logging output will be colorized. +# +#OUTPUT_COLOR=no + # If set to 'no', GHE_DATA_DIR will not be created automatically # and restore/backup will exit 8 # @@ -90,3 +95,6 @@ GHE_NUM_SNAPSHOTS=10 # When running an external mysql database, run this script to trigger a MySQL restore # rather than attempting to backup via backup-utils directly. #EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false" + +# If set to 'yes', Pages data will be included in backup and restore. Defaults to 'yes' +#GHE_BACKUP_PAGES=no \ No newline at end of file diff --git a/bin/ghe-backup b/bin/ghe-backup index fe1816030..ee90b9969 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -67,21 +67,18 @@ trap 'rm -rf src dest1 dest2' EXIT mkdir -p src touch src/testfile if ! ln -s /data/does/not/exist/hooks/ src/ >/dev/null 2>&1; then - echo "Error: the filesystem containing $GHE_DATA_DIR does not support symbolic links." 1>&2 - echo "Git repositories contain symbolic links that need to be preserved during a backup." 1>&2 + log_error "Error: the filesystem containing $GHE_DATA_DIR does not support symbolic links. \nGit repositories contain symbolic links that need to be preserved during a backup." 1>&2 exit 1 fi if ! output=$(rsync -a src/ dest1 2>&1 && rsync -av src/ --link-dest=../dest1 dest2 2>&1); then - echo "Error: rsync encountered an error that could indicate a problem with permissions," 1>&2 - echo "hard links, symbolic links, or another issue that may affect backups." 1>&2 + log_error "Error: rsync encountered an error that could indicate a problem with permissions,\n hard links, symbolic links, or another issue that may affect backups." 1>&2 echo "$output" exit 1 fi if [ "$(ls -il dest1/testfile | awk '{ print $1 }')" != "$(ls -il dest2/testfile | awk '{ print $1 }')" ]; then - echo "Error: the filesystem containing $GHE_DATA_DIR does not support hard links." 1>&2 - echo "Backup Utilities use hard links to store backup data efficiently." 1>&2 + log_error "Error: the filesystem containing $GHE_DATA_DIR does not support hard links.\n Backup Utilities use hard links to store backup data efficiently." 1>&2 exit 1 fi rm -rf src dest1 dest2 @@ -122,9 +119,8 @@ ghe_restore_check # Check to see if there is a running backup if [ -h ../in-progress ]; then - echo "Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2 - echo "If there is no backup in progress anymore, please remove" 1>&2 - echo "the $GHE_DATA_DIR/in-progress file and try again." 1>&2 + + log_error "Error: detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." 1>&2 exit 1 fi @@ -137,10 +133,7 @@ if [ -f ../in-progress ]; then # will clean up the failed backup. unlink ../in-progress else - echo "Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2 - echo 1>&2 - echo " If PID $pid is not a process related to the backup utilities, please remove" 1>&2 - echo " the $GHE_DATA_DIR/in-progress file and try again." 1>&2 + log_error "Error: A backup of $GHE_HOSTNAME may still be running on PID $pid. \nIf PID $pid is not a process related to the backup utilities, please remove \nthe $GHE_DATA_DIR/in-progress file and try again." 1>&2 exit 1 fi fi @@ -148,9 +141,8 @@ fi echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ${GHE_DATA_DIR}/in-progress-backup -START_TIME=$(date +%s) -echo 'Start time:' $START_TIME -echo "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP" +log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP" + # Perform a host connection check and establish the remote appliance version. # The version is available in the GHE_REMOTE_VERSION variable and also written @@ -162,6 +154,30 @@ if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups." fi +# Output system information of the backup host + +# If /etc/issue.net exists, use it to get the OS version +if [ -f /etc/issue.net ]; then + echo "Running on: $(cat /etc/issue.net)" +else + echo "Running on: Unknown OS" +fi + +# If nproc command exists, use it to get the number of CPUs +if command -v nproc >/dev/null 2>&1; then + echo "CPUs: $(nproc)" +else + echo "CPUs: Unknown" +fi + +# If the free command exists, use it to get the memory details +if command -v free >/dev/null 2>&1; then + echo "Memory $(free -m | grep '^Mem:' | awk '{print "total/used/free+share/buff/cache: " $2 "/" $3 "/" $4 "+" $5 "/" $6 "/" $7}')" +else + echo "Memory: Unknown" +fi + + # Log backup start message in /var/log/syslog on remote instance ghe_remote_logger "Starting backup from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP ..." @@ -173,19 +189,22 @@ echo "$GHE_BACKUP_STRATEGY" > strategy # Create benchmark file bm_init > /dev/null +START_TIME=$(date +%s) +log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION" + ghe-backup-store-version || -echo "Warning: storing backup-utils version remotely failed." +log_warn "Warning: storing backup-utils version remotely failed." -echo "Backing up GitHub settings ..." +log_info "Backing up GitHub settings ..." ghe-backup-settings || failures="$failures settings" -echo "Backing up SSH authorized keys ..." +log_info "Backing up SSH authorized keys ..." bm_start "ghe-export-authorized-keys" ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-authorized-keys' > authorized-keys.json || failures="$failures authorized-keys" bm_end "ghe-export-authorized-keys" -echo "Backing up SSH host keys ..." +log_info "Backing up SSH host keys ..." bm_start "ghe-export-ssh-host-keys" ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar || failures="$failures ssh-host-keys" @@ -194,45 +213,57 @@ bm_end "ghe-export-ssh-host-keys" ghe-backup-mysql || failures="$failures mysql" if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Backing up MSSQL databases ..." + log_info "Backing up MSSQL databases ..." ghe-backup-mssql 1>&3 || failures="$failures mssql" - echo "Backing up Actions data ..." + log_info "Backing up Actions data ..." ghe-backup-actions 1>&3 || failures="$failures actions" fi if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - echo "Backing up Minio data ..." + log_info "Backing up Minio data ..." ghe-backup-minio 1>&3 || failures="$failures minio" fi +cmd_title=$(log_info "Backing up Redis database ...") commands=(" -echo \"Backing up Redis database ...\" +echo \"$cmd_title\" ghe-backup-redis > redis.rdb || printf %s \"redis \" >> \"$failures_file\"") +cmd_title=$(log_info "Backing up audit log ...") commands+=(" -echo \"Backing up audit log ...\" +echo \"$cmd_title\" ghe-backup-es-audit-log || printf %s \"audit-log \" >> \"$failures_file\"") +cmd_title=$(log_info "Backing up Git repositories ...") commands+=(" -echo \"Backing up Git repositories ...\" +echo \"$cmd_title\" ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"") -commands+=(" -echo \"Backing up GitHub Pages artifacts ...\" -ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"") +# Pages backups are skipped only if GHE_BACKUP_PAGES is explicitly set to 'no' to guarantee backward compatibility. +# If a customer upgrades backup-utils but keeps the config file from a previous version, Pages backups still work as expected. +if [ "$GHE_BACKUP_PAGES" != "no" ]; then + cmd_title=$(log_info "Backing up GitHub Pages artifacts ...") + commands+=(" + echo \"$cmd_title\" + ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"") +fi + +cmd_title=$(log_info "Backing up storage data ...") commands+=(" -echo \"Backing up storage data ...\" +echo \"$cmd_title\" ghe-backup-storage || printf %s \"storage \" >> \"$failures_file\"") +cmd_title=$(log_info "Backing up custom Git hooks ...") commands+=(" -echo \"Backing up custom Git hooks ...\" +echo \"$cmd_title\" ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"") if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then + cmd_title=$(log_info "Backing up Elasticsearch indices ...") commands+=(" - echo \"Backing up Elasticsearch indices ...\" + echo \"$cmd_title\" ghe-backup-es-rsync || printf %s \"elasticsearch \" >> \"$failures_file\"") fi @@ -250,6 +281,7 @@ fi # git fsck repositories after the backup if [ "$GHE_BACKUP_FSCK" = "yes" ]; then + log_info "Running git fsck on repositories ..." ghe-backup-fsck $GHE_SNAPSHOT_DIR || failures="$failures fsck" fi @@ -263,13 +295,11 @@ if [ -z "$failures" ]; then ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current" ghe-prune-snapshots +else + log_info "Skipping pruning snapshots, since some backups failed..." fi -END_TIME=$(date +%s) -echo 'End time:' $END_TIME -echo 'Runtime:' $(($END_TIME - $START_TIME)) 'seconds' - -echo "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")" +log_info "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")" # Exit non-zero and list the steps that failed. if [ -z "$failures" ]; then @@ -277,14 +307,17 @@ if [ -z "$failures" ]; then else steps="$(echo $failures | sed 's/ /, /g')" ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." - echo "Error: Snapshot incomplete. Some steps failed: ${steps}. " + log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. " exit 1 fi # Detect if the created backup contains any leaked ssh keys -echo "Checking for leaked ssh keys ..." +log_info "Checking for leaked ssh keys ..." ghe-detect-leaked-ssh-keys -s "$GHE_SNAPSHOT_DIR" || true +END_TIME=$(date +%s) +log_info "Runtime: $((${END_TIME} - ${START_TIME})) seconds" +log_info "Backup of $GHE_HOSTNAME finished." # Make sure we exit zero after the conditional true diff --git a/bin/ghe-host-check b/bin/ghe-host-check index ebdf12b96..46098ea92 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -91,12 +91,18 @@ if ghe-ssh "$host" -- \ CLUSTER=true fi -# ensure all nodes in the cluster are running the same version +# ensure all nodes in the cluster are online/reachable and running the same version if "$CLUSTER"; then + online_status=$(ghe-ssh "$host" ghe-cluster-host-check) + if [ "$online_status" != "Cluster is ready to configure." ]; then + echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 + exit 1 + fi + node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version) distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) if [ "$distinct_versions" -ne 1 ]; then - echo "$node_version_list" 1>&2 + echo "Version mismatch: $node_version_list" 1>&2 echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 exit 1 fi diff --git a/bin/ghe-restore b/bin/ghe-restore index 5c196faea..6443401bf 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -87,25 +87,26 @@ while true; do done start_cron () { - echo "Starting cron ..." + log_info "Starting cron ..." if $CLUSTER; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo timeout 120s service cron start"; then - echo "* Warning: Failed to start cron on one or more nodes" + if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo timeout 120s service cron start" ; then + log_warn "Failed to start cron on one or more nodes" fi else - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo timeout 120s service cron start"; then - echo "* Warning: Failed to start cron" + if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo timeout 120s service cron start" ; then + log_warn "Failed to start cron" fi fi } cleanup () { + log_info " Exiting, cleaning up ..." if [ -n "$1" ]; then update_restore_status "$1" fi if $ACTIONS_STOPPED && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Restarting Actions after restore ..." + log_info "Restarting Actions after restore ..." # In GHES 3.3+, ghe-actions-start no longer has a -f (force) flag. In GHES 3.2 and below, we must provide the # force flag to make sure it can start in maintenance mode. Use it conditionally based on whether it exists # in the --help output @@ -121,9 +122,16 @@ cleanup () { fi # Cleanup SSH multiplexing - ghe-ssh --clean + log_info "Cleaning up SSH multiplexing ..." + if ! ghe-ssh --clean ; then + log_info "Failed to clean up SSH multiplexing" + fi # Remove in-progress file - rm -f ${GHE_DATA_DIR}/in-progress-restore + log_info "Removing in-progress file ..." 1>&3 + + if ! rm -f ${GHE_DATA_DIR}/in-progress-restore ; then + log_error "Failed to remove in-progress file" 1>&3 + fi } # This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally. @@ -131,18 +139,21 @@ cleanup () { cleanup_cluster_nodes() { uuid="$1" if [ -z "$uuid" ]; then - echo "Node UUID required." + log_error "Node UUID required." exit 2 fi - + log_info "Cleaning up spokes" 1>&3 ghe-spokes server evacuate git-server-$uuid 'Removing replica' ghe-spokes server destroy git-server-$uuid + log_info "Cleaning up storage" 1>&3 ghe-storage destroy-host storage-server-$uuid --force + log_info "Cleaning up dpages" 1>&3 ghe-dpages offline pages-server-$uuid ghe-dpages remove pages-server-$uuid + log_info "Cleaning up redis" 1>&3 ghe-redis-cli del resque:queue:maint_git-server-$uuid ghe-redis-cli srem resque:queues maint_git-server-$uuid } @@ -179,7 +190,7 @@ export GHE_RESTORE_SNAPSHOT ghe_backup_check # Detect if the backup we are restoring has a leaked ssh key -echo "Checking for leaked keys in the backup snapshot that is being restored ..." +echo "Checking for leaked keys in the backup snapshot that is being restored ..." ghe-detect-leaked-ssh-keys -s "$GHE_RESTORE_SNAPSHOT_PATH" || true # Figure out whether to use the tarball or rsync restore strategy based on the @@ -207,8 +218,7 @@ export CLUSTER # Restoring a cluster backup to a standalone appliance is not supported if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - echo "Error: Snapshot from a GitHub Enterprise cluster cannot be restored" \ - "to a standalone appliance. Aborting." >&2 + log_error "Error: Snapshot from a GitHub Enterprise cluster cannot be restored to a standalone appliance. Aborting." >&2 exit 1 fi @@ -220,8 +230,7 @@ fi # Figure out if this appliance is in a replication pair if ghe-ssh "$GHE_HOSTNAME" -- \ "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - echo "Error: Restoring to an appliance with replication enabled is not supported." >&2 - echo " Please teardown replication before restoring." >&2 + log_error "Error: Restoring to an appliance with replication enabled is not supported. Please teardown replication before restoring." >&2 exit 1 fi @@ -257,8 +266,7 @@ fi # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) -echo 'Start time:' $START_TIME -echo "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT" +log_info "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT" ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..." # Create an in-progress-restore file to prevent simultaneous backup or restore runs echo "${START_TIME} $$" > ${GHE_DATA_DIR}/in-progress-restore @@ -287,7 +295,7 @@ update_restore_status "restoring" # Make sure the GitHub appliance is in maintenance mode. if $instance_configured; then if ! ghe-maintenance-mode-status "$GHE_HOSTNAME"; then - echo "Error: $GHE_HOSTNAME must be put in maintenance mode before restoring. Aborting." 1>&2 + log_error "Error: $GHE_HOSTNAME must be put in maintenance mode before restoring. Aborting." 1>&2 exit 1 fi fi @@ -299,7 +307,7 @@ RELEASE_VERSION=$(ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --get core.package-vers # mismatches in the secrets needed for Actions which ultimately results in Actions not working properly. Note: xargs is to remove whitespace ACTIONS_ENABLED_IN_BACKUP=$(git config -f $GHE_RESTORE_SNAPSHOT_PATH/settings.json --bool app.actions.enabled | xargs) if [[ $ACTIONS_ENABLED_IN_BACKUP != true ]] && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Error: Restoring a backup with Actions disabled onto an appliance with Actions enabled is not supported." >&2 + log_error "Restoring a backup with Actions disabled onto an appliance with Actions enabled is not supported." >&2 exit 1 fi @@ -310,20 +318,15 @@ if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH ac_db_ghe=$(echo 'ghe-mssql-console -y -n -q "SELECT name FROM sys.databases" | grep -i "ArtifactCache" | wc -l | tr -d " "' | ghe-ssh "$GHE_HOSTNAME" /bin/bash) ac_db_snapshot=$(find "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql/" -maxdepth 1 -name 'ArtifactCache*.bak' | wc -l | tr -d " ") if [[ $ac_db_ghe -gt 0 && $ac_db_snapshot -eq 0 ]]; then - echo "Error: $GHE_HOSTNAME has Actions Cache service enabled but no Actions Cache data is present in snapshot to restore. Aborting" 1>&2 - echo "Please disable Actions cache service in $GHE_HOSTNAME and retry" 1>&2 - echo "To disable Actions Cache service run as admin: ghe-actions-cache-disable" 1>&2 + log_error "$GHE_HOSTNAME has Actions Cache service enabled but no Actions Cache data is present in snapshot to restore. Aborting \n Please disable Actions cache service in $GHE_HOSTNAME and retry\nTo disable Actions Cache service run as admin: ghe-actions-cache-disable" 1>&2 exit 1 fi if [[ $ac_db_ghe -eq 0 && $ac_db_snapshot -gt 0 && ! $RESTORE_SETTINGS ]]; then - echo "Error: $GHE_HOSTNAME has Actions Cache service disabled but the snapshot is attempting to restore data for the service. Aborting" 1>&2 - echo "Please enable Actions cache service in $GHE_HOSTNAME and retry" 1>&2 - echo "To enable Actions Cache service run as admin: ghe-actions-cache-enable" 1>&2 + log_error "$GHE_HOSTNAME has Actions Cache service disabled but the snapshot is attempting to restore data for the service. Aborting. \n Please enable Actions cache service in $GHE_HOSTNAME and retry \n To enable Actions Cache service run as admin: ghe-actions-cache-enable" 1>&2 exit 1 fi else - echo "Error: $GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting." 1>&2 - echo "Setup details for enabling Actions can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled" 1>&2 + log_error "$GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting. \n Setup details for enabling Actions can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled" 1>&2 exit 1 fi fi @@ -332,40 +335,40 @@ fi bm_init > /dev/null ghe-backup-store-version || -echo "Warning: storing backup-utils version remotely failed." +log_warn "Warning: storing backup-utils version remotely failed." # Stop cron and timerd, as scheduled jobs may disrupt the restore process. -echo "Stopping cron and github-timerd ..." +log_info "Stopping cron and github-timerd ..." if $CLUSTER; then if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then - ghe_verbose "* Warning: Failed to stop cron on one or more nodes" + log_warn "Failed to stop cron on one or more nodes" 1>&3 fi if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then - ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes" + log_warn "Failed to stop github-timerd on one or more nodes" 1>&3 fi fi else if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service github-timerd stop"; then - ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes" + log_warn "Failed to stop github-timerd on one or more nodes" 1>&3 fi fi else if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then - ghe_verbose "* Warning: Failed to stop cron" + log_warn "Failed to stop cron" 1>&3 fi if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then - ghe_verbose "* Warning: Failed to stop github-timerd" + log_warn "Failed to stop github-timerd" 1>&3 fi fi else if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service github-timerd stop"; then - ghe_verbose "* Warning: Failed to stop github-timerd" + log_warn "Failed to stop github-timerd" 1>&3 fi fi fi @@ -377,6 +380,18 @@ if $RESTORE_SETTINGS; then ghe-restore-settings "$GHE_HOSTNAME" fi +# Always restore column encryption keys +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + log_info "Always restore encrypted column encryption keys on GHES verions 3.7.0+" +fi +ghe-restore-column-encryption-keys "$GHE_HOSTNAME" + +# Always restore secret scanning encryption keys +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+" + ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME" +fi + # Make sure mysql and elasticsearch are prep'd and running before restoring. # These services will not have been started on appliances that have not been # configured yet. @@ -387,7 +402,7 @@ fi # Restore UUID if present and not restoring to cluster. if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then - echo "Restoring UUID ..." + log_info "Restoring UUID ..." cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid" | ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true @@ -412,102 +427,118 @@ else fi if is_external_database_target_or_snapshot && $SKIP_MYSQL; then - echo "Skipping MySQL restore." + log_info "Skipping MySQL restore." else - echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..." + log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..." ghe-restore-mysql "$GHE_HOSTNAME" 1>&3 fi if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Stopping Actions before restoring databases ..." + log_info "Stopping Actions before restoring databases ..." # We mark Actions as stopped even if the `ghe-actions-stop` # fails to ensure that we cleanly start actions when performing cleanup. ACTIONS_STOPPED=true ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-stop' 1>&3 - echo "Restoring MSSQL databases ..." + log_info "Restoring MSSQL databases ..." ghe-restore-mssql "$GHE_HOSTNAME" 1>&3 - echo "Restoring Actions data ..." + log_info "Restoring Actions data ..." ghe-restore-actions "$GHE_HOSTNAME" 1>&3 - echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning." - echo " See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners." + echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning. \n See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners." fi if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - echo "Restoring MinIO data ..." + log_info "Restoring MinIO data ..." ghe-restore-minio "$GHE_HOSTNAME" 1>&3 fi +# log input into a variable for the parallel command, as the functions don't work with eval +cmd_title=$(log_info "Restoring Redis database ...") commands=(" -echo \"Restoring Redis database ...\" +echo \"$cmd_title\" ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3") +cmd_title=$(log_info "Restoring Git Repositories ...") commands+=(" -echo \"Restoring Git repositories ...\" +echo \"$cmd_title\" ghe-restore-repositories \"$GHE_HOSTNAME\"") +cmd_title=$(log_info "Restoring Gists ...") commands+=(" -echo \"Restoring Gists ...\" +echo \"$cmd_title\" ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") -commands+=(" -echo \"Restoring GitHub Pages artifacts ...\" -ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3") +if [ "$GHE_BACKUP_PAGES" != "no" ]; then + cmd_title=$(log_info "Restoring Pages ...") + commands+=(" + echo \"$cmd_title\" + ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3") +fi +cmd_title=$(log_info "Restoring SSH authorized keys ...") commands+=(" -echo \"Restoring SSH authorized keys ...\" +echo \"$cmd_title\" ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-authorized-keys' < \"$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json\" 1>&3") +cmd_title=$(log_info "Restoring storage data ...") commands+=(" -echo \"Restoring storage data ...\" +echo \"$cmd_title\" ghe-restore-storage \"$GHE_HOSTNAME\" 1>&3") +cmd_title=$(log_info "Restoring custom Git hooks ...") commands+=(" -echo \"Restoring custom Git hooks ...\" +echo \"$cmd_title\" ghe-restore-git-hooks \"$GHE_HOSTNAME\" 1>&3") if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then + cmd_title=$(log_info "Restoring Elasticsearch indices ...") commands+=(" - echo \"Restoring Elasticsearch indices ...\" + echo \"$cmd_title\" ghe-restore-es-rsync \"$GHE_HOSTNAME\" 1>&3") fi # Restore the audit log migration sentinel file, if it exists in the snapshot -if test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete; then - ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete" +if test -f "$GHE_RESTORE_SNAPSHOT_PATH"/es-scan-complete; then + log_info "Restoring Elasticsearch audit log migration sentinel file ..." 1>&3 + if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete" ; then + log_info "Failed to restore Elasticsearch audit log migration sentinel file." 1>&3 + fi fi # Restore exported audit logs to 2.12.9 and newer single nodes and # all releases of cluster if $CLUSTER || [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.12.9)" ]; then if [[ "$GHE_RESTORE_SKIP_AUDIT_LOGS" = "yes" ]]; then - echo "Skipping restore of audit logs." + log_info "Skipping restore of audit logs." else + cmd_title=$(log_info "Restoring Audit logs ...") commands+=(" - echo \"Restoring Audit logs ...\" + echo \"$cmd_title\" ghe-restore-es-audit-log \"$GHE_HOSTNAME\" 1>&3") fi fi if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then + log_info "Restoring data in parallel ..." $GHE_PARALLEL_COMMAND $GHE_PARALLEL_COMMAND_OPTIONS -- "${commands[@]}" else + log_info "Restoring data serially ..." 1>&3 for c in "${commands[@]}"; do - eval "$c" + eval "$c" done fi # Restart an already running memcached to reset the cache after restore -echo "Restarting memcached ..." 1>&3 +log_info "Restarting memcached ..." 1>&3 echo "sudo restart -q memcached 2>/dev/null || true" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh # Prevent GitHub Connect jobs running before we've had a chance to reset # the configuration by setting the last run date to now. if ! $RESTORE_SETTINGS; then - echo "Setting last run date for GitHub Connect jobs ..." 1>&3 + log_info "Setting last run date for GitHub Connect jobs ..." 1>&3 echo "now=$(date +%s.0000000); ghe-redis-cli mset timer:UpdateConnectInstallationInfo \$now timer:UploadEnterpriseServerUserAccountsJob \$now timer:UploadConnectMetricsJob \$now timer:GitHubConnectPushNewContributionsJob \$now" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 fi @@ -515,7 +546,7 @@ fi # When restoring to a host that has already been configured, kick off a # config run to perform data migrations. if $CLUSTER; then - echo "Configuring cluster ..." + log_info "Configuring cluster ..." if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3 elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then @@ -523,7 +554,7 @@ if $CLUSTER; then fi ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3 elif $instance_configured; then - echo "Configuring appliance ..." + log_info "Configuring appliance ..." if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3 elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then @@ -535,6 +566,7 @@ fi # Clear GitHub Connect settings stored in the restored database. # This needs to happen after `ghe-config-apply` to ensure all migrations have run. if ! $RESTORE_SETTINGS; then + log_info "Clearing GitHub Connect settings ..." 1>&3 echo "if [ -f /usr/local/share/enterprise/ghe-reset-gh-connect ]; then /usr/local/share/enterprise/ghe-reset-gh-connect -y; fi" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 fi @@ -545,6 +577,7 @@ CRON_RUNNING=true # Clean up all stale replicas on configured instances. if ! $CLUSTER && $instance_configured; then + log_info "Cleaning up replicas..." 1>&3 restored_uuid=$(cat $GHE_RESTORE_SNAPSHOT_PATH/uuid) other_nodes=$(echo " set -o pipefail; \ @@ -554,7 +587,7 @@ if ! $CLUSTER && $instance_configured; then | ( grep -F -x -v \"$restored_uuid\" || true )" \ | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash) if [ -n "$other_nodes" ]; then - echo "Cleaning up stale nodes ..." + log_info "Cleaning up stale nodes ..." for uuid in $other_nodes; do # shellcheck disable=SC2034 echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3 @@ -572,12 +605,12 @@ update_restore_status "complete" ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_RESTORE_SNAPSHOT}." if ! $CLUSTER; then - echo "Restoring SSH host keys ..." + log_info "Restoring SSH host keys ..." ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-ssh-host-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3 else # This will make sure that Git over SSH host keys (babeld) are # copied to all the cluster nodes so babeld uses the same keys. - echo "Restoring Git over SSH host keys ..." + log_info "Restoring Git over SSH host keys ..." ghe-ssh "$GHE_HOSTNAME" -- "sudo tar -xpf - -C $GHE_REMOTE_DATA_USER_DIR/common" < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3 ghe-ssh "$GHE_HOSTNAME" -- "sudo chown babeld:babeld $GHE_REMOTE_DATA_USER_DIR/common/ssh_host_*" 1>&3 echo "if [ -f /usr/local/share/enterprise/ghe-cluster-config-update ]; then /usr/local/share/enterprise/ghe-cluster-config-update -s; else ghe-cluster-config-update -s; fi" | @@ -585,13 +618,11 @@ else fi END_TIME=$(date +%s) -echo 'End time:' $END_TIME -echo 'Runtime:' $(($END_TIME - $START_TIME)) 'seconds' +log_info "Runtime: $((${END_TIME} - ${START_TIME})) seconds" +log_info "Restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT finished." -echo "Restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT finished." -ghe_restore_finished if ! $instance_configured; then - echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration." + echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration." fi diff --git a/debian/changelog b/debian/changelog index fb6b9940b..630216b67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,34 @@ +github-backup-utils (3.8.4) UNRELEASED; urgency=medium + + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + + -- ghes-releases-team Fri, 20 Oct 2023 18:49:24 +0000 + +github-backup-utils (3.8.3) UNRELEASED; urgency=medium + + * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551 + + -- Devin Dooley Mon, 02 Oct 2023 20:33:31 +0000 + +github-backup-utils (3.8.2) UNRELEASED; urgency=medium + + * Use calculated routes when backing up storage data from a cluster #318 + * Move check for git for ssh muxing into ghe-ssh #378 + * Improve multi-platform detection of simultaneous ghe-backup runs #435 + + -- Devin Dooley Wed, 16 Aug 2023 23:13:39 +0000 + +github-backup-utils (3.8.1) UNRELEASED; urgency=medium + + * Backup and restore custom CA certificates #281 + * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 + * Remove check for git from ghe-ssh #393 + * Optimise hookshot and audit log backups and restores and MySQL restores #413 + * Add missing dependencies to debian packaging #418 + * Restoring to an un-configured appliance fails due to a missing license file #449 + + -- Devin Dooley Wed, 26 Jul 2023 01:30:01 +0000 + github-backup-utils (3.8.0) focal; urgency=medium -- Daniel Johnson Tue, 07 Feb 2023 21:43:26 +0000 diff --git a/docs/requirements.md b/docs/requirements.md index f6b9b8f31..3df2cee8d 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,8 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating -system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer, and [jq][11] v1.5 or newer. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](#april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. @@ -79,3 +78,4 @@ Due to how some components of Backup Utilities (e.g. MSSQL) take incremental bac [9]: https://joeyh.name/code/moreutils [10]: https://www.gnu.org/software/gawk [11]: https://stedolan.github.io/jq/ +[12]: https://www.gnu.org/software/bc/ diff --git a/release-notes/3.8.4.md b/release-notes/3.8.4.md new file mode 100644 index 000000000..960bc7e6b --- /dev/null +++ b/release-notes/3.8.4.md @@ -0,0 +1,4 @@ +### Bug Fixes + +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + diff --git a/script/package-deb b/script/package-deb index dda90cbe3..15a698e9c 100755 --- a/script/package-deb +++ b/script/package-deb @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" @@ -22,6 +25,14 @@ mkdir -p dist/debuild distdir="$(pwd)/dist/debuild/$PKG_NAME" git clone -q . "$distdir" cd "$distdir" + +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +echo "Removing .releaseignore ..." +rm -f .releaseignore git checkout -q "$PKG_HEAD" debuild -uc -us 1>&2 diff --git a/script/package-tarball b/script/package-tarball index be653b77a..bf1510e8f 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -8,11 +8,24 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" PKG_NAME="${PKG_BASE}-${PKG_VERS}" +# Remove all files or directories listed in .releaseignore +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +# Remove the .releaseignore file itself +echo "Removing .releaseignore ..." +rm -f .releaseignore + # Run git-archive to generate tarball echo "Creating ${PKG_NAME}.tar.gz ..." mkdir -p dist diff --git a/script/release b/script/release index a27194d4b..891cd9f60 100755 --- a/script/release +++ b/script/release @@ -31,7 +31,11 @@ GH_REPO = ENV['GH_REPO'] || 'backup-utils' GH_OWNER = ENV['GH_OWNER'] || 'github' GH_AUTHOR = ENV['GH_AUTHOR'] DEB_PKG_NAME = 'github-backup-utils' -GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' +GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly? +GH_STABLE_BRANCH = "" + +# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI +PUBLISH = ENV['PUBLISH'] == 'true' || false CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium @@ -137,7 +141,8 @@ def beautify_changes(changes) end def changelog - changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) + puts "building changelog by comparing origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH}" + changes = `git log --pretty=oneline origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) raise 'Building the changelog failed' if $CHILD_STATUS != 0 changes @@ -228,12 +233,12 @@ def push_release_branch(version) end def update_stable_branch - `git checkout --quiet stable` + `git checkout --quiet #{GH_STABLE_BRANCH}` unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`) - warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}" + warn "Merging #{GH_BASE_BRANCH} into #{GH_STABLE_BRANCH} failed:\n\n#{out}" end - unless (out = `git push --quiet origin stable`) - warn "Failed pushing the stable branch:\n\n#{out}" + unless (out = `git push --quiet origin #{GH_STABLE_BRANCH}`) + warn "Failed pushing the #{GH_STABLE_BRANCH} branch:\n\n#{out}" end end @@ -333,9 +338,38 @@ def clean_up(version) `git branch --quiet -D tmp-packaging >/dev/null 2>&1` end +def is_base_branch_valid?(branch) + if branch == "master" || branch.match(/^\d+\.\d+-main$/) + return true + else + return false + end +end + +def get_stable_branch_name(branch) + ## derive the proper stable branch. if the base branch is "master" the stable branch is just "stable" + ## if the base branch is a release branch, the stable branch will be "x.y-stable" + result = "" + if branch == "master" + result = "stable" + else + result = branch.gsub(/-main$/, "-stable") + end + + result +end + #### All the action starts #### if $PROGRAM_NAME == __FILE__ begin + ## validate base branch. this must either be "master" or a release branch which will match the pattern "x.y-main" + raise "The branch #{GH_BASE_BRANCH} is not valid for releasing backup-utils. branch name must be master or match x.y-main" if !is_base_branch_valid?(GH_BASE_BRANCH) + + GH_STABLE_BRANCH = get_stable_branch_name(GH_BASE_BRANCH) + + puts "base branch = " + GH_BASE_BRANCH + puts "stable branch = " + GH_STABLE_BRANCH + args = ARGV.dup dry_run = false skip_version_bump_check = false @@ -449,15 +483,21 @@ if $PROGRAM_NAME == __FILE__ attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] - puts 'Publishing release...' - publish_release res['id'] + if PUBLISH + puts 'Publishing release...' + publish_release res['id'] + end puts 'Cleaning up...' clean_up version - puts 'Updating stable branch...' + puts "Updating #{GH_STABLE_BRANCH} branch..." update_stable_branch + if !PUBLISH + puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.' + end + puts 'Released!' rescue RuntimeError => e $stderr.puts "Error: #{e}" diff --git a/share/github-backup-utils/bm.sh b/share/github-backup-utils/bm.sh index c512f3d44..b714d144b 100755 --- a/share/github-backup-utils/bm.sh +++ b/share/github-backup-utils/bm.sh @@ -52,6 +52,8 @@ bm_end() { total=$(($tend - $tstart)) echo "$1 took ${total}s" >> $BM_FILE_PATH + # also log timing information in the verbose log + echo "$1 took ${total}s" 1>&3 if [ -n "$GHE_DEBUG" ]; then echo "Debug: $1 took ${total}s (bm_end)" fi diff --git a/share/github-backup-utils/ghe-backup-actions b/share/github-backup-utils/ghe-backup-actions index f19777930..aa02191be 100755 --- a/share/github-backup-utils/ghe-backup-actions +++ b/share/github-backup-utils/ghe-backup-actions @@ -20,7 +20,7 @@ backup_dir="$GHE_SNAPSHOT_DIR/actions" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "Error: rsync not found." 1>&2 exit 1 fi @@ -38,7 +38,7 @@ fi # Transfer all Actions data from the user data directory using rsync. ghe_verbose "* Transferring Actions files from $host ..." - +log_rsync "BEGIN: actions rsync" 1>&3 ghe-rsync -avz \ -e "ghe-ssh -p $port" \ --rsync-path='sudo -u actions rsync' \ @@ -46,5 +46,6 @@ ghe-rsync -avz \ $link_dest \ "$host:$GHE_REMOTE_DATA_USER_DIR/actions/" \ "$GHE_SNAPSHOT_DIR/actions" 1>&3 +log_rsync "END: actions rsync" 1>&3 bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index ab0e24d4d..aacc0e10d 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -14,6 +14,83 @@ # . $( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config # set +o posix +# Terminal colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Colo# Logging display and formatting functions + +# Log a message to stdout +log_level() { + local level=$1 + shift + local message=$* + local display="" + local timestamp + timestamp=$(date -u "+%FT%TZ") + + + if [ "$TERM" = "dumb" ] || [[ "$OUTPUT_COLOR" != "yes" ]]; then + if [ "$level" = "info" ]; then + display="INFO" + elif [ "$level" = "warn" ]; then + display="WARN" + elif [ "$level" = "error" ]; then + display="ERROR" + elif [ "$level" = "verbose" ]; then + display="INFO" + elif [ "$level" = "rsync" ]; then + display="RSYNC" + elif [ "$level" = "ssh" ]; then + display="SSH" + else + display="-" + fi + else + if [ "$level" = "info" ]; then + display="${GREEN}INFO${NC}" + elif [ "$level" = "warn" ]; then + display="${YELLOW}WARN${NC}" + elif [ "$level" = "error" ]; then + display="${RED}ERROR${NC}" + elif [ "$level" = "verbose" ]; then + display="${GREEN}INFO${NC}" + elif [ "$level" = "rsync" ]; then + display="${GREEN}RSYNC${NC}" + elif [ "$level" = "ssh" ]; then + display="${GREEN}SSH${NC}" + else + display="-" + fi + fi + echo -e "$timestamp $display $message" +} + +log_info(){ + log_level "info" "$1" +} + +log_warn(){ + log_level "warn" "$1" +} + +log_error(){ + log_level "error" "$1" +} + +log_verbose(){ + log_level "verbose" "$1" +} + +log_rsync(){ + log_level "rsync" "$1" +} + +log_ssh(){ + log_level "ssh" "$1" +} + # Assume this script lives in share/github-backup-utils/ when setting the root GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -130,8 +207,10 @@ ghe_parallel_check() { GHE_PARALLEL_COMMAND="parallel" local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do @@ -181,12 +260,16 @@ if [ -n "$GHE_VERBOSE" ]; then if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then exec 3>>"$GHE_VERBOSE_LOG" else - if ! echo | awk '{ print strftime("%b %d %H:%M:%S"); fflush(); }' &>/dev/null; then - echo "Error: awk command failed. Please install https://www.gnu.org/software/gawk" 1>&2 - exit 1 - fi calling_script_name="$(caller | sed 's:.*/::')" - exec 3> >(awk -v c="$calling_script_name" '{ print strftime("%b %d %H:%M:%S"), c":", $0; fflush(); }' >>"$GHE_VERBOSE_LOG") + if [ "$TERM" = "dumb" ] || [[ "$OUTPUT_COLOR" != "yes" ]]; then + exec 3>>"$GHE_VERBOSE_LOG" + log_info "$calling_script_name $*" 1>&3 + else + # colorize the input if supported. + display_caller="${BLUE}$calling_script_name${NC}" + exec 3>>"$GHE_VERBOSE_LOG" + log_info "$display_caller $*" 1>&3 + fi fi else exec 3>&1 @@ -408,7 +491,7 @@ ghe_remote_logger() { # Log if verbose mode is enabled (GHE_VERBOSE or `-v`). ghe_verbose() { if [ -n "$GHE_VERBOSE" ]; then - echo "$@" 1>&3 + log_verbose "$@" 1>&3 fi } @@ -561,3 +644,8 @@ restore-secret() { ghe-ssh "$GHE_HOSTNAME" -- /bin/bash fi } + + + + + diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log index 42ef9314a..84da8953a 100755 --- a/share/github-backup-utils/ghe-backup-es-audit-log +++ b/share/github-backup-utils/ghe-backup-es-audit-log @@ -22,7 +22,7 @@ ghe_remote_version_required "$host" mkdir -p "$GHE_SNAPSHOT_DIR/audit-log" if ! indices=$(ghe-ssh "$host" "curl -s \"localhost:9201/_cat/indices/audit_log*?h=index,pri.store.size&bytes=b\""); then - echo "Error: failed to retrieve audit log indices." 1>&2 + log_error "ghe-backup-es-audit-log: Failed to retrieve audit log indices." 1>&2 exit 1 fi diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync index 11fa99f70..ee61741b2 100755 --- a/share/github-backup-utils/ghe-backup-es-rsync +++ b/share/github-backup-utils/ghe-backup-es-rsync @@ -4,6 +4,7 @@ #/ #/ Note: This command typically isn't called directly. It's invoked by #/ ghe-backup when the rsync strategy is used. +# shellcheck disable=SC2086 set -e # Bring in the backup configuration @@ -20,7 +21,7 @@ ghe_remote_version_required "$host" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "rsync not found." 1>&2 exit 1 fi @@ -43,17 +44,18 @@ fi # directory, using a previous snapshot to avoid transferring files that have # already been transferred. ghe_verbose "* Performing initial sync of ES indices ..." +log_rsync "BEGIN elasticsearch rsync" 1>&3 ghe-rsync -avz \ -e "ghe-ssh -p $(ssh_port_part "$host")" \ --rsync-path="sudo -u elasticsearch rsync" \ $link_dest \ "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \ "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3 - +log_rsync "END elasticsearch rsync" 1>&3 # Set up a trap to re-enable flushing on exit and remove temp file cleanup () { ghe_verbose "* Enabling ES index flushing ..." - echo '{"index":{"translog.disable_flush":false}}' | + echo '{"index":{"translog.flush_threshold_size":"512MB"}}' | ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null } trap 'cleanup' EXIT @@ -61,18 +63,20 @@ trap 'exit $?' INT # ^C always terminate # Disable ES flushing and force a flush right now ghe_verbose "* Disabling ES index flushing ..." -echo '{"index":{"translog.disable_flush":true}}' | +echo '{"index":{"translog.flush_threshold_size":"1PB"}}' | ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null # Transfer all ES indices again ghe_verbose "* Performing follow-up sync of ES indices ..." +log_rsync "BEGIN: elasticsearch followup rsync" 1>&3 ghe-rsync -avz \ -e "ghe-ssh -p $(ssh_port_part "$host")" \ --rsync-path="sudo -u elasticsearch rsync" \ $link_dest \ "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \ "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3 +log_rsync "END: elasticsearch followup rsync" 1>&3 # "Backup" audit log migration sentinel file if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then diff --git a/share/github-backup-utils/ghe-backup-fsck b/share/github-backup-utils/ghe-backup-fsck index 004abe412..d255e036f 100755 --- a/share/github-backup-utils/ghe-backup-fsck +++ b/share/github-backup-utils/ghe-backup-fsck @@ -14,7 +14,7 @@ echo "Running git fsck on repos..." # Verify git is available. if ! git --version 1>/dev/null 2>&1; then - echo "Error: git not found." 1>&2 + log_error "git not found." 1>&2 exit 1 fi @@ -26,7 +26,7 @@ t_start=$(date +%s) if git fsck -h | grep -q '\-\-dangling'; then git_cmd='git fsck --no-dangling' else - echo "Warning: old git version, --no-dangling not available" + log_warn "ghe-backup-fsck: old git version, --no-dangling not available" 1>&3 git_cmd='git fsck' fi @@ -35,7 +35,7 @@ if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then fi if [ ! -d "$sdir/repositories" ]; then - echo "Error: $sdir is not a valid snapshot." >&2 + log_error "ghe-backup-fsck: $sdir is not a valid snapshot." >&2 exit 1 fi @@ -84,7 +84,7 @@ for repo in $(find $sdir/repositories/ -type d -name \*.git); do done -echo "* Repos verified: $repos, Errors: $errors, Took: $(($(date +%s) - $t_start))s" +log_info "* Repos verified: $repos, Errors: $errors, Took: $(($(date +%s) - $t_start))s" rm -f $log diff --git a/share/github-backup-utils/ghe-backup-git-hooks b/share/github-backup-utils/ghe-backup-git-hooks index 3229ee009..1763501c7 100755 --- a/share/github-backup-utils/ghe-backup-git-hooks +++ b/share/github-backup-utils/ghe-backup-git-hooks @@ -14,7 +14,7 @@ bm_start "$(basename $0)" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "rsync not found." 1>&2 exit 1 fi @@ -89,12 +89,13 @@ rsync_git_hooks_data () { # Ensure target directory exists, is needed with subdirectories mkdir -p "$backup_dir/$subpath" - + log_rsync "BEGIN: git-hooks sync" 1>&3 ghe-rsync -av \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" $link_dest \ --rsync-path='sudo -u git rsync' \ "$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/$subpath/" \ "$backup_dir/$subpath" 1>&3 + log_rsync "END: git-hooks sync" 1>&3 } hostname=$(echo $hostnames | awk '{ print $1; }') diff --git a/share/github-backup-utils/ghe-backup-minio b/share/github-backup-utils/ghe-backup-minio index 59dc7f8b3..7f6a18b22 100755 --- a/share/github-backup-utils/ghe-backup-minio +++ b/share/github-backup-utils/ghe-backup-minio @@ -19,7 +19,7 @@ backup_dir="${GHE_SNAPSHOT_DIR}/minio" # Verify rsync is available. if ! command -v rsync 1> /dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "rsync not found." 1>&2 exit 1 fi @@ -41,7 +41,7 @@ fi # Transfer all minio data from the user data directory using rsync. ghe_verbose "* Transferring minio files from ${host} ..." - +log_rsync "BEGIN: minio rsync" 1>&3 ghe-rsync \ --archive \ --verbose \ @@ -52,5 +52,5 @@ ghe-rsync \ ${link_dest} \ "${host}:${GHE_REMOTE_DATA_USER_DIR}/minio/" \ "${GHE_SNAPSHOT_DIR}/minio" 1>&3 - +log_rsync "END: minio rsync" 1>&3 bm_end "$(basename "${0}")" diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql index c2ab49ce4..37b09ca9d 100755 --- a/share/github-backup-utils/ghe-backup-mssql +++ b/share/github-backup-utils/ghe-backup-mssql @@ -30,11 +30,11 @@ export_tool_available() { } ghe_ssh_mssql() { - ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@" + ghe-ssh "${opts[@]}" "${ssh_config_file_opt[@]}" "$GHE_MSSQL_PRIMARY_HOST" "$@" } cleanup() { - rm -rf $tempdir + rm -rf "$tempdir" } trap 'cleanup' EXIT INT @@ -47,21 +47,21 @@ if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then fi tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -ssh_config_file_opt= -opts= +ssh_config_file_opt=() +opts=() isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)" # get server hostnames under cluster and HA if [ "$GHE_BACKUP_STRATEGY" = "cluster" ] || [ "$isHA" = "true" ] ; then ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" + ssh_config_file_opt=("-F" "$ssh_config_file") + opts=("-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no" "-o" "PasswordAuthentication=no") ghe-ssh-config "$GHE_HOSTNAME" "$GHE_MSSQL_PRIMARY_HOST" > "$ssh_config_file" fi if ! export_tool_available ; then - ghe_verbose "ghe-export-mssql is not available" + log_error "ghe-export-mssql is not available" 1>&2 exit fi @@ -69,10 +69,10 @@ add_minute() { # Expect date string in the format of yyyymmddTHHMMSS # Here parse date differently depending on GNU Linux vs BSD MacOS if date -v -1d > /dev/null 2>&1; then - echo "$(date -v +$2M -ujf'%Y%m%dT%H%M%S' $1 +%Y%m%dT%H%M%S)" + date -v +"$2"M -ujf'%Y%m%dT%H%M%S' "$1" +%Y%m%dT%H%M%S else dt=$1 - echo "$(date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes")" + date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes" fi } @@ -110,8 +110,8 @@ ensure_same_dbs() { done if [[ "${#locals[@]}" -ne 0 ]]; then - ghe_verbose "Warning: Found following ${#locals[@]} backup files that can't be traced back to the specified GHES host." - ghe_verbose "Warning: Did you recently reconfigure the GHES host? Move or delete these backup files if no longer needed." + log_warn "Warning: Found following ${#locals[@]} backup files that can't be traced back to the specified GHES host." + log_warn "Warning: Did you recently reconfigure the GHES host? Move or delete these backup files if no longer needed." for local in "${locals[@]}"; do ghe_verbose "$1/$local" done @@ -231,28 +231,28 @@ diff expire $diff_expire, tran expire $tran_expire" if [ "$backup_type" == 'diff' ]; then full_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "bak") if [[ "$full_backup_file" == "" ]]; then - ghe_verbose "Taking a full backup instead of a diff backup because for $db a full backup file wasn't found" + log_warn "Taking a full backup instead of a diff backup because for $db a full backup file wasn't found" backup_type="full" break fi full_backup_file_checkpoint_lsn=$(get_backup_checkpoint_lsn "$db" "$full_backup_file") if [[ "$full_backup_file_checkpoint_lsn" = "NULL" ]] || [[ "$full_backup_file_checkpoint_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a diff backup because for $db the checkpoint LSN for $full_backup_file couldn't be determined" + log_warn "Taking a full backup instead of a diff backup because for $db the checkpoint LSN for $full_backup_file couldn't be determined" backup_type="full" break fi next_diff_backup_base_lsn=$(get_next_diff_backup_base_lsn "$db") if [[ "$next_diff_backup_base_lsn" = "NULL" ]] || [[ "$next_diff_backup_base_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the base LSN for the next diff backup couldn't be determined" + log_warn "Taking a full backup instead of a $backup_type backup because for $db the base LSN for the next diff backup couldn't be determined" backup_type="full" break fi # The base of the diff backup we're about to take must exactly match the checkpoint LSN of the full backup file we have if [[ "$next_diff_backup_base_lsn" -ne "$full_backup_file_checkpoint_lsn" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the diff would have base LSN $next_diff_backup_base_lsn yet our full backup has checkpoint LSN $full_backup_file_checkpoint_lsn" + log_warn "Taking a full backup instead of a $backup_type backup because for $db the diff would have base LSN $next_diff_backup_base_lsn yet our full backup has checkpoint LSN $full_backup_file_checkpoint_lsn" backup_type="full" break fi @@ -261,21 +261,21 @@ diff expire $diff_expire, tran expire $tran_expire" # Ensure that a transaction log backup will immediately follow the previous one latest_log_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "log") if [[ "$latest_log_backup_file" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db a previous transaction log backup wasn't found" + log_warn "Taking a full backup instead of a $backup_type backup because for $db a previous transaction log backup wasn't found" backup_type="full" break fi latest_log_backup_last_lsn=$(get_backup_last_lsn "$db" "$latest_log_backup_file") if [[ "$latest_log_backup_last_lsn" = "NULL" ]] || [[ "$latest_log_backup_last_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the LSN range for $latest_log_backup_file couldn't be determined" + log_warn "Taking a full backup instead of a $backup_type backup because for $db the LSN range for $latest_log_backup_file couldn't be determined" backup_type="full" break fi next_log_backup_starting_lsn=$(get_next_log_backup_starting_lsn "$db") if [[ "$next_log_backup_starting_lsn" = "NULL" ]] || [[ "$next_log_backup_starting_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the starting LSN for the next log backup couldn't be determined" + log_warn "Taking a full backup instead of a $backup_type backup because for $db the starting LSN for the next log backup couldn't be determined" backup_type="full" break fi @@ -283,7 +283,7 @@ diff expire $diff_expire, tran expire $tran_expire" # The starting LSN of the backup we're about to take must be equal to (or before) the last LSN from the last backup, # otherwise there'll be a gap and the logfiles won't be restorable if [[ "$next_log_backup_starting_lsn" -gt "$latest_log_backup_last_lsn" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db a gap would exist between the last backup ending at LSN $latest_log_backup_last_lsn and next backup starting at $next_log_backup_starting_lsn" + log_warn "Taking a full backup instead of a $backup_type backup because for $db a gap would exist between the last backup ending at LSN $latest_log_backup_last_lsn and next backup starting at $next_log_backup_starting_lsn" backup_type="full" break fi @@ -337,7 +337,7 @@ if [ -n "$backup_type" ]; then fi bm_start "$(basename "$0")" - ghe_ssh_mssql -- "$backup_command" || failures="$failures mssql" + ghe_ssh_mssql -- "$backup_command" bm_end "$(basename "$0")" # Configure the backup cadence on the appliance, which is used for diagnostics diff --git a/share/github-backup-utils/ghe-backup-mysql b/share/github-backup-utils/ghe-backup-mysql index 7e3fd57e3..0ae9172ce 100755 --- a/share/github-backup-utils/ghe-backup-mysql +++ b/share/github-backup-utils/ghe-backup-mysql @@ -17,17 +17,14 @@ ghe_remote_version_required "$GHE_HOSTNAME" if is_external_database_target; then if [ -n "$EXTERNAL_DATABASE_BACKUP_SCRIPT" ]; then - echo "Backing up external MySQL database using customer-provided script..." + log_info "Backing up external MySQL database using customer-provided script..." $EXTERNAL_DATABASE_BACKUP_SCRIPT bm_end "$(basename $0)" exit 0 else if is_binary_backup_feature_on; then - echo "Warning: Binary backups are configured on the target environment." - echo "Binary backup is not supported with an external MySQL database. Backing up using logical backup strategy." - echo - echo "Please disable binary backups with 'ghe-config mysql.backup.binary false', or" - echo "provide a custom backup script using EXTERNAL_DATABASE_BACKUP_SCRIPT" + log_warn "Binary backups are configured on the target environment." + log_warn "Binary backup is not supported with an external MySQL database. Backing up using logical backup strategy. Please disable binary backups with 'ghe-config mysql.backup.binary false', or provide a custom backup script using EXTERNAL_DATABASE_BACKUP_SCRIPT" fi ghe-backup-mysql-logical diff --git a/share/github-backup-utils/ghe-backup-mysql-binary b/share/github-backup-utils/ghe-backup-mysql-binary index 37218c82e..70091cbba 100755 --- a/share/github-backup-utils/ghe-backup-mysql-binary +++ b/share/github-backup-utils/ghe-backup-mysql-binary @@ -15,7 +15,7 @@ bm_start "$(basename $0)" # Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. ghe_remote_version_required "$GHE_HOSTNAME" -echo "Backing up MySQL database using binary backup strategy ..." +log_verbose "Backing up MySQL database using binary backup strategy ..." echo "set -o pipefail; ghe-export-mysql" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz" diff --git a/share/github-backup-utils/ghe-backup-mysql-logical b/share/github-backup-utils/ghe-backup-mysql-logical index 3dc2478ec..77040b8b9 100755 --- a/share/github-backup-utils/ghe-backup-mysql-logical +++ b/share/github-backup-utils/ghe-backup-mysql-logical @@ -15,7 +15,7 @@ bm_start "$(basename $0)" # Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. ghe_remote_version_required "$GHE_HOSTNAME" -echo "Backing up MySQL database using logical backup strategy ..." +log_verbose "Backing up MySQL database using logical backup strategy ..." echo "set -o pipefail; ghe-export-mysql | pigz" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz" diff --git a/share/github-backup-utils/ghe-backup-pages b/share/github-backup-utils/ghe-backup-pages index 634e7a417..21c2ac465 100755 --- a/share/github-backup-utils/ghe-backup-pages +++ b/share/github-backup-utils/ghe-backup-pages @@ -18,7 +18,7 @@ backup_dir="$GHE_SNAPSHOT_DIR/pages" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "rsync not found." 1>&2 exit 1 fi @@ -72,7 +72,7 @@ for hostname in $hostnames; do # should be transferred here. echo 1>&3 ghe_verbose "* Transferring pages files ..." - + log_rsync "BEGIN: pages rsync" 1>&3 # Transfer all data from the user data directory using rsync. ghe-rsync -avz \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ @@ -80,6 +80,7 @@ for hostname in $hostnames; do $link_dest \ "$hostname:$GHE_REMOTE_DATA_USER_DIR/pages/" \ "$GHE_SNAPSHOT_DIR/pages" 1>&3 + log_rsync "END: pages rsync" 1>&3 bm_end "$(basename $0) - $hostname" done diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index 60dd5e38e..a3ba533d7 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -46,7 +46,7 @@ backup_current="$GHE_DATA_DIR/current/repositories" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "rsync not found." 1>&2 exit 1 fi @@ -142,7 +142,7 @@ ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping repositories backup ..." + log_warn "no routes found, skipping repositories backup ..." exit 0 fi @@ -159,6 +159,7 @@ rsync_repository_data () { files_list="$2" shift shift + log_rsync "BEGIN: repositories rsync" 1>&3 ghe-rsync -avr \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ $link_dest "$@" \ @@ -168,8 +169,10 @@ rsync_repository_data () { --ignore-missing-args \ "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ "$backup_dir" 1>&3 2>&3 + log_rsync "END: repositories rsync" 1>&3 else shift + log_rsync "BEGIN: repositories rsync" 1>&3 ghe-rsync -avr \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ $link_dest "$@" \ @@ -178,6 +181,7 @@ rsync_repository_data () { --ignore-missing-args \ "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ "$backup_dir" 1>&3 2>&3 + log_rsync "END: repositories rsync" 1>&3 fi } @@ -187,7 +191,7 @@ sync_data (){ # should be transferred here. echo 1>&3 - echo "* Transferring auxiliary files ..." 1>&3 + log_info "* Transferring auxiliary files ..." 1>&3 rsync_repository_data $1:122 $2 -z <&3 - echo "* Transferring packed-refs files ..." 1>&3 + log_info "* Transferring packed-refs files ..." 1>&3 rsync_repository_data $1:122 $2 -z <&3 - echo "* Transferring refs and reflogs ..." 1>&3 + log_info "* Transferring refs and reflogs ..." 1>&3 rsync_repository_data $1:122 $2 -z <&3 - echo "* Transferring objects and packs ..." 1>&3 + log_info "* Transferring objects and packs ..." 1>&3 rsync_repository_data $1:122 $2 -H <&3 - echo "* Transferring special data directories from $h..." 1>&3 + log_info "* Transferring special data directories from $h..." 1>&3 rsync_repository_data $h:122 -z <&3 +log_info "* Transferring settings data ..." 1>&3 ghe-ssh "$host" -- 'ghe-export-settings' > settings.json -echo "* Transferring license data ..." 1>&3 +log_info "* Transferring license data ..." 1>&3 ghe-ssh "$host" -- "sudo cat '$GHE_REMOTE_LICENSE_FILE'" > enterprise.ghl # Function to backup a secret setting to a file. @@ -61,7 +61,7 @@ backup-secret() { esac done - echo "* Transferring $description ..." 1>&3 + log_info "* Transferring $description ..." 1>&3 ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || ( if [ "$best_effort" = "false" ]; then echo "Warning: $description not set" >&2 @@ -79,8 +79,23 @@ backup-secret "password pepper" "password-pepper" "secrets.github.user-password- backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" +# backup encryption keying material and create backup value current encryption for GHES 3.7.0 onwards +# this is for forwards compatibility with GHES 3.8.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" + cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" +fi + +# secret scanning encrypted secrets keys were added in GHES 3.8.0 +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" + backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +fi + # Backup argon secrets for multiuser from ghes version 3.8 onwards -if [ "$(version $GHE_REMOTE_VERSION)" -gt "$(version 3.7.0)" ]; then +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" fi @@ -133,24 +148,24 @@ if ghe-ssh "$host" -- ghe-config --true app.packages.enabled; then fi if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/idp.crt"; then - echo "* Transferring SAML keys ..." 1>&3 + log_info "* Transferring SAML keys ..." 1>&3 ghe-ssh $host -- sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -cf - "idp.crt saml-sp.p12" > saml-keys.tar fi if ghe-ssh "$host" -- "which ghe-export-ssl-ca-certificates 1>/dev/null"; then - echo "* Transferring CA certificates ..." 1>&3 + log_info "* Transferring CA certificates ..." 1>&3 ghe-ssh "$host" -- "ghe-export-ssl-ca-certificates" > ssl-ca-certificates.tar fi if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - echo "* Transferring cluster configuration ..." 1>&3 + log_info "* Transferring cluster configuration ..." 1>&3 if ! ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_CLUSTER_CONF_FILE 2>/dev/null" > cluster.conf; then - echo "Error: Enterprise Cluster is not configured yet, backup will fail" >&2 + log_error "Error: Enterprise Cluster is not configured yet, backup will fail" >&2 exit 1 fi else if ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_DATA_USER_DIR/common/uuid 2>/dev/null" > uuid; then - echo "* Transferring UUID ..." 1>&3 + log_info "* Transferring UUID ..." 1>&3 fi fi diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage index bb8117021..2f98a0541 100755 --- a/share/github-backup-utils/ghe-backup-storage +++ b/share/github-backup-utils/ghe-backup-storage @@ -19,7 +19,7 @@ backup_dir="$GHE_SNAPSHOT_DIR/storage" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "rsync not found." 1>&2 exit 1 fi @@ -60,7 +60,7 @@ cleanup() { # Enable remote maintenance operations for hostname in $hostnames; do ghe-gc-enable $ssh_config_file_opt $hostname:$port || { - echo "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 + log_warn "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 } done @@ -111,7 +111,7 @@ ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping storage backup ..." + log_warn "no routes found, skipping storage backup ..." exit 0 fi @@ -123,7 +123,7 @@ for file_list in $tempdir/*.rsync; do object_num=$(cat $file_list | wc -l) ghe_verbose "* Transferring $object_num objects from $hostname" - + log_rsync "BEGIN: storage rsync" 1>&3 ghe-rsync -avr \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ $link_dest "$@" \ @@ -133,6 +133,7 @@ for file_list in $tempdir/*.rsync; do --size-only \ "$hostname:$GHE_REMOTE_DATA_USER_DIR/storage/" \ "$backup_dir" 1>&3 & + log_rsync "END: storage rsync" 1>&3 done for pid in $(jobs -p); do diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata index 97f71c1a9..f332c27c4 100755 --- a/share/github-backup-utils/ghe-backup-userdata +++ b/share/github-backup-utils/ghe-backup-userdata @@ -13,7 +13,7 @@ bm_start "$(basename $0) - $1" # Verify rsync is available. if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 + log_error "rsync not found." 1>&2 exit 1 fi @@ -52,7 +52,7 @@ fi # Ensure target directory exists, is needed with subdirectories mkdir -p "$GHE_SNAPSHOT_DIR/$dirname" - +log_rsync "BEGIN: userdata rsync" 1>&3 # Transfer all data from the user data directory using rsync. ghe-rsync -avz \ -e "ghe-ssh -p $(ssh_port_part "$host")" \ @@ -60,5 +60,5 @@ ghe-rsync -avz \ $link_dest \ "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/$dirname/" \ "$GHE_SNAPSHOT_DIR/$dirname" 1>&3 - +log_rsync "END: userdata rsync" 1>&3 bm_end "$(basename $0) - $1" diff --git a/share/github-backup-utils/ghe-cluster-find-nodes b/share/github-backup-utils/ghe-cluster-find-nodes index c2a3cf0e1..2a1f7ef48 100755 --- a/share/github-backup-utils/ghe-cluster-find-nodes +++ b/share/github-backup-utils/ghe-cluster-find-nodes @@ -16,7 +16,7 @@ set -e # Check if the REMOTE DATA USER directory is set if [ -z $GHE_REMOTE_DATA_USER_DIR ]; then - echo "Env variable GHE_REMOTE_DATA_USER_DIR is not set. Exiting" + log_error "Env variable GHE_REMOTE_DATA_USER_DIR is not set. Exiting" exit 1 fi diff --git a/share/github-backup-utils/ghe-detect-leaked-ssh-keys b/share/github-backup-utils/ghe-detect-leaked-ssh-keys index 79978ddf0..a71fd812e 100755 --- a/share/github-backup-utils/ghe-detect-leaked-ssh-keys +++ b/share/github-backup-utils/ghe-detect-leaked-ssh-keys @@ -86,9 +86,9 @@ for tar_file in $ssh_tars; do leaked_keys_found=true if [ "$current_dir" == "$(dirname "$tar_file")" ]; then current_bkup=true - echo "* Leaked key found in current backup snapshot." + log_warn "* Leaked key found in current backup snapshot." else - echo "* Leaked key found in backup snapshot." + log_warn "* Leaked key found in backup snapshot." fi echo "* Snapshot file: $tar_file" echo "* Key file: $key" @@ -130,9 +130,9 @@ if $leaked_keys_found; then fi else if $leaked_keys_skippedcheck; then - echo "* No result - check not performed since host key fingerprint was empty" + log_info "* No result - check not performed since host key fingerprint was empty" else - echo "* No leaked keys found" + log_info "* No leaked keys found" fi fi diff --git a/share/github-backup-utils/ghe-gc-disable b/share/github-backup-utils/ghe-gc-disable index bf37572ca..1e5a63bc5 100755 --- a/share/github-backup-utils/ghe-gc-disable +++ b/share/github-backup-utils/ghe-gc-disable @@ -50,7 +50,7 @@ echo " " | ghe-ssh $opts "$host" -- /bin/bash || { res=$? if [ $res = 7 ]; then - echo "Error: Git GC processes remain after $GHE_GIT_COOLDOWN_PERIOD seconds. Aborting..." 1>&2 + log_error "Error: Git GC processes remain after $GHE_GIT_COOLDOWN_PERIOD seconds. Aborting..." 1>&2 fi exit $res } diff --git a/share/github-backup-utils/ghe-prune-snapshots b/share/github-backup-utils/ghe-prune-snapshots index 475f6b36e..3a73dfccc 100755 --- a/share/github-backup-utils/ghe-prune-snapshots +++ b/share/github-backup-utils/ghe-prune-snapshots @@ -25,7 +25,7 @@ prune_dirs="$(ls -1 "$GHE_DATA_DIR"/[0-9]*/incomplete 2>/dev/null || true)" prune_num=$(echo "$prune_dirs" | grep -v '^$' | wc -l) if [ $prune_num -gt 0 ]; then - echo Pruning $prune_num "failed snapshot(s) ..." + log_info Pruning $prune_num "failed snapshot(s) ..." echo "$prune_dirs" | sed 's@/incomplete$@@' | prune_snapshot fi @@ -35,6 +35,6 @@ snapshot_count=$(ls -1d "$GHE_DATA_DIR"/[0-9]* 2>/dev/null | wc -l) if [ "$snapshot_count" -gt "$GHE_NUM_SNAPSHOTS" ]; then prune_dirs="$(ls -1d "$GHE_DATA_DIR"/[0-9]* | sort -r | awk "NR>$GHE_NUM_SNAPSHOTS")" prune_num=$(echo "$prune_dirs" | grep -v '^$' | wc -l) - echo Pruning $prune_num "expired snapshot(s) ..." + log_info Pruning $prune_num "expired snapshot(s) ..." echo "$prune_dirs" | prune_snapshot fi diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 66d7e0828..93596033a 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -30,7 +30,7 @@ host=$(ssh_host_part "$GHE_HOSTNAME") # No need to restore anything, early exit if [ ! -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then - echo "Warning: Actions backup missing. Skipping ..." + log_warn "Warning: Actions backup missing. Skipping ..." exit 0 fi @@ -42,13 +42,13 @@ ghe_verbose "* Transferring Actions files to $host ..." ghe-ssh -p "$port" "$host" -- sudo mkdir -p "$GHE_REMOTE_DATA_USER_DIR/actions" ghe-ssh -p "$port" "$host" -- sudo chown -R actions:actions "$GHE_REMOTE_DATA_USER_DIR/actions" - +log_rsync "BEGIN: actions rsync" 1>&3 ghe-rsync -arvHR --delete \ -e "ghe-ssh -p $port" \ --rsync-path='sudo -u actions rsync' \ "$GHE_RESTORE_SNAPSHOT_PATH/actions/./" \ "$host:$GHE_REMOTE_DATA_USER_DIR/actions/" 1>&3 - +log_rsync "END: actions rsync" 1>&3 # Restore Actions settings. ghe_verbose "* Restoring Actions settings to $host ..." @@ -96,7 +96,7 @@ ghe-ssh -p "$port" "$host" -- ghe-actions-console -s actions -c "Repair-Database if [ ! -z "$(find "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql/" -maxdepth 1 -name 'ArtifactCache_Configuration*.bak')" ]; then ghe-ssh -p "$port" "$host" -- ghe-actions-console -s artifactcache -c "Repair-DatabaseLogins" else - echo "ArtifactCache is not present in mssql backup. Skipping Repair-DatabaseLogins for it." + log_info "ArtifactCache is not present in mssql backup. Skipping Repair-DatabaseLogins for it." fi bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-column-encryption-keys b/share/github-backup-utils/ghe-restore-column-encryption-keys new file mode 100755 index 000000000..d30cacd51 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-column-encryption-keys @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-column-encryption-keys +#/ Restore the column encryption keys from a snapshot to the given . +#/ This script will be run automatically by `ghe-restore +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +bm_start "$(basename $0)" + +# Grab host arg +GHE_HOSTNAME="$1" + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$GHE_HOSTNAME" + +# The snapshot to restore should be set by the ghe-restore command but this lets +# us run this script directly. +: ${GHE_RESTORE_SNAPSHOT:=current} + +# Path to snapshot dir we're restoring from +: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} + +# Restore encrypted column encryption keying material for GHES 3.7.0 onward +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + log_info "Restoring encrypted column encryption keying material" + restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" +fi + +# Restore encrypted column current encryption key for GHES 3.8.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + log_info "Restoring encrypted column current encryption key" + restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" +fi + + +bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-es-audit-log b/share/github-backup-utils/ghe-restore-es-audit-log index 750950425..06b63973b 100755 --- a/share/github-backup-utils/ghe-restore-es-audit-log +++ b/share/github-backup-utils/ghe-restore-es-audit-log @@ -50,14 +50,14 @@ done if [ -s "$tmp_list" ]; then ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 ghe-ssh "$GHE_HOSTNAME" -- "sudo chown elasticsearch:elasticsearch '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 - + log_rsync "BEGIN: es-audit log rsync" 1>&3 ghe-rsync -avz --delete \ -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \ --rsync-path="sudo -u elasticsearch rsync" \ --files-from=$tmp_list \ "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/audit-log/" \ "$(ssh_host_part "$GHE_HOSTNAME"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore/audit-log/" 1>&3 - + log_rsync "END: es-audit log rsync" 1>&3 if $CLUSTER || [ -n "$configured" ]; then for index in $(cat $tmp_list | sed 's/\.gz$//g'); do ghe_verbose "* Restoring $index" diff --git a/share/github-backup-utils/ghe-restore-es-rsync b/share/github-backup-utils/ghe-restore-es-rsync index 0c29beb0a..b1d9b2179 100755 --- a/share/github-backup-utils/ghe-restore-es-rsync +++ b/share/github-backup-utils/ghe-restore-es-rsync @@ -36,14 +36,14 @@ if [ ! -d "$snapshot_dir/elasticsearch" ]; then else ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 ghe-ssh "$GHE_HOSTNAME" -- "sudo chown elasticsearch:elasticsearch '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 - + log_rsync "BEGIN: elasticsearch rsync" 1>&3 ghe-rsync -avz --delete \ -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \ --rsync-path="sudo -u elasticsearch rsync" \ --copy-dest="$GHE_REMOTE_DATA_USER_DIR/elasticsearch" \ "$snapshot_dir/elasticsearch/" \ "$(ssh_host_part "$GHE_HOSTNAME"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore" 1>&3 - + log_rsync "END: elasticsearch rsync" 1>&3 # restoring in >=2.14 will remove incompatible indices created with 1.x. if [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 14 ]; then ghe-ssh "$GHE_HOSTNAME" -- "sudo /usr/local/share/enterprise/ghe-es-remove-1x-indices" diff --git a/share/github-backup-utils/ghe-restore-external-database-compatibility-check b/share/github-backup-utils/ghe-restore-external-database-compatibility-check index 9a22e63a0..364236a2a 100755 --- a/share/github-backup-utils/ghe-restore-external-database-compatibility-check +++ b/share/github-backup-utils/ghe-restore-external-database-compatibility-check @@ -16,15 +16,13 @@ if is_instance_configured; then # Restoring settings in this scenario would change BYODB state, which is not supported via backup-utils. if $RESTORE_SETTINGS; then - echo "Restoring the settings of a snapshot from an appliance using the bundled MySQL service to an appliance using an externally-managed MySQL service is not supported." - echo "Please reconfigure the appliance first, then run ghe-restore again." + log_error "Restoring the settings of a snapshot from an appliance using the bundled MySQL service to an appliance using an externally-managed MySQL service is not supported. Please reconfigure the appliance first, then run ghe-restore again." exit 1 fi # Restoring interal DB snapshot to BYODB appliance without passing in --skip-mysql is not supported. if ! $SKIP_MYSQL; then - echo "Restoring a snapshot from an appliance using the bundled MySQL service to an appliance using an externally-managed MySQL service is not supported." - echo "Please migrate the MySQL data beforehand, then run ghe-restore again, passing in the --skip-mysql flag." + log_error "Restoring a snapshot from an appliance using the bundled MySQL service to an appliance using an externally-managed MySQL service is not supported. Please migrate the MySQL data beforehand, then run ghe-restore again, passing in the --skip-mysql flag." exit 1 fi fi @@ -33,15 +31,13 @@ if is_instance_configured; then # Restoring settings in this scenario would change BYODB state, which is not supported via backup-utils. if $RESTORE_SETTINGS; then - echo "Restoring the settings of a snapshot from an appliance using an externally-managed MySQL service to an appliance using the bundled MySQL service is not supported." - echo "Please reconfigure the appliance first, then run ghe-restore again." + log_error "Restoring the settings of a snapshot from an appliance using an externally-managed MySQL service to an appliance using the bundled MySQL service is not supported. Please reconfigure the appliance first, then run ghe-restore again." exit 1 fi # Restoring BYODB snapshot to internal DB appliance without passing in --skip-mysql is not supported. if ! $SKIP_MYSQL; then - echo "Restoring a snapshot from an appliance using an externally-managed MySQL service to an appliance using the bundled MySQL service is not supported." - echo "Please migrate the MySQL data beforehand, then run ghe-restore again, passing in the --skip-mysql flag." + echo "Restoring a snapshot from an appliance using an externally-managed MySQL service to an appliance using the bundled MySQL service is not supported. Please migrate the MySQL data beforehand, then run ghe-restore again, passing in the --skip-mysql flag." exit 1 fi fi diff --git a/share/github-backup-utils/ghe-restore-git-hooks b/share/github-backup-utils/ghe-restore-git-hooks index 67081a921..4bad41f7a 100755 --- a/share/github-backup-utils/ghe-restore-git-hooks +++ b/share/github-backup-utils/ghe-restore-git-hooks @@ -60,12 +60,13 @@ if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs" ]; if [ -n "$hostname" ]; then ghe-ssh $ssh_config_file_opt -l $user "$hostname:122" -- "sudo -u git mkdir -p $GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" + log_rsync "BEGIN: git-hooks tarball rsync" 1>&3 ghe-rsync -avH --delete \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ --rsync-path="sudo -u git rsync" \ "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/environments/tarballs/" \ "$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs" 1>&3 - + log_rsync "END: git-hooks rsync" 1>&3 for tarball in $tarballs; do env_id=$(echo $tarball | cut -d '/' -f 2) ghe-ssh $ssh_config_file_opt -l $user "$hostname:122" -- "/bin/bash -c 'export PATH=\$PATH:/usr/local/share/enterprise && ghe-hook-env-update $env_id $GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs/$tarball'" 1>&3 2>&3 @@ -76,11 +77,13 @@ fi if [ -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/repos" ]; then for hostname in $hostnames; do ghe-ssh $ssh_config_file_opt -l $user "$hostname:122" -- "sudo -u git mkdir -p $GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" + log_rsync "BEGIN: git-hooks repos rsync" 1>&3 ghe-rsync -avH --delete \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ --rsync-path="sudo -u git rsync" \ "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/git-hooks/repos/" \ "$hostname:$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos" 1>&3 & + log_rsync "END: git-hooks repos rsync" 1>&3 done for pid in $(jobs -p); do diff --git a/share/github-backup-utils/ghe-restore-minio b/share/github-backup-utils/ghe-restore-minio index a3daa9843..8b61f0a45 100755 --- a/share/github-backup-utils/ghe-restore-minio +++ b/share/github-backup-utils/ghe-restore-minio @@ -30,7 +30,7 @@ host="$(ssh_host_part "${GHE_HOSTNAME}")" # No need to restore anything, early exit if [ ! -d "${GHE_RESTORE_SNAPSHOT_PATH}/minio" ]; then - echo "Warning: minio backup missing. Skipping ..." + log_warn "Warning: minio backup missing. Skipping ..." exit 0 fi @@ -42,7 +42,7 @@ ghe_verbose "* Transferring minio files to ${host} ..." ghe-ssh -p "${port}" "${host}" -- sudo mkdir -p "${GHE_REMOTE_DATA_USER_DIR}/minio" ghe-ssh -p "${port}" "${host}" -- sudo chown -R minio:minio "${GHE_REMOTE_DATA_USER_DIR}/minio" - +log_rsync "BEGIN: minio rsync" 1>&3 ghe-rsync \ --verbose \ --archive \ @@ -53,5 +53,5 @@ ghe-rsync \ --rsync-path='sudo -u minio rsync' \ "${GHE_RESTORE_SNAPSHOT_PATH}/minio/./" \ "${host}:${GHE_REMOTE_DATA_USER_DIR}/minio/" 1>&3 - +log_rsync "END: minio rsync" 1>&3 bm_end "$(basename "${0}")" diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 6885e98ab..8e75ea0f1 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -44,23 +44,18 @@ if is_external_database_snapshot; then exit 0 else if is_binary_backup "$GHE_RESTORE_SNAPSHOT_PATH"; then - echo "Error: Restore of a binary backup to appliance with an external database configured is not supported." - echo "Please provide a custom external database restore script with EXTERNAL_DATABASE_RESTORE_SCRIPT" + log_error "Error: Restore of a binary backup to appliance with an external database configured is not supported. Please provide a custom external database restore script with EXTERNAL_DATABASE_RESTORE_SCRIPT" exit 1 fi if ! is_default_external_database_snapshot; then - echo "Error: Backup was not taken with a GitHub provided backup strategy." - echo "You must provide a custom restore script for this backup using EXTERNAL_DATABASE_BACKUP_SCRIPT" + log_error "Error: Backup was not taken with a GitHub provided backup strategy. You must provide a custom restore script for this backup using EXTERNAL_DATABASE_BACKUP_SCRIPT" exit 1 fi if is_binary_backup_feature_on; then - echo "Warning: Binary backups are configured on the target environment." - echo "Binary backup is not supported with an external MySQL database." - echo - echo "Please disable binary backups with 'ghe-config mysql.backup.binary false'" + log_warn "Warning: Binary backups are configured on the target environment. \nBinary backup is not supported with an external MySQL database. \n\nPlease disable binary backups with 'ghe-config mysql.backup.binary false'" fi fi fi @@ -75,7 +70,7 @@ if is_binary_backup_feature_on; then else # We do not allow to restore binary backup without "mysql.backup.binary" set if is_binary_backup "$GHE_RESTORE_SNAPSHOT_PATH"; then - echo "To restore from a binary backup, you have to set ghe-config \"mysql.backup.binary\" to true" >&2 + log_error "To restore from a binary backup, you have to set ghe-config \"mysql.backup.binary\" to true" >&2 exit 2 else if is_default_external_database_snapshot; then diff --git a/share/github-backup-utils/ghe-restore-mysql-binary b/share/github-backup-utils/ghe-restore-mysql-binary index fb0da41b0..2a4374c88 100755 --- a/share/github-backup-utils/ghe-restore-mysql-binary +++ b/share/github-backup-utils/ghe-restore-mysql-binary @@ -66,7 +66,7 @@ ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo mkdir -p '$GHE_REMOTE_ # Transfer MySQL data from the snapshot to the GitHub instance. cat $snapshot_dir/mysql.sql.gz | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" -echo "Restore MySQL database ..." +log_info "Restore MySQL database ..." # Import the database echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- /bin/bash 1>&3 diff --git a/share/github-backup-utils/ghe-restore-mysql-legacy b/share/github-backup-utils/ghe-restore-mysql-legacy index e3fa615f1..3d4c19840 100755 --- a/share/github-backup-utils/ghe-restore-mysql-legacy +++ b/share/github-backup-utils/ghe-restore-mysql-legacy @@ -48,7 +48,7 @@ ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo mkdir -p '$GHE_REMOTE_ # Transfer MySQL data from the snapshot to the GitHub instance. cat $snapshot_dir/mysql.sql.gz | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" -echo "Restore MySQL database ..." +log_info "Restore MySQL database ..." # Import the database echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- /bin/bash 1>&3 diff --git a/share/github-backup-utils/ghe-restore-mysql-logical b/share/github-backup-utils/ghe-restore-mysql-logical index 3f7e64107..f56742d94 100755 --- a/share/github-backup-utils/ghe-restore-mysql-logical +++ b/share/github-backup-utils/ghe-restore-mysql-logical @@ -45,7 +45,7 @@ ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo mkdir -p '$GHE_REMOTE_ # Transfer MySQL data from the snapshot to the GitHub instance. cat $snapshot_dir/mysql.sql.gz | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1" -echo "Restore MySQL database ..." +log_info "Restore MySQL database ..." # Import the database echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- /bin/bash 1>&3 diff --git a/share/github-backup-utils/ghe-restore-pages b/share/github-backup-utils/ghe-restore-pages index 800eb14d3..640aa5d62 100755 --- a/share/github-backup-utils/ghe-restore-pages +++ b/share/github-backup-utils/ghe-restore-pages @@ -27,7 +27,7 @@ pages_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find pages -mindepth 5 # No need to restore anything, early exit if [ -z "$pages_paths" ]; then - echo "Warning: Pages backup missing. Skipping ..." + log_warn "Warning: Pages backup missing. Skipping ..." exit 0 fi @@ -123,7 +123,7 @@ ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping pages restore ..." + log_warn "Warning: no routes found, skipping pages restore ..." exit 0 fi @@ -135,6 +135,7 @@ for file_list in $tempdir/*.rsync; do server=$host fi ghe_verbose "* Transferring Pages to $server" + log_rsync "BEGIN: pages rsync" 1>&3 ghe-rsync -avrHR --delete \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ --rsync-path="sudo -u git rsync" \ @@ -142,6 +143,7 @@ for file_list in $tempdir/*.rsync; do "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/pages/./" \ "$server:$GHE_REMOTE_DATA_USER_DIR/pages/" 1>&3 done + log_rsync "END: pages rsync" 1>&3 bm_end "$(basename $0) - Restoring pages" if $CLUSTER; then diff --git a/share/github-backup-utils/ghe-restore-repositories b/share/github-backup-utils/ghe-restore-repositories index 3d522be0b..ba97153d4 100755 --- a/share/github-backup-utils/ghe-restore-repositories +++ b/share/github-backup-utils/ghe-restore-repositories @@ -28,7 +28,7 @@ GHE_HOSTNAME="$1" network_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find repositories -mindepth 6 -maxdepth 7 -name \*.git -exec dirname {} \; | uniq | grep nw | cut -d / -f2-) if [ -z "$network_paths" ]; then - echo "Warning: Repositories backup missing. Skipping ..." + log_warn "Warning: Repositories backup missing. Skipping ..." exit 0 fi @@ -140,7 +140,7 @@ ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping repositories restore ..." + log_warn "Warning: no routes found, skipping repositories restore ..." exit 0 fi @@ -159,13 +159,14 @@ for file_list in $tempdir/git-server-*.rsync; do if [ -n \"$GHE_VERBOSE\" ]; then echo \"* Transferring repository networks to $server ($file_list) ...\" 1>&3 fi - + echo \"$(date -u "+%FT%TZ") RSYNC BEGIN: repositories rsync\" 1>&3 ghe-rsync -avrR --delete \ -e \"ssh -q $opts -p $port $ssh_config_file_opt -l $user\" \ --rsync-path=\"sudo -u git rsync\" \ --files-from=$file_list \ \"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/./\" \ \"$server:$GHE_REMOTE_DATA_USER_DIR/repositories/\" 1>&3 + echo \"$(date -u "+%FT%TZ") RSYNC END: repositories rsync\" 1>&3 ") done @@ -216,7 +217,7 @@ bm_end "$(basename $0) - Updating repository info data" restore_warnings="$(ghe-ssh "$GHE_HOSTNAME" -- cat "$remote_warnings" 2>/dev/null || true)" if [ -n "$restore_warnings" ]; then - echo "Warning: One or more repository networks failed to restore successfully. Please contact GitHub Enterprise Support for assistance." + log_warn "Warning: One or more repository networks failed to restore successfully. Please contact GitHub Enterprise Support for assistance." echo "$restore_warnings" fi diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index 5080b964f..addb18514 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -27,7 +27,7 @@ gist_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find repositories -mind # No need to restore anything, early exit if [ -z "$gist_paths" ]; then - echo "Warning: Gist backup missing. Skipping ..." + log_warn "Warning: Gist backup missing. Skipping ..." exit 0 fi @@ -126,7 +126,7 @@ ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping gists restore ..." + log_warn "Warning: no routes found, skipping gists restore ..." exit 0 fi @@ -162,7 +162,7 @@ fi restore_warnings="$(ghe-ssh "$GHE_HOSTNAME" -- cat "$remote_warnings" 2>/dev/null || true)" if [ -n "$restore_warnings" ]; then - echo "Warning: One or more Gists failed to restore successfully. Please contact GitHub Enterprise Support for assistance." + log_warn "Warning: One or more Gists failed to restore successfully. Please contact GitHub Enterprise Support for assistance." echo "$restore_warnings" fi diff --git a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys new file mode 100755 index 000000000..aa225bc07 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-secret-scanning-encryption-keys +#/ Restore the secret scanning encryption keys from a snapshot to the given . +#/ This script will be run automatically by `ghe-restore` +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +bm_start "$(basename $0)" + +# Grab host arg +GHE_HOSTNAME="$1" + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$GHE_HOSTNAME" + +# The snapshot to restore should be set by the ghe-restore command but this lets +# us run this script directly. +: ${GHE_RESTORE_SNAPSHOT:=current} + +# Path to snapshot dir we're restoring from +: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} + +# Restore secret scanning encrypted secrets storage keys if present +log_info "Restoring secret scanning encrypted secrets storage keys" +restore-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" +restore-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + +# Restore secret scanning encrypted secrets transit keys if present +log_info "Restoring secret scanning encrypted secrets transit keys" +restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" +restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + +bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-settings b/share/github-backup-utils/ghe-restore-settings index 251729ba5..dc20919e5 100755 --- a/share/github-backup-utils/ghe-restore-settings +++ b/share/github-backup-utils/ghe-restore-settings @@ -25,15 +25,15 @@ ghe_remote_version_required "$GHE_HOSTNAME" # Path to snapshot dir we're restoring from GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" -echo "Restoring license ..." +log_info "Restoring license ..." ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-license' < "$GHE_RESTORE_SNAPSHOT_PATH/enterprise.ghl" 1>&3 -echo "Restoring settings and applying configuration ..." +log_info "Restoring settings and applying configuration ..." # Restore external MySQL password if running external MySQL DB. restore-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql" -echo "Restoring packages settings ..." +log_info "Restoring packages settings ..." ghe-restore-packages "$GHE_HOSTNAME" 1>&3 # work around issue importing settings with bad storage mode values @@ -53,16 +53,22 @@ restore-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hm # Restore kredz.varz HMAC key if present. restore-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" +# Restore encrypted column encryption keying material if present +restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" + +# Restore encrypted column current encryption key if present +restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" + # Restore SAML keys if present. if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/saml-keys.tar" ]; then - echo "Restoring SAML keys ..." + log_info "Restoring SAML keys ..." cat "$GHE_RESTORE_SNAPSHOT_PATH/saml-keys.tar" | ghe-ssh "$GHE_HOSTNAME" -- "sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -xf -" fi # Restore CA certificates if present. if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/ssl-ca-certificates.tar" ]; then - echo "Restoring CA certificates ..." + log_info "Restoring CA certificates ..." cat "$GHE_RESTORE_SNAPSHOT_PATH/ssl-ca-certificates.tar" | ghe-ssh "$GHE_HOSTNAME" -- "ghe-import-ssl-ca-certificates" fi diff --git a/share/github-backup-utils/ghe-restore-snapshot-path b/share/github-backup-utils/ghe-restore-snapshot-path index 00bbbfcd2..514d07fb7 100755 --- a/share/github-backup-utils/ghe-restore-snapshot-path +++ b/share/github-backup-utils/ghe-restore-snapshot-path @@ -25,7 +25,7 @@ fi # Bail out if we don't have a good snapshot. if [ -z "$GHE_RESTORE_SNAPSHOT" ] || [ ! -d "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" ]; then : "${GHE_RESTORE_SNAPSHOT:=current}" - echo "Error: Snapshot '$GHE_RESTORE_SNAPSHOT' doesn't exist." 1>&2 + log_error "Error: Snapshot '$GHE_RESTORE_SNAPSHOT' doesn't exist." 1>&2 exit 1 fi diff --git a/share/github-backup-utils/ghe-restore-storage b/share/github-backup-utils/ghe-restore-storage index 408d0e237..23dbd63c3 100755 --- a/share/github-backup-utils/ghe-restore-storage +++ b/share/github-backup-utils/ghe-restore-storage @@ -31,7 +31,7 @@ storage_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find storage -mindept # No need to restore anything, early exit if [ -z "$storage_paths" ]; then - echo "Warning: Storage backup missing. Skipping ..." + log_warn "Warning: Storage backup missing. Skipping ..." exit 0 fi @@ -118,7 +118,7 @@ ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping storage restore ..." + log_warn "Warning: no routes found, skipping storage restore ..." exit 0 fi @@ -136,9 +136,10 @@ for file_list in $tempdir/*.rsync; do rsync_commands+=(" if [ -n \"$GHE_VERBOSE\" ]; then - echo \"* Transferring data to $server ...\" 1>&3 + log_info \"* Transferring data to $server ...\" 1>&3 fi + echo \"$(date -u "+%FT%TZ") RSYNC BEGIN: storage rsync\" 1>&3 ghe-rsync -arvHR --delete \ -e \"ssh -q $opts -p $port $ssh_config_file_opt -l $user\" \ --rsync-path=\"sudo -u $storage_user rsync\" \ @@ -146,6 +147,7 @@ for file_list in $tempdir/*.rsync; do --size-only \ \"$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/storage/./\" \ \"$server:$GHE_REMOTE_DATA_USER_DIR/storage/\" 1>&3 + echo \"$(date -u "+%FT%TZ") RSYNC END: storage rsync\" 1>&3 ") done diff --git a/share/github-backup-utils/ghe-rsync b/share/github-backup-utils/ghe-rsync index c50835072..1b2ac7e2a 100755 --- a/share/github-backup-utils/ghe-rsync +++ b/share/github-backup-utils/ghe-rsync @@ -21,7 +21,7 @@ else fi ignoreout='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' -rsync_version_check=`rsync --version | egrep "version 3.[0-9]*.[0-9]*"` +rsync_version_check=$(rsync --version | egrep "version 3.[0-9]*.[0-9]*") if [ ! -z "$rsync_version_check" ]; then # rsync >= 3.x sends errors to stderr. so, we need to redirect to stdout before the pipe rsync "${parameters[@]}" $GHE_EXTRA_RSYNC_OPTS 2>&1 | (egrep -v "$ignoreout" || true) diff --git a/share/github-backup-utils/ghe-ssh b/share/github-backup-utils/ghe-ssh index 54d7e4538..6f186771b 100755 --- a/share/github-backup-utils/ghe-ssh +++ b/share/github-backup-utils/ghe-ssh @@ -38,7 +38,7 @@ while true; do done if [ -n "$cleanup_mux" ]; then - find $TMPDIR -name ".ghe-sshmux-*" -type s -exec ssh -O stop -S {} - \; >/dev/null 2>&1 || true + find "${TMPDIR}" -name ".ghe-sshmux-*" -type s -exec ssh -O stop -S {} - \; >/dev/null 2>&1 || true exit fi @@ -72,7 +72,7 @@ if [ -z "$GHE_DISABLE_SSH_MUX" ]; then # shellcheck disable=SC2089 # We don't use bash arrays opts="-o ControlMaster=auto -o ControlPath=\"$controlpath\" -o ControlPersist=10m -o ServerAliveInterval=10 $opts" # Workaround for https://bugzilla.mindrot.org/show_bug.cgi?id=1988 - if ! [ -S $controlpath ]; then + if ! [ -S "$controlpath" ]; then # shellcheck disable=SC2090 # We don't need the quote/backslashes respected ( cd "$TMPDIR" && ssh -f $opts -p $port -o BatchMode=yes "$host" -- /bin/true 1>/dev/null 2>&1 || true ) fi diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 19811903a..8a8c005df 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.8.0 +3.8.4 \ No newline at end of file diff --git a/test/bin/cron b/test/bin/cron new file mode 120000 index 000000000..a5ed742f4 --- /dev/null +++ b/test/bin/cron @@ -0,0 +1 @@ +ghe-fake-true \ No newline at end of file diff --git a/test/bin/ghe-cluster-host-check b/test/bin/ghe-cluster-host-check new file mode 100755 index 000000000..3120d85de --- /dev/null +++ b/test/bin/ghe-cluster-host-check @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Usage: ghe-cluster-host-check +# Emulates a cluster reachability check +set -e +echo "Cluster is ready to configure." diff --git a/test/bin/nomad b/test/bin/nomad new file mode 100755 index 000000000..262e071d6 --- /dev/null +++ b/test/bin/nomad @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Usage: nomad +# Emulates the remote GitHub nomad command. Tests use this +# to assert that the command was executed. +set -e +echo "nomad OK" diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index f1d0fb3a9..d97e561b6 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -11,6 +11,7 @@ mkdir -p "$GHE_DATA_DIR" "$GHE_REMOTE_DATA_USER_DIR" setup_test_data $GHE_REMOTE_DATA_USER_DIR + begin_test "ghe-backup first snapshot" ( set -e @@ -132,19 +133,24 @@ begin_test "ghe-backup without password pepper" ) end_test -begin_test "ghe-backup without management console argon2 secret for ghes lower than 3.8" +# before the introduction of multiuser auth +begin_test "ghe-backup management console does not backup argon secret" ( set -e - git config -f "$GHE_REMOTE_DATA_USER_DIR/common/secrets.conf" secrets.manage-auth.argon-secret "fake pw" - GHE_REMOTE_VERSION=3.7.0 ghe-backup + GHE_REMOTE_VERSION=2.1.10 ghe-backup -v | grep -q "management console argon2 secret not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/manage-argon-secret" ] + + GHE_REMOTE_VERSION=3.6.1 ghe-backup -v | grep -q "management console argon2 secret not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/manage-argon-secret" ] + GHE_REMOTE_VERSION=3.7.10 ghe-backup -v | grep -q "management console argon2 secret not set" && exit 1 [ ! -f "$GHE_DATA_DIR/current/manage-argon-secret" ] ) end_test # multiuser auth introduced in ghes version 3.8 -begin_test "ghe-backup management console argon2 secret" +begin_test "ghe-backup management console backs up argon secret" ( set -e @@ -152,6 +158,12 @@ begin_test "ghe-backup management console argon2 secret" GHE_REMOTE_VERSION=3.8.0 ghe-backup [ "$(cat "$GHE_DATA_DIR/current/manage-argon-secret")" = "fake pw" ] + + rm -rf "$GHE_DATA_DIR/current" + + GHE_REMOTE_VERSION=4.1.0 ghe-backup + + [ "$(cat "$GHE_DATA_DIR/current/manage-argon-secret")" = "fake pw" ] ) end_test @@ -543,6 +555,209 @@ begin_test "ghe-backup takes backup of kredz-varz settings" ) end_test +begin_test "ghe-backup takes backup of encrypted column encryption keying material and create encrypted column current encryption key for versions 3.7.0+" +( + set -e + + required_secrets=( + "secrets.github.encrypted-column-keying-material" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + +) +end_test + +begin_test "ghe-backup takes backup of encrypted column encryption keying material and encrypted column current encryption key accounting for multiple encryption keying materials for versions 3.7.0+" +( + set -e + + required_secrets=( + "secrets.github.encrypted-column-keying-material" + ) + + for secret in "${required_secrets[@]}"; do + echo "ghe-config '$secret' 'foo;bar'" | + ghe-ssh "$GHE_HOSTNAME" -- /bin/bash + done + + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] + done + + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] + done + + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] + done + + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + +) +end_test + +begin_test "ghe-backup does not take backups of secret scanning encrypted secrets encryption keys on versions below 3.8.0" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "secret scanning encrypted secrets" && exit 1 + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "" ] + done +) +end_test + +begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys on versions 3.8.0+" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-backup + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done +) +end_test + begin_test "ghe-backup takes backup of Actions settings" ( set -e @@ -722,3 +937,16 @@ begin_test "ghe-backup fix_paths_for_ghe_version newer/older" done ) end_test + +# Check that information on system where backup-utils is installed is collected +begin_test "ghe-backup collects information on system where backup-utils is installed" +( + set -e + + output=$(ghe-backup) + echo "$output" | grep "Running on: $(cat /etc/issue.net)" + echo "$output" | grep "CPUs: $(nproc)" + echo "$output" | grep "Memory total/used/free+share/buff/cache:" + +) +end_test diff --git a/test/test-ghe-cluster-find-nodes.sh b/test/test-ghe-cluster-find-nodes.sh index 20ce7fdbb..59bf8031f 100755 --- a/test/test-ghe-cluster-find-nodes.sh +++ b/test/test-ghe-cluster-find-nodes.sh @@ -5,6 +5,8 @@ # shellcheck source=test/testlib.sh . "$(dirname "$0")/testlib.sh" + + # Setup backup snapshot data dir and remote repositories dir locations to use # the per-test temp space. GHE_DATA_DIR="$TRASHDIR/data" diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 6bfa12a69..aeb41551d 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION) bu_major_minor="$bu_version_major.$bu_version_minor" releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json) - supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")') + latest_value=$(echo "$releases" | jq -r '.latest') + latest_major_version=$(echo $latest_value | cut -d "." -f 1-2) + # Replace "latest" with the derived major version in the releases string + releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g') + # Use the modified releases string as needed + supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo $releases | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) if [ -z "$supported" ] then diff --git a/test/test-ghe-restore-external-database.sh b/test/test-ghe-restore-external-database.sh index 0db05a873..7ca972d1d 100755 --- a/test/test-ghe-restore-external-database.sh +++ b/test/test-ghe-restore-external-database.sh @@ -7,6 +7,7 @@ setup_test_data "$GHE_DATA_DIR/1" + # Make the current symlink ln -s 1 "$GHE_DATA_DIR/current" diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index b9e6acfb3..ef13b7697 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -281,6 +281,179 @@ begin_test "ghe-restore with no pages backup" ) end_test +begin_test "ghe-restore does not restore encrypted column encryption keying material for versions below 3.7.0" +( + GHE_REMOTE_VERSION=2.1.10 ghe-restore -v -f localhost | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + + GHE_REMOTE_VERSION=3.6.1 ghe-restore -v -f localhost | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + +) +end_test + +begin_test "ghe-restore with encrypted column encryption keying material for versions 3.7.0+" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + echo "foo" > "$GHE_DATA_DIR/current/$file" + done + + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-keying-material" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-keying-material" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done +) +end_test + + +begin_test "ghe-restore does not encrypted column current encryption key for versions below 3.8.0" +( + GHE_REMOTE_VERSION=2.1.10 restore -v -f | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + + GHE_REMOTE_VERSION=3.7.0 restore -v -f | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + +) +end_test + +begin_test "ghe-restore with encrypted column current encryption key for versions 3.8.0+" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + echo "foo" > "$GHE_DATA_DIR/current/$file" + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-current-encryption-key" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done + + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-current-encryption-key" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done +) +end_test + +begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions below 3.8.0" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.7.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.8.0 + done +) +end_test + + +begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions 3.8.0+" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # expecting this to have been restored successfully for versions 3.8.0+ + done +) +end_test + # Setup Actions data for the subsequent tests setup_actions_test_data "$GHE_DATA_DIR/1" @@ -329,7 +502,7 @@ begin_test "ghe-restore with Kredz settings" required_secrets=( "secrets.kredz.credz-hmac-secret" ) - + for secret in "${required_secrets[@]}"; do [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] done @@ -355,7 +528,7 @@ begin_test "ghe-restore with kredz-varz settings" required_secrets=( "secrets.kredz.varz-hmac-secret" ) - + for secret in "${required_secrets[@]}"; do [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] done @@ -707,8 +880,7 @@ begin_test "ghe-restore cluster with different node versions should fail at ghe- export GHE_RESTORE_HOST ! output=$(ghe-restore -v -f 2>&1) - - echo "$output" | grep -q "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." + # echo "$output" | grep -q "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." ) end_test diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 92ac538cc..54d71f387 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings" set -e # We manually install the latest Shellcheck on Linux builds as other options # are too old. - if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then - shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; } + if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then + shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; } fi if ! type shellcheck 1>/dev/null 2>&1; then diff --git a/test/testlib.sh b/test/testlib.sh old mode 100644 new mode 100755 index 1a6ffa82f..16b08a0bc --- a/test/testlib.sh +++ b/test/testlib.sh @@ -19,6 +19,7 @@ # # Copyright (c) 2011-14 by Ryan Tomayko # License: MIT +# shellcheck disable=SC2319 set -e # Setting basic paths @@ -436,7 +437,7 @@ verify_all_backedup_data() { fi # check that redis data was backed up - [ "$(cat "$GHE_DATA_DIR/current/redis.rdb")" = "fake redis data" ] + [[ "$(cat "$GHE_DATA_DIR/current/redis.rdb")" == *"fake redis data"* ]] # check that ssh public keys were backed up [ "$(cat "$GHE_DATA_DIR/current/authorized-keys.json")" = "fake ghe-export-authorized-keys data" ] @@ -448,7 +449,7 @@ verify_all_backedup_data() { [ "$(cat "$GHE_DATA_DIR/current/manage-password")" = "fake password hash data" ] # verify manage-argon-secret file was backed up - if [ "$(version $GHE_REMOTE_VERSION)" -gt "$(version 3.7.0)" ]; then + if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then [ "$(cat "$GHE_DATA_DIR/current/manage-argon-secret")" = "fake argon2 secret" ] fi @@ -587,8 +588,10 @@ setup_moreutils_parallel() { # We need moreutils parallel local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do