From 9fa6423b568a9fd9cc4ea6173660f4ab922733ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Thu, 10 Aug 2023 15:34:41 +0000 Subject: [PATCH 01/48] Remove duplicate definitions in ghe-backup-config (#480) While investigating an unrelated issue, I noticed that some definitions in ghe-backup-config are present twice. The reason for this appears to be a faulty merge conflict resolution [1], as both parent commits [2, 3] only have a single copy of these definitions but the merge commit has duplicates. It seems that an unwieldy conflict came up while merging the progress-indicator with the master branch, as a result of which both the old and new locations of these definitions were accidentally kept, causing the duplication. This removes one copy of each definition to avoid confusion and potential future bugs. [1] 1eaf80941c84c21f148ea68783415a920d32ce15 [2] a8e7eca745a738c4c02cba4ce85f51bfb97bfe6b [3] 8aaccc3c8b6599d591239d22cb789572df05465a --- share/github-backup-utils/ghe-backup-config | 38 +-------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index b61f5a8c0..2e176b510 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -35,7 +35,7 @@ if [ -n "$GHE_SHOW_VERSION" ]; then fi # Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage -# shellcheck disable=SC2120 # the script name is always referenced +# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's print_usage() { grep '^#/' <"$0" | cut -c 4- exit "${1:-1}" @@ -51,10 +51,6 @@ else done fi -# Add the bin and share/github-backup-utils dirs to PATH -PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" -# shellcheck source=share/github-backup-utils/bm.sh -. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" # Save off GHE_HOSTNAME from the environment since we want it to override the # backup.config value when set. GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" @@ -150,35 +146,6 @@ log_ssh(){ log_level "ssh" "$1" } -# Assume this script lives in share/github-backup-utils/ when setting the root -GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -# Get the version from the version file. -BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")" - -# If a version check was requested, show the current version and exit -if [ -n "$GHE_SHOW_VERSION" ]; then - echo "GitHub backup-utils v$BACKUP_UTILS_VERSION" - exit 0 -fi - -# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage -# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's -print_usage() { - grep '^#/' <"$0" | cut -c 4- - exit "${1:-1}" -} - -if [ -n "$GHE_SHOW_HELP" ]; then - print_usage -else - for a in "$@"; do - if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then - print_usage - fi - done -fi - # Add the bin and share/github-backup-utils dirs to PATH PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" # shellcheck source=share/github-backup-utils/bm.sh @@ -187,9 +154,6 @@ PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" . "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-incremental-backup-restore" # shellcheck source=share/github-backup-utils/track-progress . "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress" -# Save off GHE_HOSTNAME from the environment since we want it to override the -# backup.config value when set. -GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" ghe_restore_check() { From ccfe3155aedd582608e81c1e351eedbf46c76e42 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Fri, 11 Aug 2023 02:33:46 +0000 Subject: [PATCH 02/48] Merge pull request #486 from github/fix-es-backup-failures Remove "exit on error" mode from helper script track-progress --- share/github-backup-utils/track-progress | 1 - test/test-shellcheck.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index bf50c42aa..d88b3704f 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,6 +1,5 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -set -e # Current version is working solely with backups progress(){ diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 92ac538cc..231d78edb 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -49,7 +49,7 @@ begin_test "shellopts: set -e set on all scripts" # Check all executable scripts checked into the repo, except bm.sh, ghe-backup-config, ghe-rsync and the dummy test scripts set +x cd $BASE_PATH - git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do + git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|track-progress|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do if head -n1 "$script" | grep -E -w "sh|bash" >/dev/null 2>&1; then grep -q "set -e" $script || echo $script >> $results || true fi From b7be13e0490c582ad231f1e7b1b7427462840934 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:34:27 +0000 Subject: [PATCH 03/48] Merge pull request #499 from github/shellcheck-stable Change shellcheck to use stable, not latest --- .github/workflows/main.yml | 6 +++--- test/test-shellcheck.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 809e9df50..da7695ad1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,9 +17,9 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man - wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz" - tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz" - sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck + wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" + tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" + sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck if: matrix.os != 'macos-latest' - name: Install Dependencies (macOS) run: | diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 231d78edb..82c0fbb4f 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings" set -e # We manually install the latest Shellcheck on Linux builds as other options # are too old. - if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then - shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; } + if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then + shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; } fi if ! type shellcheck 1>/dev/null 2>&1; then From bd113a7e5c5c9fb4f8c830e1d0f2109a2ae7dc29 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Thu, 17 Aug 2023 08:33:00 +0000 Subject: [PATCH 04/48] transfer_size are estimated data transfer sizes (#488) * Update bin/ghe-host-check --- bin/ghe-host-check | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 9f6ca2603..1c3252645 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -184,7 +184,8 @@ SKIP_MSG echo " - Recommended Disk requirement is $recommended_disk_req MB" 1>&2 echo "" 1>&2 - printf '### Data Transfer Sizes + printf '### Estimated Data Transfer Sizes + - repositories: %d MB - pages: %d MB - elasticsearch: %d MB @@ -192,7 +193,8 @@ SKIP_MSG - minio: %d MB - mysql: %d MB - actions: %d MB - - mssql: %d MB\n' \ + - mssql: %d MB +\n' \ "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2 if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then From 616181687b7132de176cafa28432ecd875c58b81 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Thu, 17 Aug 2023 15:34:32 +0000 Subject: [PATCH 05/48] Adding host online check (#492) * adding host online check * fix test --------- Co-authored-by: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> --- bin/ghe-host-check | 12 +++++++++--- test/bin/ghe-cluster-host-check | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100755 test/bin/ghe-cluster-host-check diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 9f6ca2603..96adff4e2 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -91,13 +91,19 @@ if ghe-ssh "$host" -- \ CLUSTER=true fi -# ensure all nodes in the cluster are running the same version +# ensure all nodes in the cluster are online/reachable and running the same version if "$CLUSTER"; then + online_status=$(ghe-ssh "$host" ghe-cluster-host-check) + if [ "$online_status" != "Cluster is ready to configure." ]; then + echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 + exit 1 + fi + node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version) distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) if [ "$distinct_versions" -ne 1 ]; then - echo "$node_version_list" 1>&2 - echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&3 + echo "Version mismatch: $node_version_list" 1>&2 + echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 exit 1 fi fi diff --git a/test/bin/ghe-cluster-host-check b/test/bin/ghe-cluster-host-check new file mode 100755 index 000000000..3120d85de --- /dev/null +++ b/test/bin/ghe-cluster-host-check @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Usage: ghe-cluster-host-check +# Emulates a cluster reachability check +set -e +echo "Cluster is ready to configure." From 703b899ee0a9a5673d7600bf906c845f2d717aaf Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Thu, 17 Aug 2023 22:04:44 +0000 Subject: [PATCH 06/48] Host key fix (#513) --- bin/ghe-backup | 66 ++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 2043e2fad..86ee33dd7 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -54,38 +54,7 @@ export CALLING_SCRIPT="ghe-backup" # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" -# Setup progress tracking -init-progress -export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total -export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type -export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress - -OPTIONAL_STEPS=0 -# Backup actions+mssql -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) -fi - -# Backup fsck -if [ "$GHE_BACKUP_FSCK" = "yes" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -# Backup minio -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi -# Backup pages -if [ "$GHE_BACKUP_PAGES" != "no" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -186,9 +155,44 @@ fi # Perform a host connection check and establish the remote appliance version. # The version is available in the GHE_REMOTE_VERSION variable and also written # to a version file in the snapshot directory itself. +# ghe_remote_version_required should be run before any other instances of ghe-ssh +# to ensure that there are no problems with host key verification. ghe_remote_version_required echo "$GHE_REMOTE_VERSION" > version +# Setup progress tracking +init-progress +export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +export PROGRESS_TYPE="Backup" +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +export PROGRESS=0 # Used to track progress of backup +echo "$PROGRESS" > /tmp/backup-utils-progress + +OPTIONAL_STEPS=0 +# Backup actions+mssql +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) +fi + +# Backup fsck +if [ "$GHE_BACKUP_FSCK" = "yes" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup minio +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup pages +if [ "$GHE_BACKUP_PAGES" != "no" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + # check that incremental settings are valid if set is_inc=$(is_incremental_backup_feature_on) From 64486dff33191c57fd85b6dd9dfe2a6c9784e496 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Mon, 21 Aug 2023 16:05:10 +0000 Subject: [PATCH 07/48] Merge pull request #490 from github/djj/typo-fix Update docs/requirements.md to fix broken link --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index f3f7d67f6..1320eb583 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](#april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. From a658a3d4aa4af36561c50526ac731372c5000ff0 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Thu, 24 Aug 2023 15:05:50 +0000 Subject: [PATCH 08/48] Merge pull request #528 from github/suppress-secret-scanning-encrypted-secrets-warning Only backup secret scanning secrets on GHES versions 3.8.0+ --- share/github-backup-utils/ghe-backup-settings | 11 +++--- test/test-ghe-backup.sh | 34 +++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 3a05feba7..aac02dde7 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -86,10 +86,13 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" fi -backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" -backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" -backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" -backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +# secret scanning encrypted secrets keys were added in GHES 3.8.0 +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" + backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +fi # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 745538264..fb3f9db6a 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -772,7 +772,7 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ) end_test -begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys" +begin_test "ghe-backup does not take backups of secret scanning encrypted secrets encryption keys on versions below 3.8.0" ( set -e @@ -787,7 +787,37 @@ begin_test "ghe-backup takes backup of secret scanning encrypted secrets encrypt ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" done - ghe-backup + GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "secret scanning encrypted secrets" && exit 1 + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "" ] + done +) +end_test + +begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys on versions 3.8.0+" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-backup required_files=( "secret-scanning-encrypted-secrets-current-storage-key" From f85535eb055f67c7dfd32f8cf36d560038659b2b Mon Sep 17 00:00:00 2001 From: bonsohi Date: Wed, 30 Aug 2023 17:39:31 +0000 Subject: [PATCH 09/48] Bump version: 3.10.0 [ci skip] --- debian/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/debian/changelog b/debian/changelog index f5caddb85..5f223ce8d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,8 @@ +github-backup-utils (3.10.0) UNRELEASED; urgency=medium + + + -- Balwinder Sohi Wed, 30 Aug 2023 17:39:31 +0000 + github-backup-utils (3.10.0) UNRELEASED; urgency=medium * Remove -o option from ps use #341 From 7cd059bb2222b83dfbffe9efb7c4754b4a0ce29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Thu, 31 Aug 2023 21:34:50 +0000 Subject: [PATCH 10/48] Merge pull request #527 from github/pluehne/find-parallel-in-more-locations Find parallel in more locations --- share/github-backup-utils/ghe-backup-config | 2 ++ test/testlib.sh | 2 ++ 2 files changed, 4 insertions(+) mode change 100644 => 100755 test/testlib.sh diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 2e176b510..5b7f1c3d3 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -217,8 +217,10 @@ ghe_parallel_check() { GHE_PARALLEL_COMMAND="parallel" local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do diff --git a/test/testlib.sh b/test/testlib.sh old mode 100644 new mode 100755 index 5dd5ac451..8e130eb75 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -600,8 +600,10 @@ setup_moreutils_parallel() { # We need moreutils parallel local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do From 65647be04550d5288a8aa216657e9e667aef44c9 Mon Sep 17 00:00:00 2001 From: Dax Amin Date: Thu, 31 Aug 2023 22:34:45 +0000 Subject: [PATCH 11/48] Merge pull request #534 from github/pluehne/allow-manually-triggering-testing-workflow Allow manually triggering testing workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da7695ad1..3e4445915 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,6 @@ name: Test and build -on: [pull_request] +on: [pull_request, workflow_dispatch] jobs: build: From 99ba2b7761d3586078a5a3bec191922d30798861 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Wed, 6 Sep 2023 22:05:33 +0000 Subject: [PATCH 12/48] Merge pull request #557 from github/hao/unit-test-fix Run backup-utils unit tests in serial --- .github/workflows/main.yml | 10 ---------- script/cibuild | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e4445915..50583737b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,9 +6,6 @@ jobs: build: strategy: matrix: - # macos-latest references are kept here for historical purposes. removed macos-latest from the - #matrix as it is not a typical case for users and causes a lot of friction with other linux-based - # installs. Recommend developing on codespaces or using an ubuntu container. os: ['ubuntu-22.04', 'ubuntu-20.04'] fail-fast: false runs-on: ${{ matrix.os }} @@ -20,12 +17,6 @@ jobs: wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck - if: matrix.os != 'macos-latest' - - name: Install Dependencies (macOS) - run: | - brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget - brew install moreutils gawk - if: matrix.os == 'macos-latest' - name: Get Sources uses: actions/checkout@v3 - name: Test @@ -35,4 +26,3 @@ jobs: shell: bash - name: Build (Linux) run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc - if: matrix.os != 'macos-latest' diff --git a/script/cibuild b/script/cibuild index 7f57f444d..73dc9ae0a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -5,7 +5,7 @@ set -e # Enable verbose logging of ssh commands export GHE_VERBOSE_SSH=true -if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then +if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then exit 1 fi From 8ae0e258bdc1d45a5c9554701772deb8ae24a743 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:25:51 +0000 Subject: [PATCH 13/48] Merge pull request #545 from github/fix-ghe-unsupported-version-test Fix GHE unsupported version check --- test/test-ghe-host-check.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 6bfa12a69..aeb41551d 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION) bu_major_minor="$bu_version_major.$bu_version_minor" releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json) - supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")') + latest_value=$(echo "$releases" | jq -r '.latest') + latest_major_version=$(echo $latest_value | cut -d "." -f 1-2) + # Replace "latest" with the derived major version in the releases string + releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g') + # Use the modified releases string as needed + supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo $releases | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) if [ -z "$supported" ] then From 248c1ae162393eed72395f9eda27618c6bc9ced2 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Tue, 12 Sep 2023 09:05:18 +0000 Subject: [PATCH 14/48] add post backup cleanup and move progress to folder (#558) --- bin/ghe-backup | 8 ++++---- bin/ghe-backup-progress | 8 ++++---- bin/ghe-restore | 6 +++--- share/github-backup-utils/ghe-backup-config | 13 +++++++++++-- share/github-backup-utils/track-progress | 11 +++++------ test/test-ghe-backup.sh | 2 ++ test/testlib.sh | 6 ++++++ 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 86ee33dd7..d8074156e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -163,11 +163,11 @@ echo "$GHE_REMOTE_VERSION" > version # Setup progress tracking init-progress export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress OPTIONAL_STEPS=0 # Backup actions+mssql @@ -191,7 +191,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then fi PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total # check that incremental settings are valid if set is_inc=$(is_incremental_backup_feature_on) diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress index 2f4b267fb..7ab36e084 100755 --- a/bin/ghe-backup-progress +++ b/bin/ghe-backup-progress @@ -31,7 +31,7 @@ while true; do done check_for_progress_file() { - if [ ! -f /tmp/backup-utils-progress-info ]; then + if [ ! -f /tmp/backup-utils-progress/info ]; then echo "No progress file found. Has a backup or restore been started?" exit 1 fi @@ -39,18 +39,18 @@ check_for_progress_file() { if [ -n "$ONCE" ]; then check_for_progress_file - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info else check_for_progress_file clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info while true; do if read -r -t 1 -n 1; then clear exit ; else clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info fi done fi diff --git a/bin/ghe-restore b/bin/ghe-restore index 92ae72443..7c9b95709 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -315,11 +315,11 @@ fi export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) init-progress -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Restore" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of restore -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 5b7f1c3d3..d3f8b05aa 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -652,12 +652,21 @@ restore-secret() { #initialize progress tracking by clearing out the temp files used to track init-progress() { - rm -f /tmp/backup-utils-progress* + if [ -d /tmp/backup-utils-progress ]; then + rm -rf /tmp/backup-utils-progress/* + else + mkdir /tmp/backup-utils-progress + fi + touch /tmp/backup-utils-progress/total + touch /tmp/backup-utils-progress/type + touch /tmp/backup-utils-progress/progress + touch /tmp/backup-utils-progress/info + chmod -R 777 /tmp/backup-utils-progress } #increase total count of progress increment-progress-total-count() { ((PROGRESS_TOTAL += $1)) - echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total } diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index d88b3704f..a560ba540 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,13 +1,12 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -# Current version is working solely with backups progress(){ - PROGRESS=$(cat /tmp/backup-utils-progress) - PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress-total) - PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) + PROGRESS=$(cat /tmp/backup-utils-progress/progress) + PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total) + PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type) PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) - echo $((PROGRESS + 1)) > /tmp/backup-utils-progress - echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info + echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress + echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info } diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index fb3f9db6a..3f3cd25ed 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -47,6 +47,8 @@ begin_test "ghe-backup subsequent snapshot" [ "$first_snapshot" != "$this_snapshot" ] verify_all_backedup_data + + verify_progress_cleanup_process ) end_test diff --git a/test/testlib.sh b/test/testlib.sh index 8e130eb75..5bd36d489 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -494,6 +494,12 @@ verify_all_backedup_data() { verify_common_data } +# A unified method to make sure post backup, the cleanup process works +verify_progress_cleanup_process() { + set -e + sudo -u nobody rm -rf /tmp/backup-utils-progress/* +} + # A unified method to check everything restored when performing a full restore # during testing. verify_all_restored_data() { From e448e956500742dbb07962793c5238e9d252b029 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:36:14 +0000 Subject: [PATCH 15/48] Merge pull request #569 from github/check-datadir-is-nfs Add NFS check for GHE_DATA_DIR --- bin/ghe-host-check | 7 +++++++ docs/requirements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 104885114..9e8d6466b 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/docs/requirements.md b/docs/requirements.md index 1320eb583..bba51c252 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,6 +57,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From f0c337019c7bb3543650b7d506e630e9cb8d5477 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Fri, 15 Sep 2023 16:36:26 +0000 Subject: [PATCH 16/48] fix backup.config typo (#587) Fixing https://github.com/github/ghes/issues/7329 --- backup.config-example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.config-example b/backup.config-example index 688cc374b..d1ea977e2 100644 --- a/backup.config-example +++ b/backup.config-example @@ -26,7 +26,7 @@ GHE_NUM_SNAPSHOTS=10 # performed before the next full backup is created. # For example, if `GHE_INCREMENTAL_BACKUP_MAX` is set to 14, backup-utils will # run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle. -#GHE_INCREMENTAL_BACKUP_MAX=14 +#GHE_INCREMENTAL_MAX_BACKUPS=14 # If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check # disk space validation and software version checks on the backup-host will be disabled. From fe080dc5263a45219916f3536cf8362bc9471e2d Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Tue, 26 Sep 2023 22:10:04 +0000 Subject: [PATCH 17/48] fix mkdir issues or progress tracking (#612) Fixing https://github.com/github/ghes/issues/7409 --- share/github-backup-utils/ghe-backup-config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index d3f8b05aa..bdf77cc86 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -654,14 +654,15 @@ restore-secret() { init-progress() { if [ -d /tmp/backup-utils-progress ]; then rm -rf /tmp/backup-utils-progress/* - else - mkdir /tmp/backup-utils-progress fi + + mkdir -p /tmp/backup-utils-progress + chmod -R 777 /tmp/backup-utils-progress + touch /tmp/backup-utils-progress/total touch /tmp/backup-utils-progress/type touch /tmp/backup-utils-progress/progress touch /tmp/backup-utils-progress/info - chmod -R 777 /tmp/backup-utils-progress } From 43f79427b484b35c25303af41de09eac0ab514b0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:21:35 -0500 Subject: [PATCH 18/48] Merge pull request #568 from github/543-workflow-build-release-part-2 Build and Release Workflow Part 2 --- .github/linters/.yaml-lint.yml | 53 +++++++++ .github/workflows/build-and-release.yml | 139 ++++++++++++++++++++++++ .github/workflows/lint.yml | 2 + .releaseignore | 2 + script/package-deb | 11 ++ script/package-tarball | 13 +++ 6 files changed, 220 insertions(+) create mode 100644 .github/linters/.yaml-lint.yml create mode 100644 .github/workflows/build-and-release.yml create mode 100644 .releaseignore diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..030c37f04 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,53 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable \ No newline at end of file diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 000000000..90b44086e --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,139 @@ +--- +name: Build and Release + +on: + workflow_dispatch: + inputs: + gh-token: + description: 'GitHub Token - used to create a commit in the backup-utils repo' + required: true + type: string + version: + description: 'Version - patch version of the release (e.g. x.y.z)' + required: true + type: string + draft: + description: 'Draft - true if the release should be a draft' + required: true + type: boolean + default: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils + ref: master + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: master + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "${{ github.actor }}" + commit_user_email: "ghes-releases-team@github.com" + commit_options: "--allow-empty" + skip_dirty_check: true + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ github.event.inputs.gh-token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ steps.empty-commit.outputs.commit_hash }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" + + + + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d15f123bb..8e878da40 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +--- name: Lint Code Base on: @@ -19,3 +20,4 @@ jobs: env: VALIDATE_ALL_CODEBASE: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILTER_REGEX_EXCLUDE: .*release-notes/.* diff --git a/.releaseignore b/.releaseignore new file mode 100644 index 000000000..bb2f6b31c --- /dev/null +++ b/.releaseignore @@ -0,0 +1,2 @@ +ownership.yaml +.github \ No newline at end of file diff --git a/script/package-deb b/script/package-deb index dda90cbe3..15a698e9c 100755 --- a/script/package-deb +++ b/script/package-deb @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" @@ -22,6 +25,14 @@ mkdir -p dist/debuild distdir="$(pwd)/dist/debuild/$PKG_NAME" git clone -q . "$distdir" cd "$distdir" + +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +echo "Removing .releaseignore ..." +rm -f .releaseignore git checkout -q "$PKG_HEAD" debuild -uc -us 1>&2 diff --git a/script/package-tarball b/script/package-tarball index be653b77a..bf1510e8f 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -8,11 +8,24 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" PKG_NAME="${PKG_BASE}-${PKG_VERS}" +# Remove all files or directories listed in .releaseignore +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +# Remove the .releaseignore file itself +echo "Removing .releaseignore ..." +rm -f .releaseignore + # Run git-archive to generate tarball echo "Creating ${PKG_NAME}.tar.gz ..." mkdir -p dist From f962e107ba45401b170c4b4bb7ccaa8aaa7b3e18 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 27 Sep 2023 08:51:06 -0400 Subject: [PATCH 19/48] Manual backport of #568 - resolved conflict From 4df64e100db8759f44519d34d4dbbb586153a64a Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 2 Oct 2023 11:37:39 -0700 Subject: [PATCH 20/48] Revert "Backport 569 for 3.10: Add NFS check for GHE_DATA_DIR" --- bin/ghe-host-check | 7 ------- docs/requirements.md | 2 -- 2 files changed, 9 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 9e8d6466b..104885114 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,13 +168,6 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" - #Check if GHE_DATA_DIR is NFS mounted - fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true - if [ "$fs_info" == "nfs" ]; then - echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 - echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 - fi - #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/docs/requirements.md b/docs/requirements.md index bba51c252..1320eb583 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,8 +57,6 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. -Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. - ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From a55a89da1cbb3256e87b12f3b52c9de649f71e9d Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:24:55 +0000 Subject: [PATCH 21/48] Merge pull request #569 from github/check-datadir-is-nfs Add NFS check for GHE_DATA_DIR --- bin/ghe-host-check | 7 +++++++ docs/requirements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 104885114..9e8d6466b 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/docs/requirements.md b/docs/requirements.md index 1320eb583..bba51c252 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,6 +57,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From dc37c05660152735c9fe3143cbe243bed9550121 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 2 Oct 2023 20:38:31 +0000 Subject: [PATCH 22/48] Bump version: 3.10.1 [ci skip] --- debian/changelog | 5 +++++ share/github-backup-utils/version | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 5f223ce8d..d670b30af 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,8 @@ +github-backup-utils (3.10.1) UNRELEASED; urgency=medium + + + -- Devin Dooley Mon, 02 Oct 2023 20:38:31 +0000 + github-backup-utils (3.10.0) UNRELEASED; urgency=medium diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 30291cba2..f870be23b 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.10.0 +3.10.1 From 8c5fe49fe6699d16dc5879a7df763ab8963864bb Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Mon, 9 Oct 2023 19:19:20 +0000 Subject: [PATCH 23/48] Merge pull request #579 from github/hao/integration-actions Add integration tests to backup-utils-private --- .github/actions/proxy-janky-build/action.yml | 33 ++++ .github/actions/proxy-janky-build/go.mod | 7 + .github/actions/proxy-janky-build/go.sum | 9 + .github/actions/proxy-janky-build/main.go | 180 +++++++++++++++++++ .github/workflows/integration-tests.yml | 41 +++++ 5 files changed, 270 insertions(+) create mode 100644 .github/actions/proxy-janky-build/action.yml create mode 100644 .github/actions/proxy-janky-build/go.mod create mode 100644 .github/actions/proxy-janky-build/go.sum create mode 100644 .github/actions/proxy-janky-build/main.go create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/actions/proxy-janky-build/action.yml b/.github/actions/proxy-janky-build/action.yml new file mode 100644 index 000000000..988c4a289 --- /dev/null +++ b/.github/actions/proxy-janky-build/action.yml @@ -0,0 +1,33 @@ +name: 'Trigger a CI Job on Janky' +description: 'Action to trigger and poll a Janky CI job' +inputs: + janky-token: + description: 'Token for making request to Janky' + required: true + job-name: + description: 'The name of the job to run' + required: true + branch-name: + description: 'The name of the branch to use' + required: true + force: + description: 'Force the job to run even if it is already passed' + required: false + envVars: + description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3' + required: false +runs: + using: 'composite' + steps: + - uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260 + with: + go-version: '1.21' + - run: | + go run main.go \ + -token ${{ inputs.janky-token }} \ + -job ${{ inputs.job-name }} \ + -branch ${{ inputs.branch-name }} \ + -force ${{ inputs.force }} \ + -envVars ${{ inputs.envVars }} + shell: bash + working-directory: .github/actions/proxy-janky-build diff --git a/.github/actions/proxy-janky-build/go.mod b/.github/actions/proxy-janky-build/go.mod new file mode 100644 index 000000000..b7560d7b9 --- /dev/null +++ b/.github/actions/proxy-janky-build/go.mod @@ -0,0 +1,7 @@ +module github.com/github/enterprise2/actions/proxy-janky-build + +go 1.21 + +require github.com/hashicorp/go-retryablehttp v0.7.2 + +require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/.github/actions/proxy-janky-build/go.sum b/.github/actions/proxy-janky-build/go.sum new file mode 100644 index 000000000..5c59c1d2e --- /dev/null +++ b/.github/actions/proxy-janky-build/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go new file mode 100644 index 000000000..e0342f121 --- /dev/null +++ b/.github/actions/proxy-janky-build/main.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +// Define our Janky Response Structs +type JankyBuildStruct struct { + Result string + Url string +} +type JankyStatusStruct struct { + Id string + Green bool + Completed bool + StartedAt string + CompletedAt string + Sha string + BuildableName string +} + +const ( + pollWaitTime = 10 * time.Second + jankyPollTimeout = 5 * time.Hour + jankyHttpRetryMax = 5 + jankyUrl = "https://janky.githubapp.com" +) + +func main() { + // Parse command-line arguments + job := flag.String("job", "", "Name of the Janky job") + token := flag.String("token", "", "Name of the Janky token") + branch := flag.String("branch", "", "Name of the Git branch") + force := flag.String("force", "false", "Force a build even if one is already passed") + envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") + flag.Parse() + + // Validate command-line arguments + if *job == "" || *token == "" || *branch == "" { + log.Fatal("job, token and branch flags must be specified") + } + + // Set up the token + request payload + authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token)) + type buildRequestObject struct { + BuildableName string `json:"buildable_name"` + BranchName string `json:"branch_name"` + Force string `json:"force"` + EnvVars map[string]string `json:"env_vars"` + } + + requestBody := buildRequestObject{ + BuildableName: *job, + BranchName: *branch, + Force: *force, + } + + // Parse the envVars flag into a map and add to the request payload + fmt.Println("Environment Variables:") + fmt.Println(*envVars) + if *envVars != "" { + envVarsMap := make(map[string]string) + for _, envVar := range strings.Split(*envVars, ",") { + envVarSplit := strings.Split(envVar, "=") + envVarsMap[envVarSplit[0]] = envVarSplit[1] + } + requestBody.EnvVars = envVarsMap + } + + payloadBytes, err := json.Marshal(requestBody) + if err != nil { + log.Fatal("Failed to marshal the JSON payload!\n" + err.Error()) + } + + // Send build request to Janky + buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes)) + if err != nil { + log.Fatal("Failed to create build request!\n" + err.Error()) + } + buildRequest.Header.Set("Content-Type", "application/json") + buildRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient := retryablehttp.NewClient() //nolint:all + retryClient.RetryMax = jankyHttpRetryMax + retryClient.Logger = nil // disable debug logging + client := retryClient.StandardClient() // uses *http.Client + resp, err := client.Do(buildRequest) + if err != nil { + log.Fatal("Failed to send build request!\n" + err.Error()) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatal("Error reading build response!\n" + err.Error()) + } + + // Check if the build was triggered successfully + if resp.StatusCode == 404 { + log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.") + } + if resp.StatusCode != 201 { + log.Fatal("Failed to trigger build! Got exception: " + string(body)) + } + + // Parse the build request response + var buildResponse JankyBuildStruct + json.Unmarshal(body, &buildResponse) + log.Println("Succesfully triggered janky!\n" + buildResponse.Result) + + // Parse the request response for the buildId + r, err := regexp.Compile("/[0-9]+/") + if err != nil { + log.Fatal("Failed to trigger build!\n" + err.Error()) + } + buildId := strings.Trim(r.FindString(buildResponse.Result), "/") + + // Setup our second HTTP client for reuse in during status polling + jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status" + statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil) + if err != nil { + log.Fatal("Failed to create status request!\n" + err.Error()) + } + statusRequest.Header.Set("Content-Type", "application/json") + statusRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient2 := retryablehttp.NewClient() //nolint:all + retryClient2.RetryMax = jankyHttpRetryMax + retryClient2.Logger = nil // disable debug logging + client2 := retryClient2.StandardClient() // uses *http.Client + + // Wait for a completed status from Janky or break the loop after a certain amount of time + timeout := time.NewTimer(jankyPollTimeout) + poll := time.NewTicker(pollWaitTime) + +jobLoop: + for { + select { + case <-timeout.C: + log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours") + case <-poll.C: + // Send build status request to Janky + statusResponse, err := client2.Do(statusRequest) + if err != nil { + log.Fatal("Failed to send status request!\n" + err.Error()) + } + defer statusResponse.Body.Close() + statusBody, err := io.ReadAll(statusResponse.Body) + if err != nil { + log.Fatal("Error reading status response!\n" + err.Error()) + } + + // Parse the status response for a green completed build + var jankyStatusResponse JankyStatusStruct + json.Unmarshal(statusBody, &jankyStatusResponse) + //fmt.Println("Janky Status Response:") + //fmt.Println(string(statusBody)) + if jankyStatusResponse.Completed && jankyStatusResponse.Green { + log.Println("Janky build Succeeded!") + break jobLoop + } + if jankyStatusResponse.Completed && !jankyStatusResponse.Green { + log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url) + } + + // wait for a bit and try again + log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]") + continue + } + } +} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..804c6d5c5 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,41 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'Branch that would be merged into' + required: true + source-branch: + description: 'Branch that would be merged' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.TARGET_BRANCH }}' + force : "true" + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }}" From d357bfbb64adf0d8c697d392c094b5ad31e1a378 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Wed, 20 Sep 2023 19:52:34 -0600 Subject: [PATCH 24/48] Manual backport of #597 - resolved conflict --- .github/workflows/integration-tests.yml | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..8727233ea --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,42 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'Branch that would be merged into' + required: true + source-branch: + description: 'Branch that would be merged' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' + force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" From db4c881f6bf46ea5c7a5ee6454f5950e84bc4140 Mon Sep 17 00:00:00 2001 From: David Daly Date: Thu, 12 Oct 2023 09:01:45 +0000 Subject: [PATCH 25/48] Merge pull request #459 from github/ddivad195/container-prefix-fix fix: backup and restore actions container prefix --- share/github-backup-utils/ghe-backup-settings | 1 + share/github-backup-utils/ghe-restore-actions | 8 ++ test/test-ghe-restore.sh | 88 +++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index aac02dde7..24e946007 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -122,6 +122,7 @@ if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" + backup-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 93596033a..303a2abd2 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -85,6 +85,14 @@ restore-secret "Actions Launch service private key" "actions-launch-app-app-priv restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" +# Restore storage container prefix, but only if it exists, and the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings +if [[ $RESTORE_SETTINGS == "true" ]]; then + if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + else + log_warn "Actions storage container prefix not present in backup. Skipping ..." + fi +fi # Setup the database logins. ghe_verbose "* Restoring database logins and users to $host ..." diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index ef13b7697..7ed345ee0 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -622,6 +622,94 @@ begin_test "ghe-restore with Actions settings" ) end_test +begin_test "ghe-restore with Actions settings passing -c" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + enable_actions + + required_files=( + "actions-config-db-login" + "actions-config-db-password" + "actions-framework-access-token" + "actions-url-signing-hmac-key-primary" + "actions-url-signing-hmac-key-secondary" + "actions-oauth-s2s-signing-cert" + "actions-oauth-s2s-signing-key" + "actions-oauth-s2s-signing-cert-thumbprint" + "actions-primary-encryption-cert-thumbprint" + "actions-aad-cert-thumbprint" + "actions-delegated-auth-cert-thumbprint" + "actions-runtime-service-principal-cert" + "actions-s2s-encryption-cert" + "actions-secondary-encryption-cert-thumbprint" + "actions-service-principal-cert" + "actions-sps-validation-cert-thumbprint" + "actions-storage-container-prefix" + + "actions-launch-secrets-private-key" + "actions-launch-deployer-hmac" + "actions-launch-client-id" + "actions-launch-client-secret" + "actions-launch-receiver-webhook-secret" + "actions-launch-app-private-key" + "actions-launch-app-public-key" + "actions-launch-app-id" + "actions-launch-app-relay-id" + "actions-launch-action-runner-secret" + "actions-launch-azp-app-cert" + "actions-launch-app-app-private-key" + + ) + + for file in "${required_files[@]}"; do + echo "foo" > "$GHE_DATA_DIR/current/$file" + done + + ghe-restore -v -f -c localhost + + required_secrets=( + "secrets.actions.ConfigurationDatabaseSqlLogin" + "secrets.actions.ConfigurationDatabaseSqlPassword" + "secrets.actions.FrameworkAccessTokenKeySecret" + "secrets.actions.UrlSigningHmacKeyPrimary" + "secrets.actions.UrlSigningHmacKeySecondary" + "secrets.actions.OAuthS2SSigningCert" + "secrets.actions.OAuthS2SSigningKey" + "secrets.actions.OAuthS2SSigningCertThumbprint" + "secrets.actions.PrimaryEncryptionCertificateThumbprint" + "secrets.actions.AADCertThumbprint" + "secrets.actions.DelegatedAuthCertThumbprint" + "secrets.actions.RuntimeServicePrincipalCertificate" + "secrets.actions.S2SEncryptionCertificate" + "secrets.actions.SecondaryEncryptionCertificateThumbprint" + "secrets.actions.ServicePrincipalCertificate" + "secrets.actions.SpsValidationCertThumbprint" + "secrets.actions.storage.container-prefix" + "secrets.launch.actions-secrets-private-key" + "secrets.launch.deployer-hmac-secret" + "secrets.launch.client-id" + "secrets.launch.client-secret" + "secrets.launch.receiver-webhook-secret" + "secrets.launch.app-private-key" + "secrets.launch.app-public-key" + "secrets.launch.app-id" + "secrets.launch.app-relay-id" + "secrets.launch.action-runner-secret" + "secrets.launch.token-oauth-key" + "secrets.launch.token-oauth-cert" + "secrets.launch.azp-app-cert" + "secrets.launch.azp-app-private-key" + + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done +) +end_test + begin_test "ghe-restore stops and starts Actions" ( set -e From d538ae8e0ecd3177e936f70a0a6cc6d62f44f805 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:18:24 +0000 Subject: [PATCH 26/48] Merge pull request #599 from github/hao/fix-issues-for-actions Fix issue with actions --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fa0d356cb..08c70e1da 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,8 +15,8 @@ on: # Get target and source branch from different variables depending on how it was triggered env: - TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' - SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' jobs: integration-tests: From 2ea39e5da94e117cbf133e46ab447f05f3fd01bf Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Mon, 25 Sep 2023 16:04:55 -0600 Subject: [PATCH 27/48] Manual backport of #602 - resolved conflict --- .github/pull_request_template.md | 40 +++++++++++++++++++++++++ .github/workflows/integration-tests.yml | 27 ++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ade3e5c58 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,40 @@ + + + + +# PR Details + +## Description + +## Testing + + + +## Ownership + + +## Related Links + diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 08c70e1da..057d5db82 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,6 +37,31 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.SOURCE_BRANCH }}' - force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" + + # Cluster integration tests are optional based on label and PR titles + cluster-integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-cluster-binary-backup + - enterprise2-backup-utils-cluster-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + if: | + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.title, '[cluster]') || + contains(github.event.pull_request.labels.*.name, 'cluster') + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" \ No newline at end of file From ff6c1a6ffe972609fc1708e12a6bbb5872b59e93 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Sat, 14 Oct 2023 01:49:18 +0000 Subject: [PATCH 28/48] Merge pull request #613 from github/jianghao0718-patch-4 Fix description for target and source branches for integration tests CI --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 057d5db82..d442f9555 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,10 +7,10 @@ on: workflow_dispatch: inputs: target-branch: - description: 'Branch that would be merged into' + description: 'enterprise2 branch to test against' required: true source-branch: - description: 'Branch that would be merged' + description: 'backup-utils-private topic branch to test' required: true # Get target and source branch from different variables depending on how it was triggered From 574a06304f248eda9a79fac4ad28d155b55b3985 Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:29:24 +0000 Subject: [PATCH 29/48] Merge pull request #663 from github/enterprise-3.10-backport-662-timreimherr/actions-46-multi-repo-token-scope Backport 662 for 3.10: Implement App Token --- .github/workflows/build-and-release.yml | 76 ++++++++++++------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 90b44086e..c9b6b66e8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -4,10 +4,6 @@ name: Build and Release on: workflow_dispatch: inputs: - gh-token: - description: 'GitHub Token - used to create a commit in the backup-utils repo' - required: true - type: string version: description: 'Version - patch version of the release (e.g. x.y.z)' required: true @@ -21,22 +17,20 @@ on: jobs: build: runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" - name: Checkout backup-utils-private uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils-private + token: ${{ steps.app-token.outputs.token }} - name: Install dependencies run: | sudo apt-get update -y @@ -79,35 +73,40 @@ jobs: outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" + - name: Get major-feature from version + id: get-major-feature + run: | + echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + - name: Verify major-feature + run: | + echo "major_feature: $MAJOR_FEATURE" - name: Checkout backup-utils uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - ref: master - name: Create empty commit uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: master + branch: ${{ env.MAJOR_FEATURE }}-stable commit_message: "${{ github.event.inputs.version }} release" - commit_user_name: "${{ github.actor }}" - commit_user_email: "ghes-releases-team@github.com" + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" commit_options: "--allow-empty" + push_options: "--force" skip_dirty_check: true - - name: Checkout backup-utils + - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 @@ -120,20 +119,17 @@ jobs: - name: Create Release uses: ncipollo/release-action@v1 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} + owner: github repo: backup-utils name: | GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} + commit: ${{ env.MAJOR_FEATURE }}-stable bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true - artifactContentType: "raw" - - - - + artifactContentType: "raw" \ No newline at end of file From 52e0621a841004ff067be724a0b3aff8aa8300a3 Mon Sep 17 00:00:00 2001 From: Hubot Date: Fri, 20 Oct 2023 14:49:13 -0400 Subject: [PATCH 30/48] Update debian/changelog and release-notes/3.10.2.md for 3.10.2 release --- debian/changelog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index d670b30af..f19d04f67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +github-backup-utils (3.10.2) UNRELEASED; urgency=medium + + * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} + +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Fri, 20 Oct 2023 18:49:12 +0000 + github-backup-utils (3.10.1) UNRELEASED; urgency=medium From e6cc2aefb266a52ae7abbf0a099f4dd1a0f96438 Mon Sep 17 00:00:00 2001 From: Hubot Date: Fri, 20 Oct 2023 14:49:14 -0400 Subject: [PATCH 31/48] Create 3.10.2 release notes --- release-notes/3.10.2.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 release-notes/3.10.2.md diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md new file mode 100644 index 000000000..ab5c0aabd --- /dev/null +++ b/release-notes/3.10.2.md @@ -0,0 +1,14 @@ +## Bug Fixes + +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} + +## Changes + +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} + +## Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + From 75f959c46d1f8ee6c4eb6593b993697704f7bf9f Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:09:58 +0000 Subject: [PATCH 32/48] Merge pull request #680 from github/enterprise-3.10-backport-679-timreimherr/build-and-release-edit Backport 679 for 3.10: Remove file rename step in build-and-release workflow --- .github/workflows/build-and-release.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c9b6b66e8..edf115738 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,14 +44,6 @@ jobs: - name: Package deb run: | ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - name: Upload deb artifact uses: actions/upload-artifact@v3 with: From 8bfa911afd70aed8963482743765154556655461 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Mon, 23 Oct 2023 16:04:06 +0000 Subject: [PATCH 33/48] 3.10.2 release From e522bb3c817bac38397167b47963c027023d5cb1 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Wed, 1 Nov 2023 17:03:56 +0000 Subject: [PATCH 34/48] 3.10.2 release From 91d3b9a44b22a79c32eacaeb51bbe48a4756f025 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 24 Oct 2023 15:27:19 +0000 Subject: [PATCH 35/48] remove comments --- release-notes/3.10.2.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md index ab5c0aabd..b673422e8 100644 --- a/release-notes/3.10.2.md +++ b/release-notes/3.10.2.md @@ -1,12 +1,12 @@ ## Bug Fixes -* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. ## Changes -* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. ## Backups and Disaster Recovery From 51fd17c21bbb5dcd61bd35c436ad3deda544858a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 24 Oct 2023 15:32:04 +0000 Subject: [PATCH 36/48] update changelog also --- debian/changelog | 12 +++++------- release-notes/3.10.2.md | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/debian/changelog b/debian/changelog index f19d04f67..e627da795 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,10 @@ github-backup-utils (3.10.2) UNRELEASED; urgency=medium - * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} - -* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} - -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. + * When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + * Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. + * When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. -- ghes-releases-team Fri, 20 Oct 2023 18:49:12 +0000 diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md index b673422e8..975217afe 100644 --- a/release-notes/3.10.2.md +++ b/release-notes/3.10.2.md @@ -10,5 +10,5 @@ ## Backups and Disaster Recovery -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. +* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. From 3cfa49324185b3ae69d120a26d548344947c8ad8 Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:12 -0400 Subject: [PATCH 37/48] Update debian/changelog, version file, and release-notes/3.10.2.md for 3.10.2 release --- debian/changelog | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/debian/changelog b/debian/changelog index e627da795..cefe37417 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,21 @@ +github-backup-utils (3.10.2) UNRELEASED; urgency=medium + + ### Bug Fixes + +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + +### Changes + +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. + +### Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Wed, 01 Nov 2023 14:28:11 +0000 + github-backup-utils (3.10.2) UNRELEASED; urgency=medium * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. From b9eeefac3520f6a000bdca30b34f171a79b3199d Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:12 -0400 Subject: [PATCH 38/48] Create 3.10.2 release notes --- release-notes/3.10.2.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md index 975217afe..d21a32097 100644 --- a/release-notes/3.10.2.md +++ b/release-notes/3.10.2.md @@ -1,14 +1,14 @@ -## Bug Fixes +### Bug Fixes -* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. -## Changes +### Changes * Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. -## Backups and Disaster Recovery +### Backups and Disaster Recovery -* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. From ce2af32fdf19aebf02944ba73ddb471395deee89 Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:13 -0400 Subject: [PATCH 39/48] Update version file to 3.10.2 --- share/github-backup-utils/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index f870be23b..e86ec35a8 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.10.1 +3.10.2 \ No newline at end of file From 4cd2f920b10eb5d1a3ea8b1eb4c33d3164f8632a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 1 Nov 2023 14:32:16 +0000 Subject: [PATCH 40/48] no need to update changelog --- debian/changelog | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/debian/changelog b/debian/changelog index cefe37417..e627da795 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,21 +1,3 @@ -github-backup-utils (3.10.2) UNRELEASED; urgency=medium - - ### Bug Fixes - -* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. - -### Changes - -* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. - -### Backups and Disaster Recovery - -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. - - -- ghes-releases-team Wed, 01 Nov 2023 14:28:11 +0000 - github-backup-utils (3.10.2) UNRELEASED; urgency=medium * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. From 3585d89f6c60b22249a044b42081e505338ce653 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Mon, 23 Oct 2023 16:04:06 +0000 Subject: [PATCH 41/48] 3.10.2 release From e179b2e93ca28b5af5382822c27d9ecc335d6763 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Wed, 1 Nov 2023 17:03:56 +0000 Subject: [PATCH 42/48] 3.10.2 release From 821c5cb820a58603ee5c6556aefe038e0b0ca35b Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Wed, 17 Jan 2024 20:09:59 +0000 Subject: [PATCH 43/48] 3.10.3 release From da132c8f94b03c08a3ec9d9b5739c58ec3ba06fc Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Fri, 19 Jan 2024 21:09:47 +0000 Subject: [PATCH 44/48] 3.10.3 release From b1ce42f20fb3395c4cc633cd80e40270b79f11f3 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Thu, 21 Mar 2024 16:48:02 +0000 Subject: [PATCH 45/48] 3.10.4 release From e09c5f9f093017ee60739315cb4db21be1399c08 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Fri, 19 Apr 2024 04:45:08 +0000 Subject: [PATCH 46/48] 3.10.5 release From bb907fba267e37c6c49dbb7c4eddbf1f79074a59 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Fri, 2 Aug 2024 20:25:33 +0000 Subject: [PATCH 47/48] 3.10.6 release From 16637daf745e00cb323f44698d3cd7baecffa8bf Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Tue, 24 Sep 2024 16:41:15 +0000 Subject: [PATCH 48/48] 3.10.7 release