From 7cd059bb2222b83dfbffe9efb7c4754b4a0ce29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Thu, 31 Aug 2023 21:34:50 +0000 Subject: [PATCH 01/29] Merge pull request #527 from github/pluehne/find-parallel-in-more-locations Find parallel in more locations --- share/github-backup-utils/ghe-backup-config | 2 ++ test/testlib.sh | 2 ++ 2 files changed, 4 insertions(+) mode change 100644 => 100755 test/testlib.sh diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 2e176b510..5b7f1c3d3 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -217,8 +217,10 @@ ghe_parallel_check() { GHE_PARALLEL_COMMAND="parallel" local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do diff --git a/test/testlib.sh b/test/testlib.sh old mode 100644 new mode 100755 index 5dd5ac451..8e130eb75 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -600,8 +600,10 @@ setup_moreutils_parallel() { # We need moreutils parallel local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do From 65647be04550d5288a8aa216657e9e667aef44c9 Mon Sep 17 00:00:00 2001 From: Dax Amin Date: Thu, 31 Aug 2023 22:34:45 +0000 Subject: [PATCH 02/29] Merge pull request #534 from github/pluehne/allow-manually-triggering-testing-workflow Allow manually triggering testing workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da7695ad1..3e4445915 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,6 @@ name: Test and build -on: [pull_request] +on: [pull_request, workflow_dispatch] jobs: build: From 99ba2b7761d3586078a5a3bec191922d30798861 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Wed, 6 Sep 2023 22:05:33 +0000 Subject: [PATCH 03/29] Merge pull request #557 from github/hao/unit-test-fix Run backup-utils unit tests in serial --- .github/workflows/main.yml | 10 ---------- script/cibuild | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e4445915..50583737b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,9 +6,6 @@ jobs: build: strategy: matrix: - # macos-latest references are kept here for historical purposes. removed macos-latest from the - #matrix as it is not a typical case for users and causes a lot of friction with other linux-based - # installs. Recommend developing on codespaces or using an ubuntu container. os: ['ubuntu-22.04', 'ubuntu-20.04'] fail-fast: false runs-on: ${{ matrix.os }} @@ -20,12 +17,6 @@ jobs: wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck - if: matrix.os != 'macos-latest' - - name: Install Dependencies (macOS) - run: | - brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget - brew install moreutils gawk - if: matrix.os == 'macos-latest' - name: Get Sources uses: actions/checkout@v3 - name: Test @@ -35,4 +26,3 @@ jobs: shell: bash - name: Build (Linux) run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc - if: matrix.os != 'macos-latest' diff --git a/script/cibuild b/script/cibuild index 7f57f444d..73dc9ae0a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -5,7 +5,7 @@ set -e # Enable verbose logging of ssh commands export GHE_VERBOSE_SSH=true -if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then +if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then exit 1 fi From 8ae0e258bdc1d45a5c9554701772deb8ae24a743 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:25:51 +0000 Subject: [PATCH 04/29] Merge pull request #545 from github/fix-ghe-unsupported-version-test Fix GHE unsupported version check --- test/test-ghe-host-check.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 6bfa12a69..aeb41551d 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION) bu_major_minor="$bu_version_major.$bu_version_minor" releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json) - supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")') + latest_value=$(echo "$releases" | jq -r '.latest') + latest_major_version=$(echo $latest_value | cut -d "." -f 1-2) + # Replace "latest" with the derived major version in the releases string + releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g') + # Use the modified releases string as needed + supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo $releases | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) if [ -z "$supported" ] then From 248c1ae162393eed72395f9eda27618c6bc9ced2 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Tue, 12 Sep 2023 09:05:18 +0000 Subject: [PATCH 05/29] add post backup cleanup and move progress to folder (#558) --- bin/ghe-backup | 8 ++++---- bin/ghe-backup-progress | 8 ++++---- bin/ghe-restore | 6 +++--- share/github-backup-utils/ghe-backup-config | 13 +++++++++++-- share/github-backup-utils/track-progress | 11 +++++------ test/test-ghe-backup.sh | 2 ++ test/testlib.sh | 6 ++++++ 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 86ee33dd7..d8074156e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -163,11 +163,11 @@ echo "$GHE_REMOTE_VERSION" > version # Setup progress tracking init-progress export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress OPTIONAL_STEPS=0 # Backup actions+mssql @@ -191,7 +191,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then fi PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total # check that incremental settings are valid if set is_inc=$(is_incremental_backup_feature_on) diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress index 2f4b267fb..7ab36e084 100755 --- a/bin/ghe-backup-progress +++ b/bin/ghe-backup-progress @@ -31,7 +31,7 @@ while true; do done check_for_progress_file() { - if [ ! -f /tmp/backup-utils-progress-info ]; then + if [ ! -f /tmp/backup-utils-progress/info ]; then echo "No progress file found. Has a backup or restore been started?" exit 1 fi @@ -39,18 +39,18 @@ check_for_progress_file() { if [ -n "$ONCE" ]; then check_for_progress_file - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info else check_for_progress_file clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info while true; do if read -r -t 1 -n 1; then clear exit ; else clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info fi done fi diff --git a/bin/ghe-restore b/bin/ghe-restore index 92ae72443..7c9b95709 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -315,11 +315,11 @@ fi export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) init-progress -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Restore" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of restore -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 5b7f1c3d3..d3f8b05aa 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -652,12 +652,21 @@ restore-secret() { #initialize progress tracking by clearing out the temp files used to track init-progress() { - rm -f /tmp/backup-utils-progress* + if [ -d /tmp/backup-utils-progress ]; then + rm -rf /tmp/backup-utils-progress/* + else + mkdir /tmp/backup-utils-progress + fi + touch /tmp/backup-utils-progress/total + touch /tmp/backup-utils-progress/type + touch /tmp/backup-utils-progress/progress + touch /tmp/backup-utils-progress/info + chmod -R 777 /tmp/backup-utils-progress } #increase total count of progress increment-progress-total-count() { ((PROGRESS_TOTAL += $1)) - echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total } diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index d88b3704f..a560ba540 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,13 +1,12 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -# Current version is working solely with backups progress(){ - PROGRESS=$(cat /tmp/backup-utils-progress) - PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress-total) - PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) + PROGRESS=$(cat /tmp/backup-utils-progress/progress) + PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total) + PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type) PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) - echo $((PROGRESS + 1)) > /tmp/backup-utils-progress - echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info + echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress + echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info } diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index fb3f9db6a..3f3cd25ed 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -47,6 +47,8 @@ begin_test "ghe-backup subsequent snapshot" [ "$first_snapshot" != "$this_snapshot" ] verify_all_backedup_data + + verify_progress_cleanup_process ) end_test diff --git a/test/testlib.sh b/test/testlib.sh index 8e130eb75..5bd36d489 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -494,6 +494,12 @@ verify_all_backedup_data() { verify_common_data } +# A unified method to make sure post backup, the cleanup process works +verify_progress_cleanup_process() { + set -e + sudo -u nobody rm -rf /tmp/backup-utils-progress/* +} + # A unified method to check everything restored when performing a full restore # during testing. verify_all_restored_data() { From e448e956500742dbb07962793c5238e9d252b029 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:36:14 +0000 Subject: [PATCH 06/29] Merge pull request #569 from github/check-datadir-is-nfs Add NFS check for GHE_DATA_DIR --- bin/ghe-host-check | 7 +++++++ docs/requirements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 104885114..9e8d6466b 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/docs/requirements.md b/docs/requirements.md index 1320eb583..bba51c252 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,6 +57,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From f0c337019c7bb3543650b7d506e630e9cb8d5477 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Fri, 15 Sep 2023 16:36:26 +0000 Subject: [PATCH 07/29] fix backup.config typo (#587) Fixing https://github.com/github/ghes/issues/7329 --- backup.config-example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.config-example b/backup.config-example index 688cc374b..d1ea977e2 100644 --- a/backup.config-example +++ b/backup.config-example @@ -26,7 +26,7 @@ GHE_NUM_SNAPSHOTS=10 # performed before the next full backup is created. # For example, if `GHE_INCREMENTAL_BACKUP_MAX` is set to 14, backup-utils will # run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle. -#GHE_INCREMENTAL_BACKUP_MAX=14 +#GHE_INCREMENTAL_MAX_BACKUPS=14 # If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check # disk space validation and software version checks on the backup-host will be disabled. From fe080dc5263a45219916f3536cf8362bc9471e2d Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Tue, 26 Sep 2023 22:10:04 +0000 Subject: [PATCH 08/29] fix mkdir issues or progress tracking (#612) Fixing https://github.com/github/ghes/issues/7409 --- share/github-backup-utils/ghe-backup-config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index d3f8b05aa..bdf77cc86 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -654,14 +654,15 @@ restore-secret() { init-progress() { if [ -d /tmp/backup-utils-progress ]; then rm -rf /tmp/backup-utils-progress/* - else - mkdir /tmp/backup-utils-progress fi + + mkdir -p /tmp/backup-utils-progress + chmod -R 777 /tmp/backup-utils-progress + touch /tmp/backup-utils-progress/total touch /tmp/backup-utils-progress/type touch /tmp/backup-utils-progress/progress touch /tmp/backup-utils-progress/info - chmod -R 777 /tmp/backup-utils-progress } From 43f79427b484b35c25303af41de09eac0ab514b0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:21:35 -0500 Subject: [PATCH 09/29] Merge pull request #568 from github/543-workflow-build-release-part-2 Build and Release Workflow Part 2 --- .github/linters/.yaml-lint.yml | 53 +++++++++ .github/workflows/build-and-release.yml | 139 ++++++++++++++++++++++++ .github/workflows/lint.yml | 2 + .releaseignore | 2 + script/package-deb | 11 ++ script/package-tarball | 13 +++ 6 files changed, 220 insertions(+) create mode 100644 .github/linters/.yaml-lint.yml create mode 100644 .github/workflows/build-and-release.yml create mode 100644 .releaseignore diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..030c37f04 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,53 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable \ No newline at end of file diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 000000000..90b44086e --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,139 @@ +--- +name: Build and Release + +on: + workflow_dispatch: + inputs: + gh-token: + description: 'GitHub Token - used to create a commit in the backup-utils repo' + required: true + type: string + version: + description: 'Version - patch version of the release (e.g. x.y.z)' + required: true + type: string + draft: + description: 'Draft - true if the release should be a draft' + required: true + type: boolean + default: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils + ref: master + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: master + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "${{ github.actor }}" + commit_user_email: "ghes-releases-team@github.com" + commit_options: "--allow-empty" + skip_dirty_check: true + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ github.event.inputs.gh-token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ steps.empty-commit.outputs.commit_hash }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" + + + + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d15f123bb..8e878da40 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +--- name: Lint Code Base on: @@ -19,3 +20,4 @@ jobs: env: VALIDATE_ALL_CODEBASE: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILTER_REGEX_EXCLUDE: .*release-notes/.* diff --git a/.releaseignore b/.releaseignore new file mode 100644 index 000000000..bb2f6b31c --- /dev/null +++ b/.releaseignore @@ -0,0 +1,2 @@ +ownership.yaml +.github \ No newline at end of file diff --git a/script/package-deb b/script/package-deb index dda90cbe3..15a698e9c 100755 --- a/script/package-deb +++ b/script/package-deb @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" @@ -22,6 +25,14 @@ mkdir -p dist/debuild distdir="$(pwd)/dist/debuild/$PKG_NAME" git clone -q . "$distdir" cd "$distdir" + +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +echo "Removing .releaseignore ..." +rm -f .releaseignore git checkout -q "$PKG_HEAD" debuild -uc -us 1>&2 diff --git a/script/package-tarball b/script/package-tarball index be653b77a..bf1510e8f 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -8,11 +8,24 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" PKG_NAME="${PKG_BASE}-${PKG_VERS}" +# Remove all files or directories listed in .releaseignore +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +# Remove the .releaseignore file itself +echo "Removing .releaseignore ..." +rm -f .releaseignore + # Run git-archive to generate tarball echo "Creating ${PKG_NAME}.tar.gz ..." mkdir -p dist From f962e107ba45401b170c4b4bb7ccaa8aaa7b3e18 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 27 Sep 2023 08:51:06 -0400 Subject: [PATCH 10/29] Manual backport of #568 - resolved conflict From 4df64e100db8759f44519d34d4dbbb586153a64a Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 2 Oct 2023 11:37:39 -0700 Subject: [PATCH 11/29] Revert "Backport 569 for 3.10: Add NFS check for GHE_DATA_DIR" --- bin/ghe-host-check | 7 ------- docs/requirements.md | 2 -- 2 files changed, 9 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 9e8d6466b..104885114 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,13 +168,6 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" - #Check if GHE_DATA_DIR is NFS mounted - fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true - if [ "$fs_info" == "nfs" ]; then - echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 - echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 - fi - #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/docs/requirements.md b/docs/requirements.md index bba51c252..1320eb583 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,8 +57,6 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. -Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. - ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From a55a89da1cbb3256e87b12f3b52c9de649f71e9d Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Mon, 2 Oct 2023 19:24:55 +0000 Subject: [PATCH 12/29] Merge pull request #569 from github/check-datadir-is-nfs Add NFS check for GHE_DATA_DIR --- bin/ghe-host-check | 7 +++++++ docs/requirements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 104885114..9e8d6466b 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/docs/requirements.md b/docs/requirements.md index 1320eb583..bba51c252 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,6 +57,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From dc37c05660152735c9fe3143cbe243bed9550121 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 2 Oct 2023 20:38:31 +0000 Subject: [PATCH 13/29] Bump version: 3.10.1 [ci skip] --- debian/changelog | 5 +++++ share/github-backup-utils/version | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 5f223ce8d..d670b30af 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,8 @@ +github-backup-utils (3.10.1) UNRELEASED; urgency=medium + + + -- Devin Dooley Mon, 02 Oct 2023 20:38:31 +0000 + github-backup-utils (3.10.0) UNRELEASED; urgency=medium diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 30291cba2..f870be23b 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.10.0 +3.10.1 From 8c5fe49fe6699d16dc5879a7df763ab8963864bb Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Mon, 9 Oct 2023 19:19:20 +0000 Subject: [PATCH 14/29] Merge pull request #579 from github/hao/integration-actions Add integration tests to backup-utils-private --- .github/actions/proxy-janky-build/action.yml | 33 ++++ .github/actions/proxy-janky-build/go.mod | 7 + .github/actions/proxy-janky-build/go.sum | 9 + .github/actions/proxy-janky-build/main.go | 180 +++++++++++++++++++ .github/workflows/integration-tests.yml | 41 +++++ 5 files changed, 270 insertions(+) create mode 100644 .github/actions/proxy-janky-build/action.yml create mode 100644 .github/actions/proxy-janky-build/go.mod create mode 100644 .github/actions/proxy-janky-build/go.sum create mode 100644 .github/actions/proxy-janky-build/main.go create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/actions/proxy-janky-build/action.yml b/.github/actions/proxy-janky-build/action.yml new file mode 100644 index 000000000..988c4a289 --- /dev/null +++ b/.github/actions/proxy-janky-build/action.yml @@ -0,0 +1,33 @@ +name: 'Trigger a CI Job on Janky' +description: 'Action to trigger and poll a Janky CI job' +inputs: + janky-token: + description: 'Token for making request to Janky' + required: true + job-name: + description: 'The name of the job to run' + required: true + branch-name: + description: 'The name of the branch to use' + required: true + force: + description: 'Force the job to run even if it is already passed' + required: false + envVars: + description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3' + required: false +runs: + using: 'composite' + steps: + - uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260 + with: + go-version: '1.21' + - run: | + go run main.go \ + -token ${{ inputs.janky-token }} \ + -job ${{ inputs.job-name }} \ + -branch ${{ inputs.branch-name }} \ + -force ${{ inputs.force }} \ + -envVars ${{ inputs.envVars }} + shell: bash + working-directory: .github/actions/proxy-janky-build diff --git a/.github/actions/proxy-janky-build/go.mod b/.github/actions/proxy-janky-build/go.mod new file mode 100644 index 000000000..b7560d7b9 --- /dev/null +++ b/.github/actions/proxy-janky-build/go.mod @@ -0,0 +1,7 @@ +module github.com/github/enterprise2/actions/proxy-janky-build + +go 1.21 + +require github.com/hashicorp/go-retryablehttp v0.7.2 + +require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/.github/actions/proxy-janky-build/go.sum b/.github/actions/proxy-janky-build/go.sum new file mode 100644 index 000000000..5c59c1d2e --- /dev/null +++ b/.github/actions/proxy-janky-build/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go new file mode 100644 index 000000000..e0342f121 --- /dev/null +++ b/.github/actions/proxy-janky-build/main.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +// Define our Janky Response Structs +type JankyBuildStruct struct { + Result string + Url string +} +type JankyStatusStruct struct { + Id string + Green bool + Completed bool + StartedAt string + CompletedAt string + Sha string + BuildableName string +} + +const ( + pollWaitTime = 10 * time.Second + jankyPollTimeout = 5 * time.Hour + jankyHttpRetryMax = 5 + jankyUrl = "https://janky.githubapp.com" +) + +func main() { + // Parse command-line arguments + job := flag.String("job", "", "Name of the Janky job") + token := flag.String("token", "", "Name of the Janky token") + branch := flag.String("branch", "", "Name of the Git branch") + force := flag.String("force", "false", "Force a build even if one is already passed") + envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") + flag.Parse() + + // Validate command-line arguments + if *job == "" || *token == "" || *branch == "" { + log.Fatal("job, token and branch flags must be specified") + } + + // Set up the token + request payload + authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token)) + type buildRequestObject struct { + BuildableName string `json:"buildable_name"` + BranchName string `json:"branch_name"` + Force string `json:"force"` + EnvVars map[string]string `json:"env_vars"` + } + + requestBody := buildRequestObject{ + BuildableName: *job, + BranchName: *branch, + Force: *force, + } + + // Parse the envVars flag into a map and add to the request payload + fmt.Println("Environment Variables:") + fmt.Println(*envVars) + if *envVars != "" { + envVarsMap := make(map[string]string) + for _, envVar := range strings.Split(*envVars, ",") { + envVarSplit := strings.Split(envVar, "=") + envVarsMap[envVarSplit[0]] = envVarSplit[1] + } + requestBody.EnvVars = envVarsMap + } + + payloadBytes, err := json.Marshal(requestBody) + if err != nil { + log.Fatal("Failed to marshal the JSON payload!\n" + err.Error()) + } + + // Send build request to Janky + buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes)) + if err != nil { + log.Fatal("Failed to create build request!\n" + err.Error()) + } + buildRequest.Header.Set("Content-Type", "application/json") + buildRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient := retryablehttp.NewClient() //nolint:all + retryClient.RetryMax = jankyHttpRetryMax + retryClient.Logger = nil // disable debug logging + client := retryClient.StandardClient() // uses *http.Client + resp, err := client.Do(buildRequest) + if err != nil { + log.Fatal("Failed to send build request!\n" + err.Error()) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatal("Error reading build response!\n" + err.Error()) + } + + // Check if the build was triggered successfully + if resp.StatusCode == 404 { + log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.") + } + if resp.StatusCode != 201 { + log.Fatal("Failed to trigger build! Got exception: " + string(body)) + } + + // Parse the build request response + var buildResponse JankyBuildStruct + json.Unmarshal(body, &buildResponse) + log.Println("Succesfully triggered janky!\n" + buildResponse.Result) + + // Parse the request response for the buildId + r, err := regexp.Compile("/[0-9]+/") + if err != nil { + log.Fatal("Failed to trigger build!\n" + err.Error()) + } + buildId := strings.Trim(r.FindString(buildResponse.Result), "/") + + // Setup our second HTTP client for reuse in during status polling + jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status" + statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil) + if err != nil { + log.Fatal("Failed to create status request!\n" + err.Error()) + } + statusRequest.Header.Set("Content-Type", "application/json") + statusRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient2 := retryablehttp.NewClient() //nolint:all + retryClient2.RetryMax = jankyHttpRetryMax + retryClient2.Logger = nil // disable debug logging + client2 := retryClient2.StandardClient() // uses *http.Client + + // Wait for a completed status from Janky or break the loop after a certain amount of time + timeout := time.NewTimer(jankyPollTimeout) + poll := time.NewTicker(pollWaitTime) + +jobLoop: + for { + select { + case <-timeout.C: + log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours") + case <-poll.C: + // Send build status request to Janky + statusResponse, err := client2.Do(statusRequest) + if err != nil { + log.Fatal("Failed to send status request!\n" + err.Error()) + } + defer statusResponse.Body.Close() + statusBody, err := io.ReadAll(statusResponse.Body) + if err != nil { + log.Fatal("Error reading status response!\n" + err.Error()) + } + + // Parse the status response for a green completed build + var jankyStatusResponse JankyStatusStruct + json.Unmarshal(statusBody, &jankyStatusResponse) + //fmt.Println("Janky Status Response:") + //fmt.Println(string(statusBody)) + if jankyStatusResponse.Completed && jankyStatusResponse.Green { + log.Println("Janky build Succeeded!") + break jobLoop + } + if jankyStatusResponse.Completed && !jankyStatusResponse.Green { + log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url) + } + + // wait for a bit and try again + log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]") + continue + } + } +} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..804c6d5c5 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,41 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'Branch that would be merged into' + required: true + source-branch: + description: 'Branch that would be merged' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.TARGET_BRANCH }}' + force : "true" + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }}" From d357bfbb64adf0d8c697d392c094b5ad31e1a378 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Wed, 20 Sep 2023 19:52:34 -0600 Subject: [PATCH 15/29] Manual backport of #597 - resolved conflict --- .github/workflows/integration-tests.yml | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..8727233ea --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,42 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'Branch that would be merged into' + required: true + source-branch: + description: 'Branch that would be merged' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' + force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" From db4c881f6bf46ea5c7a5ee6454f5950e84bc4140 Mon Sep 17 00:00:00 2001 From: David Daly Date: Thu, 12 Oct 2023 09:01:45 +0000 Subject: [PATCH 16/29] Merge pull request #459 from github/ddivad195/container-prefix-fix fix: backup and restore actions container prefix --- share/github-backup-utils/ghe-backup-settings | 1 + share/github-backup-utils/ghe-restore-actions | 8 ++ test/test-ghe-restore.sh | 88 +++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index aac02dde7..24e946007 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -122,6 +122,7 @@ if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" + backup-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 93596033a..303a2abd2 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -85,6 +85,14 @@ restore-secret "Actions Launch service private key" "actions-launch-app-app-priv restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" +# Restore storage container prefix, but only if it exists, and the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings +if [[ $RESTORE_SETTINGS == "true" ]]; then + if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + else + log_warn "Actions storage container prefix not present in backup. Skipping ..." + fi +fi # Setup the database logins. ghe_verbose "* Restoring database logins and users to $host ..." diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index ef13b7697..7ed345ee0 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -622,6 +622,94 @@ begin_test "ghe-restore with Actions settings" ) end_test +begin_test "ghe-restore with Actions settings passing -c" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + enable_actions + + required_files=( + "actions-config-db-login" + "actions-config-db-password" + "actions-framework-access-token" + "actions-url-signing-hmac-key-primary" + "actions-url-signing-hmac-key-secondary" + "actions-oauth-s2s-signing-cert" + "actions-oauth-s2s-signing-key" + "actions-oauth-s2s-signing-cert-thumbprint" + "actions-primary-encryption-cert-thumbprint" + "actions-aad-cert-thumbprint" + "actions-delegated-auth-cert-thumbprint" + "actions-runtime-service-principal-cert" + "actions-s2s-encryption-cert" + "actions-secondary-encryption-cert-thumbprint" + "actions-service-principal-cert" + "actions-sps-validation-cert-thumbprint" + "actions-storage-container-prefix" + + "actions-launch-secrets-private-key" + "actions-launch-deployer-hmac" + "actions-launch-client-id" + "actions-launch-client-secret" + "actions-launch-receiver-webhook-secret" + "actions-launch-app-private-key" + "actions-launch-app-public-key" + "actions-launch-app-id" + "actions-launch-app-relay-id" + "actions-launch-action-runner-secret" + "actions-launch-azp-app-cert" + "actions-launch-app-app-private-key" + + ) + + for file in "${required_files[@]}"; do + echo "foo" > "$GHE_DATA_DIR/current/$file" + done + + ghe-restore -v -f -c localhost + + required_secrets=( + "secrets.actions.ConfigurationDatabaseSqlLogin" + "secrets.actions.ConfigurationDatabaseSqlPassword" + "secrets.actions.FrameworkAccessTokenKeySecret" + "secrets.actions.UrlSigningHmacKeyPrimary" + "secrets.actions.UrlSigningHmacKeySecondary" + "secrets.actions.OAuthS2SSigningCert" + "secrets.actions.OAuthS2SSigningKey" + "secrets.actions.OAuthS2SSigningCertThumbprint" + "secrets.actions.PrimaryEncryptionCertificateThumbprint" + "secrets.actions.AADCertThumbprint" + "secrets.actions.DelegatedAuthCertThumbprint" + "secrets.actions.RuntimeServicePrincipalCertificate" + "secrets.actions.S2SEncryptionCertificate" + "secrets.actions.SecondaryEncryptionCertificateThumbprint" + "secrets.actions.ServicePrincipalCertificate" + "secrets.actions.SpsValidationCertThumbprint" + "secrets.actions.storage.container-prefix" + "secrets.launch.actions-secrets-private-key" + "secrets.launch.deployer-hmac-secret" + "secrets.launch.client-id" + "secrets.launch.client-secret" + "secrets.launch.receiver-webhook-secret" + "secrets.launch.app-private-key" + "secrets.launch.app-public-key" + "secrets.launch.app-id" + "secrets.launch.app-relay-id" + "secrets.launch.action-runner-secret" + "secrets.launch.token-oauth-key" + "secrets.launch.token-oauth-cert" + "secrets.launch.azp-app-cert" + "secrets.launch.azp-app-private-key" + + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done +) +end_test + begin_test "ghe-restore stops and starts Actions" ( set -e From d538ae8e0ecd3177e936f70a0a6cc6d62f44f805 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:18:24 +0000 Subject: [PATCH 17/29] Merge pull request #599 from github/hao/fix-issues-for-actions Fix issue with actions --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fa0d356cb..08c70e1da 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,8 +15,8 @@ on: # Get target and source branch from different variables depending on how it was triggered env: - TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' - SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' jobs: integration-tests: From 2ea39e5da94e117cbf133e46ab447f05f3fd01bf Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Mon, 25 Sep 2023 16:04:55 -0600 Subject: [PATCH 18/29] Manual backport of #602 - resolved conflict --- .github/pull_request_template.md | 40 +++++++++++++++++++++++++ .github/workflows/integration-tests.yml | 27 ++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ade3e5c58 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,40 @@ + + + + +# PR Details + +## Description + +## Testing + + + +## Ownership + + +## Related Links + diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 08c70e1da..057d5db82 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,6 +37,31 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.SOURCE_BRANCH }}' - force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" + + # Cluster integration tests are optional based on label and PR titles + cluster-integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-cluster-binary-backup + - enterprise2-backup-utils-cluster-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + if: | + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.title, '[cluster]') || + contains(github.event.pull_request.labels.*.name, 'cluster') + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" \ No newline at end of file From ff6c1a6ffe972609fc1708e12a6bbb5872b59e93 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Sat, 14 Oct 2023 01:49:18 +0000 Subject: [PATCH 19/29] Merge pull request #613 from github/jianghao0718-patch-4 Fix description for target and source branches for integration tests CI --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 057d5db82..d442f9555 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,10 +7,10 @@ on: workflow_dispatch: inputs: target-branch: - description: 'Branch that would be merged into' + description: 'enterprise2 branch to test against' required: true source-branch: - description: 'Branch that would be merged' + description: 'backup-utils-private topic branch to test' required: true # Get target and source branch from different variables depending on how it was triggered From 574a06304f248eda9a79fac4ad28d155b55b3985 Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:29:24 +0000 Subject: [PATCH 20/29] Merge pull request #663 from github/enterprise-3.10-backport-662-timreimherr/actions-46-multi-repo-token-scope Backport 662 for 3.10: Implement App Token --- .github/workflows/build-and-release.yml | 76 ++++++++++++------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 90b44086e..c9b6b66e8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -4,10 +4,6 @@ name: Build and Release on: workflow_dispatch: inputs: - gh-token: - description: 'GitHub Token - used to create a commit in the backup-utils repo' - required: true - type: string version: description: 'Version - patch version of the release (e.g. x.y.z)' required: true @@ -21,22 +17,20 @@ on: jobs: build: runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" - name: Checkout backup-utils-private uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils-private + token: ${{ steps.app-token.outputs.token }} - name: Install dependencies run: | sudo apt-get update -y @@ -79,35 +73,40 @@ jobs: outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" + - name: Get major-feature from version + id: get-major-feature + run: | + echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + - name: Verify major-feature + run: | + echo "major_feature: $MAJOR_FEATURE" - name: Checkout backup-utils uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - ref: master - name: Create empty commit uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: master + branch: ${{ env.MAJOR_FEATURE }}-stable commit_message: "${{ github.event.inputs.version }} release" - commit_user_name: "${{ github.actor }}" - commit_user_email: "ghes-releases-team@github.com" + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" commit_options: "--allow-empty" + push_options: "--force" skip_dirty_check: true - - name: Checkout backup-utils + - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 @@ -120,20 +119,17 @@ jobs: - name: Create Release uses: ncipollo/release-action@v1 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} + owner: github repo: backup-utils name: | GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} + commit: ${{ env.MAJOR_FEATURE }}-stable bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true - artifactContentType: "raw" - - - - + artifactContentType: "raw" \ No newline at end of file From 52e0621a841004ff067be724a0b3aff8aa8300a3 Mon Sep 17 00:00:00 2001 From: Hubot Date: Fri, 20 Oct 2023 14:49:13 -0400 Subject: [PATCH 21/29] Update debian/changelog and release-notes/3.10.2.md for 3.10.2 release --- debian/changelog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index d670b30af..f19d04f67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +github-backup-utils (3.10.2) UNRELEASED; urgency=medium + + * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} + +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Fri, 20 Oct 2023 18:49:12 +0000 + github-backup-utils (3.10.1) UNRELEASED; urgency=medium From e6cc2aefb266a52ae7abbf0a099f4dd1a0f96438 Mon Sep 17 00:00:00 2001 From: Hubot Date: Fri, 20 Oct 2023 14:49:14 -0400 Subject: [PATCH 22/29] Create 3.10.2 release notes --- release-notes/3.10.2.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 release-notes/3.10.2.md diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md new file mode 100644 index 000000000..ab5c0aabd --- /dev/null +++ b/release-notes/3.10.2.md @@ -0,0 +1,14 @@ +## Bug Fixes + +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} + +## Changes + +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} + +## Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + From 75f959c46d1f8ee6c4eb6593b993697704f7bf9f Mon Sep 17 00:00:00 2001 From: Tim Reimherr <16481702+timreimherr@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:09:58 +0000 Subject: [PATCH 23/29] Merge pull request #680 from github/enterprise-3.10-backport-679-timreimherr/build-and-release-edit Backport 679 for 3.10: Remove file rename step in build-and-release workflow --- .github/workflows/build-and-release.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c9b6b66e8..edf115738 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,14 +44,6 @@ jobs: - name: Package deb run: | ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - name: Upload deb artifact uses: actions/upload-artifact@v3 with: From 50439b0e4d355eb11fd4eef0d27bcc74a47adc78 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 24 Oct 2023 15:27:19 +0000 Subject: [PATCH 24/29] remove comments --- release-notes/3.10.2.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md index ab5c0aabd..b673422e8 100644 --- a/release-notes/3.10.2.md +++ b/release-notes/3.10.2.md @@ -1,12 +1,12 @@ ## Bug Fixes -* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. ## Changes -* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. ## Backups and Disaster Recovery From 53bf8e76fdb350b1e59b3cd1fe4c6e9090739468 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 24 Oct 2023 15:32:04 +0000 Subject: [PATCH 25/29] update changelog also --- debian/changelog | 12 +++++------- release-notes/3.10.2.md | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/debian/changelog b/debian/changelog index f19d04f67..e627da795 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,10 @@ github-backup-utils (3.10.2) UNRELEASED; urgency=medium - * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. {% comment %} https://github.com/github/backup-utils-private/pull/510, https://github.com/github/backup-utils-private/pull/492 {% endcomment %} -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. {% comment %} https://github.com/github/backup-utils-private/pull/573, https://github.com/github/backup-utils-private/pull/558 {% endcomment %} -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. {% comment %} https://github.com/github/backup-utils-private/pull/653, https://github.com/github/backup-utils-private/pull/459 {% endcomment %} - -* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. {% comment %} https://github.com/github/backup-utils-private/pull/547, https://github.com/github/backup-utils-private/pull/527 {% endcomment %} - -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. + * When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + * Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. + * When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. -- ghes-releases-team Fri, 20 Oct 2023 18:49:12 +0000 diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md index b673422e8..975217afe 100644 --- a/release-notes/3.10.2.md +++ b/release-notes/3.10.2.md @@ -10,5 +10,5 @@ ## Backups and Disaster Recovery -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. +* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. From db0b1afdc1133041504ba27d0251b4165a3c71b7 Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:12 -0400 Subject: [PATCH 26/29] Update debian/changelog, version file, and release-notes/3.10.2.md for 3.10.2 release --- debian/changelog | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/debian/changelog b/debian/changelog index e627da795..cefe37417 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,21 @@ +github-backup-utils (3.10.2) UNRELEASED; urgency=medium + + ### Bug Fixes + +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + +### Changes + +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. + +### Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Wed, 01 Nov 2023 14:28:11 +0000 + github-backup-utils (3.10.2) UNRELEASED; urgency=medium * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. From 97034c60bafe23edd9870f74f980101693952217 Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:12 -0400 Subject: [PATCH 27/29] Create 3.10.2 release notes --- release-notes/3.10.2.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md index 975217afe..d21a32097 100644 --- a/release-notes/3.10.2.md +++ b/release-notes/3.10.2.md @@ -1,14 +1,14 @@ -## Bug Fixes +### Bug Fixes -* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. -## Changes +### Changes * Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. -## Backups and Disaster Recovery +### Backups and Disaster Recovery -* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. From 76a06495469f6277e4ed054a319430f22e06e5ec Mon Sep 17 00:00:00 2001 From: Hubot Date: Wed, 1 Nov 2023 10:28:13 -0400 Subject: [PATCH 28/29] Update version file to 3.10.2 --- share/github-backup-utils/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index f870be23b..e86ec35a8 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.10.1 +3.10.2 \ No newline at end of file From 34bc2bbdc59b0ee649d3d320ebf7f599dc221743 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 1 Nov 2023 14:32:16 +0000 Subject: [PATCH 29/29] no need to update changelog --- debian/changelog | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/debian/changelog b/debian/changelog index cefe37417..e627da795 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,21 +1,3 @@ -github-backup-utils (3.10.2) UNRELEASED; urgency=medium - - ### Bug Fixes - -* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. -* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. - -### Changes - -* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. - -### Backups and Disaster Recovery - -When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. - - -- ghes-releases-team Wed, 01 Nov 2023 14:28:11 +0000 - github-backup-utils (3.10.2) UNRELEASED; urgency=medium * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail.