diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index 53028715a..000000000
--- a/.dockerignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.git/
-.github/
-.gitattributes
-.gitignore
diff --git a/.gitattributes b/.gitattributes
deleted file mode 100644
index 9afc93bcc..000000000
--- a/.gitattributes
+++ /dev/null
@@ -1,2 +0,0 @@
-.gitignore export-ignore
-.gitattributes export-ignore
diff --git a/.github/workflows/close-pulls.yml b/.github/workflows/close-pulls.yml
new file mode 100644
index 000000000..8a95bccc0
--- /dev/null
+++ b/.github/workflows/close-pulls.yml
@@ -0,0 +1,21 @@
+---
+ name: Auto Close Pull Requests
+
+ on:
+ schedule:
+ - cron: '0 * * * *'
+
+ jobs:
+ close-fork-pulls:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Close Pull Requests
+ uses: peter-evans/close-fork-pulls@v2
+ with:
+ comment: |
+ As of 2023-11-30 we have stopped the 2-way sync between this repository and our internal repository, so that our internal repository becomes the source of truth for the backup-utils source code. With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository.
+
+ Customers should no longer open pull requests in this repository. These pull requests will not be reviewed or merged. We will automatically close all PRs opened in this repository.
+
+ Customers cannot open issues in this repository. Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils. This will ensure all customer requests are handled consistently.
\ No newline at end of file
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
deleted file mode 100644
index 4a38ba869..000000000
--- a/.github/workflows/lint.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-name: Lint Code Base
-
-on:
- pull_request:
- branches-ignore:
- - 'master'
-
-jobs:
- build:
- name: Lint Code Base
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Code
- uses: actions/checkout@v2
- - name: Lint Code Base
- uses: docker://github/super-linter:v2.1.1
- env:
- VALIDATE_ALL_CODEBASE: false
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index f10f64e5e..000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Test and build
-
-on: [pull_request]
-
-jobs:
- build:
- strategy:
- matrix:
- os: ['ubuntu-20.04', 'ubuntu-18.04', 'macos-latest']
- fail-fast: false
- runs-on: ${{ matrix.os }}
- steps:
- - name: Install Dependencies (Linux)
- run: |
- sudo apt-get update -y
- sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man
- wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz"
- tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz"
- sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck
- if: matrix.os != 'macos-latest'
- - name: Install Dependencies (macOS)
- run: |
- brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget
- brew install moreutils gawk
- if: matrix.os == 'macos-latest'
- - name: Get Sources
- uses: actions/checkout@v2
- - name: Test
- run: |
- export PATH="$PATH:/snap/bin"
- make test
- shell: bash
- - name: Build (Linux)
- run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc
- if: matrix.os != 'macos-latest'
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 246384181..000000000
--- a/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-/backup.config
-/data
-/dist
diff --git a/.ruby-version b/.ruby-version
deleted file mode 100644
index 197c4d5c2..000000000
--- a/.ruby-version
+++ /dev/null
@@ -1 +0,0 @@
-2.4.0
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index 042d7c71a..000000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1,12 +0,0 @@
-# Contributing
-
-Looking to contribute something to this project? That is great, we always appreciate pull requests! Here's how you can help:
-
-1. Fork the project to your account.
-2. Clone the fork (`git clone https://github.com/[username]/backup-utils.git`).
-3. Create a new feature branch (`git checkout -b my-feature-branch`).
-4. Add and then commit your changes (`git commit -am "Add a new backup endpoint."`).
-5. Push your feature branch to GitHub.com (`git push -u origin my-feature-branch`).
-6. Open a [Pull Request](https://github.com/github/backup-utils/compare/) and wait for our feedback.
-
-Have a look at the [styleguide](https://github.com/github/backup-utils/tree/master/STYLEGUIDE.md) to make sure your code style is consistent with the code in this repository.
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 00067263c..000000000
--- a/Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM debian:stretch-slim
-
-RUN apt-get -q -y update && \
- apt-get install -y --no-install-recommends \
- tar \
- rsync \
- ca-certificates \
- ssh \
- git \
- moreutils \
- gawk \
- && rm -rf /var/lib/apt/lists/*
-
-COPY ./ /backup-utils/
-WORKDIR /backup-utils
-
-RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init
-
-ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"]
-CMD ["ghe-host-check"]
diff --git a/Dockerfile.alpine b/Dockerfile.alpine
deleted file mode 100644
index 2c21bf2cd..000000000
--- a/Dockerfile.alpine
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM alpine:latest
-
-RUN apk --update --no-cache add \
- tar \
- rsync \
- ca-certificates \
- openssh \
- git \
- bash \
- gawk \
- procps \
- coreutils
-
-COPY ./ /backup-utils/
-WORKDIR /backup-utils
-
-RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init
-
-ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"]
-CMD ["ghe-host-check"]
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index 4d231b456..000000000
--- a/LICENSE
+++ /dev/null
@@ -1,20 +0,0 @@
-Copyright (c) 2014 GitHub Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 7ca5cd24c..000000000
--- a/Makefile
+++ /dev/null
@@ -1,26 +0,0 @@
-SHELL = /bin/sh
-
-test: info
- @script/cibuild --no-package
-
-info:
- @echo This is github/backup-utils
- @echo shell is $(shell ls -l $(SHELL) | sed 's@.*/bin/sh@/bin/sh@')
- @rsync --version | head -1
- @echo
-
-dist:
- @script/package-tarball
-
-deb:
- @script/package-deb
-
-clean:
- rm -rf dist
-
-# List pull requests that need to be merged into stable
-# (helpful for the release notes)
-pending-prs:
- @git log stable...master | grep "Merge pull request"
-
-.PHONY: test info dist clean pending-prs
diff --git a/README.md b/README.md
index 7d38558c2..c0ceedb0e 100644
--- a/README.md
+++ b/README.md
@@ -3,33 +3,33 @@
This repository includes backup and recovery utilities for
[GitHub Enterprise Server][1].
-**UPDATE**: The new parallel backup and restore beta feature will require [GNU awk](https://www.gnu.org/software/gawk) and [moreutils](https://joeyh.name/code/moreutils) to be installed. Note that on some distributions/platforms, the `moreutils-parallel` package is separate from `moreutils` and must be installed on its own.
+**Note**: The parallel backup and restore feature will require [GNU awk](https://www.gnu.org/software/gawk) and [moreutils](https://joeyh.name/code/moreutils) to be installed. Note that on some distributions/platforms, the `moreutils-parallel` package is separate from `moreutils` and must be installed on its own.
**Note**: the [GitHub Enterprise Server version requirements][2] have
changed starting with Backup Utilities v2.13.0, released on 27 March 2018.
-### Features
+## Features
Backup Utilities implement a number of advanced capabilities for backup
hosts, built on top of the backup and restore features already included in
GitHub Enterprise Server.
- - Complete GitHub Enterprise Server backup and recovery system via two simple
+- Complete GitHub Enterprise Server backup and recovery system via two simple
utilities: `ghe-backup` and `ghe-restore`.
- - Online backups. The GitHub appliance need not be put in maintenance mode for
+- Online backups. The GitHub appliance need not be put in maintenance mode for
the duration of the backup run.
- - Incremental backup of Git repository data. Only changes since the last
+- Incremental backup of Git repository data. Only changes since the last
snapshot are transferred, leading to faster backup runs and lower network
bandwidth and machine utilization.
- - Efficient snapshot storage. Only data added since the previous snapshot
+- Efficient snapshot storage. Only data added since the previous snapshot
consumes new space on the backup host.
- - Multiple backup snapshots with configurable retention periods.
- - Backup commands run under the lowest CPU/IO priority on the GitHub appliance,
+- Multiple backup snapshots with configurable retention periods.
+- Backup commands run under the lowest CPU/IO priority on the GitHub appliance,
reducing performance impact while backups are in progress.
- - Runs under most Linux/Unix environments.
- - MIT licensed, open source software maintained by GitHub, Inc.
+- Runs under most Linux/Unix environments.
+- MIT licensed, open source software maintained by GitHub, Inc.
-### Documentation
+## Documentation
- **[Requirements](docs/requirements.md)**
- **[Backup host requirements](docs/requirements.md#backup-host-requirements)**
@@ -41,15 +41,86 @@ GitHub Enterprise Server.
- **[Backup snapshot file structure](docs/backup-snapshot-file-structure.md)**
- **[How does Backup Utilities differ from a High Availability replica?](docs/faq.md)**
- **[Docker](docs/docker.md)**
+- **[Releases](https://github.com/github/enterprise-releases/blob/master/docs/release-backup-utils.md)**
-### Support
+## Support
+
+If you have a question related to your specific GitHub Enterprise Server setup, would like assistance with
+backup site setup or recovery, or would like to report a bug or a feature request, please contact our [Enterprise support team][3].
+
+
+## Repository updates - November 2023
+
+In October 2023 we announced a number of changes to this repository.
+These changes will improve our (GitHub’s) ability to ship enhancements and new features to backup-utils,
+as well as simplify how GitHub Enterprise Server customers interact with backup-utils.
+
+Our process for shipping new versions of backup-utils prior to November 2023 involved a 2-way sync between this repository and an internal repository.
+This 2-way sync became significantly more problematic once we started regularly shipping patches in alignment with GitHub Enterprise Server.
+
+As of 2023-11-30 we have stopped this 2-way sync so that our internal repository becomes the source of truth for the backup-utils source code.
+With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils.
+You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository.
+
+This change has not affected the functionality of the backup-utils tool or a customer’s ability to backup or restore their GitHub Enterprise Server instance.
+
+### Details
+
+There are three specific areas that have been affected by us stop the 2-way sync between our internal repository and this public repository on 2023-11-30:
+
+1. **Pull requests**: Customers should no longer open pull requests in this repository.
+These pull requests will not be reviewed or merged.
+This is necessary because we will no longer be syncing changes between this repository and our internal repository.
+2. **Issues**: Customers cannot open issues in this repository.
+Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils.
+This will ensure all customer requests are handled consistently.
+3. **Installing/upgrading backup-utils**: Customers will not be able to use a clone of the repository to install and upgrade backup-utils.
+Customers will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases)
+(either as a Debian package or as an archive file - see below for details on how to incorporate this change).
+
+### Timeline
+
+Below is the two phase timeline we will follow to roll out the changes described above:
+
+* **Phase 1 (rolled out on 2023-11-30):** We have closed all open pull requests and issues (after reviewing each one and porting them to our internal repository if merited),
+and updated the repository settings so that new issues cannot be opened. Also, we have stopped syncing code from our internal repository to this repository.
+ * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository.
+ But the code will not be updated in the repository; you can access updated versions of backup-utils via the [release page](https://github.com/github/backup-utils/releases).
+* **Phase 2 (rolling out 2024-02-20):** The backup-utils code will be removed and the repository will be used to host documentation for backup-utils.
+After this date, you will no longer be able to clone a working copy of backup-utils from the repository.
+Instead, you will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases).
+
+### Updating your backup-utils upgrade process
+
+#### Clone of repository
+
+If your current process for upgrading backup-utils involves a clone of the repository, you will need to modify your process to download a new version of backup-utils and set it up.
+
+For example, you could download the v3.10.0 (github-backup-utils-v3.10.0.tar.gz) artifact from the [releases page](https://github.com/github/backup-utils/releases/tag/v3.10.0) with:
+
+```shell
+\$ wget https://github.com/github/backup-utils/releases/download/v3.10.0/github-backup-utils-v3.10.0.tar.gz
+```
+And then extract it:
+
+```shell
+\$ tar xzvf github-backup-utils-v3.10.0.tar.gz
+```
+
+This will give you a new folder, `github-backup-utils-v3.10.0`, which contains the code for version 3.10.0 of backup-utils. Once you copy over your backup.config file from a previous installation of backup-utils your new version of backup-utils will be ready to use.
+
+#### Docker
+
+For customers that currently use Docker to create a backup-utils image, their existing process may need updating as a result of this change. Previously customers could execute this command to build a Docker image of backup-utils:
+
+```shell
+\$ docker build github.com/github/backup-utils
+```
+
+This will not work after phase 2 roles out. You will need to update your process to first download an archive from the [release page](https://github.com/github/backup-utils/releases), extract it, and then build the Dockerfile inside the extracted directory.
-If you find a bug or would like to request a feature in Backup Utilities, please
-open an issue or pull request on this repository. If you have a question related
-to your specific GitHub Enterprise Server setup or would like assistance with
-backup site setup or recovery, please contact our [Enterprise support team][3]
-instead.
[1]: https://github.com/enterprise
[2]: docs/requirements.md#github-enterprise-version-requirements
[3]: https://support.github.com/
+
diff --git a/RELEASING.md b/RELEASING.md
deleted file mode 100644
index 0bcb66557..000000000
--- a/RELEASING.md
+++ /dev/null
@@ -1,66 +0,0 @@
-# Making a Backup Utilities release
-
-Starting with Backup Utilities v2.13.0, all major releases will follow GitHub Enterprise Server releases and the version support is inline with that of the [GitHub Enterprise Server upgrade requirements](https://help.github.com/enterprise/admin/guides/installation/about-upgrade-requirements/) and as such, support is limited to three versions of GitHub Enterprise Server: the version that corresponds with the version of Backup Utilities, and the two releases prior to it.
-
-For example, Backup Utilities 2.13.0 can be used to backup and restore all patch releases from 2.11.0 to the latest patch release of GitHub Enterprise 2.13. Backup utilities 2.14.0 will be released when GitHub Enterprise 2.14.0 is released and will then be used to backup all releases of GitHub Enterprise from 2.12.0 to the latest patch release of GitHub Enterprise 2.14.
-
-There is no need to align Backup Utilities patch releases with GitHub Enterprise Server patch releases.
-
-When making a `.0` release, you will need to specify the minimum supported version of GitHub Enterprise Server that that release supports.
-
-Only repo administrator is allowed to run the release script, otherwise it will fail.
-
-## Pre-release Actions
-
-Prior to making a release,
-
-1. Sync any changes that have been merged to backup-utils-private into this repository.
-1. Go through the list of open pull requests and merge any that are ready for merging.
-1. Go through the list of closed pull requests since the last release and ensure those that should be included in the release notes:
- - have a "bug", "enhancement" or "feature" label,
- - have a title that clearly describes the changes in that pull request. Reword if necessary.
-1. Perform a dry run (add `--dry-run` to one of the commands below) and verify the version strings are going to be changed and verify the release notes.
-
-## Automatic Process from chatops (internal to GitHub only)
-
-Coming :soon:
-
-## Automatic Process from CLI
-
-1. Install the Debian `devscripts` and `moreutils` packages:
- `sudo apt-get install devscripts moreutils`
-2. Generate a PAT through github.com with access to the `github/backup-utils` repository. This will be used for the `GH_RELEASE_TOKEN` environment variable in the next step.
-3. Run...
- - Feature release:
- `GH_AUTHOR="Bob Smith " GH_RELEASE_TOKEN=your-amazing-secure-token script/release 2.13.0 2.11.0`
- - Patch release:
- `GH_AUTHOR="Bob Smith " GH_RELEASE_TOKEN=your-amazing-secure-token script/release 2.13.1`
-
-## Manual Process
-
-In the event you can't perform the automatic process, or a problem is encountered with the automatic process, these are the manual steps you need to perform for a release.
-
-1. Install the Debian `devscripts` and `moreutils` packages:
- `sudo apt-get install devscripts moreutils`
-2. Add a new version and release notes to the `debian/changelog` file:
- `dch --newversion 2.13.0 --release-heuristic log`
- You can use `make pending-prs` to craft the release notes.
-3. Rev the `share/github-backup-utils/version` file. If this is a feature release, update `supported_minimum_version=` in `bin/ghe-host-check` too.
-4. Commit your changes.
-5. Tag the release: `git tag v2.13.0`
-6. Build that tarball package: `make dist`
-7. Build the deb package: `make deb`. All the tests should pass.
-8. Draft a new release at https://github.com/github/backup-utils/releases, including the release notes and attaching the tarball and deb packages.
- The dist tarball you should upload has the revision in the file name, i.e. something like `github-backup-utils-v2.13.0.tar.gz`
-9. Push the head of the release to the 'stable' branch.
-
-## Post-release Actions
-
-Immediately after making a release using one of the methods above, verify the release has succeeded by checking:
-
-- latest release at https://github.com/github/backup-utils/releases is correct,
-- release at https://github.com/github/backup-utils/releases is linked to the vX.Y.Z tag,
-- release has the notes you expect to see,
-- asset download links for the latest release at https://github.com/github/backup-utils/releases all download the correct version of Backup Utilities,
-- the stable branch is inline with master - https://github.com/github/backup-utils/compare/stable...master.
-- sync this repository to backup-utils-private
diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md
deleted file mode 100644
index f2f6c8782..000000000
--- a/STYLEGUIDE.md
+++ /dev/null
@@ -1,250 +0,0 @@
-## Bash Style Guide
-
-If you've not done much Bash development before you may find these debugging tips useful: http://wiki.bash-hackers.org/scripting/debuggingtips.
-
----
-##### Scripts must start with `#!/usr/bin/env bash`
-
----
-##### Use `set -e`
-
-If the return value of a command can be ignored, suffix it with `|| true`:
-
-```bash
-set -e
-command_that_might_fail || true
-command_that_should_not_fail
-```
-
-Note that ignoring an exit status with `|| true` is not a good practice though. Generally speaking, it's better to handle the error.
-
----
-##### Avoid manually checking exit status with `$?`
-
-Rely on `set -e` instead:
-
-```bash
-cmd
-if [ $? -eq 0 ]; then
- echo worked
-fi
-```
-
-should be written as:
-
-```bash
-set -e
-if cmd; then
- echo worked
-fi
-```
-
----
-##### Include a usage, description and optional examples
-
-Use this format:
-
-```bash
-#!/usr/bin/env bash
-#/ Usage: ghe-this-is-my-script [options]
-#/
-#/ This is a brief description of the script's purpose.
-#/
-#/ OPTIONS:
-#/ -h | --help Show this message.
-#/ -l | --longopt An option.
-#/ -c Another option.
-#/
-#/ EXAMPLES: (optional section but nice to have when not trivial)
-#/
-#/ This will do foo and bar:
-#/ $ ghe-this-is-my-script --longopt foobar -c 2
-#/
-set -e
-```
-
-If there are no options or required arguments, the `OPTIONS` section can be ignored.
-
----
-##### Customer-facing scripts must accept both -h and --help arguments
-
-They should also print the usage information and exit 2.
-
-For example:
-
-```bash
-#!/usr/bin/env bash
-#/ Usage: ghe-this-is-my-script [options]
-#/
-#/ This is a brief description of the script's purpose.
-set -e
-
-if [ "$1" = "--help" -o "$1" = "-h" ]; then
- grep '^#/' <"$0" | cut -c 4-
- exit 2
-fi
-
-```
-
----
-##### Avoid Bash arrays
-
-Main issues:
-
-* Portability
-* Important bugs in Bash versions < 4.3
-
----
-##### Use `test` or `[` whenever possible
-
-```bash
-test -f /etc/passwd
-test -f /etc/passwd -a -f /etc/group
-if [ "string" = "string" ]; then
- true
-fi
-```
-
----
-##### Scripts may use `[[` for advanced bash features
-
-```bash
-if [[ "$(hostname)" = *.iad.github.net ]]; then
- true
-fi
-```
-
----
-##### Scripts may use Bash for loops
-
-Preferred:
-
-```bash
-for i in $(seq 0 9); do
-done
-```
-
-or:
-
-```bash
-for ((n=0; n<10; n++)); do
-done
-```
-
----
-##### Use `$[x+y*z]` for mathematical expressions
-
-```bash
-local n=1
-let n++
-n=$[n+1] # preferred
-n=$[$n+1]
-n=$((n+1))
-n=$(($n+1))
-```
-
----
-##### Use variables sparingly
-
-Short paths and other constants should be repeated liberally throughout code since they
-can be search/replaced easily if they ever change.
-
-```bash
-DATA_DB_PATH=/data/user/db
-mkdir -p $DATA_DB_PATH
-rsync $DATA_DB_PATH remote:$DATA_DB_PATH
-```
-
-versus the much more readable:
-
-```bash
-mkdir -p /data/user/db
-rsync /data/user/db remote:/data/user/db
-```
-
----
-##### Use lowercase and uppercase variable names
-
-Use lowercase variables for locals and internal veriables, and uppercase for variables inherited or exported via the environment
-
-```bash
-#!/usr/bin/env bash
-#/ Usage: [DEBUG=0] process_repo
-nwo=$1
-[ -n $DEBUG ] && echo "** processing $nwo" >&2
-
-export GIT_DIR=/data/repos/$nwo.git
-git rev-list
-```
-
----
-##### Use `${var}` for interpolation only when required
-
-```bash
-greeting=hello
-echo $greeting
-echo ${greeting}world
-```
-
----
-##### Use functions sparingly, opting for small/simple/sequential scripts instead whenever possible
-
-When defining functions, use the following style:
-
-```bash
-my_function() {
- local arg1=$1
- [ -n $arg1 ] || return
- ...
-}
-```
-
----
-##### Use `< /etc/foo # interpolated before ssh
- chmod 0600 /etc/foo
-eof
-```
-
----
-##### Quote variables that could reasonably have a space now or in the future
-
-```bash
-if [ ! -z "$packages" ]; then
- true
-fi
-```
-
----
-##### Use two space indentation
-
----
-##### Scripts should not produce errors or warnings when checked with ShellCheck
-
-Use inline comments to disable specific tests, and explain why the test has been disabled.
-
-```bash
-hexToAscii() {
- # shellcheck disable=SC2059 # $1 needs to be interpreted as a formatted string
- printf "\x$1"
-}
-```
-
-### Testing
-
-See [the style guide](https://github.com/github/backup-utils/blob/master/test/STYLEGUIDE.md)
diff --git a/backup.config-example b/backup.config-example
deleted file mode 100644
index 61ccbb055..000000000
--- a/backup.config-example
+++ /dev/null
@@ -1,96 +0,0 @@
-# GitHub Enterprise Server backup configuration file
-
-# The hostname of the GitHub Enterprise Server appliance to back up. The host
-# must be reachable via SSH from the backup host.
-GHE_HOSTNAME="github.example.com"
-
-# Path to where backup data is stored. By default this is the "data"
-# directory next to this file but can be set to an absolute path
-# elsewhere for backing up to a separate partition / mount point.
-GHE_DATA_DIR="data"
-
-# The number of backup snapshots to retain. Old snapshots are pruned after each
-# successful ghe-backup run. This option should be tuned based on the frequency
-# of scheduled backup runs. If backups are scheduled hourly, snapshots will be
-# available for the past N hours; if backups are scheduled daily, snapshots will
-# be available for the past N days ...
-GHE_NUM_SNAPSHOTS=10
-
-# The hostname of the GitHub appliance to restore. If you've set up a separate
-# GitHub appliance to act as a standby for recovery, specify its IP or hostname
-# here. The host to restore to may also be specified directly when running
-# ghe-restore so use of this variable isn't strictly required.
-#
-#GHE_RESTORE_HOST="github-standby.example.com"
-
-# If set to 'yes', ghe-restore will omit the restore of audit logs.
-#
-#GHE_RESTORE_SKIP_AUDIT_LOGS=no
-
-# When verbose output is enabled with `-v`, it's written to stdout by default. If
-# you'd prefer it to be written to a separate file, set this option.
-#
-#GHE_VERBOSE_LOG="/var/log/backup-verbose.log"
-
-# Any extra options passed to the SSH command.
-# In a single instance environment, nothing is required by default.
-# In a clustering environment, "-i abs-path-to-ssh-private-key" is required.
-#
-#GHE_EXTRA_SSH_OPTS=""
-
-# Any extra options passed to the rsync command. Nothing required by default.
-#
-#GHE_EXTRA_RSYNC_OPTS=""
-
-# If set to 'no', GHE_DATA_DIR will not be created automatically
-# and restore/backup will exit 8
-#
-#GHE_CREATE_DATA_DIR=yes
-
-# If set to 'yes', git fsck will run on the repositories
-# and print some additional info.
-#
-# WARNING: do not enable this, only useful for debugging/development
-#GHE_BACKUP_FSCK=no
-
-# Cadence of MSSQL backups
-# ,, all in minutes
-# e.g.
-# - Full backup every week (10080 minutes)
-# - Differential backup every day (1440 minutes)
-# - Transactionlog backup every 15 minutes
-#
-#GHE_MSSQL_BACKUP_CADENCE=10080,1440,15
-
-# If set to 'yes', ghe-backup jobs will run in parallel. Defaults to 'no'.
-#
-# WARNING: this feature is in beta.
-#GHE_PARALLEL_ENABLED=yes
-
-# Sets the maximum number of jobs to run in parallel. Defaults to the number
-# of available processing units on the machine.
-#
-# WARNING: this feature is in beta.
-#GHE_PARALLEL_MAX_JOBS=2
-
-# Sets the maximum number of rsync jobs to run in parallel. Defaults to the
-# configured GHE_PARALLEL_MAX_JOBS, or the number of available processing
-# units on the machine.
-#
-# WARNING: this feature is in beta.
-# GHE_PARALLEL_RSYNC_MAX_JOBS=3
-
-# When jobs are running in parallel wait as needed to avoid starting new jobs
-# when the system's load average is not below the specified percentage. Defaults to
-# unrestricted.
-#
-# WARNING: this feature is in beta.
-#GHE_PARALLEL_MAX_LOAD=50
-
-# When running an external mysql database, run this script to trigger a MySQL backup
-# rather than attempting to backup via backup-utils directly.
-#EXTERNAL_DATABASE_BACKUP_SCRIPT="/bin/false"
-
-# When running an external mysql database, run this script to trigger a MySQL restore
-# rather than attempting to backup via backup-utils directly.
-#EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false"
diff --git a/bin/ghe-backup b/bin/ghe-backup
deleted file mode 100755
index 8d9be333c..000000000
--- a/bin/ghe-backup
+++ /dev/null
@@ -1,274 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup [-hv] [--version]
-#/
-#/ Take snapshots of all GitHub Enterprise data, including Git repository data,
-#/ the MySQL database, instance settings, GitHub Pages data, etc.
-#/
-#/ OPTIONS:
-#/ -v | --verbose Enable verbose output.
-#/ -h | --help Show this message.
-#/ --version Display version information.
-#/
-
-set -e
-
-# Parse arguments
-while true; do
- case "$1" in
- -h|--help)
- export GHE_SHOW_HELP=true
- shift
- ;;
- --version)
- export GHE_SHOW_VERSION=true
- shift
- ;;
- -v|--verbose)
- export GHE_VERBOSE=true
- shift
- ;;
- -*)
- echo "Error: invalid argument: '$1'" 1>&2
- exit 1
- ;;
- *)
- break
- ;;
- esac
-done
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
-
-# Check to make sure moreutils parallel is installed and working properly
-ghe_parallel_check
-
-# Used to record failed backup steps
-failures=
-failures_file="$(mktemp -t backup-utils-backup-failures-XXXXXX)"
-
-# CPU and IO throttling to keep backups from thrashing around.
-export GHE_NICE=${GHE_NICE:-"nice -n 19"}
-export GHE_IONICE=${GHE_IONICE:-"ionice -c 3"}
-
-# Create the timestamped snapshot directory where files for this run will live,
-# change into it, and mark the snapshot as incomplete by touching the
-# 'incomplete' file. If the backup succeeds, this file will be removed
-# signifying that the snapshot is complete.
-mkdir -p "$GHE_SNAPSHOT_DIR"
-cd "$GHE_SNAPSHOT_DIR"
-touch "incomplete"
-
-# Exit early if the snapshot filesystem doesn't support hard links, symlinks and
-# if rsync doesn't support hardlinking of dangling symlinks
-trap 'rm -rf src dest1 dest2' EXIT
-mkdir src
-touch src/testfile
-if ! ln -s /data/does/not/exist/hooks/ src/ >/dev/null 2>&1; then
- echo "Error: the filesystem containing $GHE_DATA_DIR does not support symbolic links." 1>&2
- echo "Git repositories contain symbolic links that need to be preserved during a backup." 1>&2
- exit 1
-fi
-
-if ! output=$(rsync -a src/ dest1 2>&1 && rsync -av src/ --link-dest=../dest1 dest2 2>&1); then
- echo "Error: rsync encountered an error that could indicate a problem with permissions," 1>&2
- echo "hard links, symbolic links, or another issue that may affect backups." 1>&2
- echo "$output"
- exit 1
-fi
-
-if [ "$(ls -il dest1/testfile | awk '{ print $1 }')" != "$(ls -il dest2/testfile | awk '{ print $1 }')" ]; then
- echo "Error: the filesystem containing $GHE_DATA_DIR does not support hard links." 1>&2
- echo "Backup Utilities use hard links to store backup data efficiently." 1>&2
- exit 1
-fi
-rm -rf src dest1 dest2
-
-# To prevent multiple backup runs happening at the same time, we create a
-# in-progress file with the timestamp and pid of the backup process,
-# giving us a form of locking.
-#
-# Set up a trap to remove the in-progress file if we exit for any reason but
-# verify that we are the same process before doing so.
-#
-# The cleanup trap also handles disabling maintenance mode on the appliance if
-# it was automatically enabled.
-cleanup () {
- if [ -f ../in-progress ]; then
- progress=$(cat ../in-progress)
- snapshot=$(echo "$progress" | cut -d ' ' -f 1)
- pid=$(echo "$progress" | cut -d ' ' -f 2)
- if [ "$snapshot" = "$GHE_SNAPSHOT_TIMESTAMP" ] && [ "$$" = $pid ]; then
- unlink ../in-progress
- fi
- fi
-
- rm -rf "$failures_file"
-
- # Cleanup SSH multiplexing
- ghe-ssh --clean
-}
-
-# Setup exit traps
-trap 'cleanup' EXIT
-trap 'exit $?' INT # ^C always terminate
-
-if [ -h ../in-progress ]; then
- echo "Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2
- echo "If there is no backup in progress anymore, please remove" 1>&2
- echo "the $GHE_DATA_DIR/in-progress file." 1>&2
- exit 1
-fi
-
-if [ -f ../in-progress ]; then
- progress=$(cat ../in-progress)
- snapshot=$(echo "$progress" | cut -d ' ' -f 1)
- pid=$(echo "$progress" | cut -d ' ' -f 2)
- if ! ps -p "$pid" >/dev/null 2>&1; then
- # We can safely remove in-progress, ghe-prune-snapshots
- # will clean up the failed backup.
- unlink ../in-progress
- else
- echo "Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2
- echo "If PID $pid is not a process related to the backup utilities, please remove" 1>&2
- echo "the $GHE_DATA_DIR/in-progress file and try again." 1>&2
- exit 1
- fi
-fi
-
-echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress
-
-echo "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP"
-
-# Perform a host connection check and establish the remote appliance version.
-# The version is available in the GHE_REMOTE_VERSION variable and also written
-# to a version file in the snapshot directory itself.
-ghe_remote_version_required
-echo "$GHE_REMOTE_VERSION" > version
-
-if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
- echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."
-fi
-
-# Log backup start message in /var/log/syslog on remote instance
-ghe_remote_logger "Starting backup from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP ..."
-
-export GHE_BACKUP_STRATEGY=${GHE_BACKUP_STRATEGY:-$(ghe-backup-strategy)}
-
-# Record the strategy with the snapshot so we will know how to restore.
-echo "$GHE_BACKUP_STRATEGY" > strategy
-
-# Create benchmark file
-bm_init > /dev/null
-
-ghe-backup-store-version ||
-echo "Warning: storing backup-utils version remotely failed."
-
-echo "Backing up GitHub settings ..."
-ghe-backup-settings || failures="$failures settings"
-
-echo "Backing up SSH authorized keys ..."
-bm_start "ghe-export-authorized-keys"
-ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-authorized-keys' > authorized-keys.json ||
-failures="$failures authorized-keys"
-bm_end "ghe-export-authorized-keys"
-
-echo "Backing up SSH host keys ..."
-bm_start "ghe-export-ssh-host-keys"
-ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar ||
-failures="$failures ssh-host-keys"
-bm_end "ghe-export-ssh-host-keys"
-
-ghe-backup-mysql || failures="$failures mysql"
-
-if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
- echo "Backing up MSSQL databases ..."
- ghe-backup-mssql 1>&3 || failures="$failures mssql"
-
- echo "Backing up Actions data ..."
- ghe-backup-actions 1>&3 || failures="$failures actions"
-fi
-
-if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
- echo "Backing up Minio data ..."
- ghe-backup-minio 1>&3 || failures="$failures minio"
-fi
-
-commands=("
-echo \"Backing up Redis database ...\"
-ghe-backup-redis > redis.rdb || printf %s \"redis \" >> \"$failures_file\"")
-
-commands+=("
-echo \"Backing up audit log ...\"
-ghe-backup-es-audit-log || printf %s \"audit-log \" >> \"$failures_file\"")
-
-commands+=("
-echo \"Backing up Git repositories ...\"
-ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"")
-
-commands+=("
-echo \"Backing up GitHub Pages artifacts ...\"
-ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"")
-
-commands+=("
-echo \"Backing up storage data ...\"
-ghe-backup-storage || printf %s \"storage \" >> \"$failures_file\"")
-
-commands+=("
-echo \"Backing up custom Git hooks ...\"
-ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"")
-
-if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
- commands+=("
- echo \"Backing up Elasticsearch indices ...\"
- ghe-backup-es-rsync || printf %s \"elasticsearch \" >> \"$failures_file\"")
-fi
-
-if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
- $GHE_PARALLEL_COMMAND $GHE_PARALLEL_COMMAND_OPTIONS -- "${commands[@]}"
-else
- for c in "${commands[@]}"; do
- eval "$c"
- done
-fi
-
-if [ -s "$failures_file" ]; then
- failures="$failures $(cat "$failures_file")"
-fi
-
-# git fsck repositories after the backup
-if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
- ghe-backup-fsck $GHE_SNAPSHOT_DIR || failures="$failures fsck"
-fi
-
-# If everything was successful, mark the snapshot as complete, update the
-# current symlink to point to the new snapshot and prune expired and failed
-# snapshots.
-if [ -z "$failures" ]; then
- rm "incomplete"
-
- rm -f "../current"
- ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
-
- ghe-prune-snapshots
-fi
-
-echo "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")"
-
-# Exit non-zero and list the steps that failed.
-if [ -z "$failures" ]; then
- ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP successfully."
-else
- steps="$(echo $failures | sed 's/ /, /g')"
- ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}."
- echo "Error: Snapshot incomplete. Some steps failed: ${steps}. "
- exit 1
-fi
-
-# Detect if the created backup contains any leaked ssh keys
-echo "Checking for leaked ssh keys ..."
-ghe-detect-leaked-ssh-keys -s "$GHE_SNAPSHOT_DIR" || true
-
-# Make sure we exit zero after the conditional
-true
diff --git a/bin/ghe-host-check b/bin/ghe-host-check
deleted file mode 100755
index 4d934386c..000000000
--- a/bin/ghe-host-check
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-host-check [-h] [--version] []
-#/
-#/ Verify connectivity with the GitHub Enterprise Server host.
-#/
-#/ OPTIONS:
-#/ -h | --help Show this message.
-#/ --version Display version information.
-#/ The GitHub Enterprise Server host to check. When no
-#/ is provided, the $GHE_HOSTNAME configured in
-#/ backup.config is assumed.
-#/
-
-set -e
-
-while true; do
- case "$1" in
- -h | --help)
- export GHE_SHOW_HELP=true
- shift
- ;;
- --version)
- export GHE_SHOW_VERSION=true
- shift
- ;;
- -*)
- echo "Error: invalid argument: '$1'" 1>&2
- exit 1
- ;;
- *)
- break
- ;;
- esac
-done
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-backup-config"
-
-# Use the host provided on the command line if provided, or fallback on the
-# $GHE_HOSTNAME configured in backup.config when not present.
-host="${1:-$GHE_HOSTNAME}"
-
-# Options to pass to SSH during connection check
-options="
- -o PasswordAuthentication=no
- -o ConnectTimeout=5
- -o ConnectionAttempts=1
-"
-
-# Split host:port into parts
-port=$(ssh_port_part "$host")
-hostname=$(ssh_host_part "$host")
-
-set +e
-# ghe-negotiate-version verifies if the target is a Github Enterprise Server instance
-output=$(echo "ghe-negotiate-version backup-utils $BACKUP_UTILS_VERSION" | ghe-ssh -o BatchMode=no $options $host -- /bin/sh 2>&1)
-rc=$?
-set -e
-
-if [ $rc -ne 0 ]; then
- case $rc in
- 255)
- if echo "$output" | grep -i "port 22: Network is unreachable\|port 22: connection refused\|port 22: no route to host\|ssh_exchange_identification: Connection closed by remote host\|Connection timed out during banner exchange\|port 22: Connection timed out" >/dev/null; then
- exec "$(basename $0)" "$hostname:122"
- fi
-
- echo "$output" 1>&2
- echo "Error: ssh connection with '$host' failed" 1>&2
- echo "Note that your SSH key needs to be setup on $host as described in:" 1>&2
- echo "* https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access" 1>&2
- ;;
- 101)
- echo "Error: couldn't read GitHub Enterprise Server fingerprint on '$host' or this isn't a GitHub appliance." 1>&2
- ;;
- 1)
- if [ "${port:-22}" -eq 22 ] && echo "$output" | grep "use port 122" >/dev/null; then
- exec "$(basename $0)" "$hostname:122"
- else
- echo "$output" 1>&2
- fi
- ;;
-
- esac
- exit $rc
-fi
-
-CLUSTER=false
-if ghe-ssh "$host" -- \
- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
- CLUSTER=true
-fi
-
-# ensure all nodes in the cluster are running the same version
-if "$CLUSTER"; then
- node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version)
- distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l)
- if [ "$distinct_versions" -ne 1 ]; then
- echo "$node_version_list" 1>&2
- echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2
- exit 1
- fi
-fi
-
-version=$(echo "$output" | grep "GitHub Enterprise" | awk '{print $NF}')
-
-if [ -z "$version" ]; then
- echo "Error: failed to parse version on '$host' or this isn't a GitHub appliance." 1>&2
- exit 2
-fi
-
-# Block restoring snapshots to older releases of GitHub Enterprise Server
-if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then
- snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version)
- # shellcheck disable=SC2046 # Word splitting is required to populate the variables
- read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version)
- if [ "$(version $GHE_REMOTE_VERSION)" -lt "$(version $snapshot_version_major.$snapshot_version_minor.0)" ]; then
- echo "Error: Snapshot can not be restored to an older release of GitHub Enterprise Server." >&2
- exit 1
- fi
-fi
-
-if [ -z "$GHE_ALLOW_REPLICA_BACKUP" ]; then
- if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state 2>/dev/null || true)" = "replica" ]; then
- echo "Error: high availability replica detected." 1>&2
- echo "Backup Utilities should be used to backup from the primary node in" 1>&2
- echo "high availability environments to ensure consistent and reliable backups." 1>&2
- exit 1
- fi
-fi
-
-# backup-utils 2.13 onwards limits support to the current and previous two releases
-# of GitHub Enterprise Server.
-supported_minimum_version="3.3.0"
-
-if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
- supported=1
-fi
-
-if [ -z "$supported" ]; then
- echo "Error: unsupported release of GitHub Enterprise Server detected." 1>&2
- echo "Backup Utilities v$BACKUP_UTILS_VERSION requires GitHub Enterprise Server v$supported_minimum_version or newer." 1>&2
- echo "Please update your GitHub Enterprise Server appliance or use an older version of Backup Utilities." 1>&2
- exit 1
-fi
-
-echo "Connect $hostname:$port OK (v$version)"
diff --git a/bin/ghe-restore b/bin/ghe-restore
deleted file mode 100755
index c06f85071..000000000
--- a/bin/ghe-restore
+++ /dev/null
@@ -1,578 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-restore [-cfhv] [--version] [--skip-mysql] [-s ] []
-#/
-#/ Restores a GitHub instance from local backup snapshots.
-#/
-#/ Note that the GitHub Enterprise host must be reachable and your SSH key must
-#/ be setup as described in the following help article:
-#/
-#/
-#/
-#/ OPTIONS:
-#/ -c | --config Restore appliance settings and license in addition to
-#/ datastores. Settings are not restored by default to
-#/ prevent overwriting different configuration on the
-#/ restore host.
-#/ -f | --force Don't prompt for confirmation before restoring.
-#/ -h | --help Show this message.
-#/ -v | --verbose Enable verbose output.
-#/ --skip-mysql Skip MySQL restore steps. Only applicable to external databases.
-#/ --version Display version information and exit.
-#/
-#/ -s Restore from the snapshot with the given id. Available
-#/ snapshots may be listed under the data directory.
-#/
-#/ The is the hostname or IP of the GitHub Enterprise
-#/ instance. The may be omitted when the
-#/ GHE_RESTORE_HOST config variable is set in backup.config.
-#/ When a argument is provided, it always overrides
-#/ the configured restore host.
-#/
-
-set -e
-
-# Parse arguments
-: ${RESTORE_SETTINGS:=false}
-export RESTORE_SETTINGS
-
-: ${FORCE:=false}
-export FORCE
-
-: ${SKIP_MYSQL:=false}
-export SKIP_MYSQL
-
-while true; do
- case "$1" in
- --skip-mysql)
- SKIP_MYSQL=true
- shift
- ;;
- -f|--force)
- FORCE=true
- shift
- ;;
- -s)
- snapshot_id="$(basename "$2")"
- shift 2
- ;;
- -c|--config)
- RESTORE_SETTINGS=true
- shift
- ;;
- -h|--help)
- export GHE_SHOW_HELP=true
- shift
- ;;
- --version)
- export GHE_SHOW_VERSION=true
- shift
- ;;
- -v|--verbose)
- export GHE_VERBOSE=true
- shift
- ;;
- -*)
- echo "Error: invalid argument: '$1'" 1>&2
- exit 1
- ;;
- *)
- if [ -n "$1" ]; then
- GHE_RESTORE_HOST_OPT="$1"
- shift
- else
- break
- fi
- ;;
- esac
-done
-
-start_cron () {
- echo "Starting cron ..."
- if $CLUSTER; then
- if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron start"; then
- echo "* Warning: Failed to start cron on one or more nodes"
- fi
- else
- if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron start"; then
- echo "* Warning: Failed to start cron"
- fi
- fi
-}
-
-cleanup () {
- if [ -n "$1" ]; then
- update_restore_status "$1"
- fi
-
- if $ACTIONS_STOPPED && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
- echo "Restarting Actions after restore ..."
- # In GHES 3.3+, ghe-actions-start no longer has a -f (force) flag. In GHES 3.2 and below, we must provide the
- # force flag to make sure it can start in maintenance mode. Use it conditionally based on whether it exists
- # in the --help output
- if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start --help' | grep -q force ; then
- ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start -f' 1>&3
- else
- ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start' 1>&3
- fi
- fi
-
- if ! $CRON_RUNNING; then
- start_cron
- fi
-
- # Cleanup SSH multiplexing
- ghe-ssh --clean
-}
-
-# This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally.
-# shellcheck disable=SC2034
-cleanup_cluster_nodes() {
- uuid="$1"
- if [ -z "$uuid" ]; then
- echo "Node UUID required."
- exit 2
- fi
-
- ghe-spokes server evacuate git-server-$uuid 'Removing replica'
- ghe-spokes server destroy git-server-$uuid
-
- ghe-storage destroy-host storage-server-$uuid --force
-
- ghe-dpages offline pages-server-$uuid
- ghe-dpages remove pages-server-$uuid
-
- ghe-redis-cli del resque:queue:maint_git-server-$uuid
- ghe-redis-cli srem resque:queues maint_git-server-$uuid
-}
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
-
-# Check to make sure moreutils parallel is installed and working properly
-ghe_parallel_check
-
-# Grab the host arg
-GHE_HOSTNAME="${GHE_RESTORE_HOST_OPT:-$GHE_RESTORE_HOST}"
-
-# Hostname without any port suffix
-hostname=$(echo "$GHE_HOSTNAME" | cut -f 1 -d :)
-
-# Show usage with no
-[ -z "$GHE_HOSTNAME" ] && print_usage
-
-# Flag to indicate if this script has stopped Actions.
-ACTIONS_STOPPED=false
-
-# ghe-restore-snapshot-path validates it exists, determines what current is,
-# and if there's any problem, exit for us
-GHE_RESTORE_SNAPSHOT_PATH="$(ghe-restore-snapshot-path "$snapshot_id")"
-GHE_RESTORE_SNAPSHOT=$(basename "$GHE_RESTORE_SNAPSHOT_PATH")
-export GHE_RESTORE_SNAPSHOT
-
-# Detect if the backup we are restoring has a leaked ssh key
-echo "Checking for leaked keys in the backup snapshot that is being restored ..."
-ghe-detect-leaked-ssh-keys -s "$GHE_RESTORE_SNAPSHOT_PATH" || true
-
-# Figure out whether to use the tarball or rsync restore strategy based on the
-# strategy file written in the snapshot directory.
-GHE_BACKUP_STRATEGY=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/strategy")
-
-# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
-ghe_remote_version_required "$GHE_HOSTNAME"
-
-# Figure out if this instance has been configured or is entirely new.
-instance_configured=false
-if is_instance_configured; then
- instance_configured=true
-else
- RESTORE_SETTINGS=true
-fi
-
-# Figure out if we're restoring into cluster
-CLUSTER=false
-if ghe-ssh "$GHE_HOSTNAME" -- \
- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
- CLUSTER=true
-fi
-export CLUSTER
-
-# Restoring a cluster backup to a standalone appliance is not supported
-if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
- echo "Error: Snapshot from a GitHub Enterprise cluster cannot be restored" \
- "to a standalone appliance. Aborting." >&2
- exit 1
-fi
-
-# Ensure target appliance and restore snapshot are a compatible combination with respect to BYODB
-if ! ghe-restore-external-database-compatibility-check; then
- exit 1
-fi
-
-# Figure out if this appliance is in a replication pair
-if ghe-ssh "$GHE_HOSTNAME" -- \
- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then
- echo "Error: Restoring to an appliance with replication enabled is not supported." >&2
- echo " Please teardown replication before restoring." >&2
- exit 1
-fi
-
-# Prompt to verify the restore host given is correct. Restoring overwrites
-# important data on the destination appliance that cannot be recovered. This is
-# mostly to prevent accidents where the backup host is given to restore instead
-# of a separate restore host since they're used in such close proximity.
-if $instance_configured && ! $FORCE; then
- echo
- echo "WARNING: All data on GitHub Enterprise appliance $hostname ($GHE_REMOTE_VERSION)"
- echo " will be overwritten with data from snapshot ${GHE_RESTORE_SNAPSHOT}."
- echo
-
- if is_external_database_snapshot && $RESTORE_SETTINGS; then
- echo "WARNING: This operation will also restore the external MySQL connection configuration,"
- echo " which may be dangerous if the GHES appliance the snapshot was taken from is still online."
- echo
- fi
-
- prompt_for_confirmation "Please verify that this is the correct restore host before continuing."
-fi
-
-# Prompt to verify that restoring BYODB snapshot to unconfigured instance
-# will result in BYODB connection information being restored as well.
-if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then
- echo
- echo "WARNING: This operation will also restore the external MySQL connection configuration,"
- echo " which may be dangerous if the GHES appliance the snapshot was taken from is still online."
- echo
-
- prompt_for_confirmation "Please confirm this before continuing."
-fi
-
-# Log restore start message locally and in /var/log/syslog on remote instance
-START_TIME=$(date +%s)
-echo 'Start time:' $START_TIME
-echo "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT"
-ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..."
-
-# Keep other processes on the VM or cluster in the loop about the restore status.
-#
-# Other processes will look for these states:
-# "restoring" - restore is currently in progress
-# "failed" - restore has failed
-# "complete" - restore has completed successfully
-update_restore_status () {
- if $CLUSTER; then
- echo "ghe-cluster-each -- \"echo '$1' | sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null\"" |
- ghe-ssh "$GHE_HOSTNAME" /bin/bash
- else
- echo "$1" |
- ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null"
- fi
-}
-
-CRON_RUNNING=true
-# Update remote restore state file and setup failure trap
-trap "cleanup failed" EXIT
-update_restore_status "restoring"
-
-# Make sure the GitHub appliance is in maintenance mode.
-if $instance_configured; then
- if ! ghe-maintenance-mode-status "$GHE_HOSTNAME"; then
- echo "Error: $GHE_HOSTNAME must be put in maintenance mode before restoring. Aborting." 1>&2
- exit 1
- fi
-fi
-
-# Get GHES release version in major.minor format
-RELEASE_VERSION=$(ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --get core.package-version' | cut -d '.' -f 1,2)
-
-# Make sure the GitHub appliance has Actions enabled if the snapshot contains Actions data.
-# If above is true, also check if ac is present in appliance then snapshot should also contains ac databases
-if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then
- if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
- ac_db_ghe=$(echo 'ghe-mssql-console -y -n -q "SELECT name FROM sys.databases" | grep -i "ArtifactCache" | wc -l | tr -d " "' | ghe-ssh "$GHE_HOSTNAME" /bin/bash)
- ac_db_snapshot=$(find "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql/" -maxdepth 1 -name 'ArtifactCache*.bak' | wc -l | tr -d " ")
- if [[ $ac_db_ghe -gt 0 && $ac_db_snapshot -eq 0 ]]; then
- echo "Error: $GHE_HOSTNAME has Actions Cache service enabled but no Actions Cache data is present in snapshot to restore. Aborting" 1>&2
- echo "Please disable Actions cache service in $GHE_HOSTNAME and retry" 1>&2
- echo "To disable Actions Cache service run as admin: ghe-actions-cache-disable" 1>&2
- exit 1
- fi
- if [[ $ac_db_ghe -eq 0 && $ac_db_snapshot -gt 0 && ! $RESTORE_SETTINGS ]]; then
- echo "Error: $GHE_HOSTNAME has Actions Cache service disabled but the snapshot is attempting to restore data for the service. Aborting" 1>&2
- echo "Please enable Actions cache service in $GHE_HOSTNAME and retry" 1>&2
- echo "To enable Actions Cache service run as admin: ghe-actions-cache-enable" 1>&2
- exit 1
- fi
- else
- echo "Error: $GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting." 1>&2
- echo "Setup details for enabling Actions can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled" 1>&2
- exit 1
- fi
-fi
-
-# Create benchmark file
-bm_init > /dev/null
-
-ghe-backup-store-version ||
-echo "Warning: storing backup-utils version remotely failed."
-
-# Stop cron and timerd, as scheduled jobs may disrupt the restore process.
-echo "Stopping cron and github-timerd ..."
-if $CLUSTER; then
- if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then
- ghe_verbose "* Warning: Failed to stop cron on one or more nodes"
- fi
-
- if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
- if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
- if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
- ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes"
- fi
- fi
- else
- if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service github-timerd stop"; then
- ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes"
- fi
- fi
-else
- if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then
- ghe_verbose "* Warning: Failed to stop cron"
- fi
-
- if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
- if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then
- if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then
- ghe_verbose "* Warning: Failed to stop github-timerd"
- fi
- fi
- else
- if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service github-timerd stop"; then
- ghe_verbose "* Warning: Failed to stop github-timerd"
- fi
- fi
-fi
-CRON_RUNNING=false
-
-# Restore settings and license if restoring to an unconfigured appliance or when
-# specified manually.
-if $RESTORE_SETTINGS; then
- ghe-restore-settings "$GHE_HOSTNAME"
-fi
-
-# Make sure mysql and elasticsearch are prep'd and running before restoring.
-# These services will not have been started on appliances that have not been
-# configured yet.
-if ! $CLUSTER; then
- echo "sudo ghe-service-ensure-mysql && sudo ghe-service-ensure-elasticsearch" |
- ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
-fi
-
-# Restore UUID if present and not restoring to cluster.
-if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
- echo "Restoring UUID ..."
- cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid" |
- ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null"
- ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true
- ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
- fi
-
-if is_external_database_snapshot; then
- appliance_strategy="external"
- backup_snapshot_strategy="external"
-else
- if is_binary_backup_feature_on; then
- appliance_strategy="binary"
- else
- appliance_strategy="logical"
- fi
-
- if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then
- backup_snapshot_strategy="binary"
- else
- backup_snapshot_strategy="logical"
- fi
-fi
-
-if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
- echo "Skipping MySQL restore."
-else
- echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
- ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
-fi
-
-if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
- echo "Stopping Actions before restoring databases ..."
- # We mark Actions as stopped even if the `ghe-actions-stop`
- # fails to ensure that we cleanly start actions when performing cleanup.
- ACTIONS_STOPPED=true
- ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-stop' 1>&3
-
- echo "Restoring MSSQL databases ..."
- ghe-restore-mssql "$GHE_HOSTNAME" 1>&3
-
- echo "Restoring Actions data ..."
- ghe-restore-actions "$GHE_HOSTNAME" 1>&3
- echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning."
- echo " See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners."
-fi
-
-if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
- echo "Restoring MinIO data ..."
- ghe-restore-minio "$GHE_HOSTNAME" 1>&3
-fi
-
-commands=("
-echo \"Restoring Redis database ...\"
-ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3")
-
-commands+=("
-echo \"Restoring Git repositories ...\"
-ghe-restore-repositories \"$GHE_HOSTNAME\"")
-
-commands+=("
-echo \"Restoring Gists ...\"
-ghe-restore-repositories-gist \"$GHE_HOSTNAME\"")
-
-commands+=("
-echo \"Restoring GitHub Pages artifacts ...\"
-ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3")
-
-commands+=("
-echo \"Restoring SSH authorized keys ...\"
-ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-authorized-keys' < \"$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json\" 1>&3")
-
-commands+=("
-echo \"Restoring storage data ...\"
-ghe-restore-storage \"$GHE_HOSTNAME\" 1>&3")
-
-commands+=("
-echo \"Restoring custom Git hooks ...\"
-ghe-restore-git-hooks \"$GHE_HOSTNAME\" 1>&3")
-
-if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then
- commands+=("
- echo \"Restoring Elasticsearch indices ...\"
- ghe-restore-es-rsync \"$GHE_HOSTNAME\" 1>&3")
-fi
-
-# Restore the audit log migration sentinel file, if it exists in the snapshot
-if test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete; then
- ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"
-fi
-
-# Restore exported audit logs to 2.12.9 and newer single nodes and
-# all releases of cluster
-if $CLUSTER || [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.12.9)" ]; then
- if [[ "$GHE_RESTORE_SKIP_AUDIT_LOGS" = "yes" ]]; then
- echo "Skipping restore of audit logs."
- else
- commands+=("
- echo \"Restoring Audit logs ...\"
- ghe-restore-es-audit-log \"$GHE_HOSTNAME\" 1>&3")
- fi
-
-fi
-
-if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then
- $GHE_PARALLEL_COMMAND $GHE_PARALLEL_COMMAND_OPTIONS -- "${commands[@]}"
-else
- for c in "${commands[@]}"; do
- eval "$c"
- done
-fi
-
-# Restart an already running memcached to reset the cache after restore
-echo "Restarting memcached ..." 1>&3
-echo "sudo restart -q memcached 2>/dev/null || true" |
- ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
-
-# Prevent GitHub Connect jobs running before we've had a chance to reset
-# the configuration by setting the last run date to now.
-if ! $RESTORE_SETTINGS; then
- echo "Setting last run date for GitHub Connect jobs ..." 1>&3
- echo "now=$(date +%s.0000000); ghe-redis-cli mset timer:UpdateConnectInstallationInfo \$now timer:UploadEnterpriseServerUserAccountsJob \$now timer:UploadConnectMetricsJob \$now timer:GitHubConnectPushNewContributionsJob \$now" |
- ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
-fi
-
-# When restoring to a host that has already been configured, kick off a
-# config run to perform data migrations.
-if $CLUSTER; then
- echo "Configuring cluster ..."
- if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
- ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3
- elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
- ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
- fi
- ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
-elif $instance_configured; then
- echo "Configuring appliance ..."
- if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
- ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3
- elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
- ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
- fi
- ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3
-fi
-
-# Clear GitHub Connect settings stored in the restored database.
-# This needs to happen after `ghe-config-apply` to ensure all migrations have run.
-if ! $RESTORE_SETTINGS; then
- echo "if [ -f /usr/local/share/enterprise/ghe-reset-gh-connect ]; then /usr/local/share/enterprise/ghe-reset-gh-connect -y; fi" |
- ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
-fi
-
-# Start cron. Timerd will start automatically as part of the config run.
-start_cron
-CRON_RUNNING=true
-
-# Clean up all stale replicas on configured instances.
-if ! $CLUSTER && $instance_configured; then
- restored_uuid=$(cat $GHE_RESTORE_SNAPSHOT_PATH/uuid)
- other_nodes=$(echo "
- set -o pipefail; \
- ghe-spokes server show --json \
- | jq -r '.[] | select(.host | contains(\"git-server\")).host' \
- | sed 's/^git-server-//g' \
- | ( grep -F -x -v \"$restored_uuid\" || true )" \
- | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash)
- if [ -n "$other_nodes" ]; then
- echo "Cleaning up stale nodes ..."
- for uuid in $other_nodes; do
- # shellcheck disable=SC2034
- echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3
- done
- fi
-fi
-
-# Update the remote status to "complete". This has to happen before importing
-# ssh host keys because subsequent commands will fail due to the host key
-# changing otherwise.
-trap "cleanup" EXIT
-update_restore_status "complete"
-
-# Log restore complete message in /var/log/syslog on remote instance
-ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_RESTORE_SNAPSHOT}."
-
-if ! $CLUSTER; then
- echo "Restoring SSH host keys ..."
- ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-ssh-host-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3
-else
- # This will make sure that Git over SSH host keys (babeld) are
- # copied to all the cluster nodes so babeld uses the same keys.
- echo "Restoring Git over SSH host keys ..."
- ghe-ssh "$GHE_HOSTNAME" -- "sudo tar -xpf - -C $GHE_REMOTE_DATA_USER_DIR/common" < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3
- ghe-ssh "$GHE_HOSTNAME" -- "sudo chown babeld:babeld $GHE_REMOTE_DATA_USER_DIR/common/ssh_host_*" 1>&3
- echo "if [ -f /usr/local/share/enterprise/ghe-cluster-config-update ]; then /usr/local/share/enterprise/ghe-cluster-config-update -s; else ghe-cluster-config-update -s; fi" |
- ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
-fi
-
-END_TIME=$(date +%s)
-echo 'End time:' $END_TIME
-echo 'Runtime:' $(($END_TIME - $START_TIME)) 'seconds'
-
-echo "Restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT finished."
-
-if ! $instance_configured; then
- echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration."
-fi
-
diff --git a/debian/changelog b/debian/changelog
deleted file mode 100644
index 3cf6c0d25..000000000
--- a/debian/changelog
+++ /dev/null
@@ -1,610 +0,0 @@
-github-backup-utils (3.5.0) UNRELEASED; urgency=medium
-
- * Simplify complex redirects for ghe-rsync #881
- * On restore failure, restart cron on target host #883
-
- -- Bon Sohi Thu, 12 May 2022 21:06:56 +0000
-
-github-backup-utils (3.4.1) UNRELEASED; urgency=medium
-
- * Simplify complex redirects for ghe-rsync #881
- * On restore failure, restart cron on target host #883
-
- -- Donal Ellis Fri, 22 Apr 2022 04:00:00 +0000
-
-github-backup-utils (3.3.2) UNRELEASED; urgency=medium
-
- * Simplify complex redirects for ghe-rsync #881
- * On restore failure, restart cron on target host #883
-
- -- Donal Ellis Fri, 22 Apr 2022 00:53:45 +0000
-
-github-backup-utils (3.4.0) UNRELEASED; urgency=medium
-
- * Add anchor to usage doc for settings restore #865
-
- -- Steve Culver Tue, 15 Feb 2022 19:25:09 +0000
-
-github-backup-utils (3.3.1) UNRELEASED; urgency=medium
-
- * Fix compat issue with ghe-actions-start during maintenance mode #836
-
- -- Balwinder Sohi Tue, 21 Dec 2021 23:38:01 +0000
-
-github-backup-utils (3.3.0) UNRELEASED; urgency=medium
-
- -- Balwinder Sohi Wed, 08 Dec 2021 03:12:53 +0000
-
-github-backup-utils (3.3.0.rc1) UNRELEASED; urgency=medium
-
- -- Nick Iodice Tue, 09 Nov 2021 19:56:08 +0000
-
-github-backup-utils (3.2.0) UNRELEASED; urgency=medium
-
- -- Brett Westover Tue, 28 Sep 2021 16:50:00 +0000
-
-github-backup-utils (3.2.0.rc3) UNRELEASED; urgency=medium
-
- * Move GitHub Connect reset to after ghe-config-apply #783
-
- -- Brett Westover Fri, 17 Sep 2021 00:44:59 +0000
-
-github-backup-utils (3.2.0.rc1) UNRELEASED; urgency=medium
-
- * Leaked host key check: Avoid false positives from FIPS mode #748
- * Always restore user-password-secrets #762
- * Make some of the actions setting best effort #767
- * Remove reference to `ghe-cluster-cleanup-nodes` #768
- * Stop/Restart Actions in ghe-restore #769
- * Clear GitHub Connect settings when not restoring settings #770
- * GitHub Connect Reset Issue #776
-
- -- Brett Westover Fri, 10 Sep 2021 20:10:07 +0000
-
-github-backup-utils (3.2.0) UNRELEASED; urgency=medium
-
- * Leaked host key check: Avoid false positives from FIPS mode #748
- * Always restore user-password-secrets #762
- * Make some of the actions setting best effort #767
- * Remove reference to `ghe-cluster-cleanup-nodes` #768
- * Stop/Restart Actions in ghe-restore #769
- * Clear GitHub Connect settings when not restoring settings #770
-
- -- Brett Westover Thu, 09 Sep 2021 16:42:24 +0000
-
-github-backup-utils (3.1.0) UNRELEASED; urgency=medium
-
- -- Zachary Mark Thu, 03 Jun 2021 16:55:16 +0000
-
-github-backup-utils (3.1.0~rc1) UNRELEASED; urgency=medium
-
- [ Zachary Mark ]
- * Update repository backups to use ghe-gc-* for lock file management #188
- * A faster way to restore storage blobs (clusters) #212
- * Bug fix: Be more specific in restore routes globbing #715
- * fix(docker): add coreutils to get flags for date #717
- * Add backup cadence variable to the appliance #719
- * Fix is_default_external_database_snapshot function #720
-
- [ Hideki Yamane ]
- * debian/rules
- - Drop unnecessary build-indep:
- - Add manpages generation.
- * debian/changelog
- - Fix some lintian warnings for old entries.
- * debian/source/format
- - Upgrade to newer source format 3.0 (native).
- * debian/copyright
- - Format it as Machine-readable debian/copyright format 1.0
- - Add entries Twan Wolthof and me for debian/* since
- enough code was written by non-github.com employees, at least.
- * debian/control
- - Update descriptions in debian/control
- - Fix "Architecture: all", instead of "any".
- - Declare its Homepage as https://github.com/github/backup-utils
- - Add Vcs-* metadata field.
- - Specify "Rules-Requires-Root: no"
- - Drop unnecessary "Build-Depends: devscripts"
- - Set Standards-Version: 4.5.1
- - Update Maintainer as Zachary Mark
- * debian/{clean,manpages}
- - Add files to support manpages handles.
-
- -- Zachary Mark Thu, 06 May 2021 17:11:18 +0000
-
-github-backup-utils (3.0.0) UNRELEASED; urgency=medium
-
- * Fix restoring the password pepper for already configured instances #683
-
- -- Michael Dang Tue, 16 Feb 2021 22:32:25 +0000
-
-github-backup-utils (3.0.0.rc1) UNRELEASED; urgency=medium
-
- * Cleanup nomad container when restore for 2.22 #670
- * Use ghe-cluster-nomad-cleanup for cluster mode #663
- * Only run ghe-nomad-cleanup in 3.0 #662
- * Revert backup-utils gitHub env and a few more fixes #661
- * Note how to test filesystem symlink / hardlink support #660
- * stop github-timerd based on its running environment #659
- * Backup and restore password pepper #656
- * github-env -> github-env-dispatch #654
- * Rename redis-cli to ghe-redis-cli #639
-
- -- Michael Dang Thu, 14 Jan 2021 21:17:53 +0000
-
-github-backup-utils (2.22.0) UNRELEASED; urgency=medium
-
- * Added basic timing around the ghe-restore process #625
- * Improve python3 & finding moreutils parallel #627
- * Turn off POSIX for ghe-backup-config #632
- * Add parallelized restore capability to ghe-restore-storage #635
- * Update backup-utils for new features in 2.22 #641
-
- -- Hao Jiang Wed, 23 Sep 2020 15:48:54 +0000
-
-github-backup-utils (2.21.0) UNRELEASED; urgency=medium
-
- * Introduce option to skip restoring of audit logs #596
- * Beta: Execute ghe-backup tasks in parallel #597
- * Beta: Execute ghe-restore tasks in parallel #601
- * Run repositories restore in parallel #603
- * Fix mismatched `bm_start` and `bm_end` commands #607
- * remove rsync restore method used by GHES versions prior to 2.13 #608
- * Output MySQL backup strategy for clarity during backup and restore #610
-
- -- Hao Jiang Tue, 09 Jun 2020 17:59:06 +0000
-
-github-backup-utils (2.19.5) UNRELEASED; urgency=medium
-
- * In legacy mode we should use ghe-import-mysql #581
-
- -- Caine Jette Fri, 21 Feb 2020 19:13:17 +0000
-
-github-backup-utils (2.20.2) UNRELEASED; urgency=medium
-
- * In legacy mode we should use ghe-import-mysql #581
-
- -- Hao Jiang Thu, 20 Feb 2020 22:47:09 +0000
-
-github-backup-utils (2.20.1) UNRELEASED; urgency=medium
-
- * Fixes gist route calculation for GHES version 2.20
-
- -- Caine Jette Wed, 19 Feb 2020 21:47:18 +0000
-
-github-backup-utils (2.19.4) UNRELEASED; urgency=medium
-
- * Fix the way we connect to mysql master using ssh forwarding for binary backups #567
-
- -- Hao Jiang Tue, 18 Feb 2020 17:54:31 +0000
-
-github-backup-utils (2.20.0) UNRELEASED; urgency=medium
-
- * Fix `ghe-backup-repositories` performance for large instances #541
- * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551
-
- -- Alejandro Figueroa Tue, 11 Feb 2020 20:47:17 +0000
-
-github-backup-utils (2.19.3) UNRELEASED; urgency=medium
-
- * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551
-
- -- Hao Jiang Tue, 11 Feb 2020 19:31:55 +0000
-
-github-backup-utils (2.19.2) UNRELEASED; urgency=medium
-
- * Fix `ghe-backup-repositories` performance for large instances #541
-
- -- Evgenii Khramkov Fri, 31 Jan 2020 19:13:46 +0000
-
-github-backup-utils (2.19.1) UNRELEASED; urgency=medium
-
- * Cater for more explicit gist paths used in routes file #524
- * Suppress "*.rsync': No such file or directory" errors when no data to backup or restore #525
-
- -- Colin Seymour Wed, 11 Dec 2019 09:33:01 +0000
-
-github-backup-utils (2.19.0) UNRELEASED; urgency=medium
-
- * Remove temporary exclude file from the backup host not the target GHES appliance #516
- * Update Debian package depends to include git #520
-
- -- Colin Seymour Tue, 12 Nov 2019 18:57:12 +0000
-
-github-backup-utils (2.18.0) UNRELEASED; urgency=medium
-
- * Replace "sed -E" in ghe-host-check with a more portable solution #509
-
- -- Colin Seymour Tue, 20 Aug 2019 18:49:44 +0000
-
-github-backup-utils (2.17.1) UNRELEASED; urgency=medium
-
- * Redirect ghe-export-audit-logs stderr output unless using verbose output #497
-
- -- Colin Seymour Wed, 05 Jun 2019 08:43:25 +0000
-
-github-backup-utils (2.17.0) UNRELEASED; urgency=medium
-
- * Restore target is ignored when specified on the command line #476
- * Support incremental backups for MySQL-stored audit logs #485
-
- -- Colin Seymour Thu, 23 May 2019 08:20:15 +0000
-
-github-backup-utils (2.16.1) UNRELEASED; urgency=medium
-
- * Detect storage user on each cluster host being backed up or restored #472
-
- -- Colin Seymour Tue, 26 Feb 2019 16:37:04 +0000
-
-github-backup-utils (2.16.0) UNRELEASED; urgency=medium
-
- * (There was no descriptions)
-
- -- Colin Seymour Tue, 22 Jan 2019 20:25:34 +0000
-
-github-backup-utils (2.15.1) UNRELEASED; urgency=medium
-
- * Restoring to an un-configured appliance fails due to a missing license file #449
-
- -- Colin Seymour Tue, 13 Nov 2018 17:34:21 +0000
-
-github-backup-utils (2.15.0) UNRELEASED; urgency=medium
-
- * (There was no descriptions)
-
- -- Colin Seymour Tue, 16 Oct 2018 16:40:03 +0000
-
-github-backup-utils (2.15.0) UNRELEASED; urgency=medium
-
- * (There was no descriptions)
-
- -- Colin Seymour Tue, 16 Oct 2018 16:07:36 +0000
-
-github-backup-utils (2.14.3) UNRELEASED; urgency=medium
-
- * Improve multi-platform detection of simultaneous ghe-backup runs #435
-
- -- Colin Seymour Tue, 11 Sep 2018 17:03:42 +0000
-
-github-backup-utils (2.14.2) UNRELEASED; urgency=medium
-
- * Capture and display repository/gist warnings during cluster restore #423
- * Use remote tempdir when finalizing Pages routes #424
- * Use old rsync restore method for pages prior to 2.13 #426
-
- -- Colin Seymour Tue, 21 Aug 2018 13:57:20 +0000
-
-github-backup-utils (2.14.1) UNRELEASED; urgency=medium
-
- * Don't fail a backup if the Management Console password isn't set #416
- * Fix permissions issues when repeat restoring to configured cluster instance #417
- * Add missing dependencies to debian packaging #418
- * Prevent restoring snapshots to older releases #420
-
- -- Colin Seymour Tue, 07 Aug 2018 16:00:36 +0000
-
-github-backup-utils (2.14.0) UNRELEASED; urgency=medium
-
- * Disable pager and context when running git-diff #411
- * Optimise hookshot and audit log backups and restores and MySQL restores #413
-
- -- Colin Seymour Thu, 12 Jul 2018 15:11:11 +0000
-
-github-backup-utils (2.13.2) UNRELEASED; urgency=medium
-
- * Treat missing repository networks, gists, and storage objects as a non-critical error #386
- * Clean up stale HA nodes on restore #396
- * Cleanup all SSH muxes in a non blocking way #402
- * Raise an error if the current symlink doesn't exist when attempting to restore it #404
-
- -- Colin Seymour Fri, 22 Jun 2018 10:08:22 +0000
-
-github-backup-utils (2.13.1) UNRELEASED; urgency=medium
-
- * Retry with the admin ssh port on network unreachable too. #377
- * Output backup utils version on backup and restore #384
- * Check filesystem supports hardlinks #388
- * Switch back to optimised restore route calculation #389
- * Optionally log verbose output to a file instead of stdout #391
- * Switch back to optimised backup route calculation #392
- * Remove check for git from ghe-ssh #393
- * Use remote's mktemp to create temp dir on remote host #395
-
- -- Colin Seymour Wed, 09 May 2018 10:16:18 +0000
-
-github-backup-utils (2.13.0) UNRELEASED; urgency=medium
-
- * Unify the backup & restore process #375
-
- -- Colin Seymour Tue, 27 Mar 2018 16:01:43 +0000
-
-github-backup-utils (2.11.4) UNRELEASED; urgency=medium
-
- * Move check for git for ssh muxing into ghe-ssh #378
- * Make it clear the settings need to be applied after restoring to an unconfigured instance #381
-
- -- Colin Seymour Tue, 27 Mar 2018 13:20:18 +0000
-
-github-backup-utils (2.11.3) UNRELEASED; urgency=medium
-
- * Update argument parsing and help/usage consistency #320
- * Fix failures variable #353
- * Remove other snapshot contents before removing the "incomplete" file #358
- * Backup and restore the management console password #361
- * Check for git before allowing SSH multiplex #362
- * Cleanup SSH multiplexing on exit #363
- * Filter cluster nodes by role during backup and restore #367
- * Optimise route generation and finalisation during cluster restores of pages #369
- * Allow extra rsync options to override default options #370
-
- -- Colin Seymour Wed, 28 Feb 2018 16:32:07 +0000
-
-github-backup-utils (2.11.2) UNRELEASED; urgency=medium
-
- * Allow the restoration of configuration to Cluster #347
- * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348
-
- -- Colin Seymour Thu, 09 Nov 2017 12:16:23 +0000
-
-github-backup-utils (2.11.1) UNRELEASED; urgency=medium
-
- * Refresh the existing indices when restoring Elasticsearch indices to cluster #328
- * Fix failure to restore 2.9/2.10 backups to 2.11 prevented by incorrect detection of the audit log migration #333
- * Use git to generate short name for SSH multiplex control path #335
- * Remove use of --literally when computing arbitrary shasum #338
- * Remove -o option from ps use #341
-
- -- Colin Seymour Thu, 05 Oct 2017 14:47:56 +0000
-
-github-backup-utils (2.11.0) UNRELEASED; urgency=medium
-
- * Use calculated routes when backing up storage data from a cluster #318
- * Add SSH multiplexing support #321
- * Optimise route generation and finalisation during cluster restores #322
- * Prefer the SSH port specified on the command line #324
-
- -- Colin Seymour Wed, 13 Sep 2017 16:31:20 +0000
-
-github-backup-utils (2.10.0) UNRELEASED; urgency=medium
-
- * Include the user data directory in the benchmark name #311
- * Use existing Elasticsearch indices to speed up transfer during a restore #310
- * Improve detection of failures in cluster backup rsync threads #301
- * Improve redis backup robustness #306
- * Use default niceness for restores #308
- * Add additional case to SSH port detection logic #304
- * Suppress additional dd output noise #289
- * Track completeness of Elasticsearch JSON dumps #298
-
- -- Steven Honson Thu, 08 Jun 2017 09:06:16 +1000
-
-github-backup-utils (2.9.0) UNRELEASED; urgency=medium
-
- * Block restores to appliances with HA configured #291
- * Add a `--version` flag #284
- * Check backup-utils are not being run on GitHub Enterprise host #286
- * Backup and restore custom CA certificates #281
- * Hookshot backup/restores optimisations #276
-
- -- Sergio Rubio Wed, 01 Mar 2017 09:39:26 -0800
-
-github-backup-utils (2.8.3) UNRELEASED; urgency=medium
-
- * Set restore status on all cluster nodes #274
- * Fix pages backups and restores in GitHub Enterprise 11.10 #275
-
- -- Steven Honson Wed, 21 Dec 2016 21:01:20 +1100
-
-github-backup-utils (2.8.2) UNRELEASED; urgency=medium
-
- * Backup and restore the appliance UUID #272
-
- -- Sergio Rubio Thu, 17 Nov 2016 15:58:08 +0100
-
-github-backup-utils (2.8.1) UNRELEASED; urgency=medium
-
- * Stop cron and timerd during restore #269
- * Fix compatibility issue with older versions of OpenSSH #263
-
- -- Sergio Rubio Mon, 14 Nov 2016 22:04:48 +0100
-
-github-backup-utils (2.8.0) UNRELEASED; urgency=low
-
- * Adds support for GitHub Enterprise 2.8.0
- * Speedup storage restores #247
- * More portable backup-utils #260
-
- -- rubiojr Wed, 09 Nov 2016 06:35:21 -0800
-
-github-backup-utils (2.7.1) UNRELEASED; urgency=medium
-
- * Cluster: fix offline cluster node detection #250
- * Detect leaked ssh keys in backup snapshots #253
-
- -- Sergio Rubio Tue, 20 Sep 2016 20:15:12 +0200
-
-github-backup-utils (2.7.0) UNRELEASED; urgency=medium
-
- * GitHub Enterprise 2.7.0 support
-
- -- Sergio Rubio Wed, 03 Aug 2016 20:25:31 +0200
-
-github-backup-utils (2.6.4) UNRELEASED; urgency=medium
-
- * Instrument/benchmark backups #238
- * Cluster: remove restoring cluster.conf on restore #242
- * Cluster: Prevent restoring to a standalone GHE appliance #244
-
- -- Sergio Rubio Wed, 27 Jul 2016 19:15:53 +0200
-
-github-backup-utils (2.6.3) UNRELEASED; urgency=medium
-
- * Cluster: git-hooks backup fixes #235
-
- -- Sergio Rubio Wed, 29 Jun 2016 21:05:21 +0200
-
-github-backup-utils (2.6.2) UNRELEASED; urgency=medium
-
- * git-hooks fixes #231
- * Cluster: speedup repositories restore #232 (requires GitHub Enterprise
- 2.6.4)
- * Cluster: restore Git over SSH keys #230
- * Benchmark restores #219
-
- -- Sergio Rubio Wed, 22 Jun 2016 19:36:06 +0200
-
-github-backup-utils (2.6.1) UNRELEASED; urgency=medium
-
- * Cluster: faster gist restores #220
- * Cluster: faster storage restores #212
- * Cluster: fix git-hooks restores #204
-
- -- Sergio Rubio Tue, 31 May 2016 20:54:11 +0200
-
-github-backup-utils (2.6.0) UNRELEASED; urgency=medium
-
- * Adds support for GitHub Enterprise 2.6
- * Adds an extra supported location for the backup configuration #197
- * New config option to check for corrupted repositories after the backup #195
- * General improvements and bug fixes
-
- -- Sergio Rubio Tue, 26 Apr 2016 18:03:01 +0200
-
-github-backup-utils (2.5.2) UNRELEASED; urgency=medium
-
- * New configuration variable: GHE_CREATE_DATA_DIR #186
- * Require that snapshots originated from an instance running GitHub
- Enterprise 2.5.0 or above when restoring to a cluster #182
- * Prevent Git GC operations and some other maintenance jobs from running
- while repositories are being restored #179
- * Fix Solaris and SmartOS support, using Bash everywhere #177
-
- -- Sergio Rubio Wed, 30 Mar 2016 14:32:15 +0200
-
-github-backup-utils (2.5.1) UNRELEASED; urgency=medium
-
- * Fixes for cluster restores #173
- * Fix Elasticsearch backups for GitHub Enterprise <= 2.2 #175
- * Removed experimental S3 support #167
- * Remote logging output fixes #170
- * Update ghe-host-check to detect extra port 22 error #162
-
- -- Sergio Rubio Wed, 09 Mar 2016 14:44:05 +0100
-
-github-backup-utils (2.5.0) UNRELEASED; urgency=medium
-
- * Adds GitHub Enterprise 2.5 support
- * Adds GitHub Enterprise Clustering support
- * Backups and restores SAML keypairs
-
- -- Sergio Rubio Tue, 9 Feb 2016 00:02:37 +0000
-
-github-backup-utils (2.4.0) UNRELEASED; urgency=medium
-
- * Moves the in-progress detection to a separate file with PID which is
- removed if the process is no longer running after the backup. #145, #99
- * Updates the README to explain why backup-utils is useful even if you have
- the high availability replica running. #140
- * Changes the use of the --link-dest option to only occur when backing up
- populated directories. #138
- * Adds logging to /var/log/syslog on the remote GitHub Enterprise appliance
- to both ghe-backup and ghe-restore. #131
- * Restarts memcached after restoring to an already configured appliance to
- ensure it doesn't contain out-of-sync information. #130
- * Removes the temporary /data/user/repositories-nw-backup directory that
- remains after successfully migrating the repository storage layout to the
- new format used on GitHub Enterprise 2.2.0 and later after restoring a
- backup from an older release of GitHub Enterprise. #129
- * Add devscripts to Debian's build-depends for checkbashisms. #101
- * Documents the -c option which forces the restoration of the configuration
- information to an already configured appliance. #96
-
- -- Colin Seymour Tue, 20 Oct 2015 00:02:37 +0000
-
-github-backup-utils (2.2.0) UNRELEASED; urgency=medium
-
- * Adds support for the new repositories filesystem layout include in
- GitHub Enterprise v2.2. #122, #124
- * ghe-restore now performs a config run on the instance after an incremental
- restore to 11.10.x and 2.x instances. #100
- * ghe-restore now fails fast when run against a GHE instance with replication
- enabled. Replication should be disabled during a restore and then setup
- after the restore completes. #121
- * Fixes an issue with special port 122 detection failing when port is
- overridden in an ssh config file. #102
- * Removes a warning message when running ghe-backup against an instance with
- GitHub Pages disabled. #117
- * backup-utils release version numbers now track GitHub Enterprise releases
- to ease the process of determining which version of backup-utils is
- required for a given GitHub Enterprise version.
-
- -- Ryan Tomayko Wed, 29 Apr 2015 07:29:04 +0000
-
-github-backup-utils (2.0.2) UNRELEASED; urgency=medium
-
- * ghe-restore now requires that an already-configured appliance be put into
- maintenance mode manually. This is a safeguard against accidentally
- overwriting data on the wrong instance. #62, #84
- * ghe-backup and ghe-restore now run a ghe-negotiate-version program on the
- appliance to determine whether the backup-utils and GHE versions are
- compatible. #91
- * Various portability fixes for problems surfaced when running on Solaris
- and FreeBSD. #86, #87
- * Fixes an issue in ghe-backup where mysqldump failures weren't being
- reported properly. #90
- * Automated builds are now run on Travis CI. #77
-
- -- Ryan Tomayko Tue, 20 Jan 2015 16:00:00 +0000
-
-github-backup-utils (2.0.1) UNRELEASED; urgency=medium
-
- * Adds /etc/github-backup-utils/backup.config as a default config file search
- location for deb / system installs.
- * Enables SSH BatchMode for all remote command invocation except initial host
- check / version identification.
- * Fixes a bug in ghe-backup where Git GC process detection would misclassify
- long-running server processes matching /git.*gc/, causing the backup operation
- to timeout.
- * Adds a note and link to the Migrating from GitHub Enterprise v11.10.34x to
- v2.0 documentation in the README.
- * Adds example / documentation for the GHE_EXTRA_SSH_OPTS config value to the
- backup.config-example file.
-
- -- Ryan Tomayko Mon, 17 Nov 2014 12:47:22 +0000
-
-github-backup-utils (2.0.0) UNRELEASED; urgency=medium
-
- * Support for GitHub Enterprise 2.0.
- * Support for migrating from GitHub Enterprise 11.10.34x to 2.0 (including from
- VMware to AWS).
- * ghe-backup retains hardlinks present on VM in backup snapshots, saving space.
- * ghe-restore retains hardlinks present in backup snapshot when restoring to VM.
- * backup-utils now includes debian packaging support.
- * Fixes an issue with ghe-restore -s not using the snapshot specified.
- * Fixes an issue with ghe-backup not waiting for nw-repack processes to finish
- in some instances.
-
- -- Ryan Tomayko Mon, 10 Nov 2014 10:48:36 +0000
-
-github-backup-utils (1.1.0) UNRELEASED; urgency=medium
-
- * Updated documentation on minimum GitHub Enterprise version requirements for
- online and incremental backups from v11.10.341 to at least v11.10.342.
- * The ghe-restore command now prompts for confirmation of the host to restore to
- before performing any destructive operation. This is to reduce the chances of
- restoring to the wrong host. The prompt may be bypassed in automated scenarios
- by providing the --force option.
- * Added a -c option to ghe-restore for restoring base appliance settings in
- addition to primary datastores. See ghe-restore --help for more information.
- * Added a note about disabling maintenance mode on the appliance after a
- successful ghe-restore operation.
- * Added support for filesystem layout changes and upgraded server components in
- * future versions of GitHub Enterprise.
-
- -- Twan Wolthof Sat, 18 Oct 2014 19:14:47 +0000
-
-github-backup-utils (1.0.1) UNRELEASED; urgency=medium
-
- * Initial release.
-
- -- Twan Wolthof Tue, 23 Sep 2014 08:34:55 +0000
diff --git a/debian/clean b/debian/clean
deleted file mode 100644
index 0f651869c..000000000
--- a/debian/clean
+++ /dev/null
@@ -1 +0,0 @@
-debian/*.1
diff --git a/debian/compat b/debian/compat
deleted file mode 100644
index ec635144f..000000000
--- a/debian/compat
+++ /dev/null
@@ -1 +0,0 @@
-9
diff --git a/debian/control b/debian/control
deleted file mode 100644
index b2cd7c945..000000000
--- a/debian/control
+++ /dev/null
@@ -1,32 +0,0 @@
-Source: github-backup-utils
-Maintainer: Zachary Mark
-Section: misc
-Priority: optional
-Standards-Version: 4.5.1
-Build-Depends: debhelper (>= 9), git, moreutils, jq, rsync (>= 2.6.4), help2man,
-Homepage: https://github.com/github/backup-utils
-Vcs-Git: https://github.com/github/backup-utils.git
-Vcs-Browser: https://github.com/github/backup-utils
-Rules-Requires-Root: no
-
-Package: github-backup-utils
-Architecture: all
-Depends: ${misc:Depends}, rsync (>= 2.6.4), moreutils, jq, git
-Description: Backup and recovery utilities for GitHub Enterprise Server
- The backup utilities implement a number of advanced capabilities for backup
- hosts, built on top of the backup and restore features already included in
- GitHub Enterprise Server.
- .
- These advanced features include:
- - Complete GitHub Enterprise Server backup and recovery system via two simple
- utilities: `ghe-backup` and `ghe-restore`.
- - Online backups. The GitHub appliance need not be put in maintenance mode for
- the duration of the backup run.
- - Incremental backup of Git repository data. Only changes since the last
- snapshot are transferred, leading to faster backup runs and lower network
- bandwidth and machine utilization.
- - Efficient snapshot storage. Only data added since the previous snapshot
- consumes new space on the backup host.
- - Multiple backup snapshots with configurable retention periods.
- - Backup runs under the lowest CPU/IO priority on the GitHub appliance,
- reducing performance impact while backups are in progress.
diff --git a/debian/copyright b/debian/copyright
deleted file mode 100644
index 8879cfae8..000000000
--- a/debian/copyright
+++ /dev/null
@@ -1,33 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: GitHub Enterprise Server Backup Utilities
-Source: https://github.com/github/backup-utils
-
-Files: *
-Copyright: 2014-2021, GitHub Inc.
-License: MIT
-
-Files: debian/*
-Copyright: 2014-2021, GitHub Inc.
- 2014 Twan Wolthof
- 2021 Hideki Yamane
-License: MIT
-
-License: MIT
- Permission is hereby granted, free of charge, to any person obtaining
- a copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
- .
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
- .
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/debian/install b/debian/install
deleted file mode 100644
index bc992fc49..000000000
--- a/debian/install
+++ /dev/null
@@ -1,2 +0,0 @@
-bin/* usr/bin
-share/* usr/share
diff --git a/debian/manpages b/debian/manpages
deleted file mode 100644
index 0f651869c..000000000
--- a/debian/manpages
+++ /dev/null
@@ -1 +0,0 @@
-debian/*.1
diff --git a/debian/rules b/debian/rules
deleted file mode 100755
index af3d39d08..000000000
--- a/debian/rules
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/make -f
-
-VERSION=$$(cat $(CURDIR)/share/github-backup-utils/version)
-
-override_dh_auto_build:
- # generate manpages for ghe-backup, ghe-host-check and ghe-restore
- help2man $(CURDIR)/bin/ghe-backup -N -o $(CURDIR)/debian/ghe-backup.1 \
- -n "Take snapshots of all GitHub Enterprise data" \
- --version-string="ghe-backup $(VERSION)"
- help2man $(CURDIR)/bin/ghe-host-check -N -o $(CURDIR)/debian/ghe-host-check.1 \
- -n "Restores a GitHub instance from local backup snapshots" \
- --version-string="ghe-host-check $(VERSION)"
- help2man $(CURDIR)/bin/ghe-restore -N -o $(CURDIR)/debian/ghe-restore.1 \
- -n "Verify connectivity with the GitHub Enterprise Server host" \
- --version-string="ghe-restore $(VERSION)"
-
-%:
- dh $@
diff --git a/debian/source/format b/debian/source/format
deleted file mode 100644
index 89ae9db8f..000000000
--- a/debian/source/format
+++ /dev/null
@@ -1 +0,0 @@
-3.0 (native)
diff --git a/docs/README.md b/docs/README.md
index 153549d14..e84d313df 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -6,7 +6,7 @@
- **[GitHub Enterprise Server version requirements](requirements.md#github-enterprise-version-requirements)**
- **[Getting started](getting-started.md)**
- **[Using the backup and restore commands](usage.md)**
-- **[Scheduling backups](scheduling-backups.md)**
+- **[Scheduling backups & snapshot pruning](scheduling-backups.md)**
- **[Backup snapshot file structure](backup-snapshot-file-structure.md)**
- **[How does Backup Utilities differ from a High Availability replica?](faq.md)**
- **[Docker](docker.md)**
diff --git a/docs/backup-snapshot-file-structure.md b/docs/backup-snapshot-file-structure.md
index 2a950d63e..0a7032397 100644
--- a/docs/backup-snapshot-file-structure.md
+++ b/docs/backup-snapshot-file-structure.md
@@ -52,13 +52,41 @@ Actions service uses MS SQL Server as backend data store. Each snapshot includes
To save time in backup, a three-level backup strategy is implemented. Based on the `GHE_MSSQL_BACKUP_CADENCE` setting, at each snapshot, either a (**F**)ull backup, a (**D**)ifferential or a (**T**)ransaction log backup is taken.
As a result, a suite always contains following for each database: a full backup, possibly a differential backup and at least one transaction log backup. Their relationship with timeline is demonstrated below:
-```
+
+```text
M---8:00--16:00---T---8:00--16:00---W... (timeline)
F-----------------F-----------------F... (full backup)
#-----D-----D-----#-----D-----D-----#... (differential backup)
T--T--T--T--T--T--T--T--T--T--T--T--T... (transaction log backup)
```
+
To save disk space, at each snapshot, hard links are created to point to previous backup files. Only newly-created backup files are transferred from appliance to backup host. When a new full/differential backup is created, they become the new source for hard links and new base line for transaction log backups, for subsequent snapshots.
-During restore, a suite of backup files are restored in the sequence of full -> differential -> chronological transaction log.
\ No newline at end of file
+During restore, a suite of backup files are restored in the sequence of full -> differential -> chronological transaction log.
+
+## Benchmark data
+
+Benchmark data for each snapshot is stored as a log file within the `benchmarks` directory within a snapshot directory. The benchmark log can be used to determine the duration of each backup step. For example:
+
+```text
+ghe-backup-store-version took 0s
+ghe-backup-settings took 2s
+ghe-export-authorized-keys took 0s
+ghe-export-ssh-host-keys took 0s
+ghe-backup-mysql-binary took 9s
+ghe-backup-mysql took 9s
+ghe-backup-minio took 0s
+ghe-backup-redis took 1s
+ghe-backup-es-audit-log took 1s
+ghe-backup-repositories - Generating routes took 3s
+ghe-backup-repositories - Fetching routes took 0s
+ghe-backup-repositories - Processing routes took 0s
+ghe-backup-pages - hostname took 1s
+ghe-backup-pages took 1s
+ghe-backup-storage - Generating routes took 2s
+ghe-backup-storage - Fetching routes took 0s
+ghe-backup-storage - Processing routes took 0s
+ghe-backup-git-hooks took 0s
+ghe-backup-es-rsync took 2s
+```
diff --git a/docs/getting-started.md b/docs/getting-started.md
index 80e141218..ac85e5073 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -1,19 +1,15 @@
# Getting started
- 1. [Download the latest release version][1] and extract the repository using `tar`:
+ 1. [Download the latest version of backup-utils][1] and extract the repository using `tar`:
`tar -xzvf /path/to/github-backup-utils-vMAJOR.MINOR.PATCH.tar.gz`
- *or* clone the repository using Git:
-
- `git clone -b stable https://github.com/github/backup-utils.git`
-
**Note**: you will need to use [Backup Utilities v2.11.x][2] or the `legacy` branch to
backup and restore GitHub Enterprise Server 2.10 and earlier.
2. Copy the [`backup.config-example`][3] file to `backup.config` and modify as
necessary. The `GHE_HOSTNAME` value must be set to the primary GitHub Enterprise Server
- host name. Additional options are available and documented in the
+ hostname. Additional options are available and documented in the
configuration file but none are required for basic backup functionality.
As the data on a High Availability replica may be in a transient state at the time of backup,
@@ -22,7 +18,7 @@
* Backup Utilities will attempt to load the backup configuration from the following
locations, in this order:
- ```
+ ```bash
$GHE_BACKUP_CONFIG (User configurable environment variable)
$GHE_BACKUP_ROOT/backup.config (Root directory of backup-utils install)
$HOME/.github-backup-utils/backup.config
@@ -31,8 +27,8 @@
* In a clustering environment, the `GHE_EXTRA_SSH_OPTS` key must be configured
with the `-i ` SSH option.
- 3. Add the backup host's SSH key to the GitHub appliance as an *Authorized SSH
- key*. See [Adding an SSH key for shell access][4] for instructions.
+ 3. Add the backup host's SSH public key to the GitHub Enterprise Server appliance, in order to grant it administrative shell access.
+ See [Accessing the GitHub Enterprise Server administrative shell (SSH)][4] for instructions.
4. Run `bin/ghe-host-check` to verify SSH connectivity with the GitHub
appliance.
@@ -42,4 +38,4 @@
[1]: https://github.com/github/backup-utils/releases
[2]: https://github.com/github/backup-utils/releases/tag/v2.11.4
[3]: https://github.com/github/enterprise-backup-site/blob/master/backup.config-example
-[4]: https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access
+[4]: https://docs.github.com/enterprise-server/admin/configuration/configuring-your-enterprise/accessing-the-administrative-shell-ssh
diff --git a/docs/incremental-mysql-backups-and-restores.md b/docs/incremental-mysql-backups-and-restores.md
new file mode 100644
index 000000000..53543c927
--- /dev/null
+++ b/docs/incremental-mysql-backups-and-restores.md
@@ -0,0 +1,38 @@
+# Incremental MySQL Backups and Restores
+
+Customers who have large MySQL databases who wish to save storage space can use the `--incremental` flag with `ghe-backup` and `ghe-restore`.
+Using this flag performs backups for other parts of GHES as normal, but only performs a MySQL backup of the changes to the database from the previous snapshot.
+For larger databases this can conserve a lot of storage space for backups.
+
+## Configuring number of backups
+
+In your backup.config file you will need to set the variable `GHE_INCREMENTAL_MAX_BACKUPS`.
+This variable determines how many cycles of full and incremental backups will be performed before the next full backup is created.
+For example, if `GHE_INCREMENTAL_MAX_BACKUPS` is set to 14, backup-utils will run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle.
+
+Incremental backups require the previous snapshot backups before them to work.
+This means they do not follow the pruning strategy based on `GHE_NUM_SNAPSHOTS`.
+
+## Performing incremental backups
+
+To perform incremental backups:
+
+`bin/ghe-backup --incremental`
+
+the program will detect whether it needs to performa full or incremental snapshot based on what is currently in `GHE_DATA_DIR`.
+
+To see what snapshots are part of your full and incremental backups, you can reference `GHE_DATA_DIR/inc_full_backup` and `GHE_DATA_DIR/inc_snapshot_data`, respectively.
+
+## Performing incremental restores
+
+To perform incremental restores:
+
+`bin/ghe-restore --incremental -s `
+
+The program will use the MySQL folders from each previous incremental backup and the full backup to restore the database.
+
+:warning: Incremental restores require the other snapshots in the cycle to complete a restore. Erasing snapshot directories that are part of a cycle corrupts the restore and makes it impossible to restore for the MySQL database.
+
+### Previous cycles
+
+To ensure there is a rolling window of mySQL backups, incremental MySQL backups from the cycle before the current one are kept. Those snapshots are pre-pended with `inc_previous`. To perform a restore from there, just use the full directory name for the snapshot ID.
diff --git a/docs/requirements.md b/docs/requirements.md
index 94e5c5701..c87a0775a 100644
--- a/docs/requirements.md
+++ b/docs/requirements.md
@@ -5,16 +5,38 @@ storage and must have network connectivity with the GitHub Enterprise Server app
## Backup host requirements
-Backup host software requirements are modest: Linux or other modern Unix operating
-system with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, and [rsync][4] v2.6.4 or newer.
+Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer and [bc][12] v1.0.7 or newer. See below for an update on rsync.
-The new parallel backup and restore beta feature will require [GNU awk][10] and [moreutils][9] to be installed.
+The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed.
-We encourage the use of [Docker](docker.md) if your backup host doesn't meet these
-requirements, or if Docker is your preferred platform.
+We encourage the use of [Docker](docker.md), as it ensures compatible versions of the aforementioned software are available to backup-utils.
-The backup host must be able to establish outbound network connections to the
-GitHub appliance over SSH. TCP port 122 is used to backup GitHub Enterprise Server.
+The backup host must be able to establish outbound network connections to the GitHub appliance over SSH. TCP port 122 is used to backup GitHub Enterprise Server.
+
+CPU and memory requirements are dependent on the size of the GitHub Enterprise Server appliance. We recommend a minimum of 4 cores and 8GB of RAM for the host running [GitHub Enterprise Backup Utilities](https://github.com/github/backup-utils). We recommend monitoring the backup host's CPU and memory usage to ensure it is sufficient for your environment.
+
+### April 2023 Update of Rsync Requirements
+
+The [fix in rsync `3.2.5`](https://github.com/WayneD/rsync/blob/master/NEWS.md#news-for-rsync-325-14-aug-2022) for [CVE-2022-29154](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29154) can cause severe performance degradation to `backup-utils`.
+
+If you encounter this degradation you can mitigate it by using the `--trust-sender` flag, which is available in rsync >= v3.2.5.
+**Note**: If you are using backup-utils 3.9 or greater, `--trust-sender` is automatically used if your rsync version supports it and no further changes are needed.
+
+If your backup host is running rsync < v3.2.5 you may or may not need to make changes to your rsync package, depending on whether your rsync package has backported the fix for CVE-2022-29154 without also backporting the `--trust-sender` flag.
+
+If your rsync package has backported the CVE fix _and_ the `--trust-sender` flag then you don't need to change anything.
+
+However, if your rsync package has backported the CVE fix without backporting the `--trust-sender` flag then you have three options:
+
+1. Downgrade (using the package manager on your host) the rsync package to a version before the CVE fix was backported
+2. Upgrade (using the package manager on your host) the rsync package to v3.2.5 or newer
+3. Manually download rsync v3.2.5 or newer and build the rsync binary
+
+Option #3 is required if your operating system's package manager does not have access to rsync v3.2.5 or later (e.g. Ubuntu Focal).
+
+Please note that some operating systems have their own versioning scheme for packages (including `rsync`).
+If your backup host is using one of these operating systems, you will not be able to rely on a version check to determine whether you are
+affected by the `rsync` performance degredation described above.
## Storage requirements
@@ -38,28 +60,32 @@ ls -la
Using a [case sensitive][7] file system is also required to avoid conflicts.
+Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS.
+
+Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups.
+
## GitHub Enterprise Server version requirements
Starting with Backup Utilities v2.13.0, version support is inline with that of the
[GitHub Enterprise Server upgrade requirements][8] and as such, support is limited to
three versions of GitHub Enterprise Server: the version that corresponds with the version
-of Backup Utilities, and the two releases prior to it.
+of Backup Utilities, and the two versions prior to it.
For example, Backup Utilities v2.13.0 can be used to backup and restore all patch
-releases from 2.11.0 to the latest patch release of GitHub Enterprise Server 2.13.
+versions from 2.11.0 to the latest patch version of GitHub Enterprise Server 2.13.
Backup Utilities v2.14.0 will be released when GitHub Enterprise Server 2.14.0 is released
-and will then be used to backup all releases of GitHub Enterprise Server from 2.12.0
-to the latest patch release of GitHub Enterprise Server 2.14.
+and will then be used to backup all versions of GitHub Enterprise Server from 2.12.0
+to the latest patch version of GitHub Enterprise Server 2.14.
Backup Utilities v2.11.4 and earlier offer support for GitHub Enterprise Server 2.10
-and earlier releases up to GitHub Enterprise Server 2.2.0. Backup Utilities v2.11.0 and earlier
+and earlier versions up to GitHub Enterprise Server 2.2.0. Backup Utilities v2.11.0 and earlier
offer support for GitHub Enterprise Server 2.1.0 and earlier.
-**Note**: You can restore a snapshot that's at most two feature releases behind
+**Note**: You can restore a snapshot that's at most two feature versions behind
the restore target's version of GitHub Enterprise Server. For example, to restore a
snapshot of GitHub Enterprise Server 2.11, the target GitHub Enterprise Server appliance must
be running GitHub Enterprise Server 2.12.x or 2.13.x. You can't restore a snapshot from
-2.10 to 2.13, because that's three releases ahead.
+2.10 to 2.13, because that's three versions ahead.
**Note**: You _cannot_ restore a backup created from a newer version of GitHub Enterprise Server to an older version. For example, an attempt to restore a snapshot of GitHub Enterprise Server 2.21 to a GitHub Enterprise Server 2.20 environment will fail with an error of `Error: Snapshot can not be restored to an older release of GitHub Enterprise Server.`.
@@ -67,7 +93,7 @@ be running GitHub Enterprise Server 2.12.x or 2.13.x. You can't restore a snapsh
Using multiple backup hosts or backup configurations is not currently recommended.
-Due to how some components of Backup Utiltiies (e.g. MSSQL) take incremental backups, running another instance of Backup Utilities may result in unrestorable snapshots as data may be split across backup hosts. If you still wish to have multiple instances of Backup Utilties for redundancy purposes or to run at different frequencies, ensure that they share the same `GHE_DATA_DIR` backup directory.
+Due to how some components of Backup Utilities (e.g. MSSQL) take incremental backups, running another instance of Backup Utilities may result in unrestorable snapshots as data may be split across backup hosts. If you still wish to have multiple instances of Backup Utilities for redundancy purposes or to run at different frequencies, ensure that they share the same `GHE_DATA_DIR` backup directory.
[1]: https://www.gnu.org/software/bash/
[2]: https://git-scm.com/
@@ -76,6 +102,8 @@ Due to how some components of Backup Utiltiies (e.g. MSSQL) take incremental bac
[5]: https://en.wikipedia.org/wiki/Hard_link
[6]: https://en.wikipedia.org/wiki/Symbolic_link
[7]: https://en.wikipedia.org/wiki/Case_sensitivity
-[8]: https://help.github.com/enterprise/admin/guides/installation/upgrade-requirements/
+[8]: https://docs.github.com/enterprise-server/admin/monitoring-managing-and-updating-your-instance/updating-the-virtual-machine-and-physical-resources/upgrade-requirements
[9]: https://joeyh.name/code/moreutils
[10]: https://www.gnu.org/software/gawk
+[11]: https://stedolan.github.io/jq/
+[12]: https://www.gnu.org/software/bc/
diff --git a/docs/scheduling-backups.md b/docs/scheduling-backups.md
index 8d291021a..3a7b0d12c 100644
--- a/docs/scheduling-backups.md
+++ b/docs/scheduling-backups.md
@@ -1,11 +1,20 @@
-# Scheduling backups
+# Scheduling backups & snapshot pruning
Regular backups should be scheduled using `cron(8)` or similar command
scheduling service on the backup host. The backup frequency will dictate the
worst case [recovery point objective (RPO)][1] in your backup plan. We recommend
-hourly backups at the least.
+hourly backups as a starting point.
-## Example scheduling usage
+It's important to consider the duration of each backup operation on the
+GitHub Enterprise Server (GHES) appliance. Backups of large datasets or
+over slow network links can take more than an hour. Additionally,
+maintenance queues are paused during a portion of a backup runs.
+We recommend scheduling backups to allow sufficient time for jobs
+waiting in maintenance queues to process between backup runs
+
+Only one backup may be in progress at a time.
+
+## Example scheduling of backups
The following examples assume the Backup Utilities are installed under
`/opt/backup-utils`. The crontab entry should be made under the same user that
@@ -20,15 +29,35 @@ storage.
To schedule hourly backup snapshots with verbose informational output written to
a log file and errors generating an email:
- MAILTO=admin@example.com
+```shell
+MAILTO=admin@example.com
- 0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
To schedule nightly backup snapshots instead, use:
- MAILTO=admin@example.com
+```shell
+MAILTO=admin@example.com
+
+0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
+
+## Example snapshot pruning
+
+By default all expired and incomplete snapshots are deleted at the end of the main
+backup process `ghe-backup`. If pruning these snapshots takes a long time you can
+choose to disable the pruning process from the backup run and schedule it separately.
+This can be achieved by enabling the `GHE_PRUNING_SCHEDULED` option in `backup.config`.
+Please note that this option is only avilable for `backup-utils` >= `v3.10.0`.
+If this option is enabled you will need to schedule the pruning script `ghe-prune-snapshots` using `cron` or a similar command scheduling service on the backup host.
+
+To schedule daily snapshot pruning, use:
- 0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```shell
+MAILTO=admin@example.com
+0 3 * * * /opt/backup-utils/share/github-backup-utils/ghe-prune-snapshots 1>>/opt/backup-utils/prune-snapshots.log 2>&1
+```
[1]: https://en.wikipedia.org/wiki/Recovery_point_objective
diff --git a/docs/usage.md b/docs/usage.md
index 30954c01a..4324b58cc 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -2,11 +2,10 @@
After the initial backup, use the following commands:
- - The `ghe-backup` command creates incremental snapshots of repository data,
- along with full snapshots of all other pertinent data stores.
- - The `ghe-restore` command restores snapshots to the same or separate GitHub
- Enterprise appliance. You must add the backup host's SSH key to the target
- GitHub Enterprise Server appliance before using this command.
+- The `ghe-backup` command creates incremental snapshots of repository data, long with full snapshots of all other pertinent data stores.
+- The `ghe-restore` command restores snapshots to the same or separate GitHub
+Enterprise appliance. You must add the backup host's SSH key to the target
+GitHub Enterprise Server appliance before using this command.
These commands are run on the host you [installed][1] Backup Utilities on.
@@ -16,7 +15,7 @@ You can supply your own configuration file or use the example configuration file
An example configuration file with documentation on possible settings can found in [backup.config-example](../backup.config-example).
-There are a number of command line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`.
+There are a number of command-line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`.
## Example backup and restore usage
@@ -73,14 +72,19 @@ appliance at IP "5.5.5.5":
Restore of 5.5.5.5:122 from snapshot 20180326T020444 finished.
To complete the restore process, please visit https://5.5.5.5/setup/settings to review and save the appliance configuration.
-A different backup snapshot may be selected by passing the `-s` argument and the
-datestamp-named directory from the backup location.
+A different backup snapshot may be selected by passing the `-s` argument to `ghe-restore` and specifying the
+datestamp-named directory from the backup location as the value.
The `ghe-backup` and `ghe-restore` commands also have a verbose output mode
(`-v`) that lists files as they're being transferred. It's often useful to
enable when output is logged to a file.
-### Restoring settings, TLS certificate, and license
+Every time you execute `ghe-backup` we verify the storage and software setup of the host
+you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are
+met. You can disable this check using the `--skip-checks` argument or by
+adding `GHE_SKIP_CHECKS=true` to your configuration file.
+
+### Restoring settings, TLS certificate, and license
When restoring to a new GitHub Enterprise Server instance, settings, certificate, and
license data *are* restored. These settings must be reviewed and saved before
@@ -95,15 +99,26 @@ forcing settings, certificate, and license data to be overwritten with the backu
## Backup and restore with GitHub Actions enabled
GitHub Actions data on your external storage provider is not included in regular GitHub Enterprise Server
-backups, and must be backed up separately. When restoring a GitHub Enterprise Server backup with
+backups, and must be backed up separately. When restoring a GitHub Enterprise Server backup with
GitHub Actions enabled, the following steps are required:
1. Enable GitHub Actions on the replacement appliance and configure it to use the same GitHub Actions
external storage configuration as the original appliance.
-2. Put replacement appliance into maintaince mode.
+2. Put replacement appliance into maintenance mode.
3. Use `ghe-restore` to restore the backup.
4. Re-register your self-hosted runners on the replacement appliance.
Please refer to [GHES Documentation](https://docs.github.com/en/enterprise-server/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled) for more details.
+## Incremental MySQL Backups and Restores
+
+If you are interested in performing incremental backups of the MySQL data in your GitHub Enterprise Server instance, see [Incremental MySQL Backups and Restores](incremental-mysql-backups-and-restores.md) for details.
+
+## Rsync compression
+
+From backup-utils v3.11.0 onwards, we have disabled rsync compression by default to improve transfer speed and reduce CPU usage during the transfer process.
+
+If you would like to use compression with rsync, you can add `GHE_RSYNC_COMPRESSION_ENABLED=true` in your `backup.config` file.
+
[1]: https://github.com/github/backup-utils/blob/master/docs/getting-started.md
+[2]: requirements.md
diff --git a/script/cibuild b/script/cibuild
deleted file mode 100755
index 7f57f444d..000000000
--- a/script/cibuild
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Usage: script/cibuild [--no-package]
-set -e
-
-# Enable verbose logging of ssh commands
-export GHE_VERBOSE_SSH=true
-
-if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then
- exit 1
-fi
-
-# Bail out when --no-package given
-[ "$1" = "--no-package" ] && exit 0
-
-# files we'll md5sum at the end
-pkg_files=
-
-# Build the tarball
-ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
-TMPDIR="$ROOTDIR/test/tmp"
-
-echo "Building tar.gz package ..."
-if script/package-tarball 1>$TMPDIR/package-tarball.txt 2>&1; then
- pkg_files=$(grep '^Package ' < $TMPDIR/package-tarball.txt | cut -f 2 -d ' ')
-else
- echo "Packaging tar.gz failed:"
- cat $TMPDIR/package-tarball.txt | sed 's/^/ /' 1>&2
- exit 1
-fi
-
-# Skip deb packaging if debuild not installed
-if ! type debuild 1>/dev/null 2>&1; then
- echo "debuild not installed, skipping deb packaging ..."
- exit 0
-fi
-
-# Build the deb related packages
-echo "Building deb package ..."
-if DEB_BUILD_OPTIONS=nocheck script/package-deb 1>$TMPDIR/package-deb-out.txt 2>$TMPDIR/package-deb-err.txt; then
- pkg_files="$pkg_files $(cat $TMPDIR/package-deb-out.txt)"
-else
- echo "Package build failed:"
- cat $TMPDIR/package-deb-out.txt $TMPDIR/package-deb-err.txt >&2
- echo >&2
- cat dist/debuild/github-backup-utils*.build >&2
- exit 1
-fi
-
-# Generate md5sums
-md5sum $pkg_files
diff --git a/script/package-deb b/script/package-deb
deleted file mode 100755
index dda90cbe3..000000000
--- a/script/package-deb
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-# Usage: script/package-deb
-# Script to build a deb release package from the current HEAD version.
-# The package version comes from the debian/changelog file so that should
-# be edited before running this.
-set -e
-
-# Change into project root
-cd "$(dirname "$0")"/..
-
-# Basic package name and version.
-PKG_BASE="github-backup-utils"
-PKG_VERS="$(git describe --tags)"
-PKG_NAME="${PKG_BASE}-${PKG_VERS}"
-PKG_HEAD="$(git rev-parse HEAD)"
-
-# Run git-archive to generate tarball
-rm -rf dist/debuild
-trap "rm -rf dist/debuild" EXIT
-mkdir -p dist/debuild
-
-distdir="$(pwd)/dist/debuild/$PKG_NAME"
-git clone -q . "$distdir"
-cd "$distdir"
-git checkout -q "$PKG_HEAD"
-
-debuild -uc -us 1>&2
-cd ..
-files=$(ls -1 *.deb *.tar.xz *.dsc *.changes)
-mv $files ../
-for f in $files; do echo "dist/$f"; done
diff --git a/script/package-tarball b/script/package-tarball
deleted file mode 100755
index be653b77a..000000000
--- a/script/package-tarball
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Usage: script/package-tarball
-# Script to build a tarball release package from the current HEAD version.
-# The package version comes from `git-describe --tags' so the release tag should
-# be in place before this command is run.
-set -e
-
-# Change into project root
-cd "$(dirname "$0")"/..
-
-# Basic package name and version.
-PKG_BASE="github-backup-utils"
-PKG_VERS="$(git describe --tags)"
-PKG_NAME="${PKG_BASE}-${PKG_VERS}"
-
-# Run git-archive to generate tarball
-echo "Creating ${PKG_NAME}.tar.gz ..."
-mkdir -p dist
-git archive \
- --format=tar.gz \
- --prefix="$PKG_NAME/" \
- --output="dist/${PKG_NAME}.tar.gz" \
- HEAD
-
-# List archive contents for review
-gzip -dc < "dist/${PKG_NAME}.tar.gz" | tar tf -
-
-# Output location
-echo "Package dist/${PKG_NAME}.tar.gz OK"
diff --git a/script/release b/script/release
deleted file mode 100755
index 71dc31e39..000000000
--- a/script/release
+++ /dev/null
@@ -1,466 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-#/ Usage: release [--dry-run] [--skip-version-bump-check] [min_version]
-#/
-#/ Publish a backup-utils release:
-#/ * Updates the package changelog
-#/ * Bumps the backup-utils version if required
-#/ * Creates the release pull request
-#/ * Merges the release pull request
-#/ * Creates the release draft
-#/ * Tags the release
-#/ * Builds the release assets and uploads them
-#/
-#/ Notes:
-#/ * Needs GH_RELEASE_TOKEN and GH_AUTHOR set in the environment.
-#/ * Export GH_OWNER and GH_REPO if you want to use a different owner/repo
-#/ * Only pull requests labeled with bug, feature or enhancement will show up in the
-#/ release page and the changelog.
-#/ * If this is a X.Y.0 release, a minimum supported version needs to be supplied too.
-#/
-require 'json'
-require 'net/http'
-require 'time'
-require 'erb'
-require 'English'
-
-API_HOST = ENV['GH_HOST'] || 'api.github.com'
-API_PORT = 443
-GH_REPO = ENV['GH_REPO'] || 'backup-utils'
-GH_OWNER = ENV['GH_OWNER'] || 'github'
-GH_AUTHOR = ENV['GH_AUTHOR']
-DEB_PKG_NAME = 'github-backup-utils'
-GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master'
-
-CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
-
-<%- changes.each do |ch| -%>
- * <%= ch.strip.chomp %>
-<% end -%>
-
- -- <%= GH_AUTHOR %> <%= Time.now.utc.strftime("%a, %d %b %Y %H:%M:%S %z") %>
-
-'''
-
-# Override Kernel.warn
-def warn(msg)
- Kernel.warn msg unless @no_warn
-end
-
-def client(host = API_HOST, port = API_PORT)
- @http ||= begin
- c = Net::HTTP.new(host, port)
- c.use_ssl = true
- c
- end
-end
-
-def get(path)
- req = Net::HTTP::Get.new(path)
- req['Authorization'] = "token #{release_token}"
- client.request(req)
-end
-
-def post(path, body)
- req = Net::HTTP::Post.new(path)
- req['Authorization'] = "token #{release_token}"
- req.body = body
- client.request(req)
-end
-
-def post_file(path, body)
- req = Net::HTTP::Post.new(path)
- req['Authorization'] = "token #{release_token}"
- req['Content-Type'] = path.match?(/.*\.tar\.gz$/) ? 'application/tar+gzip' : 'application/vnd.debian.binary-package'
- req.body = body
- client.request(req)
-end
-
-def put(path, body)
- req = Net::HTTP::Put.new(path)
- req['Authorization'] = "token #{release_token}"
- req.body = body
- client.request(req)
-end
-
-def patch(path, body)
- req = Net::HTTP::Patch.new(path)
- req['Authorization'] = "token #{release_token}"
- req.body = body
- client.request(req)
-end
-
-def release_token
- token = ENV['GH_RELEASE_TOKEN']
- raise 'GH_RELEASE_TOKEN environment variable not set' if token.nil?
-
- token
-end
-
-# Create a lightweight tag
-def tag(name, sha)
- body = {
- "ref": "refs/tags/#{name}",
- "sha": sha
- }.to_json
- res = post("/repos/#{GH_OWNER}/#{GH_REPO}/git/refs", body)
-
- raise "Creating tag ref failed (#{res.code})" unless res.is_a? Net::HTTPSuccess
-end
-
-def bug_or_feature?(issue_hash)
- return true if issue_hash['labels'].find { |label| ['bug', 'feature', 'enhancement'].include?(label['name']) }
- false
-end
-
-def issue_from(issue)
- res = get("/repos/#{GH_OWNER}/#{GH_REPO}/issues/#{issue}")
- raise "Issue ##{issue} not found in #{GH_OWNER}/#{GH_REPO}" unless res.is_a? Net::HTTPSuccess
-
- JSON.parse(res.body)
-end
-
-def beautify_changes(changes)
- out = []
- changes.each do |chg|
- next unless chg =~ /#(\d+)/
- begin
- issue = issue_from Regexp.last_match(1)
- out << "#{issue['title']} ##{Regexp.last_match(1)}" if bug_or_feature?(issue)
- rescue => e
- warn "Warning: #{e.message}"
- end
- end
-
- out
-end
-
-def changelog
- changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip)
- raise 'Building the changelog failed' if $CHILD_STATUS != 0
-
- changes
-end
-
-def build_changelog(changes, package_name, package_version)
- ERB.new(CHANGELOG_TMPL, nil, '-').result(binding)
-end
-
-def update_changelog(changes, name, version, path = 'debian/changelog')
- raise 'debian/changelog not found' unless File.exist?(path)
- File.open("#{path}.new", 'w') do |f|
- f.puts build_changelog changes, name, version
- f.puts(File.read(path))
- end
- File.rename("#{path}.new", path)
-end
-
-def create_release(tag_name, branch, rel_name, rel_body, draft = true)
- body = {
- 'tag_name': tag_name,
- 'target_commitish': branch,
- 'name': rel_name,
- 'body': rel_body,
- 'draft': draft,
- 'prerelease': false
- }.to_json
- res = post("/repos/#{GH_OWNER}/#{GH_REPO}/releases", body)
-
- raise "Failed to create release (#{res.code})" unless res.is_a? Net::HTTPSuccess
-
- JSON.parse(res.body)
-end
-
-def publish_release(release_id)
- body = {
- 'draft': false
- }.to_json
- res = patch("/repos/#{GH_OWNER}/#{GH_REPO}/releases/#{release_id}", body)
-
- raise "Failed to update release (#{res.code})" unless res.is_a? Net::HTTPSuccess
-end
-
-def list_releases
- res = get("/repos/#{GH_OWNER}/#{GH_REPO}/releases")
- raise 'Failed to retrieve releases' unless res.is_a? Net::HTTPSuccess
-
- JSON.parse(res.body)
-end
-
-def release_available?(tag_name)
- return true if list_releases.find { |r| r['tag_name'] == tag_name }
-
- false
-end
-
-def bump_version(new_version, min_version = nil, path = 'share/github-backup-utils/version')
- current_version = Gem::Version.new(File.read(path).strip.chomp)
- if !@skip_version_bump_check && (Gem::Version.new(new_version) < current_version)
- raise "New version should be newer than #{current_version}"
- end
- File.open("#{path}.new", 'w') { |f| f.puts new_version }
- File.rename("#{path}.new", path)
-
- unless min_version.nil?
- content = File.read('bin/ghe-host-check')
- new_content = content.gsub(/supported_minimum_version="[0-9]\.[0-9]+\.0"/, "supported_minimum_version=\"#{min_version}\"")
- File.open('bin/ghe-host-check', 'w') {|file| file.puts new_content }
-
- content = File.read('test/testlib.sh')
- new_content = content.gsub(/GHE_TEST_REMOTE_VERSION:=[0-9]\.[0-9]+\.0/,"GHE_TEST_REMOTE_VERSION:=#{new_version}")
- File.open('test/testlib.sh', 'w') {|file| file.puts new_content }
- end
-end
-
-def push_release_branch(version)
- unless (out = `git checkout --quiet -b release-#{version}`)
- raise "Creating release branch failed:\n\n#{out}"
- end
-
- unless (out = `git commit --quiet -m 'Bump version: #{version} [ci skip]' debian/changelog share/github-backup-utils/version bin/ghe-host-check test/testlib.sh script/cibuild`)
- raise "Error committing changelog and version:\n\n#{out}"
- end
-
- unless (out = `git push --quiet origin release-#{version}`)
- raise "Failed pushing the release branch:\n\n#{out}"
- end
-end
-
-def update_stable_branch
- `git checkout --quiet stable`
- unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`)
- warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}"
- end
- unless (out = `git push --quiet origin stable`)
- warn "Failed pushing the stable branch:\n\n#{out}"
- end
-end
-
-def create_release_pr(version, release_body)
- body = {
- 'title': "Bump version: #{version}",
- 'body': release_body,
- 'head': "release-#{version}",
- 'base': GH_BASE_BRANCH
- }.to_json
- res = post("/repos/#{GH_OWNER}/#{GH_REPO}/pulls", body)
- raise "Creating release PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess
-
- JSON.parse(res.body)
-end
-
-def merge_pr(number, sha, version)
- body = {
- 'commit_title': "Merge pull request ##{number} from github/release-#{version}",
- 'commit_message': "Bump version: #{version}",
- 'sha': sha,
- 'merge_method': 'merge'
- }.to_json
- pr_mergeable? number
- res = put("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}/merge", body)
- raise "Merging PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess
-
- JSON.parse(res.body)
-end
-
-class RetryError < StandardError
-end
-
-def pr_mergeable?(number)
- begin
- retries ||= 5
- res = get("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}")
- raise RetryError if JSON.parse(res.body)['mergeable'].nil?
- mergeable = JSON.parse(res.body)['mergeable']
- rescue RetryError
- sleep 1
- retry unless (retries -= 1).zero?
- raise 'PR is unmergable.'
- end
-
- mergeable || false
-end
-
-def can_auth?
- !ENV['GH_RELEASE_TOKEN'].nil?
-end
-
-def repo_exists?
- res = get("/repos/#{GH_OWNER}/#{GH_REPO}")
- res.is_a? Net::HTTPSuccess
-end
-
-def can_build_deb?
- system('which debuild > /dev/null 2>&1')
-end
-
-def package_tarball
- unless (out = `script/package-tarball 2>&1`)
- raise "Failed to package tarball:\n\n#{out}"
- end
- out
-end
-
-def package_deb
- unless (out = `DEB_BUILD_OPTIONS=nocheck script/package-deb 2>&1`)
- raise "Failed to package Debian package:\n\n#{out}"
- end
- out
-end
-
-def attach_assets_to_release(upload_url, release_id, files)
- @http = nil
- client(URI(upload_url.gsub(/{.*}/, '')).host)
- begin
- files.each do |file|
- raw_file = File.open(file).read
- res = post_file("/repos/#{GH_OWNER}/#{GH_REPO}/releases/#{release_id}/assets?name=#{File.basename(file)}", raw_file)
- raise "Failed to attach #{file} to release (#{res.code})" unless res.is_a? Net::HTTPSuccess
- end
- rescue => e
- raise e
- end
- @http = nil
-end
-
-def clean_up(version)
- `git checkout --quiet #{GH_BASE_BRANCH}`
- `git fetch --quiet origin --prune`
- `git pull --quiet origin #{GH_BASE_BRANCH} --prune`
- `git branch --quiet -D release-#{version} >/dev/null 2>&1`
- `git push --quiet origin :release-#{version} >/dev/null 2>&1`
- `git branch --quiet -D tmp-packging >/dev/null 2>&1`
-end
-
-#### All the action starts ####
-if $PROGRAM_NAME == __FILE__
- begin
- args = ARGV.dup
- dry_run = false
- skip_version_bump_check = false
- if args.include?('--dry-run')
- dry_run = true
- args.delete '--dry-run'
- end
-
- if args.include?('--no-warn')
- @no_warn = true
- args.delete '--no-warn'
- end
-
- if args.include?('--skip-version-bump-check')
- @skip_version_bump_check = true
- args.delete '--skip-version-bump-check'
- end
-
- raise 'Usage: release [--dry-run] [--skip-version-bump-check] [min_version]' if args.empty?
-
- begin
- version = Gem::Version.new(args[0])
- min_version = args[1] ? args[1] : nil
- rescue ArgumentError
- raise "Error parsing version #{args[0]}"
- end
-
- raise "Minimum supported version is required for X.Y.0 releases\n\nUsage: release [--dry-run] [min_version]" if /[0-9]\.[0-9]+\.0/ =~ version.to_s && min_version.nil?
-
- raise "The repo #{GH_REPO} does not exist for #{GH_OWNER}" unless repo_exists?
-
- raise 'GH_AUTHOR environment variable is not set' if GH_AUTHOR.nil?
-
- release_changes = []
- release_changes = beautify_changes changelog if can_auth?
- release_a = false
- release_a = release_available? "v#{version}"
-
- puts "Bumping version to #{version}..."
- bump_version version, min_version
-
- if dry_run
- puts "Existing release?: #{release_a}"
- puts "New version: #{version}"
- puts "Min version: #{min_version}" unless min_version.nil?
- puts "Owner: #{GH_OWNER}"
- puts "Repo: #{GH_REPO}"
- puts "Author: #{GH_AUTHOR}"
- puts "Token: #{ENV['GH_RELEASE_TOKEN'] && 'set' || 'unset'}"
- puts "Base branch: #{GH_BASE_BRANCH}"
- puts 'Changelog:'
- if release_changes.empty?
- puts ' => No new bug fixes, enhancements or features.'
- else
- release_changes.each { |c| puts " * #{c}" }
- end
- puts "Changes:"
- puts `git diff --color`
- `git checkout -- share/github-backup-utils/version`
- `git checkout -- bin/ghe-host-check`
- `git checkout -- test/testlib.sh`
- exit
- end
-
- raise 'Unable to build Debian pkg: "debuild" not found.' unless can_build_deb?
-
- raise "Release #{version} already exists." if release_a
-
- `git fetch --quiet origin --prune`
- branches = `git branch --all | grep release-#{version}$`
- unless branches.empty?
- out = "Release branch release-#{version} already exists. "
- out += 'Branches found:'
- branches.each_line { |l| out += "\n* #{l.strip.chomp}" }
- raise out
- end
-
- puts 'Updating changelog...'
- update_changelog release_changes, DEB_PKG_NAME, version
- release_body = "Includes general improvements & bug fixes"
- release_body += " and support for GitHub Enterprise Server v#{version}" unless min_version.nil?
- release_changes.each do |c|
- release_body += "\n* #{c}"
- end
-
- puts 'Pushing release branch and creating release PR...'
- push_release_branch version
- res = create_release_pr(version, "#{release_body}\n\n/cc @github/backup-utils")
-
- puts 'Merging release PR...'
- res = merge_pr res['number'], res['head']['sha'], version
-
- puts 'Tagging and publishing release...'
- tag "v#{version}", res['sha']
-
- puts 'Creating release...'
- release_title = "GitHub Enterprise Server Backup Utilities v#{version}"
- res = create_release "v#{version}", GH_BASE_BRANCH, release_title, release_body, true
-
- # Tidy up before building tarball and deb pkg
- clean_up version
-
- puts 'Building release tarball...'
- package_tarball
-
- puts 'Building Debian pkg...'
- package_deb
-
- puts 'Attaching Debian pkg and tarball to release...'
- base_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
- attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"]
- attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"]
-
- puts 'Publishing release...'
- publish_release res['id']
-
- puts 'Cleaning up...'
- clean_up version
-
- puts 'Updating stable branch...'
- update_stable_branch
-
- puts 'Released!'
- rescue RuntimeError => e
- $stderr.puts "Error: #{e}"
- exit 1
- end
-end
diff --git a/share/github-backup-utils/bm.sh b/share/github-backup-utils/bm.sh
deleted file mode 100755
index c512f3d44..000000000
--- a/share/github-backup-utils/bm.sh
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-# bm.sh: benchmarking Bash code blocks
-#
-# Example:
-# bm_start "wget request"
-# wget --quiet https://www.google.com
-# bm_end "wget request"
-#
-# Sample output:
-# $ bash test.sh
-# wget request took 2s
-
-bm_desc_to_varname(){
- echo "__bm$(echo "$@" | tr -cd '[[:alnum:]]')"
-}
-
-bm_start()
-{
- eval "$(bm_desc_to_varname "$@")_start=$(date +%s)"
- if [ -n "$GHE_DEBUG" ]; then
- echo "Debug: $1 (bm_start)"
- fi
- bm_init > /dev/null
-}
-
-bm_init() {
- if [ -n "$BM_FILE_PATH" ]; then
- echo $BM_FILE_PATH
- return
- fi
-
- local logfile="benchmark.${BM_TIMESTAMP:-$(date +"%Y%m%dT%H%M%S")}.log"
- if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then
- export BM_FILE_PATH=$GHE_RESTORE_SNAPSHOT_PATH/benchmarks/$logfile
- else
- export BM_FILE_PATH=$GHE_SNAPSHOT_DIR/benchmarks/$logfile
- fi
-
- mkdir -p "$(dirname $BM_FILE_PATH)"
- echo $BM_FILE_PATH
-}
-
-bm_end() {
- if [ -z "$BM_FILE_PATH" ]; then
- echo "Call bm_start first" >&2
- exit 1
- fi
-
- local tend tstart total
- tend=$(date +%s)
- tstart=$(eval "echo \$$(bm_desc_to_varname "$@")_start")
- total=$(($tend - $tstart))
-
- echo "$1 took ${total}s" >> $BM_FILE_PATH
- if [ -n "$GHE_DEBUG" ]; then
- echo "Debug: $1 took ${total}s (bm_end)"
- fi
-}
diff --git a/share/github-backup-utils/ghe-backup-actions b/share/github-backup-utils/ghe-backup-actions
deleted file mode 100755
index f19777930..000000000
--- a/share/github-backup-utils/ghe-backup-actions
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-actions
-#/ Take an online, incremental snapshot of all Actions data (excluding
-#/ what is stored in MSSQL)
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Set up remote host and root backup snapshot directory based on config
-port=$(ssh_port_part "$GHE_HOSTNAME")
-host=$(ssh_host_part "$GHE_HOSTNAME")
-backup_dir="$GHE_SNAPSHOT_DIR/actions"
-
-# Verify rsync is available.
-if ! rsync --version 1>/dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "$backup_dir"
-
-# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-if [ -d "$GHE_DATA_DIR/current/actions" ] && [ "$(ls -A $GHE_DATA_DIR/current/actions)" ]; then
- link_dest="--link-dest=$GHE_DATA_DIR/current/actions"
-fi
-
-# Transfer all Actions data from the user data directory using rsync.
-ghe_verbose "* Transferring Actions files from $host ..."
-
-ghe-rsync -avz \
- -e "ghe-ssh -p $port" \
- --rsync-path='sudo -u actions rsync' \
- --exclude "mutexes" --exclude "dumps" --exclude "tmp" \
- $link_dest \
- "$host:$GHE_REMOTE_DATA_USER_DIR/actions/" \
- "$GHE_SNAPSHOT_DIR/actions" 1>&3
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config
deleted file mode 100755
index e180f8013..000000000
--- a/share/github-backup-utils/ghe-backup-config
+++ /dev/null
@@ -1,509 +0,0 @@
-#!/usr/bin/env bash
-# Usage: . ghe-backup-config
-# GitHub Enterprise backup shell configuration.
-#
-# This file is sourced by the various utilities under bin and share/github-backup-utils to
-# load in backup configuration and ensure things are configured properly.
-#
-# All commands in share/github-backup-utils/ should start with the following:
-#
-# . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
-#
-# And all commands in bin/ should start with the following:
-#
-# . $( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config
-#
-set +o posix
-# Assume this script lives in share/github-backup-utils/ when setting the root
-GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-
-# Get the version from the version file.
-BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")"
-
-# If a version check was requested, show the current version and exit
-if [ -n "$GHE_SHOW_VERSION" ]; then
- echo "GitHub backup-utils v$BACKUP_UTILS_VERSION"
- exit 0
-fi
-
-# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage
-# shellcheck disable=SC2120 # the script name is always referenced
-print_usage() {
- grep '^#/' <"$0" | cut -c 4-
- exit "${1:-1}"
-}
-
-if [ -n "$GHE_SHOW_HELP" ]; then
- print_usage
-else
- for a in "$@"; do
- if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then
- print_usage
- fi
- done
-fi
-
-# Add the bin and share/github-backup-utils dirs to PATH
-PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
-# shellcheck source=share/github-backup-utils/bm.sh
-. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh"
-
-# Save off GHE_HOSTNAME from the environment since we want it to override the
-# backup.config value when set.
-GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME"
-
-# Source in the backup config file from the copy specified in the environment
-# first and then fall back to the backup-utils root, home directory and system.
-config_found=false
-for f in "$GHE_BACKUP_CONFIG" "$GHE_BACKUP_ROOT/backup.config" \
- "$HOME/.github-backup-utils/backup.config" "/etc/github-backup-utils/backup.config"; do
- if [ -f "$f" ]; then
- GHE_BACKUP_CONFIG="$f"
- # shellcheck disable=SC1090 # This is a user-supplied value that can't be predicted
- . "$GHE_BACKUP_CONFIG"
- config_found=true
- break
- fi
-done
-
-ghe_parallel_check() {
- if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then
- return 0
- fi
-
- # Some machines may have both moreutils parallel and GNU parallel installed.
- # Check some variants to find it
- GHE_PARALLEL_COMMAND="parallel"
- local x
- for x in \
- /usr/bin/parallel.moreutils \
- /usr/bin/parallel_moreutils \
- /usr/bin/moreutils.parallel \
- /usr/bin/moreutils_parallel \
- ; do
- if [ -x "${x}" ]; then
- GHE_PARALLEL_COMMAND="${x}"
- break
- fi
- done
-
- # Check that the GHE_PARALLEL_COMMAND is pointing to moreutils parallel
- if ! $GHE_PARALLEL_COMMAND -h | grep -q "parallel \[OPTIONS\] command -- arguments"; then
- echo "Error: moreutils not found. Please install https://joeyh.name/code/moreutils" 1>&2
- exit 1
- fi
-
- if [ -n "$GHE_PARALLEL_MAX_JOBS" ]; then
- GHE_PARALLEL_COMMAND_OPTIONS="-j $GHE_PARALLEL_MAX_JOBS"
- # Default to the number of max rsync jobs to the same as GHE_PARALLEL_MAX_JOBS, if not set.
- # This is only applicable to ghe-restore-repositories currently.
- : "${GHE_PARALLEL_RSYNC_MAX_JOBS:="$GHE_PARALLEL_MAX_JOBS"}"
- fi
-
- if [ -n "$GHE_PARALLEL_RSYNC_MAX_JOBS" ]; then
- GHE_PARALLEL_RSYNC_COMMAND_OPTIONS="-j $GHE_PARALLEL_RSYNC_MAX_JOBS"
- fi
-
- if [ -n "$GHE_PARALLEL_MAX_LOAD" ]; then
- GHE_PARALLEL_COMMAND_OPTIONS+=" -l $GHE_PARALLEL_MAX_LOAD"
- GHE_PARALLEL_RSYNC_COMMAND_OPTIONS+=" -l $GHE_PARALLEL_MAX_LOAD"
- fi
-}
-
-# Check that the config file exists before we source it in.
-if ! $config_found; then
- echo "Error: No backup configuration file found. Tried:" 1>&2
- [ -n "$GHE_BACKUP_CONFIG" ] && echo " - $GHE_BACKUP_CONFIG" 1>&2
- echo " - $GHE_BACKUP_ROOT/backup.config" 1>&2
- echo " - $HOME/.github-backup-utils/backup.config" 1>&2
- echo " - /etc/github-backup-utils/backup.config" 1>&2
- exit 2
-fi
-
-# If verbose logging is enabled, redirect fd 3 to stdout or the specified log file;
-# otherwise, redirect it to /dev/null. Write verbose output to fd 3.
-if [ -n "$GHE_VERBOSE" ]; then
- if [ -n "$GHE_VERBOSE_LOG" ]; then
- if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then
- exec 3>>"$GHE_VERBOSE_LOG"
- else
- if ! echo | awk '{ print strftime("%b %d %H:%M:%S"); fflush(); }' &>/dev/null; then
- echo "Error: awk command failed. Please install https://www.gnu.org/software/gawk" 1>&2
- exit 1
- fi
- calling_script_name="$(caller | sed 's:.*/::')"
- exec 3> >(awk -v c="$calling_script_name" '{ print strftime("%b %d %H:%M:%S"), c":", $0; fflush(); }' >>"$GHE_VERBOSE_LOG")
- fi
- else
- exec 3>&1
- fi
-else
- exec 3>/dev/null
-fi
-
-# Restore saved off hostname.
-[ -n "$GHE_HOSTNAME_PRESERVE" ] && GHE_HOSTNAME="$GHE_HOSTNAME_PRESERVE"
-
-# Check that the GHE hostname is set.
-if [ -z "$GHE_HOSTNAME" ]; then
- echo "Error: GHE_HOSTNAME not set in config file." 1>&2
- exit 2
-fi
-
-# Check that the GHE data directory is set.
-if [ -z "$GHE_DATA_DIR" ]; then
- echo "Error: GHE_DATA_DIR not set in config file." 1>&2
- exit 2
-fi
-
-# Convert the data directory path to an absolute path, basing any relative
-# paths on the backup-utils root, and using readlink, if available, to
-# canonicalize the path.
-if [ "${GHE_DATA_DIR:0:1}" != "/" ]; then
- GHE_DATA_DIR="$(cd "$GHE_BACKUP_ROOT" && readlink -m "$GHE_DATA_DIR" 2>/dev/null || echo "$GHE_BACKUP_ROOT/$GHE_DATA_DIR")"
-fi
-export GHE_DATA_DIR
-
-# Assign the Release File path if it hasn't been provided (eg: by test suite)
-: "${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}"
-
-# Check that utils are not being run directly on GHE appliance.
-if [ -f "$GHE_RELEASE_FILE" ]; then
- echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2
- echo " The backup utilities should be run on a host dedicated to" 1>&2
- echo " long-term permanent storage and must have network connectivity" 1>&2
- echo " with the GitHub Enterprise appliance." 1>&2
- exit 1
-fi
-
-GHE_CREATE_DATA_DIR=${GHE_CREATE_DATA_DIR:-yes}
-
-# Check that the data directory is set and create it if it doesn't exist.
-if [ ! -d "$GHE_DATA_DIR" ] && [ "$GHE_CREATE_DATA_DIR" = "yes" ]; then
- echo "Creating the backup data directory ..." 1>&3
- mkdir -p "$GHE_DATA_DIR"
-fi
-
-if [ ! -d "$GHE_DATA_DIR" ]; then
- echo "Error: GHE_DATA_DIR $GHE_DATA_DIR does not exist." >&2
- exit 8
-fi
-
-# Set some defaults if needed.
-: ${GHE_NUM_SNAPSHOTS:=10}
-
-# Generate a backup timestamp if one has not already been generated.
-# We export the variable so the process group shares the same value.
-: ${GHE_SNAPSHOT_TIMESTAMP:=$(date +"%Y%m%dT%H%M%S")}
-export GHE_SNAPSHOT_TIMESTAMP
-
-# Set the current snapshot directory to /. This is where
-# all backups should be written for the current invocation.
-GHE_SNAPSHOT_DIR="$GHE_DATA_DIR"/"$GHE_SNAPSHOT_TIMESTAMP"
-export GHE_SNAPSHOT_DIR
-
-# The root filesystem location. This must be used so that tests can override
-# the root as a local directory location.
-: ${GHE_REMOTE_ROOT_DIR:=""}
-
-# The root location of persistent data and applications on the remote side. This
-# is always "/data" for GitHub instances. Use of this variable allows
-# the location to be overridden in tests.
-: ${GHE_REMOTE_DATA_DIR:="/data"}
-
-# The root location of user data stores such as git repositories, pages sites,
-# elasticsearch indices, etc. This is "/data" under 1.x filesystem layouts and
-# "/data/user" under the 2.x filesystem layout. The location is adjusted
-# dynamically in ghe_remote_version_config() immediately after obtaining the
-# remote version. Utilities that transfer data in and out of the appliance
-# should use this variable to ensure proper behavior under different versions.
-: ${GHE_REMOTE_DATA_USER_DIR:="$GHE_REMOTE_DATA_DIR"}
-
-# The location of the license file on the remote side. This is always
-# "/data/enterprise/enterprise.ghl" for GitHub instances. Use of this variable
-# allows the location to be overridden in tests.
-: ${GHE_REMOTE_LICENSE_FILE:="$GHE_REMOTE_DATA_DIR/enterprise/enterprise.ghl"}
-
-# The number of seconds to wait for in progress git-gc processes to complete
-# before starting the sync of git data. See share/github-backup-utils/ghe-backup-repositories-rsync
-# for more information. Default: 10 minutes.
-: ${GHE_GIT_COOLDOWN_PERIOD:=600}
-
-# Set "true" to get verbose logging of all ssh commands on stderr
-: ${GHE_VERBOSE_SSH:=false}
-
-# The location of the cluster configuration file file on the remote side.
-# This is always "/data/user/common/cluster.conf" for GitHub Cluster instances.
-# Use of this variable allows the location to be overridden in tests.
-: ${GHE_REMOTE_CLUSTER_CONF_FILE:="$GHE_REMOTE_DATA_DIR/user/common/cluster.conf"}
-
-# The location of the file used to disable GC operations on the remote side.
-: ${SYNC_IN_PROGRESS_FILE:="$GHE_REMOTE_DATA_USER_DIR/repositories/.sync_in_progress"}
-
-# Base path for temporary directories and files.
-: ${TMPDIR:="/tmp"}
-
-# Backup cadence for MS SQL. Determines the kind of backup taken, either full, differential,
-# or transaction log, based on when the last backup of that kind was taken. This defaults to
-# taking a full backup once a week, a differential backup once a day, and transaction logs every
-# 15 minutes.
-: ${GHE_MSSQL_BACKUP_CADENCE:=10080,1440,15}
-
-###############################################################################
-### Dynamic remote version config
-
-# Adjusts remote paths based on the version of the remote appliance. This is
-# called immediately after the remote version is obtained by
-# ghe_remote_version_required(). Child processes inherit the values set here.
-ghe_remote_version_config() {
- GHE_REMOTE_DATA_USER_DIR="$GHE_REMOTE_DATA_DIR/user"
- export GHE_REMOTE_DATA_DIR GHE_REMOTE_DATA_USER_DIR
- export GHE_REMOTE_LICENSE_FILE
-}
-
-###############################################################################
-### Utility functions
-
-# If we don't have a readlink command, parse ls -l output.
-if ! type readlink 1>/dev/null 2>&1; then
- readlink() {
- if [ -x "$1" ]; then
- ls -ld "$1" | sed 's/.*-> //'
- else
- return 1
- fi
- }
-fi
-
-# Run ghe-host-check and establish the version of the remote GitHub instance in
-# the exported GHE_REMOTE_VERSION variable. If the remote version has already
-# been established then don't perform the host check again. Utilities in share/github-backup-utils
-# that need the remote version should use this function instead of calling
-# ghe-host-check directly to reduce ssh roundtrips. The top-level ghe-backup and
-# ghe-restore commands establish the version for all subcommands.
-ghe_remote_version_required() {
- if [ -z "$GHE_REMOTE_VERSION" ]; then
- _out=$(ghe-host-check "$@")
- echo "$_out"
-
- # override hostname w/ ghe-host-check output because the port could have
- # been autodetected to 122.
- GHE_HOSTNAME=$(echo "$_out" | sed 's/Connect \(.*:[0-9]*\) OK.*/\1/')
- export GHE_HOSTNAME
-
- GHE_REMOTE_VERSION=$(echo "$_out" | sed 's/.*(\(.*\))/\1/')
- export GHE_REMOTE_VERSION
-
- ghe_parse_remote_version "$GHE_REMOTE_VERSION"
- ghe_remote_version_config "$GHE_REMOTE_VERSION"
- fi
- true
-}
-
-# Parse a version string into major, minor and patch parts and echo.
-ghe_parse_version() {
- local version_major version_minor version_patch
- version_major=$(echo "${1#v}" | cut -f 1 -d .)
- version_minor=$(echo "$1" | cut -f 2 -d .)
- version_patch=$(echo "$1" | cut -f 3 -d .)
- version_patch=${version_patch%%[a-zA-Z]*}
-
- echo "$version_major $version_minor $version_patch"
-}
-# Parse major, minor, and patch parts of the remote appliance version and store
-# in GHE_VERSION_MAJOR, GHE_VERSION_MINOR, and GHE_VERSION_PATCH variables. All
-# parts are numeric. This is called automatically from
-# ghe_remote_version_required so shouldn't be used directly.
-#
-# Scripts use these variables to alter behavior based on what's supported on the
-# appliance version.
-ghe_parse_remote_version() {
- # shellcheck disable=SC2046 # Word splitting is required to populate the variables
- read -r GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH <<<$(ghe_parse_version $1)
- export GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH
-}
-
-# Parses the part out of a ":" or just "" string.
-# This is used primarily to break hostspecs with non-standard ports down for
-# rsync commands.
-ssh_host_part() {
- [ "${1##*:}" = "$1" ] && echo "$1" || echo "${1%:*}"
-}
-
-# Parses the part out of a ":" or just "" string.
-# This is used primarily to break hostspecs with non-standard ports down for
-# rsync commands.
-ssh_port_part() {
- if [ "${1##*:}" != "$1" ] && [ "${1##*:}" -ne "122" ]; then
- echo "Error: SSH port has to be 122 connecting to Github Enterprise Server, current value is ${1##*:} for $1." 1>&2
- exit 1
- fi
-
- echo 122
-}
-
-# Usage: ghe_remote_logger ...
-# Log a message to /var/log/syslog on the remote instance.
-# Note: Use sparingly. Remote logging requires an ssh connection per invocation.
-ghe_remote_logger() {
- echo "$@" |
- ghe-ssh "$GHE_HOSTNAME" -- logger -t backup-utils || true
-}
-
-# Usage: ghe_verbose
-# Log if verbose mode is enabled (GHE_VERBOSE or `-v`).
-ghe_verbose() {
- if [ -n "$GHE_VERBOSE" ]; then
- echo "$@" 1>&3
- fi
-}
-
-# Usage: ghe_debug OR echo | ghe_debug
-# Log if debug mode is enabled (GHE_DEBUG).
-ghe_debug() {
- [ -z "$GHE_DEBUG" ] && return
-
- if [ $# -ne 0 ]; then
- echo -e "Debug: $*" 1>&3
- elif [ -p /dev/stdin ]; then
- echo "\n" 1>&3
- while read line; do
- echo -e "Debug: $line" 1>&3
- done &2
- exit 1
- ;;
- esac
- done
-
- echo
-}
-
-# Function to restore a secret setting stored in a file.
-# restore-secret
-restore-secret() {
- if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/$2" ]; then
- echo "Restoring $1 ..."
- echo "ghe-config '$3' '$(cat "$GHE_RESTORE_SNAPSHOT_PATH/$2")'" |
- ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
- fi
-}
diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log
deleted file mode 100755
index e28525e5d..000000000
--- a/share/github-backup-utils/ghe-backup-es-audit-log
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-es-audit-log
-#/ Take a backup of audit logs in Elasticsearch.
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Set up remote host and root elastic backup directory based on config
-host="$GHE_HOSTNAME"
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "$GHE_SNAPSHOT_DIR/audit-log"
-
-if ! indices=$(ghe-ssh "$host" "curl -s \"localhost:9201/_cat/indices/audit_log*?h=index,pri.store.size&bytes=b\""); then
- echo "Error: failed to retrieve audit log indices." 1>&2
- exit 1
-fi
-
-# Exit if no indices were found
-[ -z "$indices" ] && exit
-
-# Determine if the audit log migration has occurred or is needed.
-if echo 'set -o pipefail; ! test -e /data/user/common/es-scan-complete && test -f /usr/local/share/enterprise/run-audit-log-transitions.sh' | ghe-ssh "$host" /bin/bash; then
- if echo 'set -o pipefail; echo n | /usr/local/share/enterprise/run-audit-log-transitions.sh > /dev/null 2>&1 && touch /data/user/common/es-scan-complete' | ghe-ssh "$host" /bin/bash; then
- touch $GHE_SNAPSHOT_DIR/es-scan-complete
- fi
-fi
-
-IFS=$'\n'
-for index in $indices; do
- IFS=' '
- set $index
- index_name=$1
- index_size=$2
-
- if [[ -f $GHE_DATA_DIR/current/audit-log/$index_name.gz && $(cat $GHE_DATA_DIR/current/audit-log/$index_name.gz.size 2>/dev/null || true) -eq $index_size ]]; then
- # Hard link any indices that have not changed since the last backup
- ln $GHE_DATA_DIR/current/audit-log/$index_name.gz $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz
- ln $GHE_DATA_DIR/current/audit-log/$index_name.gz.size $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size
- else
- echo "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index_name\" | gzip" | ghe-ssh "$host" -- /bin/bash > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz
- echo $index_size > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size
- fi
-done
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync
deleted file mode 100755
index 11fa99f70..000000000
--- a/share/github-backup-utils/ghe-backup-es-rsync
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-es-rsync
-#/ Take an online, incremental snapshot of Elasticsearch indices.
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup when the rsync strategy is used.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Set up remote host and root elastic backup directory based on config
-host="$GHE_HOSTNAME"
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Verify rsync is available.
-if ! rsync --version 1>/dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
-
-# Verify that the /data/elasticsearch directory exists.
-if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
- ghe_verbose "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist."
- exit 0
-fi
-
-# If we have a previous increment, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then
- link_dest="--link-dest=../../current/elasticsearch"
-fi
-
-# Transfer ES indices from a GitHub instance to the current snapshot
-# directory, using a previous snapshot to avoid transferring files that have
-# already been transferred.
-ghe_verbose "* Performing initial sync of ES indices ..."
-ghe-rsync -avz \
- -e "ghe-ssh -p $(ssh_port_part "$host")" \
- --rsync-path="sudo -u elasticsearch rsync" \
- $link_dest \
- "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
- "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
-
-# Set up a trap to re-enable flushing on exit and remove temp file
-cleanup () {
- ghe_verbose "* Enabling ES index flushing ..."
- echo '{"index":{"translog.disable_flush":false}}' |
- ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
-}
-trap 'cleanup' EXIT
-trap 'exit $?' INT # ^C always terminate
-
-# Disable ES flushing and force a flush right now
-ghe_verbose "* Disabling ES index flushing ..."
-echo '{"index":{"translog.disable_flush":true}}' |
-ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
-ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null
-
-# Transfer all ES indices again
-ghe_verbose "* Performing follow-up sync of ES indices ..."
-ghe-rsync -avz \
- -e "ghe-ssh -p $(ssh_port_part "$host")" \
- --rsync-path="sudo -u elasticsearch rsync" \
- $link_dest \
- "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
- "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
-
-# "Backup" audit log migration sentinel file
-if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then
- touch $GHE_SNAPSHOT_DIR/es-scan-complete
-fi
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-fsck b/share/github-backup-utils/ghe-backup-fsck
deleted file mode 100755
index 004abe412..000000000
--- a/share/github-backup-utils/ghe-backup-fsck
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-fsck [--print-nwo]
-#/
-#/ Run git fsck on backed up repositories.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-echo "Running git fsck on repos..."
-
-# Verify git is available.
-if ! git --version 1>/dev/null 2>&1; then
- echo "Error: git not found." 1>&2
- exit 1
-fi
-
-sdir=$1
-repos=0
-errors=0
-log=$(mktemp -t ghe-backup-fsck-XXXXXX)
-t_start=$(date +%s)
-if git fsck -h | grep -q '\-\-dangling'; then
- git_cmd='git fsck --no-dangling'
-else
- echo "Warning: old git version, --no-dangling not available"
- git_cmd='git fsck'
-fi
-
-if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then
- print_usage
-fi
-
-if [ ! -d "$sdir/repositories" ]; then
- echo "Error: $sdir is not a valid snapshot." >&2
- exit 1
-fi
-
-# shellcheck disable=SC2044 # Snapshot and repository directory names are safe for find iteration.
-for repo in $(find $sdir/repositories/ -type d -name \*.git); do
- repos=$(($repos+1))
- before_time=$(date +%s)
-
- status=$(
- set -e
-
- cd $repo
-
- nwo="-"
- if [ "$2" = "--print-nwo" ] && [ -f info/nwo ]; then
- nwo="$(cat info/nwo)"
- fi
-
- if [ ! -f objects/info/alternates ] || grep -q '^\.\.' objects/info/alternates; then
- $git_cmd >$log 2>&1 && {
- echo "OK $repo $nwo"; exit
- }
- else
- GIT_ALTERNATE_OBJECT_DIRECTORIES=../network.git/objects $git_cmd >$log 2>&1 && {
- echo "WARN $repo $nwo (alternates absolute path)"; exit
- }
- fi
-
- echo "ERROR $repo $nwo"
- )
-
- elapsed_time=$(($(date +%s) - before_time))
-
- if [[ ! "$status" =~ ^OK ]] || [ $elapsed_time -gt 5 ]; then
- echo "$status ${elapsed_time}s" 1>&3
- [ -n "$GHE_VERBOSE" ] && cat $log
- fi
-
- case "$status" in
- OK*)
- ;;
- ERROR*)
- errors=$(($errors+1))
- ;;
- esac
-
-done
-
-echo "* Repos verified: $repos, Errors: $errors, Took: $(($(date +%s) - $t_start))s"
-
-rm -f $log
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-git-hooks b/share/github-backup-utils/ghe-backup-git-hooks
deleted file mode 100755
index c7bd5429e..000000000
--- a/share/github-backup-utils/ghe-backup-git-hooks
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-git-hooks
-#/ Take an online, incremental snapshot of custom Git hooks configuration.
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Verify rsync is available.
-if ! rsync --version 1>/dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-backup_dir="$GHE_SNAPSHOT_DIR/git-hooks"
-# Location of last good backup for rsync --link-dest
-backup_current="$GHE_DATA_DIR/current/git-hooks"
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Split host:port into parts
-port=$(ssh_port_part "$GHE_HOSTNAME")
-host=$(ssh_host_part "$GHE_HOSTNAME")
-
-# Add user / -l option
-user="${host%@*}"
-[ "$user" = "$host" ] && user="admin"
-
-hostnames=$host
-ssh_config_file_opt=
-tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX)
-opts="$GHE_EXTRA_SSH_OPTS"
-
-# git server hostnames under cluster
-if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
- ssh_config_file="$tempdir/ssh_config"
- ssh_config_file_opt="-F $ssh_config_file"
- opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
- hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "git-server")
- ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"
-fi
-
-# Removes the remote sync-in-progress file on exit, re-enabling GC operations
-# on the remote instance.
-cleanup() {
- rm -rf $tempdir
-}
-trap 'cleanup' EXIT
-trap 'exit $?' INT # ^C always terminate
-
-# Transfer Git hooks data from a GitHub instance to the current snapshot
-# directory, using a previous snapshot to avoid transferring files that have
-# already been transferred. A set of rsync filter rules are provided on stdin
-# for each invocation.
-rsync_git_hooks_data () {
- port=$(ssh_port_part "$1")
- host=$(ssh_host_part "$1")
-
- subpath=$2
- shift 2
-
- # If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
- # --link-dest support. This also decreases physical space usage considerably.
- if [ -d "$backup_current/$subpath" ] && [ "$(ls -A $backup_current/$subpath)" ]; then
- subdir="git-hooks/$subpath"
- link_path=".."
- while true; do
- if [ "$(dirname $subdir)" = "." ]; then
- break
- fi
-
- if [ "$(dirname $subdir)" = "/" ]; then
- break
- fi
-
- link_path="../$link_path"
- subdir=$(dirname $subdir)
- done
-
- local link_dest="--link-dest=../${link_path}/current/git-hooks/$subpath"
- fi
-
- # Ensure target directory exists, is needed with subdirectories
- mkdir -p "$backup_dir/$subpath"
-
- ghe-rsync -av \
- -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" $link_dest \
- --rsync-path='sudo -u git rsync' \
- "$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/$subpath/" \
- "$backup_dir/$subpath" 1>&3
-}
-
-hostname=$(echo $hostnames | awk '{ print $1; }')
-if ghe-ssh $ssh_config_file_opt "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs' ]"; then
- rsync_git_hooks_data $hostname:122 environments/tarballs
-else
- ghe_verbose "git-hooks environment tarballs not found. Skipping ..."
-fi
-
-if ghe-ssh $ssh_config_file_opt "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos' ]"; then
- rsync_git_hooks_data $hostname:122 repos
-else
- ghe_verbose "git-hooks repositories not found. Skipping ..."
-fi
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-minio b/share/github-backup-utils/ghe-backup-minio
deleted file mode 100755
index 59dc7f8b3..000000000
--- a/share/github-backup-utils/ghe-backup-minio
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-minio
-#/ Take an online, incremental snapshot of all minio data
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config"
-
-bm_start "$(basename "${0}")"
-
-# Set up remote host and root backup snapshot directory based on config
-port="$(ssh_port_part "${GHE_HOSTNAME}")"
-host="$(ssh_host_part "${GHE_HOSTNAME}")"
-backup_dir="${GHE_SNAPSHOT_DIR}/minio"
-
-# Verify rsync is available.
-if ! command -v rsync 1> /dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "${host}"
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "${backup_dir}"
-
-# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-# Hilariously, this HAS to stay unquoted when you call `rsync` further
-# down because when the shell interpolates this out, `rsync` will throw
-# an absolute fit if this variable is quoted. Surprise!
-if [[ -d "${GHE_DATA_DIR}/current/minio" ]] &&
- [[ "$(ls -A "${GHE_DATA_DIR}/current/minio")" ]]; then
- link_dest="--link-dest=${GHE_DATA_DIR}/current/minio"
-fi
-
-# Transfer all minio data from the user data directory using rsync.
-ghe_verbose "* Transferring minio files from ${host} ..."
-
-ghe-rsync \
- --archive \
- --verbose \
- --compress \
- --rsh="ghe-ssh -p ${port}" \
- --rsync-path='sudo -u minio rsync' \
- --exclude=".minio.sys" \
- ${link_dest} \
- "${host}:${GHE_REMOTE_DATA_USER_DIR}/minio/" \
- "${GHE_SNAPSHOT_DIR}/minio" 1>&3
-
-bm_end "$(basename "${0}")"
diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql
deleted file mode 100755
index 18e0e6e20..000000000
--- a/share/github-backup-utils/ghe-backup-mssql
+++ /dev/null
@@ -1,324 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-mssql
-#/
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-# Set up remote host and root backup snapshot directory based on config
-backup_dir="$GHE_SNAPSHOT_DIR/mssql"
-last_mssql=
-backup_command=
-backup_type=
-full_expire=
-diff_expire=
-tran_expire=
-
-# Check if the export tool is available in this version
-export_tool_available() {
- if [ -z "$GHE_TEST_REMOTE_VERSION" ]; then
- ghe-ssh "$GHE_HOSTNAME" "test -e /usr/local/bin/ghe-export-mssql"
- else
- # Always return available for test
- return 0
- fi
-}
-
-if ! export_tool_available; then
- ghe_verbose "ghe-export-mssql is not available"
- exit
-fi
-
-add_minute() {
- # Expect date string in the format of yyyymmddTHHMMSS
- # Here parse date differently depending on GNU Linux vs BSD MacOS
- if date -v -1d > /dev/null 2>&1; then
- echo "$(date -v +$2M -ujf'%Y%m%dT%H%M%S' $1 +%Y%m%dT%H%M%S)"
- else
- dt=$1
- echo "$(date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes")"
- fi
-}
-
-find_timestamp() {
- filename="${1##*/}"
- IFS='@' read -ra parts <<< "$filename"
- datetime_part=${parts[1]:0:15}
- echo $datetime_part
-}
-
-actions_dbs() {
- all_dbs=$(echo 'set -o pipefail; ghe-mssql-console -y -n -q "SET NOCOUNT ON; SELECT name FROM sys.databases"' | ghe-ssh "$GHE_HOSTNAME" /bin/bash)
- for db in $all_dbs; do
- if [[ ! "$db" =~ ^(master|tempdb|model|msdb)$ ]] && [[ "$db" =~ ^[a-zA-Z0-9_-]+$ ]]; then
- echo "$db"
- fi
- done
-}
-
-ensure_same_dbs() {
- locals=()
- while read -r file; do
- filename=$(basename "$file")
- locals+=("$filename")
- done < <(find "$1" \( -name "*.bak" -o -name "*.diff" -o -name "*.log" \))
-
- for remote in $(actions_dbs); do
- remaining_locals=()
- for local in "${locals[@]}"; do
- if ! [[ "$local" == "$remote"* ]]; then
- remaining_locals+=("$local")
- fi
- done
- locals=("${remaining_locals[@]}")
- done
-
- if [[ "${#locals[@]}" -ne 0 ]]; then
- ghe_verbose "Warning: Found following ${#locals[@]} backup files that can't be traced back to the specified GHES host."
- ghe_verbose "Warning: Did you recently reconfigure the GHES host? Move or delete these backup files if no longer needed."
- for local in "${locals[@]}"; do
- ghe_verbose "$1/$local"
- done
-
- exit 1
- fi
-}
-
-run_query() {
- echo "set -o pipefail; ghe-mssql-console -y -n -q \"SET NOCOUNT ON; $1\"" | ghe-ssh "$GHE_HOSTNAME" /bin/bash | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
-}
-
-get_latest_backup_file() {
- backups_dir=$1
- db=$2
- ext=$3
-
- latest_full_backup=$(find "$backups_dir" -type f -name "$db*.$ext" | egrep '[0-9]{8}T[0-9]{6}' | sort | tail -n 1)
- latest_full_backup_file="${latest_full_backup##*/}"
- echo "$latest_full_backup_file"
-}
-
-get_backup_val() {
- db=$1
- filename=$2
- column=$3
- run_query "
- SELECT s.$column
- FROM msdb.dbo.backupset s
- JOIN msdb.dbo.backupmediafamily f
- ON s.media_set_id = f.media_set_id
- WHERE s.database_name = '$db' AND f.physical_device_name LIKE '%$filename'"
-}
-
-get_backup_checkpoint_lsn() {
- get_backup_val "$1" "$2" "checkpoint_lsn"
-}
-
-get_backup_last_lsn() {
- get_backup_val "$1" "$2" "last_lsn"
-}
-
-get_next_log_backup_starting_lsn() {
- db=$1
- # last_log_backup_lsn: The starting log sequence number of the next log backup
- # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-database-recovery-status-transact-sql
- run_query "
- SELECT last_log_backup_lsn
- FROM sys.database_recovery_status drs
- JOIN sys.databases db on drs.database_id = db.database_id
- WHERE db.name = '$db'"
-}
-
-get_next_diff_backup_base_lsn() {
- db=$1
- # differential_base_lsn: Base for differential backups. Data extents changed after this LSN will be included in a differential backup.
- # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-master-files-transact-sql
- run_query "
- SELECT differential_base_lsn
- FROM sys.master_files mf
- WHERE mf.name = '$db'"
-}
-
-last_mssql=$GHE_DATA_DIR/current/mssql
-
-if [ ! -d $last_mssql ] \
- || [ -z "$(find $last_mssql -type f -name '*.bak' | head -n 1)" ]; then
- ghe_verbose "Taking first full backup"
- backup_type="full"
-else
- ensure_same_dbs "$last_mssql"
-
- # Check schedule to determine backup type
- IFS=',' read -ra cadence <<< "$GHE_MSSQL_BACKUP_CADENCE"
-
- current=$(date -u +%Y%m%d%H%M%S)
-
- full=$(find "$last_mssql" -type f -name "*.bak" | head -n 1)
- full=$(find_timestamp $full)
- full_expire=$(add_minute $full ${cadence[0]})
- full_expire="${full_expire//T}"
-
- diff=$(find "$last_mssql" -type f -name "*.diff" | head -n 1)
- if [ -f "$diff" ]; then
- diff=$(find_timestamp $diff)
- diff_expire=$(add_minute $diff ${cadence[1]})
- diff_expire="${diff_expire//T}"
- else
- diff_expire=$(add_minute $full ${cadence[1]})
- diff_expire="${diff_expire//T}"
- fi
-
- tran=$(find "$last_mssql" -type f -name "*.log" | egrep '[0-9]{8}T[0-9]{6}' | sort | tail -1)
- tran=$(find_timestamp $tran)
- tran_expire=$(add_minute $tran ${cadence[2]})
- tran_expire="${tran_expire//T}"
-
- ghe_verbose "current $current, full expire $full_expire, \
-diff expire $diff_expire, tran expire $tran_expire"
-
- # Determine the type of backup to take based on expiry time
- if [ $current -gt $full_expire ]; then
- backup_type='full'
- elif [ $current -gt $diff_expire ]; then
- backup_type='diff'
- elif [ $current -gt $tran_expire ]; then
- backup_type='transaction'
- fi
-
- # Upgrade to a full backup if the diff/transaction backup might not be restorable due to other backup mechanisms interfering
- # with the transaction LSN chain or differential base LSN.
- if [ "$backup_type" == 'diff' ] || [ "$backup_type" == 'transaction' ]; then
- ghe_verbose "Checking for conflicting backups to ensure a $backup_type backup is sufficient"
-
- for db in $(actions_dbs); do
- # Ensure that a diff backup will be based on the full backup file we have (rather than one another backup mechanism took)
- if [ "$backup_type" == 'diff' ]; then
- full_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "bak")
- if [[ "$full_backup_file" == "" ]]; then
- ghe_verbose "Taking a full backup instead of a diff backup because for $db a full backup file wasn't found"
- backup_type="full"
- break
- fi
-
- full_backup_file_checkpoint_lsn=$(get_backup_checkpoint_lsn "$db" "$full_backup_file")
- if [[ "$full_backup_file_checkpoint_lsn" = "NULL" ]] || [[ "$full_backup_file_checkpoint_lsn" == "" ]]; then
- ghe_verbose "Taking a full backup instead of a diff backup because for $db the checkpoint LSN for $full_backup_file couldn't be determined"
- backup_type="full"
- break
- fi
-
- next_diff_backup_base_lsn=$(get_next_diff_backup_base_lsn "$db")
- if [[ "$next_diff_backup_base_lsn" = "NULL" ]] || [[ "$next_diff_backup_base_lsn" == "" ]]; then
- ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the base LSN for the next diff backup couldn't be determined"
- backup_type="full"
- break
- fi
-
- # The base of the diff backup we're about to take must exactly match the checkpoint LSN of the full backup file we have
- if [[ "$next_diff_backup_base_lsn" -ne "$full_backup_file_checkpoint_lsn" ]]; then
- ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the diff would have base LSN $next_diff_backup_base_lsn yet our full backup has checkpoint LSN $full_backup_file_checkpoint_lsn"
- backup_type="full"
- break
- fi
- fi
-
- # Ensure that a transaction log backup will immediately follow the previous one
- latest_log_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "log")
- if [[ "$latest_log_backup_file" == "" ]]; then
- ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db a previous transaction log backup wasn't found"
- backup_type="full"
- break
- fi
-
- latest_log_backup_last_lsn=$(get_backup_last_lsn "$db" "$latest_log_backup_file")
- if [[ "$latest_log_backup_last_lsn" = "NULL" ]] || [[ "$latest_log_backup_last_lsn" == "" ]]; then
- ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the LSN range for $latest_log_backup_file couldn't be determined"
- backup_type="full"
- break
- fi
-
- next_log_backup_starting_lsn=$(get_next_log_backup_starting_lsn "$db")
- if [[ "$next_log_backup_starting_lsn" = "NULL" ]] || [[ "$next_log_backup_starting_lsn" == "" ]]; then
- ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the starting LSN for the next log backup couldn't be determined"
- backup_type="full"
- break
- fi
-
- # The starting LSN of the backup we're about to take must be equal to (or before) the last LSN from the last backup,
- # otherwise there'll be a gap and the logfiles won't be restorable
- if [[ "$next_log_backup_starting_lsn" -gt "$latest_log_backup_last_lsn" ]]; then
- ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db a gap would exist between the last backup ending at LSN $latest_log_backup_last_lsn and next backup starting at $next_log_backup_starting_lsn"
- backup_type="full"
- break
- fi
- done
- fi
-fi
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "$backup_dir"
-
-# Use hard links to "copy" over previous applicable backups to the new snapshot folder to save disk space and time
-if [ -d $last_mssql ]; then
- for p in $last_mssql/*
- do
- [[ -e "$p" ]] || break
-
- filename="${p##*/}"
- extension="${filename##*.}"
- transfer=
-
- # Copy full backups unless we're taking a new full backup
- if [ $extension = "bak" ] && [ "$backup_type" != 'full' ]; then
- transfer=1
- fi
-
- # Copy diff backups unless we're taking a new full or diff backup
- if [ $extension = "diff" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then
- transfer=1
- fi
-
- # Copy transaction log backups unless we're taking a new full or diff backup
- if [ $extension = "log" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then
- transfer=1
- fi
-
- if [ -n "$transfer" ]; then
- ghe_verbose "Creating hard link to $filename"
- ln $last_mssql/$filename $backup_dir/$filename
- fi
- done
-fi
-
-if [ -n "$backup_type" ]; then
- ghe_verbose "Taking $backup_type backup"
-
- backup_command='ghe-export-mssql'
- if [ "$backup_type" = "diff" ]; then
- backup_command='ghe-export-mssql -d'
- elif [ "$backup_type" = "transaction" ]; then
- backup_command='ghe-export-mssql -t'
- fi
-
- bm_start "$(basename $0)"
- ghe-ssh "$GHE_HOSTNAME" -- "$backup_command" || failures="$failures mssql"
- bm_end "$(basename $0)"
-
- # Configure the backup cadence on the appliance, which is used for diagnostics
- ghe-ssh "$GHE_HOSTNAME" "ghe-config mssql.backup.cadence $GHE_MSSQL_BACKUP_CADENCE"
-
- # Transfer backup files from appliance to backup host
- appliance_dir="$GHE_REMOTE_DATA_DIR/user/mssql/backups"
- backups=$(echo "set -o pipefail; if sudo test -d \"$appliance_dir\"; then \
- sudo ls \"$appliance_dir\"; fi" | ghe-ssh "$GHE_HOSTNAME" /bin/bash)
- for b in $backups
- do
- ghe_verbose "Transferring to backup host $b"
- ghe-ssh "$GHE_HOSTNAME" "sudo cat $appliance_dir/$b" > $backup_dir/$b
- done
-fi
diff --git a/share/github-backup-utils/ghe-backup-mysql b/share/github-backup-utils/ghe-backup-mysql
deleted file mode 100755
index 7e3fd57e3..000000000
--- a/share/github-backup-utils/ghe-backup-mysql
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-mysql
-#/ Backup MySQL from a GitHub instance.
-#/
-#/ Note: This script typically isn't called directly. It's invoked by the
-#/ ghe-backup command.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
-ghe_remote_version_required "$GHE_HOSTNAME"
-
-if is_external_database_target; then
- if [ -n "$EXTERNAL_DATABASE_BACKUP_SCRIPT" ]; then
- echo "Backing up external MySQL database using customer-provided script..."
- $EXTERNAL_DATABASE_BACKUP_SCRIPT
- bm_end "$(basename $0)"
- exit 0
- else
- if is_binary_backup_feature_on; then
- echo "Warning: Binary backups are configured on the target environment."
- echo "Binary backup is not supported with an external MySQL database. Backing up using logical backup strategy."
- echo
- echo "Please disable binary backups with 'ghe-config mysql.backup.binary false', or"
- echo "provide a custom backup script using EXTERNAL_DATABASE_BACKUP_SCRIPT"
- fi
-
- ghe-backup-mysql-logical
- fi
-else
- if is_binary_backup_feature_on; then
- ghe-backup-mysql-binary
- else
- ghe-backup-mysql-logical
- fi
-fi
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-mysql-binary b/share/github-backup-utils/ghe-backup-mysql-binary
deleted file mode 100755
index 37218c82e..000000000
--- a/share/github-backup-utils/ghe-backup-mysql-binary
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-mysql-binary
-#/ Backup MySQL from a GitHub instance using binary backup strategy.
-#/
-#/ Note: This script typically isn't called directly. It's invoked by the
-#/ ghe-backup command.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
-ghe_remote_version_required "$GHE_HOSTNAME"
-
-echo "Backing up MySQL database using binary backup strategy ..."
-
-echo "set -o pipefail; ghe-export-mysql" |
-ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
-echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel"
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-mysql-logical b/share/github-backup-utils/ghe-backup-mysql-logical
deleted file mode 100755
index 3dc2478ec..000000000
--- a/share/github-backup-utils/ghe-backup-mysql-logical
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-mysql-logical
-#/ Backup MySQL from a GitHub instance using logical backup strategy.
-#/
-#/ Note: This script typically isn't called directly. It's invoked by the
-#/ ghe-backup command.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
-ghe_remote_version_required "$GHE_HOSTNAME"
-
-echo "Backing up MySQL database using logical backup strategy ..."
-
-echo "set -o pipefail; ghe-export-mysql | pigz" |
-ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
-
-if is_external_database_target; then
- echo "LOGICAL_EXTERNAL_BACKUP" > "$GHE_SNAPSHOT_DIR/logical-external-database-backup-sentinel"
-fi
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-pages b/share/github-backup-utils/ghe-backup-pages
deleted file mode 100755
index e6445c812..000000000
--- a/share/github-backup-utils/ghe-backup-pages
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-pages
-#/ Take an online, incremental snapshot of all Pages data
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Set up remote host and root backup snapshot directory based on config
-host="$GHE_HOSTNAME"
-backup_dir="$GHE_SNAPSHOT_DIR/pages"
-
-# Verify rsync is available.
-if ! rsync --version 1>/dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Split host:port into parts
-port=$(ssh_port_part "$GHE_HOSTNAME")
-host=$(ssh_host_part "$GHE_HOSTNAME")
-
-# Add user / -l option
-user="${host%@*}"
-[ "$user" = "$host" ] && user="admin"
-
-hostnames=$host
-ssh_config_file_opt=
-tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX)
-opts="$GHE_EXTRA_SSH_OPTS"
-
-# Pages server hostnames under cluster
-if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
- ssh_config_file="$tempdir/ssh_config"
- ssh_config_file_opt="-F $ssh_config_file"
- opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
- hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "pages-server")
- ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"
-fi
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "$backup_dir"
-
-# Removes the remote sync-in-progress file on exit, re-enabling GC operations
-# on the remote instance.
-cleanup() {
- rm -rf $tempdir
-}
-trap 'cleanup' EXIT INT
-
-# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/pages)" ]; then
- link_dest="--link-dest=../../current/pages"
-fi
-
-for hostname in $hostnames; do
- bm_start "$(basename $0) - $hostname"
- echo 1>&3
- ghe_verbose "* Starting backup for host: $hostname"
- # Sync all auxiliary repository data. This includes files and directories like
- # HEAD, audit_log, config, description, info/, etc. No refs or object data
- # should be transferred here.
- echo 1>&3
- ghe_verbose "* Transferring pages files ..."
-
- # Transfer all data from the user data directory using rsync.
- ghe-rsync -avz \
- -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \
- --rsync-path='sudo -u git rsync' \
- $link_dest \
- "$hostname:$GHE_REMOTE_DATA_USER_DIR/pages/" \
- "$GHE_SNAPSHOT_DIR/pages" 1>&3
- bm_end "$(basename $0) - $hostname"
-done
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-redis b/share/github-backup-utils/ghe-backup-redis
deleted file mode 100755
index b0eb44949..000000000
--- a/share/github-backup-utils/ghe-backup-redis
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-redis
-#/ Take a snapshot of all Redis data. This is needed because older versions of
-#/ the remote side ghe-export-redis command use a blocking SAVE instead of a
-#/ non-blocking BGSAVE.
-#/
-#/ Note: This script typically isn't called directly. It's invoked by the
-#/ ghe-backup command.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$GHE_HOSTNAME"
-
-# Force a redis BGSAVE, and wait for it to complete.
-ghe-ssh "$GHE_HOSTNAME" /bin/bash < /dev/null; then
- redis_cli=ghe-redis-cli
- redis_arg=--remote
- else
- redis_cli=redis-cli
- redis_arg=
- fi
- redis_host=\$(ghe-config cluster.redis-master 2>/dev/null || echo "localhost")
- timestamp=\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE)
-
- for i in \$(seq 10); do
- if ! \$redis_cli \$redis_arg -h \$redis_host BGSAVE | grep -q ERR; then
- break
- fi
- sleep 15
- done
- for n in \$(seq 3600); do
- if [ "\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE)" != "\$timestamp" ]; then
- break
- fi
- sleep 1
- done
- [ "\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE)" != "\$timestamp" ] # exits 1 if bgsave didn't work
-
- if [ "\$redis_host" != "localhost" ]; then
- ssh \$redis_host sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb'
- else
- sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb'
- fi
-EOF
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories
deleted file mode 100755
index 5185f63d9..000000000
--- a/share/github-backup-utils/ghe-backup-repositories
+++ /dev/null
@@ -1,378 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-repositories
-#/ Take an online, incremental snapshot of all Git repository data.
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# This command is designed to allow for transferring active Git repository data
-# from a GitHub instance to a backup site in a way that ensures data is
-# captured in a consistent state even when being written to.
-#
-# - All Git GC operations are disabled on the GitHub instance for the duration of
-# the backup. This removes the possibly of objects or packs being removed
-# while the backup is in progress.
-#
-# - In progress Git GC operations are given a cooldown window to complete. The
-# script will sleep for up to 60 seconds waiting for GC operations to finish.
-#
-# - Git repository data is transferred in a specific order: auxiliary files,
-# packed refs, loose refs, reflogs, and finally objects and pack files in that
-# order. This ensures that all referenced objects are captured.
-#
-# - Git GC operations are re-enabled on the GitHub instance.
-#
-# The script uses multiple runs of rsync to transfer repository files. Each run
-# includes a list of filter rules that ensure only specific types of files are
-# transferred.
-#
-# See the "FILTER RULES" and "INCLUDE/EXCLUDE PATTERN RULES" sections of the
-# rsync(1) manual for more information:
-#
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Set up remote host and root backup snapshot directory based on config
-host="$GHE_HOSTNAME"
-backup_dir="$GHE_SNAPSHOT_DIR/repositories"
-
-# Location of last good backup for rsync --link-dest
-backup_current="$GHE_DATA_DIR/current/repositories"
-
-# Verify rsync is available.
-if ! rsync --version 1>/dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Split host:port into parts
-port=$(ssh_port_part "$GHE_HOSTNAME")
-host=$(ssh_host_part "$GHE_HOSTNAME")
-
-# Add user / -l option
-user="${host%@*}"
-[ "$user" = "$host" ] && user="admin"
-
-hostnames=$host
-ssh_config_file_opt=
-tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX)
-remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX)
-routes_list=$tempdir/routes_list
-remote_routes_list=$remote_tempdir/remote_routes_list
-opts="$GHE_EXTRA_SSH_OPTS"
-
-# git server hostnames under cluster
-if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
- ssh_config_file="$tempdir/ssh_config"
- ssh_config_file_opt="-F $ssh_config_file"
- opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
- hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "git-server")
- ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"
-fi
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "$backup_dir"
-
-# Removes the remote sync-in-progress file on exit, re-enabling GC operations
-# on the remote instance.
-cleanup() {
- for pid in $(jobs -p); do
- kill -KILL $pid > /dev/null 2>&1 || true
- done
-
- # Enable remote GC operations
- for hostname in $hostnames; do
- ghe-gc-enable $ssh_config_file_opt $hostname:$port || {
- echo "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2
- }
- done
-
- ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir
- rm -rf $tempdir
-}
-trap 'cleanup' EXIT INT
-
-# Disable remote GC operations
-for hostname in $hostnames; do
- ghe-gc-disable $ssh_config_file_opt $hostname:$port
-done
-
-# If we have a previous increment, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-if [ -d "$backup_current" ]; then
- link_dest="--link-dest=../../current/repositories"
-fi
-
-# Calculate sync routes. This will store the healthy repo paths for each node
-#
-# This gets a repo path and stores the path in the $node.sync file
-# a/nw/a8/3f/02/100000855 dgit-node1 >> dgit-node1.sync
-# a/nw/a8/bc/8d/100000880 dgit-node3 >> dgit-node3.sync
-# a/nw/a5/06/81/100000659 dgit-node2 >> dgit-node2.sync
-# ...
-# One route per line.
-#
-# NOTE: The route generation is performed on the appliance as it is considerably
-# more performant than performing over an SSH pipe.
-#
-bm_start "$(basename $0) - Generating routes"
-echo "github-env ./bin/dgit-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
-ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug
-bm_end "$(basename $0) - Generating routes"
-
-bm_start "$(basename $0) - Fetching routes"
-ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list
-cat $routes_list | ghe_debug
-bm_end "$(basename $0) - Fetching routes"
-
-bm_start "$(basename $0) - Processing routes"
-if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then
- server=$host
-fi
-cat $routes_list | awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}'
-ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')"
-bm_end "$(basename $0) - Processing routes"
-
-if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then
- echo "Warning: no routes found, skipping repositories backup ..."
- exit 0
-fi
-
-# Transfer repository data from a GitHub instance to the current snapshot
-# directory, using a previous snapshot to avoid transferring files that have
-# already been transferred. A set of rsync filter rules are provided on stdin
-# for each invocation.
-rsync_repository_data () {
- port=$(ssh_port_part "$1")
- host=$(ssh_host_part "$1")
-
- #check if we are syncing from a given file list
- if [[ "$2" == *".rsync" ]]; then
- files_list="$2"
- shift
- shift
- ghe-rsync -avr \
- -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \
- $link_dest "$@" \
- --rsync-path='sudo -u git rsync' \
- --include-from=- --exclude=\* \
- --files-from="$files_list" \
- --ignore-missing-args \
- "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \
- "$backup_dir" 1>&3 2>&3
- else
- shift
- ghe-rsync -avr \
- -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \
- $link_dest "$@" \
- --rsync-path='sudo -u git rsync' \
- --include-from=- --exclude=\* \
- --ignore-missing-args \
- "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \
- "$backup_dir" 1>&3 2>&3
- fi
-}
-
-sync_data (){
- # Sync all auxiliary repository data. This includes files and directories like
- # HEAD, audit_log, config, description, info/, etc. No refs or object data
- # should be transferred here.
- echo 1>&3
-
- echo "* Transferring auxiliary files ..." 1>&3
- rsync_repository_data $1:122 $2 -z <&3
- echo "* Transferring packed-refs files ..." 1>&3
- rsync_repository_data $1:122 $2 -z <&3
- echo "* Transferring refs and reflogs ..." 1>&3
- rsync_repository_data $1:122 $2 -z <&3
- echo "* Transferring objects and packs ..." 1>&3
- rsync_repository_data $1:122 $2 -H <&3
-
-}
-
-# rsync all the repositories
-bm_start "$(basename $0) - Repo sync"
-for file_list in $tempdir/*.rsync; do
- hostname=$(basename $file_list .rsync)
-
- repo_num=$(cat $file_list | wc -l)
- ghe_verbose "* Transferring $repo_num repositories from $hostname"
-
- sync_data $hostname $file_list &
-done
-
-for pid in $(jobs -p); do
- wait $pid
-done
-bm_end "$(basename $0) - Repo sync"
-
-# Since there are no routes for special data directories, we need to do this
-# serially for all hostnames. Good candidate for future optimizations.
-
-bm_start "$(basename $0) - Special Data Directories Sync"
-for h in $hostnames; do
- # Sync __special__ data directories, including the __alambic_assets__,
- # __hookshot__, and __purgatory__ directories. The __nodeload_archives__,
- # __gitmon__, and __render__ directories are excludes since they act only as
- # caches.
- #
- # Under v2.x and greater, only the special __purgatory__ directory remains under
- # /data/repositories. All other special user data directories have been moved under
- # the /data/user directory.
- echo 1>&3
- echo "* Transferring special data directories from $h..." 1>&3
- rsync_repository_data $h:122 -z <&3
-done
-bm_end "$(basename $0) - Special Data Directories Sync"
-
-if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
- bm_start "$(basename $0) - Verifying Routes"
- cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes
- (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes
-
- git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
-
- bm_end "$(basename $0) - Verifying Routes"
-fi
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings
deleted file mode 100755
index f2e14d88d..000000000
--- a/share/github-backup-utils/ghe-backup-settings
+++ /dev/null
@@ -1,151 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-settings
-#/ Backup settings from a snapshot to the given .
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-bm_start "$(basename $0)"
-
-# Grab the host
-host="$GHE_HOSTNAME"
-
-# Create the snapshot directory if needed and change into it.
-mkdir -p "$GHE_SNAPSHOT_DIR"
-cd "$GHE_SNAPSHOT_DIR"
-
-echo "* Transferring settings data ..." 1>&3
-ghe-ssh "$host" -- 'ghe-export-settings' > settings.json
-
-echo "* Transferring license data ..." 1>&3
-ghe-ssh "$host" -- "sudo cat '$GHE_REMOTE_LICENSE_FILE'" > enterprise.ghl
-
-# Function to backup a secret setting to a file.
-# backup-secret [--best-effort]
-backup-secret() {
-
- best_effort=false
- description=""
- file=""
- setting=""
- count=0
-
- while [ $# -gt 0 ]; do
- case "$1" in
- --best-effort)
- shift 1
- best_effort=true
- ;;
- *)
- case $count in
- 0)
- description=$1
- ;;
- 1)
- file=$1
- ;;
- 2)
- setting=$1
- ;;
- *)
- >&2 echo "Too many arguments"
- ;;
- esac
- count=$((count+1))
- shift 1
- esac
- done
-
- echo "* Transferring $description ..." 1>&3
- ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || (
- if [ "$best_effort" = "false" ]; then
- echo "Warning: $description not set" >&2
- fi
- )
- if [ -n "$(cat "$file+")" ]; then
- mv "$file+" "$file"
- else
- unlink "$file+"
- fi
-}
-
-backup-secret "management console password" "manage-password" "secrets.manage"
-backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets"
-
-# Backup external MySQL password if running external MySQL DB.
-if is_service_external 'mysql'; then
- backup-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql"
-fi
-
-# Backup Actions settings.
-if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then
- backup-secret "Actions configuration database login" "actions-config-db-login" "secrets.actions.ConfigurationDatabaseSqlLogin"
- backup-secret "Actions configuration database password" "actions-config-db-password" "secrets.actions.ConfigurationDatabaseSqlPassword"
- backup-secret "Actions framework access token key secret" "actions-framework-access-token" "secrets.actions.FrameworkAccessTokenKeySecret" --best-effort
- backup-secret "Actions Url signing HMAC key primary" "actions-url-signing-hmac-key-primary" "secrets.actions.UrlSigningHmacKeyPrimary"
- backup-secret "Actions Url signing HMAC key secondary" "actions-url-signing-hmac-key-secondary" "secrets.actions.UrlSigningHmacKeySecondary"
- backup-secret "Actions OAuth S2S signing cert" "actions-oauth-s2s-signing-cert" "secrets.actions.OAuthS2SSigningCert"
- backup-secret "Actions OAuth S2S signing key" "actions-oauth-s2s-signing-key" "secrets.actions.OAuthS2SSigningKey"
- backup-secret "Actions OAuth S2S signing cert thumbprint" "actions-oauth-s2s-signing-cert-thumbprint" "secrets.actions.OAuthS2SSigningCertThumbprint"
- backup-secret "Actions primary encryption cert thumbprint" "actions-primary-encryption-cert-thumbprint" "secrets.actions.PrimaryEncryptionCertificateThumbprint"
- backup-secret "Actions AAD cert thumbprint" "actions-aad-cert-thumbprint" "secrets.actions.AADCertThumbprint" --best-effort
- backup-secret "Actions delegated auth cert thumbprint" "actions-delegated-auth-cert-thumbprint" "secrets.actions.DelegatedAuthCertThumbprint" --best-effort
- backup-secret "Actions runtime service principal cert" "actions-runtime-service-principal-cert" "secrets.actions.RuntimeServicePrincipalCertificate" --best-effort
- backup-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secrets.actions.S2SEncryptionCertificate"
- backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint"
- backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort
- backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint"
-
- backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key"
- backup-secret "Actions Launch credz HMAC key" "actions-launch-credz-hmac" "secrets.launch.credz-hmac-secret"
- backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret"
- backup-secret "Actions Launch Client id" "actions-launch-client-id" "secrets.launch.client-id"
- backup-secret "Actions Launch Client secret" "actions-launch-client-secret" "secrets.launch.client-secret"
- backup-secret "Actions Launch receiver webhook secret" "actions-launch-receiver-webhook-secret" "secrets.launch.receiver-webhook-secret"
- backup-secret "Actions Launch app private key" "actions-launch-app-private-key" "secrets.launch.app-private-key"
- backup-secret "Actions Launch app public key" "actions-launch-app-public-key" "secrets.launch.app-public-key"
- backup-secret "Actions Launch app id" "actions-launch-app-id" "secrets.launch.app-id"
- backup-secret "Actions Launch app relay id" "actions-launch-app-relay-id" "secrets.launch.app-relay-id"
- backup-secret "Actions Launch action runner secret" "actions-launch-action-runner-secret" "secrets.launch.action-runner-secret"
- backup-secret "Actions Launch service cert" "actions-launch-azp-app-cert" "secrets.launch.azp-app-cert"
- backup-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key"
-fi
-
-if ghe-ssh "$host" -- ghe-config --true app.packages.enabled; then
- backup-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key"
- backup-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key"
- backup-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket"
- backup-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url"
- backup-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type"
- backup-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string"
- backup-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name"
-fi
-
-if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/idp.crt"; then
- echo "* Transferring SAML keys ..." 1>&3
- ghe-ssh $host -- sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -cf - "idp.crt saml-sp.p12" > saml-keys.tar
-fi
-
-if ghe-ssh "$host" -- "which ghe-export-ssl-ca-certificates 1>/dev/null"; then
- echo "* Transferring CA certificates ..." 1>&3
- ghe-ssh "$host" -- "ghe-export-ssl-ca-certificates" > ssl-ca-certificates.tar
-fi
-
-if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
- echo "* Transferring cluster configuration ..." 1>&3
- if ! ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_CLUSTER_CONF_FILE 2>/dev/null" > cluster.conf; then
- echo "Error: Enterprise Cluster is not configured yet, backup will fail" >&2
- exit 1
- fi
-else
- if ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_DATA_USER_DIR/common/uuid 2>/dev/null" > uuid; then
- echo "* Transferring UUID ..." 1>&3
- fi
-fi
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage
deleted file mode 100755
index 67517512b..000000000
--- a/share/github-backup-utils/ghe-backup-storage
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-storage
-#/ Take an online, incremental snapshot of all Alambic Storage data using the
-#/ calculated routes method.
-#/
-#/ Note: This command typically isn't called directly. It's invoked by
-#/ ghe-backup.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-# Set up remote host and root backup snapshot directory based on config
-host="$GHE_HOSTNAME"
-backup_dir="$GHE_SNAPSHOT_DIR/storage"
-
-# Verify rsync is available.
-if ! rsync --version 1>/dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Split host:port into parts
-port=$(ssh_port_part "$GHE_HOSTNAME")
-host=$(ssh_host_part "$GHE_HOSTNAME")
-
-# Add user / -l option
-user="${host%@*}"
-[ "$user" = "$host" ] && user="admin"
-
-hostnames=$host
-ssh_config_file_opt=
-tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX)
-remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX)
-routes_list=$tempdir/routes_list
-remote_routes_list=$remote_tempdir/remote_routes_list
-opts="$GHE_EXTRA_SSH_OPTS"
-
-# storage server hostnames under cluster
-if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
- ssh_config_file="$tempdir/ssh_config"
- ssh_config_file_opt="-F $ssh_config_file"
- opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
- hostnames=$(ghe-cluster-nodes "$GHE_HOSTNAME" "storage-server")
- ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file"
-fi
-
-# Make sure root backup dir exists if this is the first run
-mkdir -p "$backup_dir"
-
-# Removes the remote sync-in-progress file on exit, re-enabling GC operations
-# on the remote instance.
-cleanup() {
- # Enable remote maintenance operations
- for hostname in $hostnames; do
- ghe-gc-enable $ssh_config_file_opt $hostname:$port || {
- echo "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2
- }
- done
-
- ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir
- rm -rf $tempdir
-}
-trap 'cleanup' EXIT INT
-
-# Disable remote maintenance operations
-for hostname in $hostnames; do
- ghe-gc-disable $ssh_config_file_opt $hostname:$port
-done
-
-# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-if [ -d "$GHE_DATA_DIR/current/storage" ] && [ "$(ls -A $GHE_DATA_DIR/current/storage)" ]; then
- link_dest="--link-dest=../../current/storage"
-fi
-
-# Calculate sync routes. This will store the healthy object paths for each node
-#
-# This gets a repo path and stores the path in the $node.sync file
-# a/nw/a8/3f/02/100000855 storage-server-node1 >> storage-server-node1.sync
-# a/nw/a8/bc/8d/100000880 storage-server-node3 >> storage-server-node3.sync
-# a/nw/a5/06/81/100000659 storage-server-node2 >> storage-server-node2.sync
-# ...
-#one route per line.
-#
-# NOTE: The route generation is performed on the appliance as it is considerably
-# more performant than performing over an SSH pipe.
-#
-bm_start "$(basename $0) - Generating routes"
-echo "github-env ./bin/storage-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
-ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug
-bm_end "$(basename $0) - Generating routes"
-
-bm_start "$(basename $0) - Fetching routes"
-ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list
-cat $routes_list | ghe_debug
-bm_end "$(basename $0) - Fetching routes"
-
-bm_start "$(basename $0) - Processing routes"
-if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then
- server=$host
-fi
-cat $routes_list | awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}'
-ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')"
-bm_end "$(basename $0) - Processing routes"
-
-if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then
- echo "Warning: no routes found, skipping storage backup ..."
- exit 0
-fi
-
-# rsync all the storage objects
-bm_start "$(basename $0) - Storage object sync"
-for file_list in $tempdir/*.rsync; do
- hostname=$(basename $file_list .rsync)
- storage_user=$(ghe-ssh $ssh_config_file_opt $hostname:$port -- stat -c %U /data/user/storage || echo git)
-
- object_num=$(cat $file_list | wc -l)
- ghe_verbose "* Transferring $object_num objects from $hostname"
-
- ghe-rsync -avr \
- -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \
- $link_dest "$@" \
- --rsync-path="sudo -u $storage_user rsync" \
- --files-from="$file_list" \
- --ignore-missing-args \
- --size-only \
- "$hostname:$GHE_REMOTE_DATA_USER_DIR/storage/" \
- "$backup_dir" 1>&3 &
-done
-
-for pid in $(jobs -p); do
- wait $pid
-done
-bm_end "$(basename $0) - Storage object sync"
-
-if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
- bm_start "$(basename $0) - Verifying Routes"
-
- cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes
- (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | uniq | sort | uniq) > $tempdir/destination_routes
-
- git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
-
- bm_end "$(basename $0) - Verifying Routes"
-fi
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-store-version b/share/github-backup-utils/ghe-backup-store-version
deleted file mode 100755
index 6faffff99..000000000
--- a/share/github-backup-utils/ghe-backup-store-version
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-store-version
-#/ Stores information about the used version of backup-utils on
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0)"
-
-version_info="$BACKUP_UTILS_VERSION"
-if [ -d $GHE_BACKUP_ROOT/.git ]; then
- ref=$(git --git-dir=$GHE_BACKUP_ROOT/.git rev-parse HEAD || true)
- if [ -n "$ref" ]; then
- version_info="$version_info:$ref"
- fi
-fi
-
-echo "$version_info" |
- ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1"
-
-bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-strategy b/share/github-backup-utils/ghe-backup-strategy
deleted file mode 100755
index b9a919f3c..000000000
--- a/share/github-backup-utils/ghe-backup-strategy
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-strategy
-#/
-#/ Determine the backup strategy that will be used.
-#/
-#/ The rsync strategy should be used for single VMs and all HA configurations.
-#/
-#/ The cluster strategy should be used to backup GHE clusters.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-if ghe-ssh "$GHE_HOSTNAME" -- \
- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ ! -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then
- echo "cluster"
-else
- echo "rsync"
-fi
diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata
deleted file mode 100755
index 97f71c1a9..000000000
--- a/share/github-backup-utils/ghe-backup-userdata
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-backup-userdata
-#/ Take an online, incremental snapshot of a user data directory. This is used
-#/ for a number of different simple datastores kept under /data/user on the
-#/ remote appliance, including: hookshot, alambic_assets, and pages data.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-bm_start "$(basename $0) - $1"
-
-# Verify rsync is available.
-if ! rsync --version 1>/dev/null 2>&1; then
- echo "Error: rsync not found." 1>&2
- exit 1
-fi
-
-# Grab the host and /data/user directory name.
-host="$GHE_HOSTNAME"
-dirname="$1"
-
-# Perform a host-check and establish GHE_REMOTE_XXX variables.
-ghe_remote_version_required "$host"
-
-# Verify that the user data directory exists. Bail out if not, which may be due
-# to an older version of GHE or no data has been added to this directory yet.
-ghe-ssh "$host" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0
-
-# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
-# --link-dest support. This also decreases physical space usage considerably.
-if [ -d "$GHE_DATA_DIR/current/$dirname" ] && [ "$(ls -A $GHE_DATA_DIR/current/$dirname)" ]; then
-
- subdir=$dirname
- link_path=".."
- while true; do
- if [ "$(dirname $subdir)" = "." ]; then
- break
- fi
-
- if [ "$(dirname $subdir)" = "/" ]; then
- break
- fi
-
- link_path="../$link_path"
- subdir=$(dirname $subdir)
- done
-
- link_dest="--link-dest=../${link_path}/current/$dirname"
-fi
-
-# Ensure target directory exists, is needed with subdirectories
-mkdir -p "$GHE_SNAPSHOT_DIR/$dirname"
-
-# Transfer all data from the user data directory using rsync.
-ghe-rsync -avz \
- -e "ghe-ssh -p $(ssh_port_part "$host")" \
- --rsync-path='sudo -u git rsync' \
- $link_dest \
- "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/$dirname/" \
- "$GHE_SNAPSHOT_DIR/$dirname" 1>&3
-
-bm_end "$(basename $0) - $1"
diff --git a/share/github-backup-utils/ghe-cluster-nodes b/share/github-backup-utils/ghe-cluster-nodes
deleted file mode 100755
index a4d29247b..000000000
--- a/share/github-backup-utils/ghe-cluster-nodes
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-cluster-nodes
-#/
-#/ Finds all nodes of the cluster using the config on .
-#/ If it is a 2.8 and later cluster version the results are returned as
-#/ prefix-uuid, otherwise the configured hostnames are returned.
-#/ Also filters nodes based on the prefix role.
-#/
-#/ Note: This script typically isn't called directly. It's invoked by the
-#/ ghe-backup-* and ghe-restore-* commands in cluster environments.
-set -e
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-# Check if the REMOTE DATA USER directory is set
-if [ -z $GHE_REMOTE_DATA_USER_DIR ]; then
- echo "Env variable GHE_REMOTE_DATA_USER_DIR is not set. Exiting"
- exit 1
-fi
-
-# Show usage and bail with no arguments
-[ -z "$*" ] && print_usage
-
-GHE_HOSTNAME="$1"
-prefix="$2"
-role=$(echo "$prefix" | cut -d '-' -f1)
-
-if ghe-ssh "$GHE_HOSTNAME" test -f $GHE_REMOTE_ROOT_DIR/etc/github/cluster; then
- node_uuids=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-each -r "$role" -u)
- hostnames=''
- for uuid in $node_uuids; do
- hostnames+="$prefix-$uuid "
- done
-else
- uuid=$(ghe-ssh "$GHE_HOSTNAME" cat $GHE_REMOTE_DATA_USER_DIR/common/uuid)
- hostnames="$prefix-$uuid"
-fi
-
-echo "$hostnames"
diff --git a/share/github-backup-utils/ghe-detect-leaked-ssh-keys b/share/github-backup-utils/ghe-detect-leaked-ssh-keys
deleted file mode 100755
index 79978ddf0..000000000
--- a/share/github-backup-utils/ghe-detect-leaked-ssh-keys
+++ /dev/null
@@ -1,140 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-detect-leaked-ssh-key [-s ]
-#/
-#/ This utility will check each snapshot's existing SSH host keys against the list
-#/ of known leaked SSH host keys from GitHub Enterprise packages.
-#/
-#/ OPTIONS:
-#/ -h | --help Show this message.
-#/ -s |--snapshot Scan the snapshot with the given id.
-#/ Available snapshots may be listed under the data directory.
-#/
-set -e
-
-usage() {
- grep '^#/' < "$0" | cut -c 4-
- exit 2
-}
-
-TEMPDIR=$(mktemp -d)
-
-while [ $# -gt 0 ]; do
- case "$1" in
- -h|--help)
- usage
- ;;
- -s|--snapshot)
- snapshot=$2
- shift
- ;;
- *)
- usage
- ;;
- esac
- shift
-done
-
-ppid_script=$(ps -o args= $PPID 2>/dev/null | awk '{print $2}')
-if [ -n "$ppid_script" ]; then
- ppid_name=$(basename $ppid_script)
-fi
-
-sshkeygen_multiple_hash_formats=false
-if (ssh-keygen -E 2>&1 | head -1 | grep -q 'option requires an argument'); then
- sshkeygen_multiple_hash_formats=true
-fi
-
-# Bring in the backup configuration
-# shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
-
-FINGERPRINT_BLACKLIST="${FINGERPRINT_BLACKLIST:-$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-ssh-leaked-host-keys-list.txt")}"
-
-keys="ssh_host_dsa_key.pub ssh_host_ecdsa_key.pub ssh_host_ed25519_key.pub ssh_host_rsa_key.pub"
-
-# Get all the host ssh keys tar from all snapshots directories
-if [ -n "$snapshot" ]; then
- if [ ! -d "$snapshot" ]; then
- echo "Invalid snapshot directory: $snapshot" >&2
- exit 1
- fi
- ssh_tars=$(find "$snapshot" -maxdepth 1 -type f -iname 'ssh-host-keys.tar')
-else
- ssh_tars=$(find "$GHE_DATA_DIR" -maxdepth 2 -type f -iname 'ssh-host-keys.tar')
-fi
-
-# Store the current backup snapshot folder
-if [ -L "$GHE_DATA_DIR/current" ]; then
- current_dir=$(cd "$GHE_DATA_DIR/current"; pwd -P)
-fi
-
-leaked_keys_found=false
-leaked_keys_skippedcheck=false
-current_bkup=false
-for tar_file in $ssh_tars; do
- for key in $keys; do
- if tar -tvf "$tar_file" $key &>/dev/null; then
- tar -C $TEMPDIR -xvf "$tar_file" $key &>/dev/null
- if $sshkeygen_multiple_hash_formats; then
- fingerprint=$(ssh-keygen -l -E md5 -f $TEMPDIR/$key | cut -d ' ' -f 2 | cut -f2- -d':')
- else
- fingerprint=$(ssh-keygen -lf $TEMPDIR/$key | cut -d ' ' -f 2)
- fi
- if [ -z "$fingerprint" ]; then
- leaked_keys_skippedcheck=true
- elif echo "$FINGERPRINT_BLACKLIST" | grep -q "$fingerprint"; then
- leaked_keys_found=true
- if [ "$current_dir" == "$(dirname "$tar_file")" ]; then
- current_bkup=true
- echo "* Leaked key found in current backup snapshot."
- else
- echo "* Leaked key found in backup snapshot."
- fi
- echo "* Snapshot file: $tar_file"
- echo "* Key file: $key"
- echo "* Key: $fingerprint"
- echo
- fi
- fi
- done
-done
-
-if $leaked_keys_found; then
- if echo "$ppid_name" | grep -q 'ghe-restore'; then
- echo
- echo "* The snapshot that is being restored contains a leaked SSH host key."
- echo "* We recommend rolling the SSH host keys after completing the restore."
- echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance."
- echo "* (An upgrade may be required)"
- echo
- elif echo "$ppid_name" | grep -q 'ghe-backup'; then
- echo "* The current backup contains leaked SSH host keys."
- echo "* We strongly recommend rolling your SSH host keys and making a new backup."
- echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance."
- echo "* (An upgrade may be required)"
- else
- if $current_bkup; then
- echo "* The current backup contains leaked SSH host keys."
- echo "* Current backup directory: $current_dir"
- echo "* We strongly recommend rolling your SSH host keys and making a new backup."
- echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance."
- echo "* (An upgrade may be required)"
- fi
- echo
- echo "* One or more older backup snapshots contain leaked SSH host keys."
- echo "* No immediate action is needed but when you use one of these older snapshots for a restore, "
- echo "* please make sure to roll the SSH host keys after restore."
- echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance."
- echo "* (An upgrade may be required)"
- echo
- fi
-else
- if $leaked_keys_skippedcheck; then
- echo "* No result - check not performed since host key fingerprint was empty"
- else
- echo "* No leaked keys found"
- fi
-fi
-
-# Cleanup temp dir
-rm -rf $TEMPDIR
diff --git a/share/github-backup-utils/ghe-docker-init b/share/github-backup-utils/ghe-docker-init
deleted file mode 100755
index 76e6086b8..000000000
--- a/share/github-backup-utils/ghe-docker-init
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-PATH=$PATH:/backup-utils/bin
-
-mkdir -p /etc/github-backup-utils
-
-touch /etc/github-backup-utils/backup.config
-
-env | grep ^GHE_ | sed -r "s/(.[^=]+)=(.*)/\1=\"\2\"/g" >> /etc/github-backup-utils/backup.config
-
-exec "$@"
diff --git a/share/github-backup-utils/ghe-gc-disable b/share/github-backup-utils/ghe-gc-disable
deleted file mode 100755
index bf37572ca..000000000
--- a/share/github-backup-utils/ghe-gc-disable
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-#/ Usage: ghe-gc-disable [