diff --git a/.github/actions/proxy-janky-build/action.yml b/.github/actions/proxy-janky-build/action.yml new file mode 100644 index 000000000..988c4a289 --- /dev/null +++ b/.github/actions/proxy-janky-build/action.yml @@ -0,0 +1,33 @@ +name: 'Trigger a CI Job on Janky' +description: 'Action to trigger and poll a Janky CI job' +inputs: + janky-token: + description: 'Token for making request to Janky' + required: true + job-name: + description: 'The name of the job to run' + required: true + branch-name: + description: 'The name of the branch to use' + required: true + force: + description: 'Force the job to run even if it is already passed' + required: false + envVars: + description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3' + required: false +runs: + using: 'composite' + steps: + - uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260 + with: + go-version: '1.21' + - run: | + go run main.go \ + -token ${{ inputs.janky-token }} \ + -job ${{ inputs.job-name }} \ + -branch ${{ inputs.branch-name }} \ + -force ${{ inputs.force }} \ + -envVars ${{ inputs.envVars }} + shell: bash + working-directory: .github/actions/proxy-janky-build diff --git a/.github/actions/proxy-janky-build/go.mod b/.github/actions/proxy-janky-build/go.mod new file mode 100644 index 000000000..b7560d7b9 --- /dev/null +++ b/.github/actions/proxy-janky-build/go.mod @@ -0,0 +1,7 @@ +module github.com/github/enterprise2/actions/proxy-janky-build + +go 1.21 + +require github.com/hashicorp/go-retryablehttp v0.7.2 + +require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/.github/actions/proxy-janky-build/go.sum b/.github/actions/proxy-janky-build/go.sum new file mode 100644 index 000000000..5c59c1d2e --- /dev/null +++ b/.github/actions/proxy-janky-build/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go new file mode 100644 index 000000000..e0342f121 --- /dev/null +++ b/.github/actions/proxy-janky-build/main.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +// Define our Janky Response Structs +type JankyBuildStruct struct { + Result string + Url string +} +type JankyStatusStruct struct { + Id string + Green bool + Completed bool + StartedAt string + CompletedAt string + Sha string + BuildableName string +} + +const ( + pollWaitTime = 10 * time.Second + jankyPollTimeout = 5 * time.Hour + jankyHttpRetryMax = 5 + jankyUrl = "https://janky.githubapp.com" +) + +func main() { + // Parse command-line arguments + job := flag.String("job", "", "Name of the Janky job") + token := flag.String("token", "", "Name of the Janky token") + branch := flag.String("branch", "", "Name of the Git branch") + force := flag.String("force", "false", "Force a build even if one is already passed") + envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") + flag.Parse() + + // Validate command-line arguments + if *job == "" || *token == "" || *branch == "" { + log.Fatal("job, token and branch flags must be specified") + } + + // Set up the token + request payload + authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token)) + type buildRequestObject struct { + BuildableName string `json:"buildable_name"` + BranchName string `json:"branch_name"` + Force string `json:"force"` + EnvVars map[string]string `json:"env_vars"` + } + + requestBody := buildRequestObject{ + BuildableName: *job, + BranchName: *branch, + Force: *force, + } + + // Parse the envVars flag into a map and add to the request payload + fmt.Println("Environment Variables:") + fmt.Println(*envVars) + if *envVars != "" { + envVarsMap := make(map[string]string) + for _, envVar := range strings.Split(*envVars, ",") { + envVarSplit := strings.Split(envVar, "=") + envVarsMap[envVarSplit[0]] = envVarSplit[1] + } + requestBody.EnvVars = envVarsMap + } + + payloadBytes, err := json.Marshal(requestBody) + if err != nil { + log.Fatal("Failed to marshal the JSON payload!\n" + err.Error()) + } + + // Send build request to Janky + buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes)) + if err != nil { + log.Fatal("Failed to create build request!\n" + err.Error()) + } + buildRequest.Header.Set("Content-Type", "application/json") + buildRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient := retryablehttp.NewClient() //nolint:all + retryClient.RetryMax = jankyHttpRetryMax + retryClient.Logger = nil // disable debug logging + client := retryClient.StandardClient() // uses *http.Client + resp, err := client.Do(buildRequest) + if err != nil { + log.Fatal("Failed to send build request!\n" + err.Error()) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatal("Error reading build response!\n" + err.Error()) + } + + // Check if the build was triggered successfully + if resp.StatusCode == 404 { + log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.") + } + if resp.StatusCode != 201 { + log.Fatal("Failed to trigger build! Got exception: " + string(body)) + } + + // Parse the build request response + var buildResponse JankyBuildStruct + json.Unmarshal(body, &buildResponse) + log.Println("Succesfully triggered janky!\n" + buildResponse.Result) + + // Parse the request response for the buildId + r, err := regexp.Compile("/[0-9]+/") + if err != nil { + log.Fatal("Failed to trigger build!\n" + err.Error()) + } + buildId := strings.Trim(r.FindString(buildResponse.Result), "/") + + // Setup our second HTTP client for reuse in during status polling + jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status" + statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil) + if err != nil { + log.Fatal("Failed to create status request!\n" + err.Error()) + } + statusRequest.Header.Set("Content-Type", "application/json") + statusRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient2 := retryablehttp.NewClient() //nolint:all + retryClient2.RetryMax = jankyHttpRetryMax + retryClient2.Logger = nil // disable debug logging + client2 := retryClient2.StandardClient() // uses *http.Client + + // Wait for a completed status from Janky or break the loop after a certain amount of time + timeout := time.NewTimer(jankyPollTimeout) + poll := time.NewTicker(pollWaitTime) + +jobLoop: + for { + select { + case <-timeout.C: + log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours") + case <-poll.C: + // Send build status request to Janky + statusResponse, err := client2.Do(statusRequest) + if err != nil { + log.Fatal("Failed to send status request!\n" + err.Error()) + } + defer statusResponse.Body.Close() + statusBody, err := io.ReadAll(statusResponse.Body) + if err != nil { + log.Fatal("Error reading status response!\n" + err.Error()) + } + + // Parse the status response for a green completed build + var jankyStatusResponse JankyStatusStruct + json.Unmarshal(statusBody, &jankyStatusResponse) + //fmt.Println("Janky Status Response:") + //fmt.Println(string(statusBody)) + if jankyStatusResponse.Completed && jankyStatusResponse.Green { + log.Println("Janky build Succeeded!") + break jobLoop + } + if jankyStatusResponse.Completed && !jankyStatusResponse.Green { + log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url) + } + + // wait for a bit and try again + log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]") + continue + } + } +} diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..030c37f04 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,53 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ade3e5c58 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,40 @@ + + + + +# PR Details + +## Description + +## Testing + + + +## Ownership + + +## Related Links + diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 000000000..edf115738 --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,127 @@ +--- +name: Build and Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version - patch version of the release (e.g. x.y.z)' + required: true + type: string + draft: + description: 'Draft - true if the release should be a draft' + required: true + type: boolean + default: true + +jobs: + build: + runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" + - name: Get major-feature from version + id: get-major-feature + run: | + echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + - name: Verify major-feature + run: | + echo "major_feature: $MAJOR_FEATURE" + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: ${{ env.MAJOR_FEATURE }}-stable + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" + commit_options: "--allow-empty" + push_options: "--force" + skip_dirty_check: true + - name: Checkout backup-utils-private for release notes + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ steps.app-token.outputs.token }} + owner: github + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ env.MAJOR_FEATURE }}-stable + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..efdd23ebe --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,67 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'enterprise2 branch to test against' + required: true + source-branch: + description: 'backup-utils-private topic branch to test' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" + + # Cluster integration tests are optional based on label and PR titles + cluster-integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-cluster-binary-backup + - enterprise2-backup-utils-cluster-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + if: | + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.title, '[cluster]') || + contains(github.event.pull_request.labels.*.name, 'cluster') + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 44eec8716..12b4a3df2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +--- name: Lint Code Base on: @@ -21,3 +22,4 @@ jobs: env: VALIDATE_ALL_CODEBASE: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILTER_REGEX_EXCLUDE: .*release-notes/.* diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 26ce0090a..ef3d887d7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,15 +1,12 @@ name: Test and build -on: [pull_request] +on: [pull_request, workflow_dispatch] jobs: build: strategy: matrix: - # macos-latest references are kept here for historical purposes. removed macos-latest from the - #matrix as it is not a typical case for users and causes a lot of friction with other linux-based - # installs. Recommend developing on codespaces or using an ubuntu container. os: ['ubuntu-22.04', 'ubuntu-20.04'] fail-fast: false runs-on: ${{ matrix.os }} @@ -18,15 +15,9 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man - wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz" - tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz" - sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck - if: matrix.os != 'macos-latest' - - name: Install Dependencies (macOS) - run: | - brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget - brew install moreutils gawk - if: matrix.os == 'macos-latest' + wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" + tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" + sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck - name: Get Sources uses: actions/checkout@v3 - name: Test @@ -36,4 +27,3 @@ jobs: shell: bash - name: Build (Linux) run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc - if: matrix.os != 'macos-latest' diff --git a/.releaseignore b/.releaseignore new file mode 100644 index 000000000..bb2f6b31c --- /dev/null +++ b/.releaseignore @@ -0,0 +1,2 @@ +ownership.yaml +.github \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 667739729..710cb9d52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ libssl-dev \ git \ jq \ + bc \ curl \ tar \ gzip \ @@ -54,8 +55,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ git \ openssh-client \ jq \ + bc \ moreutils \ - gawk \ + gawk \ ca-certificates \ xxhash \ && rm -rf /var/lib/apt/lists/* diff --git a/backup.config-example b/backup.config-example index 4c78a103b..8aadc98e2 100644 --- a/backup.config-example +++ b/backup.config-example @@ -16,6 +16,10 @@ GHE_DATA_DIR="data" # be available for the past N days ... GHE_NUM_SNAPSHOTS=10 +# If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check +# disk space validation and software version checks on the backup-host will be disabled. +#GHE_SKIP_CHECKS=false + # The hostname of the GitHub appliance to restore. If you've set up a separate # GitHub appliance to act as a standby for recovery, specify its IP or hostname # here. The host to restore to may also be specified directly when running diff --git a/bin/ghe-backup b/bin/ghe-backup index ad8147f82..2c020c54e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -5,9 +5,10 @@ #/ the MySQL database, instance settings, GitHub Pages data, etc. #/ #/ OPTIONS: -#/ -v | --verbose Enable verbose output. -#/ -h | --help Show this message. -#/ --version Display version information. +#/ -v | --verbose Enable verbose output. +#/ -h | --help Show this message. +#/ --version Display version information. +#/ --skip-checks Skip storage/sw version checks #/ set -e @@ -27,6 +28,10 @@ while true; do export GHE_VERBOSE=true shift ;; + --skip-checks) + export GHE_SKIP_CHECKS=true + shift + ;; -*) echo "Error: invalid argument: '$1'" 1>&2 exit 1 @@ -37,11 +42,14 @@ while true; do esac done + export CALLING_SCRIPT="ghe-backup" + # Bring in the backup configuration # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" + # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -121,7 +129,7 @@ ghe_restore_check # Check to see if there is a running backup if [ -h ../in-progress ]; then - log_error "Error: detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." 1>&2 + log_error "Detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." >&2 exit 1 fi @@ -148,9 +156,44 @@ log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERS # Perform a host connection check and establish the remote appliance version. # The version is available in the GHE_REMOTE_VERSION variable and also written # to a version file in the snapshot directory itself. +# ghe_remote_version_required should be run before any other instances of ghe-ssh +# to ensure that there are no problems with host key verification. ghe_remote_version_required echo "$GHE_REMOTE_VERSION" > version +# Setup progress tracking +init-progress +export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total +export PROGRESS_TYPE="Backup" +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type +export PROGRESS=0 # Used to track progress of backup +echo "$PROGRESS" > /tmp/backup-utils-progress/progress + +OPTIONAL_STEPS=0 +# Backup actions+mssql +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) +fi + +# Backup fsck +if [ "$GHE_BACKUP_FSCK" = "yes" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup minio +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup pages +if [ "$GHE_BACKUP_PAGES" != "no" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total + if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups." fi @@ -259,6 +302,7 @@ echo \"$cmd_title\" ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"") if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then + increment-progress-total-count 1 cmd_title=$(log_info "Backing up Elasticsearch indices ...") commands+=(" echo \"$cmd_title\" @@ -293,6 +337,8 @@ if [ -z "$failures" ]; then ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current" ghe-prune-snapshots +else + log_info "Skipping pruning snapshots, since some backups failed..." fi END_TIME=$(date +%s) @@ -306,6 +352,7 @@ else steps="${failures// /, }" ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. " + ghe_backup_finished exit 1 fi diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress new file mode 100755 index 000000000..7ab36e084 --- /dev/null +++ b/bin/ghe-backup-progress @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +#/ Usage: ghe-backup-progress [--once] +#/ Tracks the completed steps of a backup or restore operation. +#/ +#/ By default the progress is printed every continuously or until a key is pressed. +#/ Use the --once option to print the current progress once and exit. +#/ +#/ Options: +#/ --once Don't loop, just print the current progress once. +# +set -e + +while true; do + case "$1" in + -o|--once) + ONCE=1 + shift + ;; + -h|--help) + export GHE_SHOW_HELP=true + shift + ;; + -*) + echo "Unknown option: $1" >&2 + exit 1 + ;; + *) + break + ;; + esac +done + +check_for_progress_file() { + if [ ! -f /tmp/backup-utils-progress/info ]; then + echo "No progress file found. Has a backup or restore been started?" + exit 1 + fi +} + +if [ -n "$ONCE" ]; then + check_for_progress_file + cat /tmp/backup-utils-progress/info +else + check_for_progress_file + clear + cat /tmp/backup-utils-progress/info + while true; do + if read -r -t 1 -n 1; then + clear + exit ; + else + clear + cat /tmp/backup-utils-progress/info + fi + done +fi diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 252e84ee7..8d845cd40 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -91,13 +91,19 @@ if ghe-ssh "$host" -- \ CLUSTER=true fi -# ensure all nodes in the cluster are running the same version +# ensure all nodes in the cluster are online/reachable and running the same version if "$CLUSTER"; then + online_status=$(ghe-ssh "$host" ghe-cluster-host-check) + if [ "$online_status" != "Cluster is ready to configure." ]; then + echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 + exit 1 + fi + node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version) distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) if [ "$distinct_versions" -ne 1 ]; then - echo "$node_version_list" 1>&2 - echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&3 + echo "Version mismatch: $node_version_list" 1>&2 + echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 exit 1 fi fi @@ -144,7 +150,13 @@ if [ -z "$supported" ]; then exit 1 fi -if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then +if [[ "$CALLING_SCRIPT" == "ghe-backup" && "$GHE_SKIP_CHECKS" != "true" ]]; then + cat << SKIP_MSG 1>&2 +**You can disable the following storage & version checks by running ghe-backup with option "--skip-checks" +OR updating GHE_SKIP_CHECKS to 'true' in your backup.config file. + +SKIP_MSG + # Bring in the requirements file min_rsync="" min_openssh="" @@ -156,6 +168,13 @@ if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "Checking host for sufficient space for a backup..." 1>&2 available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') @@ -171,22 +190,26 @@ if [[ "$CALLING_SCRIPT" == "ghe-backup" ]]; then mssql_disk_size=$(transfer_size mssql /tmp) min_disk_req=$((repos_disk_size + pages_disk_size + es_disk_size + stor_disk_size + minio_disk_size + mysql_disk_size + actions_disk_size + mssql_disk_size)) - echo "Available space: $((available_space / (1024 ** 2))) MB" 1>&2 - echo -e "Min Disk required for this backup is at least $min_disk_req MB\n" 1>&2 - -cat <&2 -### Data Transfer Sizes -repositories: $repos_disk_size MB -pages: $pages_disk_size MB -elasticsearch: $es_disk_size MB -storage: $stor_disk_size MB -minio: $minio_disk_size MB -mysql: $mysql_disk_size MB -actions: $actions_disk_size MB -mssql: $mssql_disk_size MB -DATA_TRANSFER_SIZE - - if [[ $available_space -lt $min_disk_req ]]; then + recommended_disk_req=$((min_disk_req * 5)) + echo " - Available space: $((available_space / (1024 ** 2))) MB" 1>&2 + echo " - Min Disk required for this backup is at least $min_disk_req MB" 1>&2 + echo " - Recommended Disk requirement is $recommended_disk_req MB" 1>&2 + echo "" 1>&2 + + printf '### Estimated Data Transfer Sizes + + - repositories: %d MB + - pages: %d MB + - elasticsearch: %d MB + - storage: %d MB + - minio: %d MB + - mysql: %d MB + - actions: %d MB + - mssql: %d MB +\n' \ + "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2 + + if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2 exit 1 fi diff --git a/bin/ghe-restore b/bin/ghe-restore index c0aae6ed9..1929b5cdf 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -86,6 +86,8 @@ while true; do esac done + + start_cron () { log_info "Starting cron ..." if $CLUSTER; then @@ -164,6 +166,10 @@ cleanup_cluster_nodes() { # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" + + + + # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -265,6 +271,51 @@ if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then prompt_for_confirmation "Please confirm this before continuing." fi +# Calculate the actual amounts of steps in the restore process +# taking into account the options passed to the script and the appliance configuration +# calculate restore steps +OPTIONAL_STEPS=0 + +# Restoring UUID +if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring Actions + MSSQL +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) +fi +# Restoring minio +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring Elasticsearch +if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring audit log +if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then + if [[ "$GHE_RESTORE_SKIP_AUDIT_LOG" != "yes" ]]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) + fi +fi +# Replica cleanup +if ! $CLUSTER && $instance_configured; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi +# Restoring settings + restore-chat-integration + restore-packages +if $RESTORE_SETTINGS; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3)) +fi + +# Minimum number of steps is 7 +export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) + +init-progress +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total +export PROGRESS_TYPE="Restore" +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type +export PROGRESS=0 # Used to track progress of restore +echo "$PROGRESS" > /tmp/backup-utils-progress/progress # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) @@ -343,10 +394,11 @@ log_warn "Warning: storing backup-utils version remotely failed." # Stop cron and timerd, as scheduled jobs may disrupt the restore process. log_info "Stopping cron and github-timerd ..." if $CLUSTER; then + bm_start "$(basename $0) - Stopping cron and github-timerd on cluster" if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then log_warn "Failed to stop cron on one or more nodes" 1>&3 fi - + bm_end "$(basename $0) - Stopping cron and github-timerd on cluster" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then @@ -358,11 +410,14 @@ if $CLUSTER; then log_warn "Failed to stop github-timerd on one or more nodes" 1>&3 fi fi + else + bm_start "$(basename $0) - Stopping cron and github-timerd" + echo "$(basename $0) - Stopping cron and github-timerd" if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then log_warn "Failed to stop cron" 1>&3 fi - + bm_end "$(basename $0) - Stopping cron and github-timerd" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then @@ -374,15 +429,29 @@ else log_warn "Failed to stop github-timerd" 1>&3 fi fi + fi CRON_RUNNING=false + # Restore settings and license if restoring to an unconfigured appliance or when # specified manually. if $RESTORE_SETTINGS; then ghe-restore-settings "$GHE_HOSTNAME" fi +# Always restore column encryption keys +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + log_info "Always restore encrypted column encryption keys on GHES verions 3.7.0+" +fi +ghe-restore-column-encryption-keys "$GHE_HOSTNAME" + +# Always restore secret scanning encryption keys +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+" + ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME" +fi + # Make sure mysql and elasticsearch are prep'd and running before restoring. # These services will not have been started on appliances that have not been # configured yet. @@ -394,11 +463,15 @@ fi # Restore UUID if present and not restoring to cluster. if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then log_info "Restoring UUID ..." + + bm_start "$(basename $0) - Restore UUID" ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" <"$GHE_RESTORE_SNAPSHOT_PATH/uuid" ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft" + bm_end "$(basename $0) - Restore UUID" fi + if is_external_database_snapshot; then appliance_strategy="external" backup_snapshot_strategy="external" @@ -420,6 +493,7 @@ if is_external_database_target_or_snapshot && $SKIP_MYSQL; then log_info "Skipping MySQL restore." else log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..." + increment-progress-total-count 2 ghe-restore-mysql "$GHE_HOSTNAME" 1>&3 fi @@ -448,7 +522,7 @@ fi cmd_title=$(log_info "Restoring Redis database ...") commands=(" echo \"$cmd_title\" -ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3") +ghe-restore-redis \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"") cmd_title=$(log_info "Restoring Git Repositories ...") commands+=(" @@ -470,7 +544,7 @@ fi cmd_title=$(log_info "Restoring SSH authorized keys ...") commands+=(" echo \"$cmd_title\" -ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-authorized-keys' < \"$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json\" 1>&3") +ghe-restore-ssh-keys \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"") cmd_title=$(log_info "Restoring storage data ...") commands+=(" @@ -517,14 +591,17 @@ if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then else log_info "Restoring data serially ..." 1>&3 for c in "${commands[@]}"; do + . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/bm.sh" eval "$c" done fi # Restart an already running memcached to reset the cache after restore -log_info "Restarting memcached ..." 1>&3 +log_info "Restarting memcached ..." 1>&3 +bm_start "$(basename $0) - Restarting memcached" echo "sudo restart -q memcached 2>/dev/null || true" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh +bm_end "$(basename $0) - Restarting memcached" # Prevent GitHub Connect jobs running before we've had a chance to reset # the configuration by setting the last run date to now. @@ -538,20 +615,24 @@ fi # config run to perform data migrations. if $CLUSTER; then log_info "Configuring cluster ..." + bm_start "$(basename $0) - configure cluster" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3 elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 fi ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3 + bm_end "$(basename $0) - configure cluster" elif $instance_configured; then log_info "Configuring appliance ..." + bm_start "$(basename $0) - configure appliance" if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3 elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 fi ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3 + bm_end "$(basename $0) - configure appliance" fi # Clear GitHub Connect settings stored in the restored database. @@ -569,6 +650,7 @@ CRON_RUNNING=true # Clean up all stale replicas on configured instances. if ! $CLUSTER && $instance_configured; then log_info "Cleaning up replicas..." 1>&3 + bm_start "$(basename $0) - Cleanup replicas" restored_uuid=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid") other_nodes=$(echo " set -o pipefail; \ @@ -584,6 +666,7 @@ if ! $CLUSTER && $instance_configured; then echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3 done fi + bm_end "$(basename $0) - Cleanup replicas" fi # Update the remote status to "complete". This has to happen before importing diff --git a/debian/changelog b/debian/changelog index ee3c73176..8e260e7ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,42 @@ +github-backup-utils (3.9.4) UNRELEASED; urgency=medium + + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + +* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Fri, 20 Oct 2023 18:49:19 +0000 + +github-backup-utils (3.9.3) UNRELEASED; urgency=medium + + * Replace "sed -E" in ghe-host-check with a more portable solution #509 + * Fix `ghe-backup-repositories` performance for large instances #541 + * Turn off POSIX for ghe-backup-config #632 + + -- Devin Dooley Mon, 02 Oct 2023 20:36:46 +0000 + +github-backup-utils (3.9.2) UNRELEASED; urgency=medium + + + -- Devin Dooley Mon, 21 Aug 2023 20:51:10 +0000 + +github-backup-utils (3.9.1) UNRELEASED; urgency=medium + + * Make it clear the settings need to be applied after restoring to an unconfigured instance #381 + * Remove check for git from ghe-ssh #393 + * Use remote's mktemp to create temp dir on remote host #395 + * Enabling concurrent `ghe-backup` tasks from separate backup hosts #441 + + -- Devin Dooley Wed, 26 Jul 2023 01:46:33 +0000 + +github-backup-utils (3.9.0) UNRELEASED; urgency=medium + + * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 + * Cleanup SSH multiplexing on exit #363 + * Allow extra rsync options to override default options #370 + * Retry with the admin ssh port on network unreachable too. #377 + + -- Joseph Franks Thu, 29 Jun 2023 22:36:16 +0000 + github-backup-utils (3.9.0) UNRELEASED; urgency=medium * Set restore status on all cluster nodes #274 diff --git a/docs/requirements.md b/docs/requirements.md index e506fe281..1288846ac 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), and [jq][11] v1.5 or newer. See below for an update on rsync. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](#april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer. Ubuntu is the operating system we use to test `backup-utils` and it’s what we recommend you use too. You are welcome to use a different operating system, and we'll do our best to help you if you run into issues. But we can't guarantee that we'll be able to resolve issues that are specific to that operating system. @@ -59,6 +59,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the @@ -101,3 +103,4 @@ Due to how some components of Backup Utilities (e.g. MSSQL) take incremental bac [9]: https://joeyh.name/code/moreutils [10]: https://www.gnu.org/software/gawk [11]: https://stedolan.github.io/jq/ +[12]: https://www.gnu.org/software/bc/ diff --git a/docs/usage.md b/docs/usage.md index b4db45e73..c603c5e36 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -73,13 +73,18 @@ appliance at IP "5.5.5.5": Restore of 5.5.5.5:122 from snapshot 20180326T020444 finished. To complete the restore process, please visit https://5.5.5.5/setup/settings to review and save the appliance configuration. -A different backup snapshot may be selected by passing the `-s` argument and the -datestamp-named directory from the backup location. +A different backup snapshot may be selected by passing the `-s` argument to `ghe-restore` and specifying the +datestamp-named directory from the backup location as the value. The `ghe-backup` and `ghe-restore` commands also have a verbose output mode (`-v`) that lists files as they're being transferred. It's often useful to enable when output is logged to a file. +Every time you execute `ghe-backup` we verify the storage and software setup of the host +you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are +met. You can disable this check using the `--skip-checks` argument or by +adding `GHE_SKIP_CHECKS=true` to your configuration file. + ### Restoring settings, TLS certificate, and license When restoring to a new GitHub Enterprise Server instance, settings, certificate, and @@ -107,3 +112,4 @@ GitHub Actions enabled, the following steps are required: Please refer to [GHES Documentation](https://docs.github.com/en/enterprise-server/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled) for more details. [1]: https://github.com/github/backup-utils/blob/master/docs/getting-started.md +[2]: requirements.md diff --git a/release-notes/3.9.4.md b/release-notes/3.9.4.md new file mode 100644 index 000000000..edaf871e8 --- /dev/null +++ b/release-notes/3.9.4.md @@ -0,0 +1,8 @@ +### Bug Fixes + +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + +### Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + diff --git a/script/cibuild b/script/cibuild index 7f57f444d..73dc9ae0a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -5,7 +5,7 @@ set -e # Enable verbose logging of ssh commands export GHE_VERBOSE_SSH=true -if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then +if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then exit 1 fi diff --git a/script/package-deb b/script/package-deb index dda90cbe3..15a698e9c 100755 --- a/script/package-deb +++ b/script/package-deb @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" @@ -22,6 +25,14 @@ mkdir -p dist/debuild distdir="$(pwd)/dist/debuild/$PKG_NAME" git clone -q . "$distdir" cd "$distdir" + +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +echo "Removing .releaseignore ..." +rm -f .releaseignore git checkout -q "$PKG_HEAD" debuild -uc -us 1>&2 diff --git a/script/package-tarball b/script/package-tarball index be653b77a..bf1510e8f 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -8,11 +8,24 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" PKG_NAME="${PKG_BASE}-${PKG_VERS}" +# Remove all files or directories listed in .releaseignore +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +# Remove the .releaseignore file itself +echo "Removing .releaseignore ..." +rm -f .releaseignore + # Run git-archive to generate tarball echo "Creating ${PKG_NAME}.tar.gz ..." mkdir -p dist diff --git a/script/release b/script/release index a27194d4b..891cd9f60 100755 --- a/script/release +++ b/script/release @@ -31,7 +31,11 @@ GH_REPO = ENV['GH_REPO'] || 'backup-utils' GH_OWNER = ENV['GH_OWNER'] || 'github' GH_AUTHOR = ENV['GH_AUTHOR'] DEB_PKG_NAME = 'github-backup-utils' -GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' +GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly? +GH_STABLE_BRANCH = "" + +# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI +PUBLISH = ENV['PUBLISH'] == 'true' || false CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium @@ -137,7 +141,8 @@ def beautify_changes(changes) end def changelog - changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) + puts "building changelog by comparing origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH}" + changes = `git log --pretty=oneline origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) raise 'Building the changelog failed' if $CHILD_STATUS != 0 changes @@ -228,12 +233,12 @@ def push_release_branch(version) end def update_stable_branch - `git checkout --quiet stable` + `git checkout --quiet #{GH_STABLE_BRANCH}` unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`) - warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}" + warn "Merging #{GH_BASE_BRANCH} into #{GH_STABLE_BRANCH} failed:\n\n#{out}" end - unless (out = `git push --quiet origin stable`) - warn "Failed pushing the stable branch:\n\n#{out}" + unless (out = `git push --quiet origin #{GH_STABLE_BRANCH}`) + warn "Failed pushing the #{GH_STABLE_BRANCH} branch:\n\n#{out}" end end @@ -333,9 +338,38 @@ def clean_up(version) `git branch --quiet -D tmp-packaging >/dev/null 2>&1` end +def is_base_branch_valid?(branch) + if branch == "master" || branch.match(/^\d+\.\d+-main$/) + return true + else + return false + end +end + +def get_stable_branch_name(branch) + ## derive the proper stable branch. if the base branch is "master" the stable branch is just "stable" + ## if the base branch is a release branch, the stable branch will be "x.y-stable" + result = "" + if branch == "master" + result = "stable" + else + result = branch.gsub(/-main$/, "-stable") + end + + result +end + #### All the action starts #### if $PROGRAM_NAME == __FILE__ begin + ## validate base branch. this must either be "master" or a release branch which will match the pattern "x.y-main" + raise "The branch #{GH_BASE_BRANCH} is not valid for releasing backup-utils. branch name must be master or match x.y-main" if !is_base_branch_valid?(GH_BASE_BRANCH) + + GH_STABLE_BRANCH = get_stable_branch_name(GH_BASE_BRANCH) + + puts "base branch = " + GH_BASE_BRANCH + puts "stable branch = " + GH_STABLE_BRANCH + args = ARGV.dup dry_run = false skip_version_bump_check = false @@ -449,15 +483,21 @@ if $PROGRAM_NAME == __FILE__ attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] - puts 'Publishing release...' - publish_release res['id'] + if PUBLISH + puts 'Publishing release...' + publish_release res['id'] + end puts 'Cleaning up...' clean_up version - puts 'Updating stable branch...' + puts "Updating #{GH_STABLE_BRANCH} branch..." update_stable_branch + if !PUBLISH + puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.' + end + puts 'Released!' rescue RuntimeError => e $stderr.puts "Error: #{e}" diff --git a/share/github-backup-utils/bm.sh b/share/github-backup-utils/bm.sh index b714d144b..561bcfffb 100755 --- a/share/github-backup-utils/bm.sh +++ b/share/github-backup-utils/bm.sh @@ -49,6 +49,7 @@ bm_end() { local tend tstart total tend=$(date +%s) tstart=$(eval "echo \$$(bm_desc_to_varname "$@")_start") + total=$(($tend - $tstart)) echo "$1 took ${total}s" >> $BM_FILE_PATH @@ -57,4 +58,6 @@ bm_end() { if [ -n "$GHE_DEBUG" ]; then echo "Debug: $1 took ${total}s (bm_end)" fi + # track progress + progress "$1 took ${total}s" } diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 83533ee97..943d61e7c 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -19,9 +19,64 @@ RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' BLUE='\033[0;34m' -NC='\033[0m' # No Colo# Logging display and formatting functions +NC='\033[0m' # No Color + + +# Assume this script lives in share/github-backup-utils/ when setting the root +GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# Get the version from the version file. +BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")" + +# If a version check was requested, show the current version and exit +if [ -n "$GHE_SHOW_VERSION" ]; then + echo "GitHub backup-utils v$BACKUP_UTILS_VERSION" + exit 0 +fi + +# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage +# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's +print_usage() { + grep '^#/' <"$0" | cut -c 4- + exit "${1:-1}" +} + +if [ -n "$GHE_SHOW_HELP" ]; then + print_usage +else + for a in "$@"; do + if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then + print_usage + fi + done +fi + +# Save off GHE_HOSTNAME from the environment since we want it to override the +# backup.config value when set. +GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" + +# Source in the backup config file from the copy specified in the environment +# first and then fall back to the backup-utils root, home directory and system. +config_found=false +for f in "$GHE_BACKUP_CONFIG" "$GHE_BACKUP_ROOT/backup.config" \ + "$HOME/.github-backup-utils/backup.config" "/etc/github-backup-utils/backup.config"; do + if [ -f "$f" ]; then + GHE_BACKUP_CONFIG="$f" + # shellcheck disable=SC1090 # This is a user-supplied value that can't be predicted + . "$GHE_BACKUP_CONFIG" + config_found=true + break + fi +done + +GHE_RESTORE_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-restore") +GHE_BACKUP_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-backup") + +export GHE_RESTORE_IN_PROGRESS +export GHE_BACKUP_IN_PROGRESS + +# Logging display and formatting functions -# Log a message to stdout log_level() { local level=$1 shift @@ -91,43 +146,12 @@ log_ssh(){ log_level "ssh" "$1" } -# Assume this script lives in share/github-backup-utils/ when setting the root -GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -# Get the version from the version file. -BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")" - -# If a version check was requested, show the current version and exit -if [ -n "$GHE_SHOW_VERSION" ]; then - echo "GitHub backup-utils v$BACKUP_UTILS_VERSION" - exit 0 -fi - -# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage -# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's -print_usage() { - grep '^#/' <"$0" | cut -c 4- - exit "${1:-1}" -} - -if [ -n "$GHE_SHOW_HELP" ]; then - print_usage -else - for a in "$@"; do - if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then - print_usage - fi - done -fi - # Add the bin and share/github-backup-utils dirs to PATH PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" # shellcheck source=share/github-backup-utils/bm.sh . "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" - -# Save off GHE_HOSTNAME from the environment since we want it to override the -# backup.config value when set. -GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" +# shellcheck source=share/github-backup-utils/track-progress +. "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress" # Source in the backup config file from the copy specified in the environment # first and then fall back to the backup-utils root, home directory and system. @@ -216,8 +240,10 @@ ghe_parallel_check() { GHE_PARALLEL_COMMAND="parallel" local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do @@ -285,6 +311,7 @@ else exec 3>/dev/null fi + # Restore saved off hostname. [ -n "$GHE_HOSTNAME_PRESERVE" ] && GHE_HOSTNAME="$GHE_HOSTNAME_PRESERVE" @@ -646,6 +673,26 @@ restore-secret() { fi } +#initialize progress tracking by clearing out the temp files used to track +init-progress() { + if [ -d /tmp/backup-utils-progress ]; then + rm -rf /tmp/backup-utils-progress/* + fi + + mkdir -p /tmp/backup-utils-progress + chmod -R 777 /tmp/backup-utils-progress + + touch /tmp/backup-utils-progress/total + touch /tmp/backup-utils-progress/type + touch /tmp/backup-utils-progress/progress + touch /tmp/backup-utils-progress/info +} + +#increase total count of progress +increment-progress-total-count() { + ((PROGRESS_TOTAL += $1)) + echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total +} diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync index 00946279a..ee61741b2 100755 --- a/share/github-backup-utils/ghe-backup-es-rsync +++ b/share/github-backup-utils/ghe-backup-es-rsync @@ -4,6 +4,7 @@ #/ #/ Note: This command typically isn't called directly. It's invoked by #/ ghe-backup when the rsync strategy is used. +# shellcheck disable=SC2086 set -e # Bring in the backup configuration @@ -54,7 +55,7 @@ log_rsync "END elasticsearch rsync" 1>&3 # Set up a trap to re-enable flushing on exit and remove temp file cleanup () { ghe_verbose "* Enabling ES index flushing ..." - echo '{"index":{"translog.disable_flush":false}}' | + echo '{"index":{"translog.flush_threshold_size":"512MB"}}' | ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null } trap 'cleanup' EXIT @@ -62,7 +63,7 @@ trap 'exit $?' INT # ^C always terminate # Disable ES flushing and force a flush right now ghe_verbose "* Disabling ES index flushing ..." -echo '{"index":{"translog.disable_flush":true}}' | +echo '{"index":{"translog.flush_threshold_size":"1PB"}}' | ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql index ead33f6bc..37b09ca9d 100755 --- a/share/github-backup-utils/ghe-backup-mssql +++ b/share/github-backup-utils/ghe-backup-mssql @@ -30,11 +30,11 @@ export_tool_available() { } ghe_ssh_mssql() { - ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@" + ghe-ssh "${opts[@]}" "${ssh_config_file_opt[@]}" "$GHE_MSSQL_PRIMARY_HOST" "$@" } cleanup() { - rm -rf $tempdir + rm -rf "$tempdir" } trap 'cleanup' EXIT INT @@ -47,16 +47,16 @@ if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then fi tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -ssh_config_file_opt= -opts= +ssh_config_file_opt=() +opts=() isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)" # get server hostnames under cluster and HA if [ "$GHE_BACKUP_STRATEGY" = "cluster" ] || [ "$isHA" = "true" ] ; then ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" + ssh_config_file_opt=("-F" "$ssh_config_file") + opts=("-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no" "-o" "PasswordAuthentication=no") ghe-ssh-config "$GHE_HOSTNAME" "$GHE_MSSQL_PRIMARY_HOST" > "$ssh_config_file" fi @@ -69,10 +69,10 @@ add_minute() { # Expect date string in the format of yyyymmddTHHMMSS # Here parse date differently depending on GNU Linux vs BSD MacOS if date -v -1d > /dev/null 2>&1; then - echo "$(date -v +$2M -ujf'%Y%m%dT%H%M%S' $1 +%Y%m%dT%H%M%S)" + date -v +"$2"M -ujf'%Y%m%dT%H%M%S' "$1" +%Y%m%dT%H%M%S else dt=$1 - echo "$(date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes")" + date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes" fi } @@ -337,7 +337,7 @@ if [ -n "$backup_type" ]; then fi bm_start "$(basename "$0")" - ghe_ssh_mssql -- "$backup_command" || failures="$failures mssql" + ghe_ssh_mssql -- "$backup_command" bm_end "$(basename "$0")" # Configure the backup cadence on the appliance, which is used for diagnostics diff --git a/share/github-backup-utils/ghe-backup-pages b/share/github-backup-utils/ghe-backup-pages index 21c2ac465..b0c9f1fed 100755 --- a/share/github-backup-utils/ghe-backup-pages +++ b/share/github-backup-utils/ghe-backup-pages @@ -63,6 +63,7 @@ if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/page link_dest="--link-dest=../../current/pages" fi +count=0 for hostname in $hostnames; do bm_start "$(basename $0) - $hostname" echo 1>&3 @@ -82,6 +83,7 @@ for hostname in $hostnames; do "$GHE_SNAPSHOT_DIR/pages" 1>&3 log_rsync "END: pages rsync" 1>&3 bm_end "$(basename $0) - $hostname" + count=$((count + 1)) done - +increment-progress-total-count $count bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index a3ba533d7..e90d8c37a 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -144,6 +144,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "no routes found, skipping repositories backup ..." exit 0 +else + increment-progress-total-count 3 fi # Transfer repository data from a GitHub instance to the current snapshot @@ -377,7 +379,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." - + increment-progress-total-count 1 bm_end "$(basename $0) - Verifying Routes" fi diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index c71085a0b..aac02dde7 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -78,8 +78,21 @@ backup-secret "management console password" "manage-password" "secrets.manage" backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" -backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" -backup-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" + +# backup encryption keying material and create backup value current encryption for GHES 3.7.0 onwards +# this is for forwards compatibility with GHES 3.8.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" + cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" +fi + +# secret scanning encrypted secrets keys were added in GHES 3.8.0 +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" + backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +fi # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage index 2f98a0541..9752de21d 100755 --- a/share/github-backup-utils/ghe-backup-storage +++ b/share/github-backup-utils/ghe-backup-storage @@ -113,6 +113,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "no routes found, skipping storage backup ..." exit 0 +else + increment-progress-total-count 2 fi # rsync all the storage objects @@ -149,6 +151,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." + increment-progress-total-count 1 bm_end "$(basename $0) - Verifying Routes" fi diff --git a/share/github-backup-utils/ghe-restore-column-encryption-keys b/share/github-backup-utils/ghe-restore-column-encryption-keys new file mode 100755 index 000000000..d30cacd51 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-column-encryption-keys @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-column-encryption-keys +#/ Restore the column encryption keys from a snapshot to the given . +#/ This script will be run automatically by `ghe-restore +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +bm_start "$(basename $0)" + +# Grab host arg +GHE_HOSTNAME="$1" + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$GHE_HOSTNAME" + +# The snapshot to restore should be set by the ghe-restore command but this lets +# us run this script directly. +: ${GHE_RESTORE_SNAPSHOT:=current} + +# Path to snapshot dir we're restoring from +: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} + +# Restore encrypted column encryption keying material for GHES 3.7.0 onward +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + log_info "Restoring encrypted column encryption keying material" + restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" +fi + +# Restore encrypted column current encryption key for GHES 3.8.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + log_info "Restoring encrypted column current encryption key" + restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" +fi + + +bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-pages b/share/github-backup-utils/ghe-restore-pages index 640aa5d62..0b104863e 100755 --- a/share/github-backup-utils/ghe-restore-pages +++ b/share/github-backup-utils/ghe-restore-pages @@ -29,6 +29,8 @@ pages_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find pages -mindepth 5 if [ -z "$pages_paths" ]; then log_warn "Warning: Pages backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -125,6 +127,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping pages restore ..." exit 0 +else + increment-progress-total-count 2 fi bm_start "$(basename $0) - Restoring pages" @@ -154,6 +158,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/dpages-cluster-restore-finalize" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/ghe-restore-redis b/share/github-backup-utils/ghe-restore-redis new file mode 100755 index 000000000..1bdfe7642 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-redis @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-redis +#/ Restore redis files from an rsync snapshot. +#/ +#/ Note: This script typically isn't called directly. It's invoked by the +#/ ghe-restore command. +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" + +# Show usage and bail with no arguments +[[ -z ${*} ]] && print_usage + +# Grab host arg +GHE_HOSTNAME="${1}" + +# Grab snapshot path arg +GHE_RESTORE_SNAPSHOT_PATH="${2}" + +bm_start "$(basename "${0}")" + +ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-redis' < "$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb" 1>&3 + +bm_end "$(basename "${0}")" \ No newline at end of file diff --git a/share/github-backup-utils/ghe-restore-repositories b/share/github-backup-utils/ghe-restore-repositories index d59864196..9a749bc74 100755 --- a/share/github-backup-utils/ghe-restore-repositories +++ b/share/github-backup-utils/ghe-restore-repositories @@ -30,6 +30,8 @@ network_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find repositories -mi if [ -z "$network_paths" ]; then log_warn "Warning: Repositories backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -142,6 +144,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping repositories restore ..." exit 0 +else + increment-progress-total-count 3 fi # rsync all the repository networks to the git server where they belong. @@ -190,6 +194,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/dgit-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index addb18514..7faae5260 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -29,6 +29,8 @@ gist_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find repositories -mind if [ -z "$gist_paths" ]; then log_warn "Warning: Gist backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -128,6 +130,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping gists restore ..." exit 0 +else + increment-progress-total-count 2 fi # rsync all the gist repositories @@ -157,6 +161,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/gist-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys new file mode 100755 index 000000000..aa225bc07 --- /dev/null +++ b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-secret-scanning-encryption-keys +#/ Restore the secret scanning encryption keys from a snapshot to the given . +#/ This script will be run automatically by `ghe-restore` +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +bm_start "$(basename $0)" + +# Grab host arg +GHE_HOSTNAME="$1" + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$GHE_HOSTNAME" + +# The snapshot to restore should be set by the ghe-restore command but this lets +# us run this script directly. +: ${GHE_RESTORE_SNAPSHOT:=current} + +# Path to snapshot dir we're restoring from +: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} + +# Restore secret scanning encrypted secrets storage keys if present +log_info "Restoring secret scanning encrypted secrets storage keys" +restore-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" +restore-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + +# Restore secret scanning encrypted secrets transit keys if present +log_info "Restoring secret scanning encrypted secrets transit keys" +restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" +restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + +bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-ssh-keys b/share/github-backup-utils/ghe-restore-ssh-keys new file mode 100755 index 000000000..216803d7a --- /dev/null +++ b/share/github-backup-utils/ghe-restore-ssh-keys @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-ssh-keys +#/ Restore ssh keys from an rsync snapshot. +#/ +#/ Note: This script typically isn't called directly. It's invoked by the +#/ ghe-restore command. +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" + +# Show usage and bail with no arguments +[[ -z ${*} ]] && print_usage + +bm_start "$(basename "${0}")" + +# Grab host arg +GHE_HOSTNAME="${1}" + +# Grab snapshot path arg +GHE_RESTORE_SNAPSHOT_PATH="${2}" + +bm_start "$(basename "${0}")" + +ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-authorized-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json" 1>&3 + +bm_end "$(basename "${0}")" \ No newline at end of file diff --git a/share/github-backup-utils/ghe-restore-storage b/share/github-backup-utils/ghe-restore-storage index af6c24a23..8f0d73c48 100755 --- a/share/github-backup-utils/ghe-restore-storage +++ b/share/github-backup-utils/ghe-restore-storage @@ -33,6 +33,8 @@ storage_paths=$(cd $GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/ && find storage -mindept if [ -z "$storage_paths" ]; then log_warn "Warning: Storage backup missing. Skipping ..." exit 0 +else + increment-progress-total-count 5 fi # Perform a host-check and establish GHE_REMOTE_XXX variables. @@ -120,6 +122,8 @@ bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping storage restore ..." exit 0 +else + increment-progress-total-count 2 fi # rsync all the objects to the storage server where they belong. @@ -169,6 +173,7 @@ if $CLUSTER; then chunks=\$(find $remote_tempdir/ -name chunk\*) parallel -i /bin/sh -c "cat {} | github-env ./bin/storage-cluster-restore-finalize" -- \$chunks EOF + increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" fi diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress new file mode 100755 index 000000000..a560ba540 --- /dev/null +++ b/share/github-backup-utils/track-progress @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +#/ track-progress: track progress of backup or restore tasks + +progress(){ + + PROGRESS=$(cat /tmp/backup-utils-progress/progress) + PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total) + PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type) + PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) + echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress + echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info +} diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index a5c4c7633..ed7555c71 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.9.0 +3.9.4 \ No newline at end of file diff --git a/test/bin/ghe-cluster-host-check b/test/bin/ghe-cluster-host-check new file mode 100755 index 000000000..3120d85de --- /dev/null +++ b/test/bin/ghe-cluster-host-check @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Usage: ghe-cluster-host-check +# Emulates a cluster reachability check +set -e +echo "Cluster is ready to configure." diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 135b26170..ed383b003 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -47,6 +47,8 @@ begin_test "ghe-backup subsequent snapshot" [ "$first_snapshot" != "$this_snapshot" ] verify_all_backedup_data + + verify_progress_cleanup_process ) end_test @@ -555,7 +557,7 @@ begin_test "ghe-backup takes backup of kredz-varz settings" ) end_test -begin_test "ghe-backup takes backup of encrypted column encryption keying material" +begin_test "ghe-backup takes backup of encrypted column encryption keying material and create encrypted column current encryption key for versions 3.7.0+" ( set -e @@ -567,10 +569,44 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" done + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + ghe-backup required_files=( "encrypted-column-encryption-keying-material" + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-current-encryption-key" ) for file in "${required_files[@]}"; do @@ -580,28 +616,147 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ) end_test -begin_test "ghe-backup takes backup of encrypted column current encryption key" +begin_test "ghe-backup takes backup of encrypted column encryption keying material and encrypted column current encryption key accounting for multiple encryption keying materials for versions 3.7.0+" ( set -e required_secrets=( - "secrets.github.encrypted-column-current-encryption-key" + "secrets.github.encrypted-column-keying-material" ) for secret in "${required_secrets[@]}"; do - ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + echo "ghe-config '$secret' 'foo;bar'" | + ghe-ssh "$GHE_HOSTNAME" -- /bin/bash done + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + ghe-backup required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] + done + + required_files_current_encryption_key=( "encrypted-column-current-encryption-key" ) + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + ) + for file in "${required_files[@]}"; do - [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] done + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + + ghe-backup + + required_files=( + "encrypted-column-encryption-keying-material" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo;bar" ] + done + + required_files_current_encryption_key=( + "encrypted-column-current-encryption-key" + ) + + for file in "${required_files_current_encryption_key[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "bar" ] + done + +) +end_test + +begin_test "ghe-backup does not take backups of secret scanning encrypted secrets encryption keys on versions below 3.8.0" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "secret scanning encrypted secrets" && exit 1 + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "" ] + done +) +end_test + +begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys on versions 3.8.0+" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-backup + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done ) end_test diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 6bfa12a69..aeb41551d 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION) bu_major_minor="$bu_version_major.$bu_version_minor" releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json) - supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")') + latest_value=$(echo "$releases" | jq -r '.latest') + latest_major_version=$(echo $latest_value | cut -d "." -f 1-2) + # Replace "latest" with the derived major version in the releases string + releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g') + # Use the modified releases string as needed + supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo $releases | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) if [ -z "$supported" ] then diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index c39c042b7..ef13b7697 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -281,7 +281,18 @@ begin_test "ghe-restore with no pages backup" ) end_test -begin_test "ghe-restore with encrypted column encryption keying material" +begin_test "ghe-restore does not restore encrypted column encryption keying material for versions below 3.7.0" +( + GHE_REMOTE_VERSION=2.1.10 ghe-restore -v -f localhost | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + + GHE_REMOTE_VERSION=3.6.1 ghe-restore -v -f localhost | grep -q "encrypted column encryption keying material not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-keying-material" ] + +) +end_test + +begin_test "ghe-restore with encrypted column encryption keying material for versions 3.7.0+" ( set -e rm -rf "$GHE_REMOTE_ROOT_DIR" @@ -295,6 +306,23 @@ begin_test "ghe-restore with encrypted column encryption keying material" echo "foo" > "$GHE_DATA_DIR/current/$file" done + # GHES version 3.7.0 + GHE_REMOTE_VERSION=3.7.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-keying-material" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done + + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + ghe-restore -v -f localhost required_secrets=( "secrets.github.encrypted-column-keying-material" @@ -306,7 +334,19 @@ begin_test "ghe-restore with encrypted column encryption keying material" ) end_test -begin_test "ghe-restore with encrypted column current encryption key" + +begin_test "ghe-restore does not encrypted column current encryption key for versions below 3.8.0" +( + GHE_REMOTE_VERSION=2.1.10 restore -v -f | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + + GHE_REMOTE_VERSION=3.7.0 restore -v -f | grep -q "encrypted column current encryption key not set" && exit 1 + [ ! -f "$GHE_DATA_DIR/current/encrypted-column-current-encryption-key" ] + +) +end_test + +begin_test "ghe-restore with encrypted column current encryption key for versions 3.8.0+" ( set -e rm -rf "$GHE_REMOTE_ROOT_DIR" @@ -320,6 +360,10 @@ begin_test "ghe-restore with encrypted column current encryption key" echo "foo" > "$GHE_DATA_DIR/current/$file" done + # GHES version 3.8.0 + GHE_REMOTE_VERSION=3.8.0 + export GHE_REMOTE_VERSION + ghe-restore -v -f localhost required_secrets=( "secrets.github.encrypted-column-current-encryption-key" @@ -328,6 +372,85 @@ begin_test "ghe-restore with encrypted column current encryption key" for secret in "${required_secrets[@]}"; do [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] done + + + # GHES version 3.9.0 + GHE_REMOTE_VERSION=3.9.0 + export GHE_REMOTE_VERSION + + ghe-restore -v -f localhost + required_secrets=( + "secrets.github.encrypted-column-current-encryption-key" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done +) +end_test + +begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions below 3.8.0" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.7.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.8.0 + done +) +end_test + + +begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions 3.8.0+" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # expecting this to have been restored successfully for versions 3.8.0+ + done ) end_test diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 92ac538cc..82c0fbb4f 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings" set -e # We manually install the latest Shellcheck on Linux builds as other options # are too old. - if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then - shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; } + if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then + shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; } fi if ! type shellcheck 1>/dev/null 2>&1; then @@ -49,7 +49,7 @@ begin_test "shellopts: set -e set on all scripts" # Check all executable scripts checked into the repo, except bm.sh, ghe-backup-config, ghe-rsync and the dummy test scripts set +x cd $BASE_PATH - git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do + git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|track-progress|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do if head -n1 "$script" | grep -E -w "sh|bash" >/dev/null 2>&1; then grep -q "set -e" $script || echo $script >> $results || true fi diff --git a/test/testlib.sh b/test/testlib.sh old mode 100644 new mode 100755 index 631049e30..56b22da69 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -486,6 +486,12 @@ verify_all_backedup_data() { verify_common_data } +# A unified method to make sure post backup, the cleanup process works +verify_progress_cleanup_process() { + set -e + sudo -u nobody rm -rf /tmp/backup-utils-progress/* +} + # A unified method to check everything restored when performing a full restore # during testing. verify_all_restored_data() { @@ -588,8 +594,10 @@ setup_moreutils_parallel() { # We need moreutils parallel local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do