diff --git a/.github/actions/proxy-janky-build/action.yml b/.github/actions/proxy-janky-build/action.yml new file mode 100644 index 000000000..988c4a289 --- /dev/null +++ b/.github/actions/proxy-janky-build/action.yml @@ -0,0 +1,33 @@ +name: 'Trigger a CI Job on Janky' +description: 'Action to trigger and poll a Janky CI job' +inputs: + janky-token: + description: 'Token for making request to Janky' + required: true + job-name: + description: 'The name of the job to run' + required: true + branch-name: + description: 'The name of the branch to use' + required: true + force: + description: 'Force the job to run even if it is already passed' + required: false + envVars: + description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3' + required: false +runs: + using: 'composite' + steps: + - uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260 + with: + go-version: '1.21' + - run: | + go run main.go \ + -token ${{ inputs.janky-token }} \ + -job ${{ inputs.job-name }} \ + -branch ${{ inputs.branch-name }} \ + -force ${{ inputs.force }} \ + -envVars ${{ inputs.envVars }} + shell: bash + working-directory: .github/actions/proxy-janky-build diff --git a/.github/actions/proxy-janky-build/go.mod b/.github/actions/proxy-janky-build/go.mod new file mode 100644 index 000000000..b7560d7b9 --- /dev/null +++ b/.github/actions/proxy-janky-build/go.mod @@ -0,0 +1,7 @@ +module github.com/github/enterprise2/actions/proxy-janky-build + +go 1.21 + +require github.com/hashicorp/go-retryablehttp v0.7.2 + +require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/.github/actions/proxy-janky-build/go.sum b/.github/actions/proxy-janky-build/go.sum new file mode 100644 index 000000000..5c59c1d2e --- /dev/null +++ b/.github/actions/proxy-janky-build/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go new file mode 100644 index 000000000..e0342f121 --- /dev/null +++ b/.github/actions/proxy-janky-build/main.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +// Define our Janky Response Structs +type JankyBuildStruct struct { + Result string + Url string +} +type JankyStatusStruct struct { + Id string + Green bool + Completed bool + StartedAt string + CompletedAt string + Sha string + BuildableName string +} + +const ( + pollWaitTime = 10 * time.Second + jankyPollTimeout = 5 * time.Hour + jankyHttpRetryMax = 5 + jankyUrl = "https://janky.githubapp.com" +) + +func main() { + // Parse command-line arguments + job := flag.String("job", "", "Name of the Janky job") + token := flag.String("token", "", "Name of the Janky token") + branch := flag.String("branch", "", "Name of the Git branch") + force := flag.String("force", "false", "Force a build even if one is already passed") + envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") + flag.Parse() + + // Validate command-line arguments + if *job == "" || *token == "" || *branch == "" { + log.Fatal("job, token and branch flags must be specified") + } + + // Set up the token + request payload + authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token)) + type buildRequestObject struct { + BuildableName string `json:"buildable_name"` + BranchName string `json:"branch_name"` + Force string `json:"force"` + EnvVars map[string]string `json:"env_vars"` + } + + requestBody := buildRequestObject{ + BuildableName: *job, + BranchName: *branch, + Force: *force, + } + + // Parse the envVars flag into a map and add to the request payload + fmt.Println("Environment Variables:") + fmt.Println(*envVars) + if *envVars != "" { + envVarsMap := make(map[string]string) + for _, envVar := range strings.Split(*envVars, ",") { + envVarSplit := strings.Split(envVar, "=") + envVarsMap[envVarSplit[0]] = envVarSplit[1] + } + requestBody.EnvVars = envVarsMap + } + + payloadBytes, err := json.Marshal(requestBody) + if err != nil { + log.Fatal("Failed to marshal the JSON payload!\n" + err.Error()) + } + + // Send build request to Janky + buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes)) + if err != nil { + log.Fatal("Failed to create build request!\n" + err.Error()) + } + buildRequest.Header.Set("Content-Type", "application/json") + buildRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient := retryablehttp.NewClient() //nolint:all + retryClient.RetryMax = jankyHttpRetryMax + retryClient.Logger = nil // disable debug logging + client := retryClient.StandardClient() // uses *http.Client + resp, err := client.Do(buildRequest) + if err != nil { + log.Fatal("Failed to send build request!\n" + err.Error()) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatal("Error reading build response!\n" + err.Error()) + } + + // Check if the build was triggered successfully + if resp.StatusCode == 404 { + log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.") + } + if resp.StatusCode != 201 { + log.Fatal("Failed to trigger build! Got exception: " + string(body)) + } + + // Parse the build request response + var buildResponse JankyBuildStruct + json.Unmarshal(body, &buildResponse) + log.Println("Succesfully triggered janky!\n" + buildResponse.Result) + + // Parse the request response for the buildId + r, err := regexp.Compile("/[0-9]+/") + if err != nil { + log.Fatal("Failed to trigger build!\n" + err.Error()) + } + buildId := strings.Trim(r.FindString(buildResponse.Result), "/") + + // Setup our second HTTP client for reuse in during status polling + jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status" + statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil) + if err != nil { + log.Fatal("Failed to create status request!\n" + err.Error()) + } + statusRequest.Header.Set("Content-Type", "application/json") + statusRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient2 := retryablehttp.NewClient() //nolint:all + retryClient2.RetryMax = jankyHttpRetryMax + retryClient2.Logger = nil // disable debug logging + client2 := retryClient2.StandardClient() // uses *http.Client + + // Wait for a completed status from Janky or break the loop after a certain amount of time + timeout := time.NewTimer(jankyPollTimeout) + poll := time.NewTicker(pollWaitTime) + +jobLoop: + for { + select { + case <-timeout.C: + log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours") + case <-poll.C: + // Send build status request to Janky + statusResponse, err := client2.Do(statusRequest) + if err != nil { + log.Fatal("Failed to send status request!\n" + err.Error()) + } + defer statusResponse.Body.Close() + statusBody, err := io.ReadAll(statusResponse.Body) + if err != nil { + log.Fatal("Error reading status response!\n" + err.Error()) + } + + // Parse the status response for a green completed build + var jankyStatusResponse JankyStatusStruct + json.Unmarshal(statusBody, &jankyStatusResponse) + //fmt.Println("Janky Status Response:") + //fmt.Println(string(statusBody)) + if jankyStatusResponse.Completed && jankyStatusResponse.Green { + log.Println("Janky build Succeeded!") + break jobLoop + } + if jankyStatusResponse.Completed && !jankyStatusResponse.Green { + log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url) + } + + // wait for a bit and try again + log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]") + continue + } + } +} diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..030c37f04 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,53 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ade3e5c58 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,40 @@ + + + + +# PR Details + +## Description + +## Testing + + + +## Ownership + + +## Related Links + diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 000000000..edf115738 --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,127 @@ +--- +name: Build and Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version - patch version of the release (e.g. x.y.z)' + required: true + type: string + draft: + description: 'Draft - true if the release should be a draft' + required: true + type: boolean + default: true + +jobs: + build: + runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" + - name: Get major-feature from version + id: get-major-feature + run: | + echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + - name: Verify major-feature + run: | + echo "major_feature: $MAJOR_FEATURE" + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: ${{ env.MAJOR_FEATURE }}-stable + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" + commit_options: "--allow-empty" + push_options: "--force" + skip_dirty_check: true + - name: Checkout backup-utils-private for release notes + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ steps.app-token.outputs.token }} + owner: github + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ env.MAJOR_FEATURE }}-stable + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..d442f9555 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,67 @@ +name: Run Integration Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'enterprise2 branch to test against' + required: true + source-branch: + description: 'backup-utils-private topic branch to test' + required: true + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" + + # Cluster integration tests are optional based on label and PR titles + cluster-integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-cluster-binary-backup + - enterprise2-backup-utils-cluster-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + if: | + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.title, '[cluster]') || + contains(github.event.pull_request.labels.*.name, 'cluster') + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d15f123bb..8e878da40 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +--- name: Lint Code Base on: @@ -19,3 +20,4 @@ jobs: env: VALIDATE_ALL_CODEBASE: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILTER_REGEX_EXCLUDE: .*release-notes/.* diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da7695ad1..50583737b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,14 +1,11 @@ name: Test and build -on: [pull_request] +on: [pull_request, workflow_dispatch] jobs: build: strategy: matrix: - # macos-latest references are kept here for historical purposes. removed macos-latest from the - #matrix as it is not a typical case for users and causes a lot of friction with other linux-based - # installs. Recommend developing on codespaces or using an ubuntu container. os: ['ubuntu-22.04', 'ubuntu-20.04'] fail-fast: false runs-on: ${{ matrix.os }} @@ -20,12 +17,6 @@ jobs: wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck - if: matrix.os != 'macos-latest' - - name: Install Dependencies (macOS) - run: | - brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget - brew install moreutils gawk - if: matrix.os == 'macos-latest' - name: Get Sources uses: actions/checkout@v3 - name: Test @@ -35,4 +26,3 @@ jobs: shell: bash - name: Build (Linux) run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc - if: matrix.os != 'macos-latest' diff --git a/.releaseignore b/.releaseignore new file mode 100644 index 000000000..bb2f6b31c --- /dev/null +++ b/.releaseignore @@ -0,0 +1,2 @@ +ownership.yaml +.github \ No newline at end of file diff --git a/backup.config-example b/backup.config-example index 688cc374b..d1ea977e2 100644 --- a/backup.config-example +++ b/backup.config-example @@ -26,7 +26,7 @@ GHE_NUM_SNAPSHOTS=10 # performed before the next full backup is created. # For example, if `GHE_INCREMENTAL_BACKUP_MAX` is set to 14, backup-utils will # run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle. -#GHE_INCREMENTAL_BACKUP_MAX=14 +#GHE_INCREMENTAL_MAX_BACKUPS=14 # If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check # disk space validation and software version checks on the backup-host will be disabled. diff --git a/bin/ghe-backup b/bin/ghe-backup index 86ee33dd7..d8074156e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -163,11 +163,11 @@ echo "$GHE_REMOTE_VERSION" > version # Setup progress tracking init-progress export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress OPTIONAL_STEPS=0 # Backup actions+mssql @@ -191,7 +191,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then fi PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total # check that incremental settings are valid if set is_inc=$(is_incremental_backup_feature_on) diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress index 2f4b267fb..7ab36e084 100755 --- a/bin/ghe-backup-progress +++ b/bin/ghe-backup-progress @@ -31,7 +31,7 @@ while true; do done check_for_progress_file() { - if [ ! -f /tmp/backup-utils-progress-info ]; then + if [ ! -f /tmp/backup-utils-progress/info ]; then echo "No progress file found. Has a backup or restore been started?" exit 1 fi @@ -39,18 +39,18 @@ check_for_progress_file() { if [ -n "$ONCE" ]; then check_for_progress_file - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info else check_for_progress_file clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info while true; do if read -r -t 1 -n 1; then clear exit ; else clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info fi done fi diff --git a/bin/ghe-host-check b/bin/ghe-host-check index ddda264f3..f58dae62c 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/bin/ghe-restore b/bin/ghe-restore index 92ae72443..7c9b95709 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -315,11 +315,11 @@ fi export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) init-progress -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Restore" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of restore -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) diff --git a/debian/changelog b/debian/changelog index 5f223ce8d..e627da795 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,18 @@ +github-backup-utils (3.10.2) UNRELEASED; urgency=medium + + * When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. + * When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + * Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. + * When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- ghes-releases-team Fri, 20 Oct 2023 18:49:12 +0000 + +github-backup-utils (3.10.1) UNRELEASED; urgency=medium + + + -- Devin Dooley Mon, 02 Oct 2023 20:38:31 +0000 + github-backup-utils (3.10.0) UNRELEASED; urgency=medium diff --git a/docs/requirements.md b/docs/requirements.md index 7f501a02c..1288846ac 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -59,6 +59,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the diff --git a/release-notes/3.10.2.md b/release-notes/3.10.2.md new file mode 100644 index 000000000..d21a32097 --- /dev/null +++ b/release-notes/3.10.2.md @@ -0,0 +1,14 @@ +### Bug Fixes + +* When using GitHub Enterprise Server Backup Utilities, a host check will test the availability of all nodes before running any operations on them and output a warning if a node is unreachable. Previously if a node was unreachable the host check could silently fail. +* When using GitHub Enterprise Server Backup Utilities, user permissions assigned to the temporary progress files in `/tmp` have been updated to allow access by other users. Previously, having backup and restore operations performed by different users could cause failed deletion error messages for temporary files. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + +### Changes + +* Adds support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. + +### Backups and Disaster Recovery + +When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + diff --git a/script/cibuild b/script/cibuild index 7f57f444d..73dc9ae0a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -5,7 +5,7 @@ set -e # Enable verbose logging of ssh commands export GHE_VERBOSE_SSH=true -if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then +if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then exit 1 fi diff --git a/script/package-deb b/script/package-deb index dda90cbe3..15a698e9c 100755 --- a/script/package-deb +++ b/script/package-deb @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" @@ -22,6 +25,14 @@ mkdir -p dist/debuild distdir="$(pwd)/dist/debuild/$PKG_NAME" git clone -q . "$distdir" cd "$distdir" + +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +echo "Removing .releaseignore ..." +rm -f .releaseignore git checkout -q "$PKG_HEAD" debuild -uc -us 1>&2 diff --git a/script/package-tarball b/script/package-tarball index be653b77a..bf1510e8f 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -8,11 +8,24 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" PKG_NAME="${PKG_BASE}-${PKG_VERS}" +# Remove all files or directories listed in .releaseignore +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +# Remove the .releaseignore file itself +echo "Removing .releaseignore ..." +rm -f .releaseignore + # Run git-archive to generate tarball echo "Creating ${PKG_NAME}.tar.gz ..." mkdir -p dist diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 2e176b510..bdf77cc86 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -217,8 +217,10 @@ ghe_parallel_check() { GHE_PARALLEL_COMMAND="parallel" local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do @@ -650,12 +652,22 @@ restore-secret() { #initialize progress tracking by clearing out the temp files used to track init-progress() { - rm -f /tmp/backup-utils-progress* + if [ -d /tmp/backup-utils-progress ]; then + rm -rf /tmp/backup-utils-progress/* + fi + + mkdir -p /tmp/backup-utils-progress + chmod -R 777 /tmp/backup-utils-progress + + touch /tmp/backup-utils-progress/total + touch /tmp/backup-utils-progress/type + touch /tmp/backup-utils-progress/progress + touch /tmp/backup-utils-progress/info } #increase total count of progress increment-progress-total-count() { ((PROGRESS_TOTAL += $1)) - echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total } diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index aac02dde7..24e946007 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -122,6 +122,7 @@ if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" + backup-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 93596033a..303a2abd2 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -85,6 +85,14 @@ restore-secret "Actions Launch service private key" "actions-launch-app-app-priv restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" +# Restore storage container prefix, but only if it exists, and the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings +if [[ $RESTORE_SETTINGS == "true" ]]; then + if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + else + log_warn "Actions storage container prefix not present in backup. Skipping ..." + fi +fi # Setup the database logins. ghe_verbose "* Restoring database logins and users to $host ..." diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index d88b3704f..a560ba540 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,13 +1,12 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -# Current version is working solely with backups progress(){ - PROGRESS=$(cat /tmp/backup-utils-progress) - PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress-total) - PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) + PROGRESS=$(cat /tmp/backup-utils-progress/progress) + PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total) + PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type) PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) - echo $((PROGRESS + 1)) > /tmp/backup-utils-progress - echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info + echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress + echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info } diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 30291cba2..e86ec35a8 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.10.0 +3.10.2 \ No newline at end of file diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index fb3f9db6a..3f3cd25ed 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -47,6 +47,8 @@ begin_test "ghe-backup subsequent snapshot" [ "$first_snapshot" != "$this_snapshot" ] verify_all_backedup_data + + verify_progress_cleanup_process ) end_test diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 6bfa12a69..aeb41551d 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION) bu_major_minor="$bu_version_major.$bu_version_minor" releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json) - supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")') + latest_value=$(echo "$releases" | jq -r '.latest') + latest_major_version=$(echo $latest_value | cut -d "." -f 1-2) + # Replace "latest" with the derived major version in the releases string + releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g') + # Use the modified releases string as needed + supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo $releases | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) if [ -z "$supported" ] then diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index ef13b7697..7ed345ee0 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -622,6 +622,94 @@ begin_test "ghe-restore with Actions settings" ) end_test +begin_test "ghe-restore with Actions settings passing -c" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + enable_actions + + required_files=( + "actions-config-db-login" + "actions-config-db-password" + "actions-framework-access-token" + "actions-url-signing-hmac-key-primary" + "actions-url-signing-hmac-key-secondary" + "actions-oauth-s2s-signing-cert" + "actions-oauth-s2s-signing-key" + "actions-oauth-s2s-signing-cert-thumbprint" + "actions-primary-encryption-cert-thumbprint" + "actions-aad-cert-thumbprint" + "actions-delegated-auth-cert-thumbprint" + "actions-runtime-service-principal-cert" + "actions-s2s-encryption-cert" + "actions-secondary-encryption-cert-thumbprint" + "actions-service-principal-cert" + "actions-sps-validation-cert-thumbprint" + "actions-storage-container-prefix" + + "actions-launch-secrets-private-key" + "actions-launch-deployer-hmac" + "actions-launch-client-id" + "actions-launch-client-secret" + "actions-launch-receiver-webhook-secret" + "actions-launch-app-private-key" + "actions-launch-app-public-key" + "actions-launch-app-id" + "actions-launch-app-relay-id" + "actions-launch-action-runner-secret" + "actions-launch-azp-app-cert" + "actions-launch-app-app-private-key" + + ) + + for file in "${required_files[@]}"; do + echo "foo" > "$GHE_DATA_DIR/current/$file" + done + + ghe-restore -v -f -c localhost + + required_secrets=( + "secrets.actions.ConfigurationDatabaseSqlLogin" + "secrets.actions.ConfigurationDatabaseSqlPassword" + "secrets.actions.FrameworkAccessTokenKeySecret" + "secrets.actions.UrlSigningHmacKeyPrimary" + "secrets.actions.UrlSigningHmacKeySecondary" + "secrets.actions.OAuthS2SSigningCert" + "secrets.actions.OAuthS2SSigningKey" + "secrets.actions.OAuthS2SSigningCertThumbprint" + "secrets.actions.PrimaryEncryptionCertificateThumbprint" + "secrets.actions.AADCertThumbprint" + "secrets.actions.DelegatedAuthCertThumbprint" + "secrets.actions.RuntimeServicePrincipalCertificate" + "secrets.actions.S2SEncryptionCertificate" + "secrets.actions.SecondaryEncryptionCertificateThumbprint" + "secrets.actions.ServicePrincipalCertificate" + "secrets.actions.SpsValidationCertThumbprint" + "secrets.actions.storage.container-prefix" + "secrets.launch.actions-secrets-private-key" + "secrets.launch.deployer-hmac-secret" + "secrets.launch.client-id" + "secrets.launch.client-secret" + "secrets.launch.receiver-webhook-secret" + "secrets.launch.app-private-key" + "secrets.launch.app-public-key" + "secrets.launch.app-id" + "secrets.launch.app-relay-id" + "secrets.launch.action-runner-secret" + "secrets.launch.token-oauth-key" + "secrets.launch.token-oauth-cert" + "secrets.launch.azp-app-cert" + "secrets.launch.azp-app-private-key" + + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done +) +end_test + begin_test "ghe-restore stops and starts Actions" ( set -e diff --git a/test/testlib.sh b/test/testlib.sh old mode 100644 new mode 100755 index 53b41dfe9..fc5b0f771 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -494,6 +494,12 @@ verify_all_backedup_data() { verify_common_data } +# A unified method to make sure post backup, the cleanup process works +verify_progress_cleanup_process() { + set -e + sudo -u nobody rm -rf /tmp/backup-utils-progress/* +} + # A unified method to check everything restored when performing a full restore # during testing. verify_all_restored_data() { @@ -600,8 +606,10 @@ setup_moreutils_parallel() { # We need moreutils parallel local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do