Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 424223b

Browse files
amildahlamildahl
authored
Merge pull request #1164 from github/3.11-main
Sync 3.11-main into master
2 parents ff83a62 + 7286fda commit 424223b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+1904
-834
lines changed

‎.github/CODEOWNERS

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Backup-Utils owned by lifecycle AOR
2+
* @github/ghes-lifecycle
3+
# Actions related backups and restores
4+
# /share/github-backup-utils/*-actions @github/ghes-lifecycle @github/<TBD>
5+
# Git related backups and restores
6+
# /share/github-backup-utils/*-repositories @github/ghes-lifecycle @github/<TBD>
7+
# /share/github-backup-utils/*-git-hooks @github/ghes-lifecycle @github/<TBD>

‎.github/actions/proxy-janky-build/action.yml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: 'Trigger a CI Job on Janky'
2+
description: 'Action to trigger and poll a Janky CI job'
3+
inputs:
4+
janky-token:
5+
description: 'Token for making request to Janky'
6+
required: true
7+
job-name:
8+
description: 'The name of the job to run'
9+
required: true
10+
branch-name:
11+
description: 'The name of the branch to use'
12+
required: true
13+
force:
14+
description: 'Force the job to run even if it is already passed'
15+
required: false
16+
envVars:
17+
description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3'
18+
required: false
19+
runs:
20+
using: 'composite'
21+
steps:
22+
- uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260
23+
with:
24+
go-version: '1.21'
25+
- run: |
26+
go run main.go \
27+
-token ${{ inputs.janky-token }} \
28+
-job ${{ inputs.job-name }} \
29+
-branch ${{ inputs.branch-name }} \
30+
-force ${{ inputs.force }} \
31+
-envVars ${{ inputs.envVars }}
32+
shell: bash
33+
working-directory: .github/actions/proxy-janky-build

‎.github/actions/proxy-janky-build/go.mod

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
module github.com/github/enterprise2/actions/proxy-janky-build
2+
3+
go 1.21
4+
5+
require github.com/hashicorp/go-retryablehttp v0.7.2
6+
7+
require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect

‎.github/actions/proxy-janky-build/go.sum

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2+
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
3+
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
4+
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
5+
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
6+
github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
7+
github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
8+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
9+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

‎.github/actions/proxy-janky-build/main.go

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"encoding/base64"
6+
"encoding/json"
7+
"flag"
8+
"fmt"
9+
"io"
10+
"log"
11+
"net/http"
12+
"regexp"
13+
"strings"
14+
"time"
15+
16+
"github.com/hashicorp/go-retryablehttp"
17+
)
18+
19+
// Define our Janky Response Structs
20+
type JankyBuildStruct struct {
21+
Result string
22+
Url string
23+
}
24+
type JankyStatusStruct struct {
25+
Id string
26+
Green bool
27+
Completed bool
28+
StartedAt string
29+
CompletedAt string
30+
Sha string
31+
BuildableName string
32+
}
33+
34+
const (
35+
pollWaitTime = 10 * time.Second
36+
jankyPollTimeout = 5 * time.Hour
37+
jankyHttpRetryMax = 5
38+
jankyUrl = "https://janky.githubapp.com"
39+
)
40+
41+
func main() {
42+
// Parse command-line arguments
43+
job := flag.String("job", "", "Name of the Janky job")
44+
token := flag.String("token", "", "Name of the Janky token")
45+
branch := flag.String("branch", "", "Name of the Git branch")
46+
force := flag.String("force", "false", "Force a build even if one is already passed")
47+
envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3")
48+
flag.Parse()
49+
50+
// Validate command-line arguments
51+
if *job == "" || *token == "" || *branch == "" {
52+
log.Fatal("job, token and branch flags must be specified")
53+
}
54+
55+
// Set up the token + request payload
56+
authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token))
57+
type buildRequestObject struct {
58+
BuildableName string `json:"buildable_name"`
59+
BranchName string `json:"branch_name"`
60+
Force string `json:"force"`
61+
EnvVars map[string]string `json:"env_vars"`
62+
}
63+
64+
requestBody := buildRequestObject{
65+
BuildableName: *job,
66+
BranchName: *branch,
67+
Force: *force,
68+
}
69+
70+
// Parse the envVars flag into a map and add to the request payload
71+
fmt.Println("Environment Variables:")
72+
fmt.Println(*envVars)
73+
if *envVars != "" {
74+
envVarsMap := make(map[string]string)
75+
for _, envVar := range strings.Split(*envVars, ",") {
76+
envVarSplit := strings.Split(envVar, "=")
77+
envVarsMap[envVarSplit[0]] = envVarSplit[1]
78+
}
79+
requestBody.EnvVars = envVarsMap
80+
}
81+
82+
payloadBytes, err := json.Marshal(requestBody)
83+
if err != nil {
84+
log.Fatal("Failed to marshal the JSON payload!\n" + err.Error())
85+
}
86+
87+
// Send build request to Janky
88+
buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes))
89+
if err != nil {
90+
log.Fatal("Failed to create build request!\n" + err.Error())
91+
}
92+
buildRequest.Header.Set("Content-Type", "application/json")
93+
buildRequest.Header.Set("Authorization", "Basic "+authToken)
94+
retryClient := retryablehttp.NewClient() //nolint:all
95+
retryClient.RetryMax = jankyHttpRetryMax
96+
retryClient.Logger = nil // disable debug logging
97+
client := retryClient.StandardClient() // uses *http.Client
98+
resp, err := client.Do(buildRequest)
99+
if err != nil {
100+
log.Fatal("Failed to send build request!\n" + err.Error())
101+
}
102+
defer resp.Body.Close()
103+
body, err := io.ReadAll(resp.Body)
104+
if err != nil {
105+
log.Fatal("Error reading build response!\n" + err.Error())
106+
}
107+
108+
// Check if the build was triggered successfully
109+
if resp.StatusCode == 404 {
110+
log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.")
111+
}
112+
if resp.StatusCode != 201 {
113+
log.Fatal("Failed to trigger build! Got exception: " + string(body))
114+
}
115+
116+
// Parse the build request response
117+
var buildResponse JankyBuildStruct
118+
json.Unmarshal(body, &buildResponse)
119+
log.Println("Succesfully triggered janky!\n" + buildResponse.Result)
120+
121+
// Parse the request response for the buildId
122+
r, err := regexp.Compile("/[0-9]+/")
123+
if err != nil {
124+
log.Fatal("Failed to trigger build!\n" + err.Error())
125+
}
126+
buildId := strings.Trim(r.FindString(buildResponse.Result), "/")
127+
128+
// Setup our second HTTP client for reuse in during status polling
129+
jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status"
130+
statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil)
131+
if err != nil {
132+
log.Fatal("Failed to create status request!\n" + err.Error())
133+
}
134+
statusRequest.Header.Set("Content-Type", "application/json")
135+
statusRequest.Header.Set("Authorization", "Basic "+authToken)
136+
retryClient2 := retryablehttp.NewClient() //nolint:all
137+
retryClient2.RetryMax = jankyHttpRetryMax
138+
retryClient2.Logger = nil // disable debug logging
139+
client2 := retryClient2.StandardClient() // uses *http.Client
140+
141+
// Wait for a completed status from Janky or break the loop after a certain amount of time
142+
timeout := time.NewTimer(jankyPollTimeout)
143+
poll := time.NewTicker(pollWaitTime)
144+
145+
jobLoop:
146+
for {
147+
select {
148+
case <-timeout.C:
149+
log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours")
150+
case <-poll.C:
151+
// Send build status request to Janky
152+
statusResponse, err := client2.Do(statusRequest)
153+
if err != nil {
154+
log.Fatal("Failed to send status request!\n" + err.Error())
155+
}
156+
defer statusResponse.Body.Close()
157+
statusBody, err := io.ReadAll(statusResponse.Body)
158+
if err != nil {
159+
log.Fatal("Error reading status response!\n" + err.Error())
160+
}
161+
162+
// Parse the status response for a green completed build
163+
var jankyStatusResponse JankyStatusStruct
164+
json.Unmarshal(statusBody, &jankyStatusResponse)
165+
//fmt.Println("Janky Status Response:")
166+
//fmt.Println(string(statusBody))
167+
if jankyStatusResponse.Completed && jankyStatusResponse.Green {
168+
log.Println("Janky build Succeeded!")
169+
break jobLoop
170+
}
171+
if jankyStatusResponse.Completed && !jankyStatusResponse.Green {
172+
log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url)
173+
}
174+
175+
// wait for a bit and try again
176+
log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]")
177+
continue
178+
}
179+
}
180+
}

‎.github/linters/.yaml-lint.yml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
---
2+
###########################################
3+
# These are the rules used for #
4+
# linting all the yaml files in the stack #
5+
# NOTE: #
6+
# You can disable line with: #
7+
# # yamllint disable-line #
8+
###########################################
9+
rules:
10+
braces:
11+
level: warning
12+
min-spaces-inside: 0
13+
max-spaces-inside: 0
14+
min-spaces-inside-empty: 1
15+
max-spaces-inside-empty: 5
16+
brackets:
17+
level: warning
18+
min-spaces-inside: 0
19+
max-spaces-inside: 0
20+
min-spaces-inside-empty: 1
21+
max-spaces-inside-empty: 5
22+
colons:
23+
level: warning
24+
max-spaces-before: 0
25+
max-spaces-after: 1
26+
commas:
27+
level: warning
28+
max-spaces-before: 0
29+
min-spaces-after: 1
30+
max-spaces-after: 1
31+
comments: disable
32+
comments-indentation: disable
33+
document-end: disable
34+
document-start: disable
35+
empty-lines:
36+
level: warning
37+
max: 2
38+
max-start: 0
39+
max-end: 0
40+
hyphens:
41+
level: warning
42+
max-spaces-after: 1
43+
indentation:
44+
level: warning
45+
spaces: consistent
46+
indent-sequences: true
47+
check-multi-line-strings: false
48+
key-duplicates: enable
49+
line-length: disable
50+
new-line-at-end-of-file: disable
51+
new-lines:
52+
type: unix
53+
trailing-spaces: disable

‎.github/pull_request_template.md

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<!-- Welcome to backup-utils-private repo and Thanks for contributing!
2+
3+
Note: Merging to the master branch will include your change in a future (unreleased) version of backup-utils. If the change needs to be shipped to the current release versions it will need to be backported. For more information, see the backport guide https://github.com/github/enterprise-releases/blob/master/docs/backport-an-existing-pr.md
4+
5+
If you have any questions we can be found in the #ghes-backup-utils Slack channel.
6+
-->
7+
8+
<!--
9+
Additional notes regarding CI:
10+
- All required CIs needs to be pass before merging PR
11+
- Integration test will run against enterprise2 repo with environment variable, do not re-run directly from janky or Github CI, please use Actions to re-run the failed tests
12+
- If you are making changes impacts cluster, please add `cluster` label or `[cluster]` in your PR title so it will trigger optional cluster integration test. Those tests will take about 3 hours so relax and come back later to check the results. ;)
13+
-->
14+
15+
# PR Details
16+
17+
## Description
18+
<!--
19+
[Please fill out a brief description of the change being made]
20+
-->
21+
## Testing
22+
<!--
23+
[Please add testing done as part of this change.]
24+
-->
25+
<!-- Keep in mind that for backup-utils the following applies:
26+
- Backup-util [current version] will support
27+
- GHES [current version]
28+
- GHES [current version -1]
29+
- GHES [current version -2]
30+
- Any changes that are made to backup-utils will also need to be supported on those GHES versions above (n-2)
31+
- Please make sure those versions are tested against for this change
32+
-->
33+
34+
## Ownership
35+
<!-- [Add any relevants owners for this change]
36+
-->
37+
38+
## Related Links
39+
<!-- [Please add any related links/issues to this PR]
40+
-->

‎.github/workflows/backup.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
build:
4848
runs-on: ubuntu-latest
4949
steps:
50-
- uses: actions/checkout@v3
50+
- uses: actions/checkout@v4
5151
with:
5252
repository: github/backup-utils-private
5353
token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}"
@@ -62,7 +62,7 @@ jobs:
6262
needs: build
6363
runs-on:
6464
group: larger-hosted-public-runners
65-
labels: ubuntu-latest-xl
65+
labels: ubuntu-latest
6666
env:
6767
SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }}
6868
steps:
@@ -71,7 +71,7 @@ jobs:
7171
name: backup-utils
7272
- name: Load docker container
7373
run: docker load -i backup-utils.tar
74-
- uses: actions/checkout@v3
74+
- uses: actions/checkout@v4
7575
- name: Create backup directory
7676
run: mkdir "$HOME/ghe-backup-data"
7777
- name: set up ssh SSH_KEY
@@ -98,7 +98,7 @@ jobs:
9898
sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" .
9999
100100
- name: Login to Azure
101-
if: ${{ inputs.backup-name }} != ""
101+
if: "${{ inputs.backup-name != '' }}"
102102
run: |
103103
az login \
104104
--service-principal \
@@ -108,11 +108,11 @@ jobs:
108108
az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
109109
110110
- name: Upload backup to Azure
111-
if: ${{ inputs.backup-name }} != ""
111+
if: "${{ inputs.backup-name != '' }}"
112112
run: |
113113
az storage blob upload \
114114
--account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \
115115
--container-name "${{ secrets.AZURE_CONTAINER_NAME }}" \
116116
--name "${{ inputs.backup-name }}.tar.gz" \
117117
--file "${{ inputs.backup-name }}.tar.gz" \
118-
--connection-string "${{ secrets.CONNECTIONSTRING }}"
118+
--connection-string "${{ secrets.CONNECTIONSTRING }}"

0 commit comments

Comments
 (0)