Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1cf10d3

Browse files
manugargconallob
manugarg
authored and
conallob
committed
[cloudprober] Add support for reading files from S3 and GCS. (cloudprober#396)
* This applies to all kinds of file resources: - File based targets - TLS certificate files - OAuth token files - Cloudprober config itself.
1 parent 73341b4 commit 1cf10d3

File tree

12 files changed

+246
-39
lines changed

12 files changed

+246
-39
lines changed

‎config/config.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package config
1717
import (
1818
"bufio"
1919
"bytes"
20+
"context"
2021
"errors"
2122
"flag"
2223
"fmt"
@@ -113,7 +114,7 @@ func handleIncludes(baseDir string, content []byte) (string, error) {
113114
}
114115

115116
func readConfigFile(fileName string) (string, error) {
116-
b, err := file.ReadFile(fileName)
117+
b, err := file.ReadFile(context.Background(), fileName)
117118
if err != nil {
118119
return "", err
119120
}

‎internal/file/file.go‎

Lines changed: 55 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2020 The Cloudprober Authors.
1+
// Copyright 2020-2023 The Cloudprober Authors.
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -19,16 +19,13 @@ package file
1919

2020
import (
2121
"context"
22-
"errors"
2322
"fmt"
2423
"io"
2524
"net/http"
2625
"os"
2726
"strings"
2827
"sync"
2928
"time"
30-
31-
"golang.org/x/oauth2/google"
3229
)
3330

3431
type cacheEntry struct {
@@ -43,64 +40,99 @@ var global = struct {
4340
cache: make(map[string]cacheEntry),
4441
}
4542

46-
type readFunc func(path string) ([]byte, error)
47-
type modTimeFunc func(path string) (time.Time, error)
43+
type readFunc func(ctx context.Context, path string) ([]byte, error)
44+
type modTimeFunc func(ctx context.Context, path string) (time.Time, error)
45+
46+
var zeroTime = time.Time{}
4847

4948
var prefixToReadfunc = map[string]readFunc{
50-
"gs://": readFileFromGCS,
49+
"gs://": readFileFromGCS,
50+
"s3://": readFileFromS3,
51+
"http://": readFileFromHTTP,
52+
"https://": readFileFromHTTP,
5153
}
5254

5355
var prefixToModTimeFunc = map[string]modTimeFunc{
54-
"gs://": modTimeGCS,
56+
"gs://": gcsModTime,
57+
"s3://": s3ModTime,
58+
"http://": httpModTime,
59+
"https://": httpModTime,
5560
}
5661

57-
func readFileFromGCS(objectPath string) ([]byte, error) {
58-
hc, err := google.DefaultClient(context.Background())
59-
if err != nil {
60-
return nil, err
62+
func parseObjectURL(objectPath string) (bucket, object string, err error) {
63+
parts := strings.SplitN(objectPath, "/", 2)
64+
if len(parts) != 2 {
65+
return "", "", fmt.Errorf("invalid object URL: %s", objectPath)
6166
}
67+
return parts[0], parts[1], nil
68+
}
6269

63-
objURL := "https://storage.googleapis.com/" + objectPath
64-
res, err := hc.Get(objURL)
70+
func httpLastModified(res *http.Response) (time.Time, error) {
71+
t, err := time.Parse(time.RFC1123, res.Header.Get("Last-Modified"))
72+
if err != nil {
73+
return zeroTime, fmt.Errorf("error parsing Last-Modified header: %v", err)
74+
}
75+
return t, nil
76+
}
6577

78+
func readFileFromHTTP(ctx context.Context, fileURL string) ([]byte, error) {
79+
req, err := http.NewRequestWithContext(ctx, "GET", fileURL, nil)
80+
if err != nil {
81+
return nil, err
82+
}
83+
res, err := http.DefaultClient.Do(req)
6684
if err != nil {
6785
return nil, err
6886
}
6987

7088
if res.StatusCode != http.StatusOK {
71-
return nil, fmt.Errorf("got error while retrieving GCS object, http status: %s, status code: %d", res.Status, res.StatusCode)
89+
return nil, fmt.Errorf("got error while retrieving HTTP object, http status: %s, status code: %d", res.Status, res.StatusCode)
7290
}
7391

7492
defer res.Body.Close()
7593
return io.ReadAll(res.Body)
7694
}
7795

78-
func modTimeGCS(objectPath string) (time.Time, error) {
79-
return time.Time{}, errors.New("mod-time is not implemented for GCS files yet")
96+
func httpModTime(ctx context.Context, fileURL string) (time.Time, error) {
97+
req, err := http.NewRequestWithContext(ctx, "HEAD", fileURL, nil)
98+
if err != nil {
99+
return zeroTime, err
100+
}
101+
res, err := http.DefaultClient.Do(req)
102+
if err != nil {
103+
return zeroTime, err
104+
}
105+
106+
if res.StatusCode != http.StatusOK {
107+
return zeroTime, fmt.Errorf("got error while retrieving HTTP object, http status: %s, status code: %d", res.Status, res.StatusCode)
108+
}
109+
110+
defer res.Body.Close()
111+
return httpLastModified(res)
80112
}
81113

82114
// ReadFile returns file contents as a slice of bytes. It's similar to ioutil's
83115
// ReadFile, but includes support for files on non-disk locations. For example,
84116
// files with paths starting with gs:// are assumed to be on GCS, and are read
85117
// from GCS.
86-
func ReadFile(fname string) ([]byte, error) {
118+
func ReadFile(ctx context.Context, fname string) ([]byte, error) {
87119
for prefix, f := range prefixToReadfunc {
88120
if strings.HasPrefix(fname, prefix) {
89-
return f(fname[len(prefix):])
121+
return f(ctx, fname[len(prefix):])
90122
}
91123
}
92124
return os.ReadFile(fname)
93125
}
94126

95-
func ReadWithCache(fname string, refreshInterval time.Duration) ([]byte, error) {
127+
func ReadWithCache(ctx context.Context, fname string, refreshInterval time.Duration) ([]byte, error) {
96128
global.mu.RLock()
97129
fc, ok := global.cache[fname]
98130
global.mu.RUnlock()
99131
if ok && (time.Since(fc.lastReload) < refreshInterval) {
100132
return fc.b, nil
101133
}
102134

103-
b, err := ReadFile(fname)
135+
b, err := ReadFile(ctx, fname)
104136
if err != nil {
105137
return nil, err
106138
}
@@ -112,10 +144,10 @@ func ReadWithCache(fname string, refreshInterval time.Duration) ([]byte, error)
112144
}
113145

114146
// ModTime returns file's modified timestamp.
115-
func ModTime(fname string) (time.Time, error) {
147+
func ModTime(ctx context.Context, fname string) (time.Time, error) {
116148
for prefix, f := range prefixToModTimeFunc {
117149
if strings.HasPrefix(fname, prefix) {
118-
return f(fname[len(prefix):])
150+
return f(ctx, fname[len(prefix):])
119151
}
120152
}
121153

‎internal/file/file_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
package file
1616

1717
import (
18-
"io/ioutil"
18+
"context"
1919
"os"
2020
"testing"
2121
"time"
@@ -24,7 +24,7 @@ import (
2424
)
2525

2626
func createTempFile(t *testing.T, b []byte) string {
27-
tmpfile, err := ioutil.TempFile("", "")
27+
tmpfile, err := os.CreateTemp("", "")
2828
if err != nil {
2929
t.Fatal(err)
3030
return ""
@@ -38,7 +38,7 @@ func createTempFile(t *testing.T, b []byte) string {
3838
return tmpfile.Name()
3939
}
4040

41-
func testReadFile(path string) ([]byte, error) {
41+
func testReadFile(ctx context.Context, path string) ([]byte, error) {
4242
return []byte("content-for-" + path), nil
4343
}
4444

@@ -59,7 +59,7 @@ func TestReadFile(t *testing.T) {
5959

6060
for path, expectedContent := range testData {
6161
t.Run("ReadFile("+path+")", func(t *testing.T) {
62-
b, err := ReadFile(path)
62+
b, err := ReadFile(context.Background(), path)
6363
if err != nil {
6464
t.Fatalf("Error while reading the file: %s", path)
6565
}
@@ -85,7 +85,7 @@ func TestReadWithCache(t *testing.T) {
8585
}
8686

8787
readAndVerify := func(expectedContent string, reloadInterval time.Duration) {
88-
b, err := ReadWithCache(f.Name(), reloadInterval)
88+
b, err := ReadWithCache(context.Background(), f.Name(), reloadInterval)
8989
assert.NoError(t, err, "reading file")
9090
assert.Equal(t, expectedContent, string(b))
9191
}

‎internal/file/gcs.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright 2020-2023 The Cloudprober Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package file
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"io"
21+
"net/http"
22+
"path"
23+
"time"
24+
25+
"golang.org/x/oauth2/google"
26+
)
27+
28+
const gcsHTTPBaseURL = "https://storage.googleapis.com"
29+
30+
func gcsRequest(ctx context.Context, method, objectPath string) (*http.Response, error) {
31+
hc, err := google.DefaultClient(ctx)
32+
if err != nil {
33+
return nil, err
34+
}
35+
36+
req, err := http.NewRequestWithContext(ctx, method, path.Join(gcsHTTPBaseURL, objectPath), nil)
37+
if err != nil {
38+
return nil, err
39+
}
40+
41+
res, err := hc.Do(req)
42+
if err != nil {
43+
return nil, err
44+
}
45+
46+
if res.StatusCode != http.StatusOK {
47+
return nil, fmt.Errorf("got error while retrieving GCS object, http status: %s, status code: %d", res.Status, res.StatusCode)
48+
}
49+
50+
return res, nil
51+
}
52+
53+
func readFileFromGCS(ctx context.Context, objectPath string) ([]byte, error) {
54+
res, err := gcsRequest(ctx, "GET", objectPath)
55+
if err != nil {
56+
return nil, err
57+
}
58+
59+
defer res.Body.Close()
60+
return io.ReadAll(res.Body)
61+
}
62+
63+
func gcsModTime(ctx context.Context, objectPath string) (time.Time, error) {
64+
res, err := gcsRequest(ctx, "HEAD", objectPath)
65+
if err != nil {
66+
return zeroTime, err
67+
}
68+
69+
defer res.Body.Close()
70+
return httpLastModified(res)
71+
}

‎internal/file/s3.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Copyright 2020-2023 The Cloudprober Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
/*
16+
Package file implements utilities to read files from various backends.
17+
*/
18+
package file
19+
20+
import (
21+
"context"
22+
"fmt"
23+
"io"
24+
"time"
25+
26+
"github.com/aws/aws-sdk-go-v2/aws"
27+
"github.com/aws/aws-sdk-go-v2/config"
28+
"github.com/aws/aws-sdk-go-v2/service/s3"
29+
)
30+
31+
// readFileFromS3 reads a file using an S3 Bucket URL
32+
// s3://bucket-name/path/to/file.txt
33+
func readFileFromS3(ctx context.Context, objectPath string) ([]byte, error) {
34+
bucket, object, err := parseObjectURL(objectPath)
35+
if err != nil {
36+
return nil, err
37+
}
38+
39+
sdkConfig, err := config.LoadDefaultConfig(ctx)
40+
if err != nil {
41+
return nil, fmt.Errorf("failed to load default config: %v", err)
42+
}
43+
s3Client := s3.NewFromConfig(sdkConfig)
44+
45+
result, err := s3Client.GetObject(ctx, &s3.GetObjectInput{
46+
Bucket: aws.String(bucket),
47+
Key: aws.String(object),
48+
})
49+
50+
if err != nil {
51+
return nil, fmt.Errorf("failed to retrieve file (%s): %v", objectPath, err)
52+
}
53+
defer result.Body.Close()
54+
55+
return io.ReadAll(result.Body)
56+
}
57+
58+
func s3ModTime(ctx context.Context, objectPath string) (time.Time, error) {
59+
bucket, object, err := parseObjectURL(objectPath)
60+
if err != nil {
61+
return time.Time{}, err
62+
}
63+
64+
sdkConfig, err := config.LoadDefaultConfig(ctx)
65+
if err != nil {
66+
return time.Time{}, fmt.Errorf("failed to load default config: %v", err)
67+
}
68+
s3Client := s3.NewFromConfig(sdkConfig)
69+
70+
result, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
71+
Bucket: aws.String(bucket),
72+
Key: aws.String(object),
73+
})
74+
if err != nil {
75+
return time.Time{}, fmt.Errorf("failed to retrieve file (%s): %v", objectPath, err)
76+
}
77+
78+
return *result.LastModified, nil
79+
}

‎internal/oauth/bearer.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package oauth
1616

1717
import (
18+
"context"
1819
"encoding/json"
1920
"fmt"
2021
"os"
@@ -55,7 +56,7 @@ var tokenFunctions = struct {
5556
fromFile, fromCmd, fromGCEMetadata, fromK8sTokenFile func(c *configpb.BearerToken) (*oauth2.Token, error)
5657
}{
5758
fromFile: func(c *configpb.BearerToken) (*oauth2.Token, error) {
58-
b, err := file.ReadFile(c.GetFile())
59+
b, err := file.ReadFile(context.Background(), c.GetFile())
5960
if err != nil {
6061
return nil, err
6162
}

‎internal/oauth/oauth.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ func TokenSourceFromConfig(c *configpb.Config, l *logger.Logger) (oauth2.TokenSo
6464
return creds.TokenSource, nil
6565
}
6666

67-
jsonKey, err := file.ReadFile(f)
67+
jsonKey, err := file.ReadFile(context.Background(), f)
6868
if err != nil {
6969
return nil, fmt.Errorf("error reading Google Credentials file (%s): %v", f, err)
7070
}

0 commit comments

Comments
 (0)