diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e7526dbb..00000000 --- a/.gitmodules +++ /dev/null @@ -1,117 +0,0 @@ -[submodule "vendor/github.com/aws/aws-sdk-go"] - path = vendor/github.com/aws/aws-sdk-go - url = https://github.com/aws/aws-sdk-go -[submodule "vendor/github.com/lsegal/gucumber"] - path = vendor/github.com/lsegal/gucumber - url = https://github.com/lsegal/gucumber -[submodule "vendor/github.com/smartystreets/goconvey"] - path = vendor/github.com/smartystreets/goconvey - url = https://github.com/smartystreets/goconvey -[submodule "vendor/github.com/smartystreets/assertions"] - path = vendor/github.com/smartystreets/assertions - url = https://github.com/smartystreets/assertions -[submodule "vendor/github.com/jtolds/gls"] - path = vendor/github.com/jtolds/gls - url = https://github.com/jtolds/gls -[submodule "vendor/github.com/jacobsa/fuse"] - path = vendor/github.com/jacobsa/fuse - url = https://github.com/kahing/fusego -[submodule "vendor/github.com/Azure/go-autorest"] - path = vendor/github.com/Azure/go-autorest - url = https://github.com/Azure/go-autorest/ -[submodule "vendor/github.com/Azure/azure-pipeline-go"] - path = vendor/github.com/Azure/azure-pipeline-go - url = https://github.com/Azure/azure-pipeline-go/ -[submodule "vendor/github.com/Azure/azure-sdk-for-go"] - path = vendor/github.com/Azure/azure-sdk-for-go - url = https://github.com/Azure/azure-sdk-for-go -[submodule "vendor/github.com/Azure/azure-storage-blob-go"] - path = vendor/github.com/Azure/azure-storage-blob-go - url = https://github.com/Azure/azure-storage-blob-go/ -[submodule "vendor/github.com/google/uuid"] - path = vendor/github.com/google/uuid - url = https://github.com/google/uuid -[submodule "vendor/github.com/sirupsen/logrus"] - path = vendor/github.com/sirupsen/logrus - url = https://github.com/sirupsen/logrus -[submodule "vendor/github.com/shirou/gopsutil"] - path = vendor/github.com/shirou/gopsutil - url = https://github.com/shirou/gopsutil/ -[submodule "vendor/github.com/kardianos/osext"] - path = vendor/github.com/kardianos/osext - url = https://github.com/kardianos/osext -[submodule "vendor/github.com/mitchellh/go-homedir"] - path = vendor/github.com/mitchellh/go-homedir - url = https://github.com/mitchellh/go-homedir -[submodule "vendor/github.com/satori/go.uuid"] - path = vendor/github.com/satori/go.uuid - url = https://github.com/satori/go.uuid -[submodule "vendor/github.com/sevlyar/go-daemon"] - path = vendor/github.com/sevlyar/go-daemon - url = https://github.com/sevlyar/go-daemon -[submodule "vendor/github.com/urfave/cli"] - path = vendor/github.com/urfave/cli - url = https://github.com/urfave/cli -[submodule "vendor/contrib.go.opencensus.io/exporter/ocagent"] - path = vendor/contrib.go.opencensus.io/exporter/ocagent - url = https://github.com/census-ecosystem/opencensus-go-exporter-ocagent -[submodule "vendor/github.com/census-instrumentation/opencensus-proto"] - path = vendor/github.com/census-instrumentation/opencensus-proto - url = https://github.com/census-instrumentation/opencensus-proto/ -[submodule "vendor/github.com/mattn/go-ieproxy"] - path = vendor/github.com/mattn/go-ieproxy - url = https://github.com/mattn/go-ieproxy -[submodule "vendor/google.golang.org/grpc"] - path = vendor/google.golang.org/grpc - url = https://github.com/grpc/grpc-go -[submodule "vendor/go.opencensus.io"] - path = vendor/go.opencensus.io - url = https://github.com/census-instrumentation/opencensus-go -[submodule "vendor/github.com/dgrijalva/jwt-go"] - path = vendor/github.com/dgrijalva/jwt-go - url = https://github.com/dgrijalva/jwt-go -[submodule "vendor/github.com/dimchansky/utfbom"] - path = vendor/github.com/dimchansky/utfbom - url = https://github.com/dimchansky/utfbom -[submodule "vendor/github.com/golang/protobuf"] - path = vendor/github.com/golang/protobuf - url = https://github.com/golang/protobuf/ -[submodule "vendor/github.com/grpc-ecosystem/grpc-gateway"] - path = vendor/github.com/grpc-ecosystem/grpc-gateway - url = https://github.com/grpc-ecosystem/grpc-gateway/ -[submodule "vendor/github.com/hashicorp/golang-lru"] - path = vendor/github.com/hashicorp/golang-lru - url = https://github.com/hashicorp/golang-lru/ -[submodule "vendor/golang.org/x/crypto"] - path = vendor/golang.org/x/crypto - url = https://go.googlesource.com/crypto -[submodule "vendor/golang.org/x/net"] - path = vendor/golang.org/x/net - url = https://go.googlesource.com/net -[submodule "vendor/golang.org/x/text"] - path = vendor/golang.org/x/text - url = https://go.googlesource.com/text -[submodule "vendor/golang.org/x/sys"] - path = vendor/golang.org/x/sys - url = https://go.googlesource.com/sys -[submodule "vendor/google.golang.org/genproto"] - path = vendor/google.golang.org/genproto - url = https://github.com/google/go-genproto -[submodule "vendor/google.golang.org/api"] - path = vendor/google.golang.org/api - url = https://code.googlesource.com/google-api-go-client -[submodule "vendor/gopkg.in/ini.v1"] - path = vendor/gopkg.in/ini.v1 - url = https://gopkg.in/ini.v1 -[submodule "vendor/golang.org/x/sync"] - path = vendor/golang.org/x/sync - url = https://go.googlesource.com/sync -[submodule "vendor/gopkg.in/check.v1"] - path = vendor/gopkg.in/check.v1 - url = https://gopkg.in/check.v1 -[submodule "vendor/github.com/kr/pretty"] - path = vendor/github.com/kr/pretty - url = https://github.com/kr/pretty.git -[submodule "vendor/github.com/kr/text"] - path = vendor/github.com/kr/text - url = https://github.com/kr/text.git diff --git a/.travis.yml b/.travis.yml index 63ce2c8a..69202383 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ install: - make s3proxy.jar script: travis_retry ./test/run-tests.sh GoofysTest go: - - 1.11.5 + - 1.16 go_import_path: github.com/kahing/goofys matrix: include: @@ -21,6 +21,7 @@ matrix: - name: "AWS" env: - CLOUD=s3 + - BUCKET_OWNER=kahing.cheung@databricks.com - AWS=- - secure: "WfeYn/4OIdv2loNuvZxO6fUgv7745/EhTBmaaMkB+l/TrFn+apIFItILHVhzIL2qNSzPORtLnbcZxO86HkvMnIYHKFJ79CV1UDR/Hpu92bYrjXTF6GJWlfGarqud2trv+Q2DzIE9BhOHVvh+KuNC+B2etsE43Cqq5hN/q3QYWX1epy214K8OlMmLTc3vtaNtGycNbMVAuGjL8nI6uNdP7t8KmaJgRl2WAcTFcOJSvZ5l6bDr4U6m9VjnBHMYDQLOxmS7sXjebdHT2fudtEXXAW6qMudRFLa3LIuDKw8qqHjk/fqiYNxlN7eC78yJh6Q+sZAW4PV42/KSAacmqngCGlzoysDvEAQMuS98zFPIYs+ZPclcAOagm/Lh8kr8wywzbnXoQYjr0CaS/UOn4A3fCXtn6TGDD8DWBa+buIw9dSP4lC8sQbU6DGWI66pIwS51ppS+O7wPnA/SXw24DBQegNtueULSlHYACPJ5rnXpnp0Zje8hKtgrbGjT9LgtqMvMpCGNAjcaolyZOM+b6C2GuT6MUyu4IAIzydJsR9BK0gZ6c3y6nzMz5p7G/WrAMXiQUJ4RvcMDK4GOIecQnZfRGhFEcWgBQaKLtwh/QzcrIOigPhMNBllsWNJBaQwvp3X4E6ulWN8rI/Z4p/0bBW/YkoZgPhsMMtrOJvbwK5uDAu4=" # AWS_ACCESS_KEY_ID - secure: "G1NqxEHqattdQLH5AIoXr2cGOnLiEN2V9+LtavTAVoreY58lAYuLkdydeCklL8FPohR3cQF/c0h+91SKzdWA7QF1jB6bvTbobjG1/VuslwSpyXJIpbfQWIUizsVQy89+Ez3wCSC+ul4HPjMaVPab6wieLArZzLfZL316FwwN+n/7o7OQs7tlntjxbtd9834rwL7lGHBpw/wyozoZ7Kp+SYSMjbeG4S5yG+KwQ7fcJdgc7zCDPT+zn/fewEKqu8OKxs3UWMvWSf3DkP97R9bBD7QAzD0ZukUn8E2MB7PX1Zd9rk62+Gw2p/ItiGLjC7S/TINZUwToVRgynMbQxbKKa56z3IzqkpskxIGOEW6qm9D1vwyaVsjax8AhN9o3B9AKpOe91YdXXw85G+DMjgj7+ECuIxHQyWxaeoNenpzhuCpYG3+zZONfxjG9HPkhnT77VL73HDgBZGi4fgcQ+of7mln9kwnOqCdFBNdasJf1Jc/FgjPvw9T5d5Gzh3+F49bPHSpsk0d4VUSkjl0pWmiQXQcZKose3P07gpOtbcG9jYYc8XSjCTVTN1G53CLoxFi6kJpmhZqvd/uUjYSMm32tIiFTHSL6Tun3WPIRpkLuCsX2q9UG4vjD2T8DVuPR5abmbyrcg/SMLt86pU4X3TP7rYHPnsC9XzezJi9e6CA9Crg=" # AWS_SECRET_ACCESS_KEY diff --git a/Makefile b/Makefile index c9ff9272..4d33aee8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ run-test: s3proxy.jar ./test/run-tests.sh s3proxy.jar: - wget https://github.com/gaul/s3proxy/releases/download/s3proxy-1.7.0/s3proxy -O s3proxy.jar + wget https://github.com/gaul/s3proxy/releases/download/s3proxy-1.8.0/s3proxy -O s3proxy.jar get-deps: s3proxy.jar go get -t ./... diff --git a/README-gcs.md b/README-gcs.md new file mode 100644 index 00000000..1de323d1 --- /dev/null +++ b/README-gcs.md @@ -0,0 +1,29 @@ +# Google Cloud Storage (GCS) + + +## Prerequisite + +Service Account credentials or user authentication. Ensure that either the service account or user has the proper permissions to the Bucket / Object under GCS. + +To have a successful mount, we require users to have object listing (`storage.objects.list`) permission to the bucket. + +### Service Account credentials + +Create a service account credentials (https://cloud.google.com/iam/docs/creating-managing-service-accounts) and generate the JSON credentials file. + +### User Authentication and `gcloud` Default Authentication +User can authenticate to gcloud's default environment by first installing cloud sdk (https://cloud.google.com/sdk/) and running `gcloud auth application-default login` command. + + +## Using Goofys for GCS + +### With service account credentials file +``` +GOOGLE_APPLICATION_CREDENTIALS="/path/to/creds.json" goofys gs://[BUCKET] /path/to/mount +``` + +### With user authentication (`gcloud auth application-default login`) + +``` +goofys gs://[BUCKET] [MOUNT DIRECTORY] +``` \ No newline at end of file diff --git a/README.md b/README.md index 6bc7b107..564dbb76 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Build Status](https://travis-ci.org/kahing/goofys.svg?branch=master)](https://travis-ci.org/kahing/goofys) [![Github All Releases](https://img.shields.io/github/downloads/kahing/goofys/total.svg)](https://github.com/kahing/goofys/releases/) [![Twitter Follow](https://img.shields.io/twitter/follow/s3goofys.svg?style=social&label=Follow)](https://twitter.com/s3goofys) +[![Stack Overflow Questions](https://img.shields.io/stackexchange/stackoverflow/t/goofys?label=Stack%20Overflow%20questions)](https://stackoverflow.com/search?q=%5Bgoofys%5D+is%3Aquestion) # Overview @@ -18,7 +19,8 @@ close-to-open. # Installation -* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). You may also need to install fuse-utils first. +* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). +You may also need to install fuse too if you want to mount it on startup. * On macOS, install via [Homebrew](https://brew.sh/): @@ -100,9 +102,6 @@ List of non-POSIX behaviors/limitations: * `unlink` returns success even if file is not present * `fsync` is ignored, files are only flushed on `close` -In addition to the items above, the following are supportable but not yet implemented: - * creating files larger than 1TB - ## Compatibility with non-AWS S3 goofys has been tested with the following non-AWS S3 providers: @@ -115,6 +114,7 @@ goofys has been tested with the following non-AWS S3 providers: * Minio (limited) * OpenStack Swift * S3Proxy +* Scaleway * Wasabi Additionally, goofys also works with the following non-S3 object stores: diff --git a/api/api.go b/api/api.go index f8ce2e39..cf606aef 100644 --- a/api/api.go +++ b/api/api.go @@ -25,9 +25,11 @@ func Mount( if flags.DebugS3 { SetCloudLogLevel(logrus.DebugLevel) } + // Mount the file system. mountCfg := &fuse.MountConfig{ FSName: bucketName, + Subtype: "goofys", Options: flags.MountOptions, ErrorLogger: GetStdLogger(NewLogger("fuse"), logrus.ErrorLevel), DisableWritebackCaching: true, @@ -106,6 +108,10 @@ func Mount( if spec.Prefix != "" { bucketName += ":" + spec.Prefix } + case "gs": + config := NewGCSConfig() + bucketName = spec.Bucket + flags.Backend = config } } } diff --git a/api/common/conf_azure.go b/api/common/conf_azure.go index 824584d5..48ce3485 100644 --- a/api/common/conf_azure.go +++ b/api/common/conf_azure.go @@ -269,7 +269,7 @@ func azureFindAccount(client azblob.AccountsClient, account string) (*azblob.End return nil, "", err } - for _, acc := range *accountsRes.Value { + for _, acc := range accountsRes.Values() { if *acc.Name == account { // /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/... parts := strings.SplitN(*acc.ID, "/", 6) @@ -370,7 +370,7 @@ func AzureBlobConfig(endpoint string, location string, storageType string) (conf if key == "" { var keysRes azblob.AccountListKeysResult - keysRes, err = client.ListKeys(context.TODO(), resourceGroup, account) + keysRes, err = client.ListKeys(context.TODO(), resourceGroup, account, azblob.Kerb) if err != nil || len(*keysRes.Keys) == 0 { err = fmt.Errorf("Missing key: configure via AZURE_STORAGE_KEY "+ "or %v/config", configDir) @@ -388,14 +388,21 @@ func AzureBlobConfig(endpoint string, location string, storageType string) (conf key = *(*keysRes.Keys)[0].Value } } else { - return + if key == "" { + return + } else { + // we have the credential already, we + // can't look up the endpoint but we + // can guess that + err = nil + } } } if endpoint == "" { endpoint = "https://" + account + "." + storageType + "." + azure.PublicCloud.StorageEndpointSuffix - azbLog.Debugf("Unable to detect endpoint for account %v, using %v", + azbLog.Infof("Unable to detect endpoint for account %v, using %v", account, endpoint) } diff --git a/api/common/conf_gcs.go b/api/common/conf_gcs.go new file mode 100644 index 00000000..01543f96 --- /dev/null +++ b/api/common/conf_gcs.go @@ -0,0 +1,35 @@ +package common + +import ( + "context" + "golang.org/x/oauth2/google" +) + +type GCSConfig struct { + Credentials *google.Credentials + ChunkSize int // determine the size of chunks when uploading a file to GCS +} + +const ( + defaultGCSChunkSize int = 64 * 1024 * 1024 +) + +// NewGCSConfig returns a GCS Config. +func NewGCSConfig() *GCSConfig { + // Credentials will be checked when initializing GCS bucket to create an authenticated / unauthenticated client + // We allow nil credentials for unauthenticated client. + credentials, err := google.FindDefaultCredentials(context.Background()) + if err == nil { + // authenticated config + return &GCSConfig{ + ChunkSize: defaultGCSChunkSize, + Credentials: credentials, + } + } else { + log.Debugf("Initializing an unauthenticated config: %v", err) + // unauthenticated config + return &GCSConfig{ + ChunkSize: defaultGCSChunkSize, + } + } +} diff --git a/api/common/conf_s3.go b/api/common/conf_s3.go index de904f0e..2401119f 100644 --- a/api/common/conf_s3.go +++ b/api/common/conf_s3.go @@ -53,6 +53,8 @@ type S3Config struct { Credentials *credentials.Credentials Session *session.Session + + BucketOwner string } var s3Session *session.Session diff --git a/api/common/logger.go b/api/common/logger.go index 079f065f..5c321712 100644 --- a/api/common/logger.go +++ b/api/common/logger.go @@ -119,14 +119,14 @@ func GetLogger(name string) *LogHandle { if logger, ok := loggers[name]; ok { if name != "main" && name != "fuse" { - logger.Level = cloudLogLevel + logger.SetLevel(cloudLogLevel) } return logger } else { logger := NewLogger(name) loggers[name] = logger if name != "main" && name != "fuse" { - logger.Level = cloudLogLevel + logger.SetLevel(cloudLogLevel) } return logger } diff --git a/api/common/panic_logger.go b/api/common/panic_logger.go index 8312c21d..3c710dfa 100644 --- a/api/common/panic_logger.go +++ b/api/common/panic_logger.go @@ -27,6 +27,8 @@ type FusePanicLogger struct { Fs fuseutil.FileSystem } +var _ fuseutil.FileSystem = FusePanicLogger{} + func LogPanic(err *error) { if e := recover(); e != nil { log.Errorf("stacktrace from panic: %v \n"+string(debug.Stack()), e) @@ -36,6 +38,10 @@ func LogPanic(err *error) { } } +func (fs FusePanicLogger) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) (err error) { + defer LogPanic(&err) + return fs.Fs.BatchForget(ctx, op) +} func (fs FusePanicLogger) StatFS(ctx context.Context, op *fuseops.StatFSOp) (err error) { defer LogPanic(&err) return fs.Fs.StatFS(ctx, op) diff --git a/bench/Dockerfile.azure b/bench/Dockerfile.azure index f7fcaf7c..6fdab696 100644 --- a/bench/Dockerfile.azure +++ b/bench/Dockerfile.azure @@ -4,7 +4,7 @@ RUN apt-get update && \ apt-get -y install --no-install-recommends git fuse \ # blobfuse dependencies \ pkg-config libfuse-dev cmake libcurl4-gnutls-dev libgnutls28-dev uuid-dev \ - libgcrypt20-dev gcc g++ \ + libgcrypt20-dev libboost-all-dev gcc g++ \ # for running goofys benchmark \ curl python-setuptools python-pip gnuplot-nox imagemagick \ # finally, clean up to make image smaller \ @@ -22,7 +22,7 @@ RUN curl -O https://storage.googleapis.com/golang/go${GOVER}.linux-amd64.tar.gz ARG MAKEFLAGS=-j8 RUN git clone --depth 1 https://github.com/Azure/azure-storage-fuse.git && \ - cd azure-storage-fuse && ./build.sh > /dev/null && make -C build install && \ + cd azure-storage-fuse && bash ./build.sh > /dev/null && make -C build install && \ cd .. && rm -Rf azure-storage-fuse RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ diff --git a/bench/Dockerfile.gcs b/bench/Dockerfile.gcs new file mode 100644 index 00000000..8a51c128 --- /dev/null +++ b/bench/Dockerfile.gcs @@ -0,0 +1,44 @@ +FROM golang:1.14 AS goofys-builder + +# install goofys +WORKDIR $GOPATH/src/github.com/kahing/goofys + +COPY . . + +RUN git init +RUN git submodule update --init --recursive +RUN go build + +# install gcsfuse, the binary is added to /go/bin/gcsfuse +RUN go get -u github.com/googlecloudplatform/gcsfuse + +FROM ubuntu:18.04 +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get -y install --no-install-recommends \ + # gcsfuse dependencies \ + fuse \ + # for running goofys benchmark \ + curl python-setuptools python-pip gnuplot-nox imagemagick awscli \ + # finally, clean up to make image smaller \ + && apt-get clean + +# install catfs, required to run goofys with cache +RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ + mv catfs /usr/bin && chmod 0755 /usr/bin/catfs + +# goofys graph generation +RUN pip install numpy + +ENV PATH=$PATH:/root/go/bin + +# copy go binaries +COPY --from=goofys-builder /go/src/github.com/kahing/goofys/goofys /root/go/bin/goofys +COPY --from=goofys-builder /go/bin/gcsfuse /root/go/bin/gcsfuse + +WORKDIR /root/go/src/github.com/kahing/goofys + +# copy bench scripts +COPY bench bench + +ENTRYPOINT ["/root/go/src/github.com/kahing/goofys/bench/run_bench.sh"] diff --git a/bench/bench.sh b/bench/bench.sh index ea030006..d08cb2ef 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -5,6 +5,7 @@ : ${test:=""} : ${CACHE:="false"} : ${CLEANUP:="true"} +: ${MOUNT_LOG:="mount.log"} iter=10 @@ -49,7 +50,7 @@ else BLOBFUSE="false" fi -$cmd >& mount.log & +$cmd >& $MOUNT_LOG & PID=$! function cleanup { @@ -73,7 +74,7 @@ function cleanup { function cleanup_err { err=$? - cat mount.log + cat $MOUNT_LOG if [ $MOUNTED == 1 ]; then popd >&/dev/null || true rmdir $prefix >&/dev/null || true @@ -99,7 +100,7 @@ function wait_for_mount { done if ! grep -q $mnt /proc/mounts; then echo "$mnt not mounted by $cmd" - cat mount.log + cat $MOUNT_LOG exit 1 fi } @@ -144,7 +145,7 @@ function run_test { fusermount -u $mnt sleep 1 fi - $cmd >& mount.log & + $cmd >& $MOUNT_LOG & PID=$! wait_for_mount pushd "$prefix" >/dev/null @@ -235,7 +236,7 @@ function create_files_parallel { get_howmany $@ (for i in $(seq 1 $howmany); do - echo $i > file$i & true + (echo $i > file$i && echo "created file$i") & true done wait) } diff --git a/bench/run_bench.sh b/bench/run_bench.sh index b82e419b..63048586 100755 --- a/bench/run_bench.sh +++ b/bench/run_bench.sh @@ -24,6 +24,7 @@ dir=$(dirname $0) mkdir bench-mnt S3FS_CACHE="-ouse_cache=/tmp/cache" +GCSFUSE_CACHE="" # by default performs caching GOOFYS_CACHE="--cache /tmp/cache -o allow_other" if [ "$CACHE" == "false" ]; then @@ -51,6 +52,12 @@ if [ -f /usr/local/bin/blobfuse ]; then GOOFYS_ENDPOINT="" fi +if [ -f /root/go/bin/gcsfuse ]; then + PROG="gcsfuse" + GOOFYS_BUCKET="gs://${BUCKET}" + GOOFYS_ENDPOINT="" +fi + rm -f $dir/bench.goofys $dir/bench.$PROG $dir/bench.png $dir/bench-cached.png export BUCKET @@ -59,6 +66,7 @@ export ENDPOINT S3FS="s3fs -f -ostat_cache_expire=1 ${S3FS_CACHE} ${S3FS_ENDPOINT} $BUCKET bench-mnt" GOOFYS="goofys -f --stat-cache-ttl 1s --type-cache-ttl 1s ${GOOFYS_CACHE} ${GOOFYS_ENDPOINT} ${GOOFYS_BUCKET} bench-mnt" BLOBFUSE="blobfuse bench-mnt --container-name=$BUCKET --tmp-path=/tmp/cache" +GCSFUSE="gcsfuse --temp-dir /tmp/cache --foreground $GCSFUSE_CACHE $BUCKET bench-mnt" LOCAL="cat" iter=10 @@ -108,6 +116,10 @@ for fs in $PROG goofys; do FS=$BLOBFUSE CREATE_FS=$FS ;; + gcsfuse) + FS=$GCSFUSE + CREATE_FS=$FS + ;; cat) FS=$LOCAL CREATE_FS=$FS diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..313fdd39 --- /dev/null +++ b/go.mod @@ -0,0 +1,37 @@ +module github.com/kahing/goofys + +go 1.14 + +require ( + cloud.google.com/go/storage v1.14.0 + github.com/Azure/azure-pipeline-go v0.2.3 + github.com/Azure/azure-sdk-for-go v61.4.0+incompatible + github.com/Azure/azure-storage-blob-go v0.14.0 + github.com/Azure/go-autorest/autorest v0.11.18 + github.com/Azure/go-autorest/autorest/adal v0.9.13 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 + github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect + github.com/aws/aws-sdk-go v1.44.37 + github.com/go-ole/go-ole v1.2.5 // indirect + github.com/gofrs/uuid v4.2.0+incompatible + github.com/google/uuid v1.2.0 + github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 // indirect + github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8 + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 + github.com/mitchellh/go-homedir v1.1.0 + github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b + github.com/sevlyar/go-daemon v0.1.5 + github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 + github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 + github.com/smartystreets/assertions v1.2.0 // indirect + github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a + golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.2.0 + google.golang.org/api v0.43.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c + gopkg.in/ini.v1 v1.51.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..ccb6a0a7 --- /dev/null +++ b/go.sum @@ -0,0 +1,764 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0 h1:oqqswrt4x6b9OGBnNqdssxBl1xf0rSUNjU2BR4BZar0= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go v32.1.0+incompatible h1:BPMo+kL2AuaRMHW4hA1glEvquAmBvH5nEuMmfe3+D38= +github.com/Azure/azure-sdk-for-go v32.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v61.4.0+incompatible h1:BF2Pm3aQWIa6q9KmxyF1JYKYXtVw67vtvu2Wd54NGuY= +github.com/Azure/azure-sdk-for-go v61.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 h1:2aS42GGoGBG81HdmxlNh8Sq29TlPNW1AkexyoRcIc18= +github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 h1:8DQB8yl7aLQuP+nuR5e2RO6454OvFlSTXXaNHshc16s= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.7/go.mod h1:AkzUsqkrdmNhfP2i54HqINVQopw0CLDnvHpJ88Zz1eI= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= +github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= +github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 h1:VNVgNDgai+7LqdALaNTNKDD+EriKUN30tvQH6bgYjh8= +github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026/go.mod h1:Opf9rtYVq0eTyX+aRVmRO9hE8ERAozcdrBxWG9Q6mkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5 h1:LkLYmg9wdUlhXv9H31druPMUFPSrpN9sWMxPCTvu5dI= +github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5/go.mod h1:xtZnnLxHY6QniCrfIpTwr5h8mH8zr+jsOFj0y9cfyp4= +github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8 h1:uv+7DeBJF6+a54ZUihuXR+uTzhv2JslllK5ByILxxbg= +github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8/go.mod h1:liOmRdJd8oTwHCQ5M9JemRE3CebdlYcZWLk+ZjQeuq0= +github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= +github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff/go.mod h1:gJWba/XXGl0UoOmBQKRWCJdHrr3nE0T65t6ioaj3mLI= +github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11/go.mod h1:+DBdDyfoO2McrOyDemRBq0q9CMEByef7sYl7JH5Q3BI= +github.com/jacobsa/reqtrace v0.0.0-20150505043853-245c9e0234cb/go.mod h1:ivcmUvxXWjb27NsPEaiYK7AidlZXS7oQ5PowUS9z3I4= +github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3/go.mod h1:mPvulh9VKXvo+yOlrD4VYOOYuLdZJ36wa/5QIrtXvWs= +github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6/go.mod h1:JEWKD6V8xETMW+DEv+IQVz++f8Cn8O/X0HPeDY3qNis= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sevlyar/go-daemon v0.1.5 h1:Zy/6jLbM8CfqJ4x4RPr7MJlSKt90f00kNM1D401C+Qk= +github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= +github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 h1:wsZAb4P8F7uQSwsnxE1gk9AHCcc5U0wvyDzcLwFY0Eo= +github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 h1:cCFqi2EDJ/eRGFkhvqpkh/6t8dubmVVejQatc7BxG2A= +github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a h1:Q372CokLjPoo4bx4qmpjaxZ0yEJR05rZkeEyXcEvSTY= +github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a/go.mod h1:pLnliFgEwGWwRKy8217bLxXBDmG6pBQeZcIzf+m7U+k= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 h1:D7nTwh4J0i+5mW4Zjzn5omvlr6YBcWywE6KOcatyNxY= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0 h1:4sAyIHT6ZohtAQDoxws+ez7bROYmUlOVvsUscYCDTqA= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6 h1:4Xw2NwItrJOFR5s6PnK98PI6Bgw1LhMP1j/rO5WP0S4= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/aws_test.go b/internal/aws_test.go index e9347c7f..fc259afe 100644 --- a/internal/aws_test.go +++ b/internal/aws_test.go @@ -19,6 +19,7 @@ import ( . "gopkg.in/check.v1" "fmt" + "sync" "syscall" "time" ) @@ -56,6 +57,18 @@ func (s *AwsTest) TestBucket404(t *C) { type S3BucketEventualConsistency struct { *S3Backend + // a list of blobs ever put by this backend, we speculatively + // retry on these blobs to workaround eventual consistency + mu sync.RWMutex + blobs map[string]bool +} + +func NewS3BucketEventualConsistency(s *S3Backend) *S3BucketEventualConsistency { + return &S3BucketEventualConsistency{ + s, + sync.RWMutex{}, + make(map[string]bool), + } } func (s *S3BucketEventualConsistency) Init(key string) (err error) { @@ -75,6 +88,33 @@ func (s *S3BucketEventualConsistency) Init(key string) (err error) { return } +func (s *S3BucketEventualConsistency) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { + var err error + var res *HeadBlobOutput + + for i := 0; i < 10; i++ { + res, err = s.S3Backend.HeadBlob(param) + switch err { + case syscall.ENXIO: + s3Log.Infof("waiting for bucket") + time.Sleep((time.Duration(i) + 1) * 2 * time.Second) + case syscall.ENOENT: + s.mu.RLock() + _, ok := s.blobs[param.Key] + s.mu.RUnlock() + + if ok { + s3Log.Infof("waiting for blob: %v", param.Key) + time.Sleep((time.Duration(i) + 1) * 20 * time.Millisecond) + } + default: + return res, err + } + } + + return res, err +} + func (s *S3BucketEventualConsistency) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { for i := 0; i < 10; i++ { res, err := s.S3Backend.ListBlobs(param) @@ -136,6 +176,10 @@ func (s *S3BucketEventualConsistency) CopyBlob(param *CopyBlobInput) (*CopyBlobO } func (s *S3BucketEventualConsistency) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { + s.mu.Lock() + s.blobs[param.Key] = true + s.mu.Unlock() + for i := 0; i < 10; i++ { res, err := s.S3Backend.PutBlob(param) switch err { diff --git a/internal/backend.go b/internal/backend.go index 4da7eb55..414e291d 100644 --- a/internal/backend.go +++ b/internal/backend.go @@ -26,6 +26,7 @@ import ( ) type Capabilities struct { + // set this to true to disable parallel upload NoParallelMultipart bool MaxMultipartSize uint64 // indicates that the blob store has native support for directories @@ -162,7 +163,7 @@ type MultipartBlobCommitInput struct { Parts []*string NumParts uint32 - // for GCS + // additional backend specific data backendData interface{} } @@ -278,10 +279,41 @@ func (p sortBlobItemOutput) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (l ListBlobsInput) String() string { + return fmt.Sprintf("ListBlobsInput{Prefix: %v, Delim: %v, StartAfter: %v, MaxKeys: %v, ContToken: %v}", + NilStr(l.Prefix), + NilStr(l.Delimiter), + NilStr(l.StartAfter), + NilUint32(l.MaxKeys), + NilStr(l.ContinuationToken)) +} + +func (b GetBlobInput) String() string { + return fmt.Sprintf("GetBlobInput{Key: %s, Start: %d, Count: %d}", b.Key, b.Start, b.Count) +} + func (b BlobItemOutput) String() string { return fmt.Sprintf("%v: %v", *b.Key, b.Size) } +func (b PutBlobInput) String() string { + return fmt.Sprintf("PutBlobInput{Key: %s, Size: %d, ContentType: %s}", b.Key, NilUint64(b.Size), + NilStr(b.ContentType)) +} + +func (b CopyBlobInput) String() string { + return fmt.Sprintf("CopyBlobInput{Src: %s, Dest: %s, Size: %d}", b.Source, b.Destination, b.Size) +} + +func (b MultipartBlobBeginInput) String() string { + return fmt.Sprintf("MultipartBlobBeginInput{Key: %s, ContentType: %s}", b.Key, NilStr(b.ContentType)) +} + +func (b MultipartBlobAddInput) String() string { + return fmt.Sprintf("MultipartBlobAddInput{Key: %s, Offset: %d, Size: %d, Last: %v}", + NilStr(b.Commit.Key), b.Offset, b.Size, b.Last) +} + func (b BlobPrefixOutput) String() string { return fmt.Sprintf("%v", *b.Prefix) } diff --git a/internal/backend_adlv1.go b/internal/backend_adlv1.go index 67995b6a..3226a9ea 100644 --- a/internal/backend_adlv1.go +++ b/internal/backend_adlv1.go @@ -27,8 +27,8 @@ import ( "syscall" "time" + uuid "github.com/gofrs/uuid" "github.com/jacobsa/fuse" - uuid "github.com/satori/go.uuid" "github.com/sirupsen/logrus" adl "github.com/Azure/azure-sdk-for-go/services/datalake/store/2016-11-01/filesystem" @@ -352,8 +352,8 @@ func (b *ADLv1) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { continuationToken = param.StartAfter } - _, prefixes, items, err := b.appendToListResults(nilStr(param.Prefix), - recursive, nilStr(continuationToken), param.MaxKeys, nil, nil) + _, prefixes, items, err := b.appendToListResults(NilStr(param.Prefix), + recursive, NilStr(continuationToken), param.MaxKeys, nil, nil) if err == fuse.ENOENT { err = nil } else if err != nil { diff --git a/internal/backend_adlv2.go b/internal/backend_adlv2.go index 2eafa49a..baa6d3e3 100644 --- a/internal/backend_adlv2.go +++ b/internal/backend_adlv2.go @@ -221,6 +221,32 @@ func decodeADLv2Error(body io.Reader) (adlErr adl2.DataLakeStorageError, err err return } +func adlv2ErrLogHeaders(errCode string, resp *http.Response) { + switch errCode { + case "MissingRequiredHeader", "UnsupportedHeader": + var s strings.Builder + for k, _ := range resp.Request.Header { + s.WriteString(k) + s.WriteString(" ") + } + adl2Log.Errorf("%v, sent: %v", errCode, s.String()) + case "InvalidHeaderValue": + var s strings.Builder + for k, v := range resp.Request.Header { + if k != "Authorization" { + s.WriteString(k) + s.WriteString(":") + s.WriteString(v[0]) + s.WriteString(" ") + } + } + adl2Log.Errorf("%v, sent: %v", errCode, s.String()) + case "InvalidSourceUri": + adl2Log.Errorf("SourceUri: %v", + resp.Request.Header.Get("X-Ms-Rename-Source")) + } +} + func mapADLv2Error(resp *http.Response, err error, rawError bool) error { if resp == nil { @@ -258,20 +284,7 @@ func mapADLv2Error(resp *http.Response, err error, rawError bool) error { } adlErr, err := decodeADLv2Error(resp.Body) if err == nil { - switch *adlErr.Error.Code { - case "MissingRequiredHeader", "UnsupportedHeader": - var s strings.Builder - for k, _ := range resp.Request.Header { - s.WriteString(k) - s.WriteString(" ") - } - adl2Log.Errorf("%v, sent: %v", - *adlErr.Error.Code, - s.String()) - case "InvalidSourceUri": - adl2Log.Errorf("SourceUri: %v", - resp.Request.Header.Get("X-Ms-Rename-Source")) - } + adlv2ErrLogHeaders(*adlErr.Error.Code, resp) } case http.StatusPreconditionFailed: return syscall.EAGAIN @@ -288,9 +301,29 @@ func mapADLv2Error(resp *http.Response, err error, rawError bool) error { return syscall.EINVAL } } - } else { - return err + } else if resp.StatusCode == http.StatusOK && err != nil { + // trying to capture this error: + // autorest.DetailedError{Original:(*errors.errorString)(0xc0003eb3f0), + // PackageType:"storagedatalake.adl2PathClient", + // Method:"List", StatusCode:200, Message:"Failure + // responding to request", ServiceError:[]uint8(nil), + // Response:(*http.Response)(0xc0016517a0)} + // ("storagedatalake.adl2PathClient#List: Failure + // responding to request: StatusCode=200 -- Original + // Error: Error occurred reading http.Response#Body - + // Error = 'read tcp + // 10.20.255.49:34194->52.239.155.98:443: read: + // connection reset by peer'") + if detailedErr, ok := err.(autorest.DetailedError); ok { + if detailedErr.Method == "List" && + strings.Contains(detailedErr.Error(), + "read: connection reset by peer") { + return syscall.ECONNRESET + } + } } + + return err } func getHeader(resp *http.Response, key string) *string { @@ -341,6 +374,42 @@ func (b *ADLv2) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { return &res.HeadBlobOutput, nil } +// autorest handles retry based on request errors but doesn't retry on +// reading body. List is idempotent anyway so we can retry it here +func (b *ADLv2) listBlobs(param *ListBlobsInput, maxResults *int32) (adl2PathList, error) { + var err error + var res adl2PathList + + // autorest's DefaultMaxRetry is 3 which seems wrong. Also + // read errors are transient and should probably be retried more + for attempt := 0; attempt < 30; attempt++ { + res, err = b.client.List(context.TODO(), param.Delimiter == nil, b.bucket, + NilStr(param.Prefix), NilStr(param.ContinuationToken), maxResults, + nil, "", nil, "") + err = mapADLv2Error(res.Response.Response, err, false) + if err == nil { + break + } else if err != syscall.ECONNRESET { + return res, err + } else { + // autorest's DefaultRetryDuration is 30s but + // that's for failed requests. Read errors is + // probably more transient and should be + // retried faster + if !autorest.DelayForBackoffWithCap( + 30*time.Millisecond, + 0, + attempt, + res.Response.Response.Request.Context().Done()) { + return res, err + } + } + + } + + return res, err +} + func (b *ADLv2) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { if param.Delimiter != nil && *param.Delimiter != "/" { return nil, fuse.EINVAL @@ -351,11 +420,8 @@ func (b *ADLv2) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { maxResults = PInt32(int32(*param.MaxKeys)) } - res, err := b.client.List(context.TODO(), param.Delimiter == nil, b.bucket, - nilStr(param.Prefix), nilStr(param.ContinuationToken), maxResults, - nil, "", nil, "") + res, err := b.listBlobs(param, maxResults) if err != nil { - err = mapADLv2Error(res.Response.Response, err, false) if err == fuse.ENOENT { return &ListBlobsOutput{ RequestId: res.Response.Response.Header.Get(ADL2_REQUEST_ID), @@ -413,7 +479,7 @@ func (b *ADLv2) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { items = append(items, BlobItemOutput{ Key: &key, ETag: p.ETag, - LastModified: parseADLv2Time(nilStr(p.LastModified)), + LastModified: parseADLv2Time(NilStr(p.LastModified)), Size: uint64(p.contentLength()), }) } @@ -505,7 +571,7 @@ func (b *ADLv2) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { } res, err := b.client.Read(context.TODO(), b.bucket, param.Key, bytes, - "", nil, nilStr(param.IfMatch), "", "", "", + "", nil, NilStr(param.IfMatch), "", "", "", "", nil, "") if err != nil { return nil, mapADLv2Error(res.Response.Response, err, false) @@ -567,7 +633,7 @@ func (b *ADLv2) toADLProperties(metadata map[string]*string) string { func (b *ADLv2) create(key string, pathType adl2.PathResourceType, contentType *string, metadata map[string]*string, leaseId string) (resp autorest.Response, err error) { resp, err = b.client.Create(context.TODO(), b.bucket, key, - pathType, "", "", "", "", "", "", "", nilStr(contentType), + pathType, "", "", "", "", "", "", "", NilStr(contentType), "", "", "", "", leaseId, "", b.toADLProperties(metadata), "", "", "", "", "", "", "", "", "", "", "", nil, "") if err != nil { @@ -641,7 +707,7 @@ func (b *ADLv2) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { return nil, err } - flush, err := b.flush(param.Key, size, nilStr(param.ContentType), "") + flush, err := b.flush(param.Key, size, NilStr(param.ContentType), "") if err != nil { return nil, err } @@ -694,7 +760,7 @@ func (b *ADLv2) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBl } commitData := &ADLv2MultipartBlobCommitInput{ - ContentType: nilStr(param.ContentType), + ContentType: NilStr(param.ContentType), RenewLeaseStop: make(chan bool, 1), } diff --git a/internal/backend_azblob.go b/internal/backend_azblob.go index 4f574c00..b83fedbb 100644 --- a/internal/backend_azblob.go +++ b/internal/backend_azblob.go @@ -420,7 +420,7 @@ func nilMetadata(m map[string]*string) map[string]string { metadata := make(map[string]string) for k, v := range m { k = strings.ToLower(k) - metadata[k] = nilStr(v) + metadata[k] = NilStr(v) } return metadata } @@ -443,7 +443,7 @@ func (b *AZBlob) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { } blob := c.NewBlobURL(param.Key) - resp, err := blob.GetProperties(context.TODO(), azblob.BlobAccessConditions{}) + resp, err := blob.GetProperties(context.TODO(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -470,13 +470,6 @@ func (b *AZBlob) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { }, nil } -func nilStr(v *string) string { - if v == nil { - return "" - } else { - return *v - } -} func nilUint32(v *uint32) uint32 { if v == nil { return 0 @@ -499,11 +492,11 @@ func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { prefixes := make([]BlobPrefixOutput, 0) items := make([]BlobItemOutput, 0) - var blobItems []azblob.BlobItem + var blobItems []azblob.BlobItemInternal var nextMarker *string options := azblob.ListBlobsSegmentOptions{ - Prefix: nilStr(param.Prefix), + Prefix: NilStr(param.Prefix), MaxResults: int32(nilUint32(param.MaxKeys)), Details: azblob.BlobListingDetails{ // blobfuse (following wasb) convention uses @@ -523,7 +516,7 @@ func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { azblob.Marker{ param.ContinuationToken, }, - nilStr(param.Delimiter), + NilStr(param.Delimiter), options) if err != nil { return nil, mapAZBError(err) @@ -561,6 +554,15 @@ func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { } } + if len(blobItems) == 1 && len(blobItems[0].Name) <= len(options.Prefix) && strings.HasSuffix(options.Prefix, "/") { + // There is only 1 result and that one result does not have the desired prefix. This can + // happen if we ask for ListBlobs under /some/path/ and the result is List(/some/path). This + // means the prefix we are listing is a blob => So return empty response to indicate that + // this prefix should not be treated a directory by goofys. + // NOTE: This undesired behaviour happens only on azblob when hierarchial namespaces are + // enabled. + return &ListBlobsOutput{}, nil + } var sortItems bool for idx, _ := range blobItems { @@ -709,8 +711,7 @@ func (b *AZBlob) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { src := c.NewBlobURL(param.Source) dest := c.NewBlobURL(param.Destination) - resp, err := dest.StartCopyFromURL(context.TODO(), src.URL(), nilMetadata(param.Metadata), - azblob.ModifiedAccessConditions{}, azblob.BlobAccessConditions{}) + resp, err := dest.StartCopyFromURL(context.TODO(), src.URL(), azblob.Metadata{}, azblob.ModifiedAccessConditions{}, azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}) if err != nil { return nil, mapAZBError(err) } @@ -719,7 +720,7 @@ func (b *AZBlob) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { time.Sleep(50 * time.Millisecond) var copy *azblob.BlobGetPropertiesResponse - for copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}); err == nil; copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}) { + for copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}); err == nil; copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) { // if there's a new copy, we can only assume the last one was done if copy.CopyStatus() != azblob.CopyStatusPending || copy.CopyID() != resp.CopyID() { break @@ -751,7 +752,7 @@ func (b *AZBlob) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { ModifiedAccessConditions: azblob.ModifiedAccessConditions{ IfMatch: ifMatch, }, - }, false) + }, false, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -802,9 +803,9 @@ func (b *AZBlob) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { resp, err := blob.Upload(context.TODO(), body, azblob.BlobHTTPHeaders{ - ContentType: nilStr(param.ContentType), + ContentType: NilStr(param.ContentType), }, - nilMetadata(param.Metadata), azblob.BlobAccessConditions{}) + nilMetadata(param.Metadata), azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -841,7 +842,7 @@ func (b *AZBlob) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobA atomic.AddUint32(¶m.Commit.NumParts, 1) _, err = blob.StageBlock(context.TODO(), base64BlockId, param.Body, - azblob.LeaseAccessConditions{}, nil) + azblob.LeaseAccessConditions{}, nil, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -871,7 +872,7 @@ func (b *AZBlob) MultipartBlobCommit(param *MultipartBlobCommitInput) (*Multipar resp, err := blob.CommitBlockList(context.TODO(), parts, azblob.BlobHTTPHeaders{}, nilMetadata(param.Metadata), - azblob.BlobAccessConditions{}) + azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return nil, mapAZBError(err) } diff --git a/internal/backend_gcs.go b/internal/backend_gcs.go new file mode 100644 index 00000000..fb8efdac --- /dev/null +++ b/internal/backend_gcs.go @@ -0,0 +1,469 @@ +package internal + +import ( + "github.com/kahing/goofys/api/common" + + "bytes" + "context" + "fmt" + "io" + "net" + "path" + "strings" + "syscall" + + "cloud.google.com/go/storage" + "github.com/jacobsa/fuse" + + "golang.org/x/sync/errgroup" + syncsem "golang.org/x/sync/semaphore" + + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type GCSBackend struct { + bucketName string + config *common.GCSConfig // stores user and bucket configuration + cap Capabilities + bucket *storage.BucketHandle // provides set of methods to operate on a bucket + logger *common.LogHandle // logger for GCS backend +} + +const ( + maxListKeys int = 1000 // the max limit for number of elements during listObjects +) + +type GCSMultipartBlobCommitInput struct { + cancel context.CancelFunc // useful to abort a multipart upload in GCS + writer *storage.Writer // used to emulate mpu under GCS, which currently used a single gcsWriter +} + +// NewGCS initializes a GCS Backend. +// It creates an authenticated client or unauthenticated client based on existing credentials in the environment. +func NewGCS(bucket string, config *common.GCSConfig) (*GCSBackend, error) { + var client *storage.Client + var err error + + // TODO: storage.NewClient has automated mechanisms to set up credentials together with HTTP settings. + // Currently, we are using config.Credentials only to differentiate between creating an authenticated or + // unauthenticated client not using it to initialize a client. + + // If config.Credentials are configured, we'll get an authenticated client. + if config.Credentials != nil { + client, err = storage.NewClient(context.Background()) + } else { + // otherwise we will get an unauthenticated client. option.WithoutAuthentication() is necessary + // because the API will generate an error if it could not find credentials and this option is unset. + client, err = storage.NewClient(context.Background(), option.WithoutAuthentication()) + } + + if err != nil { + return nil, err + } + + return &GCSBackend{ + config: config, + bucketName: bucket, + bucket: client.Bucket(bucket), + cap: Capabilities{ + MaxMultipartSize: 5 * 1024 * 1024 * 1024, + Name: "gcs", + // parallel multipart upload is not supported in GCS + NoParallelMultipart: true, + }, + logger: common.GetLogger("gcs"), + }, nil +} + +// Init checks user's access to bucket. +func (g *GCSBackend) Init(key string) error { + // We will do a successful mount if the user can list on the bucket. + // This is different other backends because GCS does not differentiate between object not found and + // bucket not found. + prefix, _ := path.Split(key) + _, err := g.ListBlobs(&ListBlobsInput{ + MaxKeys: PUInt32(1), + Prefix: PString(prefix), + }) + g.logger.Debugf("INIT GCS: ListStatus = %s", getDebugResponseStatus(err)) + if err == syscall.ENXIO { + return fmt.Errorf("bucket %v does not exist", g.bucketName) + } + // Errors can be returned directly since ListBlobs converts them to syscall errors. + return err +} + +func (g *GCSBackend) Capabilities() *Capabilities { + return &g.cap +} + +// Bucket returns the GCSBackend's bucket name. +func (g *GCSBackend) Bucket() string { + return g.bucketName +} + +func getDebugResponseStatus(err error) string { + if err != nil { + return fmt.Sprintf("ERROR: %v", err) + } + return "SUCCESS" +} + +// HeadBlob gets the file object metadata. +func (g *GCSBackend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { + attrs, err := g.bucket.Object(param.Key).Attrs(context.Background()) + g.logger.Debugf("HEAD %v = %v", param.Key, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + return &HeadBlobOutput{ + BlobItemOutput: BlobItemOutput{ + Key: &attrs.Name, + ETag: &attrs.Etag, + LastModified: &attrs.Updated, + Size: uint64(attrs.Size), + StorageClass: &attrs.StorageClass, + }, + ContentType: &attrs.ContentType, + IsDirBlob: strings.HasSuffix(param.Key, "/"), + Metadata: PMetadata(attrs.Metadata), + }, nil +} + +func (g *GCSBackend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { + query := storage.Query{ + Prefix: NilStr(param.Prefix), + Delimiter: NilStr(param.Delimiter), + StartOffset: NilStr(param.StartAfter), + } + objectIterator := g.bucket.Objects(context.Background(), &query) + + // Set max keys, a number > 0 is required by the SDK. + maxKeys := int(NilUint32(param.MaxKeys)) + if maxKeys == 0 { + maxKeys = maxListKeys // follow the default JSON API mechanism to return 1000 items if maxKeys is not set. + } + + pager := iterator.NewPager(objectIterator, maxKeys, NilStr(param.ContinuationToken)) + + var entries []*storage.ObjectAttrs + nextToken, err := pager.NextPage(&entries) + g.logger.Debugf("LIST %s : %s", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + var nextContToken *string + if nextToken != "" { + nextContToken = &nextToken + } + + var prefixes []BlobPrefixOutput + var items []BlobItemOutput + for _, entry := range entries { + // if blob is a prefix, then Prefix field will be set + if entry.Prefix != "" { + prefixes = append(prefixes, BlobPrefixOutput{&entry.Prefix}) + } else if entry.Name != "" { // otherwise for actual blob, Name field will set + items = append(items, BlobItemOutput{ + Key: &entry.Name, + ETag: &entry.Etag, + LastModified: &entry.Updated, + Size: uint64(entry.Size), + StorageClass: &entry.StorageClass, + }) + } else { + log.Errorf("LIST Unknown object: %v", entry) + } + } + + return &ListBlobsOutput{ + Prefixes: prefixes, + Items: items, + NextContinuationToken: nextContToken, + IsTruncated: nextContToken != nil, + }, nil +} + +func (g *GCSBackend) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { + err := g.bucket.Object(param.Key).Delete(context.Background()) + + g.logger.Debugf("DELETE Object %v = %s ", param.Key, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + return &DeleteBlobOutput{}, nil +} + +// DeleteBlobs deletes multiple GCS blobs. +func (g *GCSBackend) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { + // The go sdk does not support batch requests: https://issuetracker.google.com/issues/142641783 + // So we're using goroutines and errorgroup to delete multiple objects + eg, rootCtx := errgroup.WithContext(context.Background()) + sem := syncsem.NewWeighted(100) + + for _, item := range param.Items { + if err := sem.Acquire(rootCtx, 1); err != nil { + return nil, err + } + curItem := item + eg.Go(func() error { + defer sem.Release(1) + return g.bucket.Object(curItem).Delete(rootCtx) + }) + } + + if err := eg.Wait(); err != nil { + return nil, mapGCSError(err) + } + + return &DeleteBlobsOutput{}, nil +} + +// RenameBlob is not supported for GCS backend. So Goofys will do a CopyBlob followed by DeleteBlob for renames. +func (g *GCSBackend) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { + return nil, syscall.ENOTSUP +} + +// CopyBlob copies a source object to another destination object under the same bucket. +func (g *GCSBackend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { + src := g.bucket.Object(param.Source) + dest := g.bucket.Object(param.Destination) + + copier := dest.CopierFrom(src) + copier.StorageClass = NilStr(param.StorageClass) + copier.Etag = NilStr(param.ETag) + copier.Metadata = NilMetadata(param.Metadata) + + _, err := copier.Run(context.Background()) + g.logger.Debugf("Copy object %s = %s ", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + return &CopyBlobOutput{}, nil +} + +// GetBlob returns a file reader for a GCS object. +func (g *GCSBackend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { + obj := g.bucket.Object(param.Key).ReadCompressed(true) + + var reader *storage.Reader + var err error + if param.Count != 0 { + reader, err = obj.NewRangeReader(context.Background(), int64(param.Start), int64(param.Count)) + } else if param.Start != 0 { + reader, err = obj.NewRangeReader(context.Background(), int64(param.Start), -1) + } else { + // If we don't limit the range, the full object will be read + reader, err = obj.NewReader(context.Background()) + } + + g.logger.Debugf("GET Blob %s = %v", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + // Caveats: the SDK's reader object doesn't provide ETag, StorageClass, and Metadata attributes within a single + // API call, hence we're not returning these information in the output. + // Relevant GitHub issue: https://github.com/googleapis/google-cloud-go/issues/2740 + return &GetBlobOutput{ + HeadBlobOutput: HeadBlobOutput{ + BlobItemOutput: BlobItemOutput{ + Key: PString(param.Key), + LastModified: &reader.Attrs.LastModified, + Size: uint64(reader.Attrs.Size), + }, + ContentType: &reader.Attrs.ContentType, + }, + Body: reader, + }, nil +} + +// PutBlob writes a file to GCS. +func (g *GCSBackend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { + // Handle nil pointer error when param.Body is nil + body := param.Body + if body == nil { + body = bytes.NewReader([]byte("")) + } + + writer := g.bucket.Object(param.Key).NewWriter(context.Background()) + writer.ContentType = NilStr(param.ContentType) + writer.Metadata = NilMetadata(param.Metadata) + // setting chunkSize to be equal to the file size will make this a single request upload + writer.ChunkSize = int(NilUint64(param.Size)) + + _, err := io.Copy(writer, body) + g.logger.Debugf("PUT Blob (to writer) %s = %s ", param, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + err = writer.Close() + g.logger.Debugf("PUT Blob (Flush) %v = %s ", param.Key, getDebugResponseStatus(err)) + if err != nil { + return nil, mapGCSError(err) + } + + attrs := writer.Attrs() + + return &PutBlobOutput{ + ETag: &attrs.Etag, + //LastModified: &attrs.Updated, // this field exist in the upstream open source goofys repo + StorageClass: &attrs.StorageClass, + }, nil +} + +// MultipartBlobBegin begins a multi part blob request. +// Under GCS backend, we'll initialize the gcsWriter object and the context for the multipart blob request here. +func (g *GCSBackend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { + ctx, cancel := context.WithCancel(context.Background()) + writer := g.bucket.Object(param.Key).NewWriter(ctx) + writer.ChunkSize = g.config.ChunkSize + writer.ContentType = NilStr(param.ContentType) + writer.Metadata = NilMetadata(param.Metadata) + + g.logger.Debugf("Multipart Blob BEGIN: %s", param) + + return &MultipartBlobCommitInput{ + Key: ¶m.Key, + Metadata: param.Metadata, + backendData: &GCSMultipartBlobCommitInput{ + writer: writer, + cancel: cancel, + }, + }, nil +} + +// MultipartBlobAdd adds part of blob to the upload request. +// Under GCS backend, we'll write that blob part into the gcsWriter. +// TODO(deka): This is a temporary implementation to allow most tests to run. +// We might change this implementation in the future. +func (g *GCSBackend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { + commitData, ok := param.Commit.backendData.(*GCSMultipartBlobCommitInput) + if !ok { + panic("Incorrect commit data type") + } + + // Handle nil pointer error when param.Body is nil + body := param.Body + if body == nil { + body = bytes.NewReader([]byte("")) + } + + n, err := io.Copy(commitData.writer, body) + g.logger.Debugf("Multipart Blob ADD %s bytesWritten: %v = %s", param, n, getDebugResponseStatus(err)) + if err != nil { + commitData.cancel() + return nil, err + } + + return &MultipartBlobAddOutput{}, nil +} + +func (g *GCSBackend) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { + commitData, ok := param.backendData.(*GCSMultipartBlobCommitInput) + if !ok { + panic("Incorrect commit data type") + } + g.logger.Debugf("Multipart Blob ABORT %v", param.Key) + commitData.cancel() + + return &MultipartBlobAbortOutput{}, nil +} + +func (g *GCSBackend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { + commitData, ok := param.backendData.(*GCSMultipartBlobCommitInput) + if !ok { + panic("Incorrect commit data type") + } + + // Flushing a writer will make GCS to fully upload the buffer + err := commitData.writer.Close() + g.logger.Debugf("Multipart Blob COMMIT %v = %s ", param.Key, getDebugResponseStatus(err)) + if err != nil { + commitData.cancel() + return nil, mapGCSError(err) + } + attrs := commitData.writer.Attrs() + + return &MultipartBlobCommitOutput{ + ETag: &attrs.Etag, + }, nil +} + +func (g *GCSBackend) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { + // No-op: GCS expires a resumable session after 7 days automatically + return &MultipartExpireOutput{}, nil +} + +func (g *GCSBackend) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { + err := g.bucket.Delete(context.Background()) + if err != nil { + return nil, mapGCSError(err) + } + return &RemoveBucketOutput{}, nil +} + +func (g *GCSBackend) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { + // Requires an authenticated credentials + err := g.bucket.Create(context.Background(), g.config.Credentials.ProjectID, nil) + if err != nil { + return nil, mapGCSError(err) + } + + return &MakeBucketOutput{}, nil +} + +func (g *GCSBackend) Delegate() interface{} { + return g +} + +// mapGCSError maps an error to syscall / fuse errors. +func mapGCSError(err error) error { + if err == nil { + return nil + } + + if err == storage.ErrObjectNotExist { + return fuse.ENOENT + } + + // this error can be returned during list operation if the bucket does not exist + if err == storage.ErrBucketNotExist { + return syscall.ENXIO + } + + if e, ok := err.(*googleapi.Error); ok { + switch e.Code { + case 409: + return fuse.EEXIST + case 404: + return fuse.ENOENT + // Retryable errors: + // https://cloud.google.com/storage/docs/json_api/v1/status-codes#429_Too_Many_Requests + // https://cloud.google.com/storage/docs/json_api/v1/status-codes#500_Internal_Server_Error + case 429, 500, 502, 503, 504: + return syscall.EAGAIN + default: + // return syscall error if it's not nil + fuseErr := mapHttpError(e.Code) + if fuseErr != nil { + return fuseErr + } + } + } + + if e, ok := err.(net.Error); ok { + if e.Timeout() { + return syscall.ETIMEDOUT + } + } + + return err +} diff --git a/internal/backend_gcs3.go b/internal/backend_gcs3.go index 7c7272f9..e3b4dc72 100644 --- a/internal/backend_gcs3.go +++ b/internal/backend_gcs3.go @@ -34,7 +34,7 @@ type GCS3 struct { *S3Backend } -type GCSMultipartBlobCommitInput struct { +type GCS3MultipartBlobCommitInput struct { Size uint64 ETag *string Prev *MultipartBlobAddInput @@ -45,7 +45,7 @@ func NewGCS3(bucket string, flags *FlagStorage, config *S3Config) (*GCS3, error) if err != nil { return nil, err } - s3Backend.Capabilities().Name = "gcs" + s3Backend.Capabilities().Name = "gcs3" s := &GCS3{S3Backend: s3Backend} s.S3Backend.gcs = true s.S3Backend.cap.NoParallelMultipart = true @@ -125,7 +125,7 @@ func (s *GCS3) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlo Metadata: param.Metadata, UploadId: &location, Parts: make([]*string, 10000), // at most 10K parts - backendData: &GCSMultipartBlobCommitInput{}, + backendData: &GCS3MultipartBlobCommitInput{}, }, nil } @@ -183,9 +183,9 @@ func (s *GCS3) uploadPart(param *MultipartBlobAddInput, totalSize uint64, last b } func (s *GCS3) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { - var commitData *GCSMultipartBlobCommitInput + var commitData *GCS3MultipartBlobCommitInput var ok bool - if commitData, ok = param.Commit.backendData.(*GCSMultipartBlobCommitInput); !ok { + if commitData, ok = param.Commit.backendData.(*GCS3MultipartBlobCommitInput); !ok { panic("Incorrect commit data type") } @@ -210,9 +210,9 @@ func (s *GCS3) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAdd } func (s *GCS3) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { - var commitData *GCSMultipartBlobCommitInput + var commitData *GCS3MultipartBlobCommitInput var ok bool - if commitData, ok = param.backendData.(*GCSMultipartBlobCommitInput); !ok { + if commitData, ok = param.backendData.(*GCS3MultipartBlobCommitInput); !ok { panic("Incorrect commit data type") } diff --git a/internal/backend_gcs_test.go b/internal/backend_gcs_test.go new file mode 100644 index 00000000..6cae756e --- /dev/null +++ b/internal/backend_gcs_test.go @@ -0,0 +1,585 @@ +package internal + +import ( + "github.com/kahing/goofys/api/common" + + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "sort" + "syscall" + + "github.com/jacobsa/fuse" + "golang.org/x/oauth2/google" + "golang.org/x/sync/errgroup" + . "gopkg.in/check.v1" +) + +type GCSTestSpec struct { + objCount int // number of non-prefix blobs uploaded to the bucket + prefixCount int // number of prefix blobs uploaded to the bucket + existingObjKey string // test setup will add a blob with this name + existingObjContent string + existingObjLen uint64 + defaultObjLen uint64 // blobs created in the setup will be of this length + nonPrefixObjects []string + prefixObjects []string + prefixes []string // these are the prefixes of the prefixObjects + metadata map[string]*string // default Metadata for all blobs + contentType *string // default Content-Type for all blobs + // will be reset in SetUpTest, optionally modified on each test, and cleaned up in TearDownTest + additionalBlobs []string +} + +type GCSBackendTest struct { + gcsBackend *GCSBackend + bucketName string + testSpec GCSTestSpec + chunkSize int +} + +var _ = Suite(&GCSBackendTest{}) + +func (s *GCSBackendTest) getGCSTestConfig() (*common.GCSConfig, error) { + config := common.NewGCSConfig() + config.ChunkSize = s.chunkSize + return config, nil +} + +func (s *GCSBackendTest) getGCSTestBackend(gcsBucket string) (*GCSBackend, error) { + config, err := s.getGCSTestConfig() + if err != nil { + return nil, err + } + spec, err := ParseBucketSpec(gcsBucket) + if err != nil { + return nil, err + } + gcsBackend, err := NewGCS(spec.Bucket, config) + + return gcsBackend, err +} + +func (s *GCSBackendTest) SetUpSuite(c *C) { + if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + c.Skip("Skipping because GOOGLE_APPLICATION_CREDENTIALS variable is unset.") + } + + bktName := "goofys-test-" + RandStringBytesMaskImprSrc(16) + s.bucketName = bktName + s.testSpec = GCSTestSpec{ + objCount: 10, + prefixCount: 5, + defaultObjLen: 16, + metadata: map[string]*string{ + "foo": PString("bar"), + "bar": PString("baz"), + }, + contentType: PString("text/plain"), + } + s.chunkSize = 1 * 1024 * 1024 + s.gcsBackend, _ = s.getGCSTestBackend(fmt.Sprintf("gs://%s", bktName)) + + _, err := s.gcsBackend.MakeBucket(&MakeBucketInput{}) + c.Assert(err, IsNil) + + errGroup, _ := errgroup.WithContext(context.Background()) + objLen := s.testSpec.defaultObjLen + + // create object without prefix + // files is stored as 00_blob, 01_blob, ... 09_blob (10 items) + for i := 0; i < s.testSpec.objCount; i++ { + objKey := fmt.Sprintf("%02d_blob", i) + objContent := RandStringBytesMaskImprSrc(int(objLen)) + + errGroup.Go(func() error { + _, err := s.gcsBackend.PutBlob(&PutBlobInput{ + Key: objKey, + Body: bytes.NewReader([]byte(objContent)), + Metadata: s.testSpec.metadata, + ContentType: s.testSpec.contentType, + }) + c.Assert(err, IsNil) + + // verify object attr after a put blob + objAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, IsNil) + c.Assert(NilStr(objAttr.Key), Equals, objKey) + c.Assert(objAttr.Size, Equals, uint64(int(objLen))) + c.Assert(NilStr(objAttr.ContentType), Equals, NilStr(s.testSpec.contentType)) + c.Assert(objAttr.Metadata, DeepEquals, s.testSpec.metadata) + + return err + }) + + // set the existing object to be the first object + if s.testSpec.existingObjKey == "" { + s.testSpec.existingObjKey = objKey + s.testSpec.existingObjContent = objContent + s.testSpec.existingObjLen = objLen + } + s.testSpec.nonPrefixObjects = append(s.testSpec.nonPrefixObjects, objKey) + } + + // create object with prefix + // file is stored as 00_prefix/blob, 01_prefix/blob, ..., 04_prefix/blob (5 items) + for i := 0; i < s.testSpec.prefixCount; i++ { + objKey := fmt.Sprintf("%02d_prefix/blob", i) + objContent := RandStringBytesMaskImprSrc(int(objLen)) + errGroup.Go(func() error { + _, err := s.gcsBackend.PutBlob(&PutBlobInput{ + Key: objKey, + Body: bytes.NewReader([]byte(objContent)), + Metadata: s.testSpec.metadata, + ContentType: s.testSpec.contentType, + }) + c.Assert(err, IsNil) + + objAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, IsNil) + c.Assert(NilStr(objAttr.Key), Equals, objKey) + c.Assert(objAttr.Size, Equals, uint64(int(objLen))) + c.Assert(NilStr(objAttr.ContentType), Equals, NilStr(s.testSpec.contentType)) + c.Assert(objAttr.Metadata, DeepEquals, s.testSpec.metadata) + return err + }) + + s.testSpec.prefixObjects = append(s.testSpec.prefixObjects, objKey) + dir, _ := path.Split(objKey) + s.testSpec.prefixes = append(s.testSpec.prefixes, dir) + } + + err = errGroup.Wait() + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TearDownSuite(c *C) { + if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + c.Skip("Skipping because GOOGLE_APPLICATION_CREDENTIALS variable is unset.") + } + out, err := s.gcsBackend.ListBlobs(&ListBlobsInput{}) + c.Assert(err, IsNil) + + var items []string + for _, item := range out.Items { + items = append(items, NilStr(item.Key)) + } + + _, err = s.gcsBackend.DeleteBlobs(&DeleteBlobsInput{Items: items}) + c.Assert(err, IsNil) + + _, err = s.gcsBackend.RemoveBucket(&RemoveBucketInput{}) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) SetUpTest(c *C) { + s.testSpec.additionalBlobs = []string{} +} + +func (s *GCSBackendTest) TearDownTest(c *C) { + _, err := s.gcsBackend.DeleteBlobs(&DeleteBlobsInput{s.testSpec.additionalBlobs}) + // successful deletion should be: nil error or not found + c.Assert(err == nil || err == syscall.ENOENT, Equals, true) +} + +func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated(c *C) { + // No error when accessing existing private bucket. + err := s.gcsBackend.Init(s.bucketName) + c.Assert(err, IsNil) + + // Not Found error when accessing nonexistent bucket. + randBktName := RandStringBytesMaskImprSrc(16) + gcsBackend, err := s.getGCSTestBackend(randBktName) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName)) + + // No error when accessing public bucket. + gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_Init_Unauthenticated(c *C) { + defaultCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + defer func() { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCredentials) + }() + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "") + + credentials, err := google.FindDefaultCredentials(context.Background()) + if credentials != nil { + c.Skip("Skipping this test because credentials still exist in the environment.") + } + + // Access error on existing private bucket. + gcsBackend, err := s.getGCSTestBackend(s.bucketName) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(15)) + c.Assert(err, Equals, syscall.EACCES) + + // Not Found error when accessing nonexistent bucket. + randBktName := RandStringBytesMaskImprSrc(16) + gcsBackend, err = s.getGCSTestBackend(randBktName) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName)) + + // No error when accessing public bucket. + gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_Init_Authenticated_ReadOnlyAccess(c *C) { + // READONLY_GOOGLE_APPLICATION_CREDENTIALS is a credential that is authorized to read-only access to the bucket + if os.Getenv("READONLY_GOOGLE_APPLICATION_CREDENTIALS") == "" { + c.Skip("Skipping because READONLY_GOOGLE_APPLICATION_CREDENTIALS is unset.") + } + + defaultCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + defer func() { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCredentials) + }() + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", os.Getenv("READONLY_GOOGLE_APPLICATION_CREDENTIALS")) + + // No error when accessing existing private bucket. + gcsBackend, err := s.getGCSTestBackend(s.bucketName) + c.Assert(err, IsNil) + err = gcsBackend.Init(s.bucketName) + c.Assert(err, IsNil) + + // Access error when trying to modify object. + _, err = gcsBackend.DeleteBlob(&DeleteBlobInput{Key: s.testSpec.existingObjKey}) + c.Assert(err, Equals, syscall.EACCES) + + // Not Found error when accessing nonexistent bucket. + randBktName := RandStringBytesMaskImprSrc(16) + gcsBackend, err = s.getGCSTestBackend(randBktName) + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, ErrorMatches, fmt.Sprintf("bucket %s does not exist", randBktName)) + + // No error when accessing public bucket. + gcsBackend, err = s.getGCSTestBackend("gcp-public-data-nexrad-l2") + c.Assert(err, IsNil) + err = gcsBackend.Init(RandStringBytesMaskImprSrc(16)) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_HeadBlob_NotExist(c *C) { + objKey := RandStringBytesMaskImprSrc(16) + _, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, Equals, fuse.ENOENT) +} + +func (s *GCSBackendTest) TestGCSBackend_GetBlob_NotExist(c *C) { + objKey := RandStringBytesMaskImprSrc(16) + _, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: objKey}) + c.Assert(err, Equals, fuse.ENOENT) +} + +func (s *GCSBackendTest) TestGCSBackend_GetFullBlob(c *C) { + fullBlob, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: s.testSpec.existingObjKey}) + c.Assert(err, IsNil) + defer fullBlob.Body.Close() + + actualContent, err := ioutil.ReadAll(fullBlob.Body) + c.Assert(err, IsNil) + c.Assert(string(actualContent), Equals, s.testSpec.existingObjContent) +} + +func (s *GCSBackendTest) TestGCSBackend_GetPartialBlob(c *C) { + var startOffset uint64 = 4 + var count uint64 = 8 + partialBlob, err := s.gcsBackend.GetBlob(&GetBlobInput{ + Key: s.testSpec.existingObjKey, + Start: startOffset, + Count: count, + }) + c.Assert(err, IsNil) + defer partialBlob.Body.Close() + + actualContent, err := ioutil.ReadAll(partialBlob.Body) + c.Assert(err, IsNil) + c.Assert(string(actualContent), Equals, s.testSpec.existingObjContent[startOffset:startOffset+count]) +} + +func (s *GCSBackendTest) TestGCSBackend_PutBlob_NilBody(c *C) { + objKey := RandStringBytesMaskImprSrc(int(s.testSpec.defaultObjLen)) + s.addToAdditionalBlobs(objKey) + + _, err := s.gcsBackend.PutBlob(&PutBlobInput{Key: objKey, Body: nil}) + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) addToAdditionalBlobs(objKey string) { + s.testSpec.additionalBlobs = append(s.testSpec.additionalBlobs, objKey) +} + +func (s *GCSBackendTest) TestGCSBackend_GetGzipEncodedBlob(c *C) { + objKey := "gzipObj" + s.addToAdditionalBlobs(objKey) + originalContent := RandStringBytesMaskImprSrc(int(s.testSpec.defaultObjLen)) + + // Goofys cannot upload a file with content-encoding: gzip. So we will use GCS sdk directly to create such blob + s.writeGzipEncodedFile(c, objKey, originalContent) + + gzipBlob, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: objKey}) + c.Assert(err, IsNil) + defer gzipBlob.Body.Close() + + actualContent, err := ioutil.ReadAll(gzipBlob.Body) + c.Assert(string(actualContent), Equals, originalContent) +} + +func (s *GCSBackendTest) writeGzipEncodedFile(c *C, objKey string, content string) { + writer := s.gcsBackend.bucket.Object(objKey).NewWriter(context.Background()) + writer.ContentEncoding = "gzip" + _, err := writer.Write([]byte(content)) + defer writer.Close() + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_NoPrefixNoDelim(c *C) { + // list all objects in bucket as items, no prefix because delim is unset + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{}) + c.Assert(err, IsNil) + + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects, s.testSpec.prefixObjects) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_NoPrefixWithDelim(c *C) { + // list all objects but separates items and prefixes + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{Delimiter: PString("/")}) + c.Assert(err, IsNil) + // should contain non prefix objects: 00_blob, ..., 09_blob and prefixes: 00_prefix/, ..., 04_prefix/ + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects, s.testSpec.prefixes) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_WithPrefixNoDelim(c *C) { + // list all objects that have a matching prefix as items + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{Prefix: PString("00")}) + c.Assert(err, IsNil) + // should contain all items under prefix: 00_blob & 00_prefix/blob + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects[:1], s.testSpec.prefixObjects[:1]) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_WithPrefixWithDelim(c *C) { + // lists objects that have a matching prefix and separates non prefix & prefixes + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Prefix: PString("00"), + Delimiter: PString("/"), + }) + c.Assert(err, IsNil) + // should contain 00_blob and 00_prefix/ + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects[:1], s.testSpec.prefixes[:1]) + c.Assert(listOutputs.NextContinuationToken, IsNil) + c.Assert(listOutputs.IsTruncated, Equals, false) +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_StartAfter(c *C) { + cutoffKey := "04_prefix/blob" + // list results all come right on or after StartAfter in lexicographical order + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Delimiter: PString("/"), + StartAfter: PString(cutoffKey), + }) + c.Assert(err, IsNil) + + actualOutputs := s.extractAllNamesFromListOutputs(listOutputs) + // contain elements from 05_blob and 04_prefix/ + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects[5:], + s.testSpec.prefixes[len(s.testSpec.prefixes)-1:]) + + // assert all items are larger or equal to StartAfter prefix (because we used delimiter) in lexicographical order + cutOffPrefix, _ := path.Split(cutoffKey) + for _, item := range actualOutputs { + c.Assert(item >= cutOffPrefix, Equals, true) + } +} + +func (s *GCSBackendTest) TestGCSBackend_ListBlobs_MaxKeysPaginate(c *C) { + totalKeys := s.testSpec.objCount + s.testSpec.prefixCount + maxKeys := 1 + + var nextContToken *string + var actualOutputs []string + + // iterate over pages of maxKeys + for i := 0; i < totalKeys-1; i++ { + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Delimiter: PString("/"), + MaxKeys: PUInt32(uint32(maxKeys)), + ContinuationToken: nextContToken, + }) + c.Assert(err, IsNil) + c.Assert(len(listOutputs.Items)+len(listOutputs.Prefixes), Equals, maxKeys) + c.Assert(listOutputs.NextContinuationToken, NotNil) + nextContToken = listOutputs.NextContinuationToken + actualOutputs = append(actualOutputs, s.extractAllNamesFromListOutputs(listOutputs)...) + } + // remaining object in the last page + listOutputs, err := s.gcsBackend.ListBlobs(&ListBlobsInput{ + Delimiter: PString("/"), + MaxKeys: PUInt32(uint32(maxKeys)), + ContinuationToken: nextContToken, + }) + actualOutputs = append(actualOutputs, s.extractAllNamesFromListOutputs(listOutputs)...) + c.Assert(err, IsNil) + c.Assert(len(listOutputs.Items)+len(listOutputs.Prefixes) <= maxKeys, Equals, true) + c.Assert(listOutputs.NextContinuationToken, IsNil) + s.checkListOutputs(c, actualOutputs, s.testSpec.nonPrefixObjects, s.testSpec.prefixes) +} + +func (s *GCSBackendTest) extractAllNamesFromListOutputs(listOutputs *ListBlobsOutput) []string { + var actualOutputs []string + for _, item := range listOutputs.Items { + actualOutputs = append(actualOutputs, NilStr(item.Key)) + } + for _, prefix := range listOutputs.Prefixes { + actualOutputs = append(actualOutputs, NilStr(prefix.Prefix)) + } + return actualOutputs +} + +// This check the actual outputs against list of expected outputs +func (s *GCSBackendTest) checkListOutputs(c *C, actualOutputs []string, expectedOutputsArgs ...[]string) { + var expectedResults []string + for _, expectedOutputs := range expectedOutputsArgs { + expectedResults = append(expectedResults, expectedOutputs...) + } + sort.Strings(expectedResults) + sort.Strings(actualOutputs) + + c.Assert(actualOutputs, DeepEquals, expectedResults) +} + +func (s *GCSBackendTest) TestGCSBackend_CopyBlob_PreserveMetadata(c *C) { + srcKey := s.testSpec.existingObjKey + destKey := RandStringBytesMaskImprSrc(16) + s.addToAdditionalBlobs(destKey) + + srcAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: srcKey}) + c.Assert(err, IsNil) + + _, err = s.gcsBackend.CopyBlob(&CopyBlobInput{ + Source: srcKey, + Destination: destKey, + }) + c.Assert(err, IsNil) + + destAttr, err := s.gcsBackend.HeadBlob(&HeadBlobInput{Key: destKey}) + c.Assert(err, IsNil) + c.Assert(NilStr(destAttr.ContentType), Equals, NilStr(srcAttr.ContentType)) + c.Assert(destAttr.Metadata, DeepEquals, srcAttr.Metadata) + c.Assert(srcAttr.ETag, Not(Equals), destAttr.ETag) + + destOut, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: srcKey}) + c.Assert(err, IsNil) + defer destOut.Body.Close() + destContent, err := ioutil.ReadAll(destOut.Body) + c.Assert(string(destContent), Equals, s.testSpec.existingObjContent) +} + +func (s *GCSBackendTest) TestGCSBackend_MultipartUpload_BeginAddCommit(c *C) { + objKey := RandStringBytesMaskImprSrc(16) + s.addToAdditionalBlobs(objKey) + + commitInput, err := s.gcsBackend.MultipartBlobBegin(&MultipartBlobBeginInput{ + Key: objKey, + }) + c.Assert(err, IsNil) + c.Assert(commitInput.NumParts, Equals, uint32(0)) + + // We will have numFullChunks+1 chunks. That last chunk is of size lastChunkSize + var numFullChunks = 2 + lastChunkSize := s.chunkSize - 1 + fileSize := uint64((numFullChunks * s.chunkSize) + lastChunkSize) + + // generate data to simulate MPU behavior + buf := new(bytes.Buffer) + data := io.LimitReader(&SeqReader{}, int64(fileSize)) + _, err = buf.ReadFrom(data) + c.Assert(err, IsNil) + reader := bytes.NewReader(buf.Bytes()) + + // add the full chunks + for i := 0; i < numFullChunks; i++ { + sectionReader := io.NewSectionReader(reader, int64(i*s.chunkSize), int64(s.chunkSize)) + addOut, err := s.gcsBackend.MultipartBlobAdd(&MultipartBlobAddInput{ + Commit: commitInput, + PartNumber: uint32(i + 1), + Body: sectionReader, + Size: uint64(s.chunkSize), + }) + c.Assert(err, IsNil) + c.Assert(addOut, NotNil) + } + + // add the remaining chunk + sectionReader := io.NewSectionReader(reader, int64(numFullChunks*s.chunkSize), int64(lastChunkSize)) + addOut, err := s.gcsBackend.MultipartBlobAdd(&MultipartBlobAddInput{ + Commit: commitInput, + PartNumber: uint32(numFullChunks + 1), + Body: sectionReader, + Size: uint64(lastChunkSize), + }) + c.Assert(addOut, NotNil) + c.Assert(err, IsNil) + + commitOut, err := s.gcsBackend.MultipartBlobCommit(commitInput) + c.Assert(err, IsNil) + c.Assert(commitOut.ETag, NotNil) + + _, err = s.gcsBackend.HeadBlob(&HeadBlobInput{Key: objKey}) + c.Assert(err, IsNil) + + // assert uploaded content is correct + blobOutput, err := s.gcsBackend.GetBlob(&GetBlobInput{Key: objKey}) + c.Assert(err, IsNil) + defer blobOutput.Body.Close() + + _, err = reader.Seek(0, io.SeekStart) // seek to the beginning of the file + c.Assert(err, IsNil) + _, err = CompareReader(reader, blobOutput.Body, 0) // assert content + c.Assert(err, IsNil) +} + +func (s *GCSBackendTest) TestGCSBackend_MultipartUpload_Abort(c *C) { + src := RandStringBytesMaskImprSrc(16) + commitInput, err := s.gcsBackend.MultipartBlobBegin(&MultipartBlobBeginInput{Key: src}) + c.Assert(err, IsNil) + + addOut, err := s.gcsBackend.MultipartBlobAdd(&MultipartBlobAddInput{ + Commit: commitInput, + PartNumber: 1, + Body: bytes.NewReader([]byte(src)), + Size: uint64(len(src)), + Last: false, + }) + c.Assert(err, IsNil) + c.Assert(addOut, NotNil) + + _, err = s.gcsBackend.MultipartBlobAbort(commitInput) + c.Assert(err, IsNil) + + _, err = s.gcsBackend.HeadBlob(&HeadBlobInput{Key: src}) + c.Assert(err, Equals, fuse.ENOENT) +} diff --git a/internal/backend_s3.go b/internal/backend_s3.go index f19f541c..5bb4c3ba 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -124,6 +124,10 @@ func (s *S3Backend) newS3() { s.setV2Signer(&s.S3.Handlers) } s.S3.Handlers.Sign.PushBack(addAcceptEncoding) + s.S3.Handlers.Build.PushFrontNamed(request.NamedHandler{ + Name: "UserAgentHandler", + Fn: request.MakeAddToUserAgentHandler("goofys", VersionNumber+"-"+VersionHash), + }) } func (s *S3Backend) detectBucketLocationByHEAD() (err error, isAws bool) { @@ -475,7 +479,7 @@ func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes stri params := &s3.UploadPartCopyInput{ Bucket: &s.bucket, Key: &to, - CopySource: aws.String(pathEscape(from)), + CopySource: aws.String(url.QueryEscape(from)), UploadId: &mpuId, CopySourceRange: &bytes, CopySourceIfMatch: srcEtag, @@ -504,7 +508,7 @@ func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes stri } func sizeToParts(size int64) (int, int64) { - const MAX_S3_MPU_SIZE = 5 * 1024 * 1024 * 1024 * 1024 + const MAX_S3_MPU_SIZE int64 = 5 * 1024 * 1024 * 1024 * 1024 if size > MAX_S3_MPU_SIZE { panic(fmt.Sprintf("object size: %v exceeds maximum S3 MPU size: %v", size, MAX_S3_MPU_SIZE)) } @@ -667,7 +671,7 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { params := &s3.CopyObjectInput{ Bucket: &s.bucket, - CopySource: aws.String(pathEscape(from)), + CopySource: aws.String(url.QueryEscape(from)), Key: ¶m.Destination, StorageClass: param.StorageClass, ContentType: s.flags.GetMimeType(param.Destination), @@ -981,7 +985,36 @@ func (s *S3Backend) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error if err != nil { return nil, mapAwsError(err) } - return &MakeBucketOutput{}, nil + + if s.config.BucketOwner != "" { + var owner s3.Tag + owner.SetKey("Owner") + owner.SetValue(s.config.BucketOwner) + + param := s3.PutBucketTaggingInput{ + Bucket: &s.bucket, + Tagging: &s3.Tagging{ + TagSet: []*s3.Tag{&owner}, + }, + } + + for i := 0; i < 10; i++ { + _, err = s.PutBucketTagging(¶m) + err = mapAwsError((err)) + switch err { + case nil: + break + case syscall.ENXIO, syscall.EINTR: + s3Log.Infof("waiting for bucket") + time.Sleep((time.Duration(i) + 1) * 2 * time.Second) + default: + s3Log.Errorf("Failed to tag bucket %v: %v", s.bucket, err) + return nil, err + } + } + } + + return &MakeBucketOutput{}, err } func (s *S3Backend) Delegate() interface{} { diff --git a/internal/cgroup.go b/internal/cgroup.go index 8f00b285..c73ff501 100644 --- a/internal/cgroup.go +++ b/internal/cgroup.go @@ -45,7 +45,7 @@ func getCgroupAvailableMem() (retVal uint64, err error) { // newer version of docker mounts the cgroup memory limit/usage files directly under // /sys/fs/cgroup/memory/ rather than /sys/fs/cgroup/memory/docker/$container_id/ - if _, err := os.Stat(filepath.Join(CGROUP_FOLDER_PREFIX, path)); os.IsExist(err) { + if _, err := os.Stat(filepath.Join(CGROUP_FOLDER_PREFIX, path)); err == nil { path = filepath.Join(CGROUP_FOLDER_PREFIX, path) } else { path = filepath.Join(CGROUP_FOLDER_PREFIX) diff --git a/internal/dir.go b/internal/dir.go index 687674f0..64e838f2 100644 --- a/internal/dir.go +++ b/internal/dir.go @@ -899,7 +899,7 @@ func (parent *Inode) Unlink(name string) (err error) { } func (parent *Inode) Create( - name string, metadata fuseops.OpMetadata) (inode *Inode, fh *FileHandle) { + name string, metadata fuseops.OpContext) (inode *Inode, fh *FileHandle) { parent.logFuse("Create", name) diff --git a/internal/file.go b/internal/file.go index a2ccbaa7..69cf5b6d 100644 --- a/internal/file.go +++ b/internal/file.go @@ -71,7 +71,7 @@ const READAHEAD_CHUNK = uint32(20 * 1024 * 1024) // NewFileHandle returns a new file handle for the given `inode` triggered by fuse // operation with the given `opMetadata` -func NewFileHandle(inode *Inode, opMetadata fuseops.OpMetadata) *FileHandle { +func NewFileHandle(inode *Inode, opMetadata fuseops.OpContext) *FileHandle { tgid, err := GetTgid(opMetadata.Pid) if err != nil { log.Debugf( @@ -186,12 +186,14 @@ func (fh *FileHandle) waitForCreateMPU() (err error) { func (fh *FileHandle) partSize() uint64 { var size uint64 - if fh.lastPartId < 1000 { + if fh.lastPartId < 500 { size = 5 * 1024 * 1024 - } else if fh.lastPartId < 2000 { + } else if fh.lastPartId < 1000 { size = 25 * 1024 * 1024 - } else { + } else if fh.lastPartId < 2000 { size = 125 * 1024 * 1024 + } else { + size = 625 * 1024 * 1024 } maxPartSize := fh.cloud.Capabilities().MaxMultipartSize diff --git a/internal/flags.go b/internal/flags.go index 4a84f49a..0cea21f4 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -77,6 +77,7 @@ COPYRIGHT: ` } +var VersionNumber string var VersionHash string func NewApp() (app *cli.App) { @@ -86,7 +87,7 @@ func NewApp() (app *cli.App) { app = &cli.App{ Name: "goofys", - Version: "0.23.1-" + VersionHash, + Version: VersionNumber + "-" + VersionHash, Usage: "Mount an S3 bucket locally", HideHelp: true, Writer: os.Stderr, diff --git a/internal/goofys.go b/internal/goofys.go index 66325737..4f4bd76f 100644 --- a/internal/goofys.go +++ b/internal/goofys.go @@ -34,6 +34,8 @@ import ( "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" + "net/http" + "github.com/sirupsen/logrus" ) @@ -54,7 +56,7 @@ type Goofys struct { umask uint32 - gcs bool + gcsS3 bool rootAttrs InodeAttributes bufferPool *BufferPool @@ -113,6 +115,8 @@ func NewBackend(bucket string, flags *FlagStorage) (cloud StorageBackend, err er } else { cloud, err = NewS3(bucket, flags, config) } + } else if config, ok := flags.Backend.(*GCSConfig); ok { + cloud, err = NewGCS(bucket, config) } else { err = fmt.Errorf("Unknown backend config: %T", flags.Backend) } @@ -196,7 +200,7 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage, log.Errorf("Unable to setup backend: %v", err) return nil } - _, fs.gcs = cloud.Delegate().(*GCS3) + _, fs.gcsS3 = cloud.Delegate().(*GCS3) randomObjectName := prefix + (RandStringBytesMaskImprSrc(32)) err = cloud.Init(randomObjectName) @@ -522,6 +526,8 @@ func mapHttpError(status int) error { return fuse.ENOENT case 405: return syscall.ENOTSUP + case http.StatusConflict: + return syscall.EINTR case 429: return syscall.EAGAIN case 500: @@ -568,13 +574,6 @@ func mapAwsError(err error) error { } } -// note that this is NOT the same as url.PathEscape in golang 1.8, -// as this preserves / and url.PathEscape converts / to %2F -func pathEscape(path string) string { - u := url.URL{Path: path} - return u.EscapedPath() -} - func (fs *Goofys) allocateInodeId() (id fuseops.InodeID) { id = fs.nextInodeID fs.nextInodeID++ @@ -877,7 +876,7 @@ func (fs *Goofys) OpenFile( in := fs.getInodeOrDie(op.Inode) fs.mu.RUnlock() - fh, err := in.OpenFile(op.Metadata) + fh, err := in.OpenFile(op.OpContext) if err != nil { return } @@ -950,7 +949,7 @@ func (fs *Goofys) FlushFile( // This check helps us with scenarios like https://github.com/kahing/goofys/issues/273 // Also see goofys_test.go:TestClientForkExec. if fh.Tgid != nil { - tgid, err := GetTgid(op.Metadata.Pid) + tgid, err := GetTgid(op.OpContext.Pid) if err != nil { fh.inode.logFuse("<-- FlushFile", fmt.Sprintf("Failed to retrieve tgid from op.Metadata.Pid. FlushFileOp:%#v, err:%v", @@ -1010,7 +1009,7 @@ func (fs *Goofys) CreateFile( parent := fs.getInodeOrDie(op.Parent) fs.mu.RUnlock() - inode, fh := parent.Create(op.Name, op.Metadata) + inode, fh := parent.Create(op.Name, op.OpContext) parent.mu.Lock() diff --git a/internal/goofys_test.go b/internal/goofys_test.go index 638db499..16b86773 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -57,12 +57,16 @@ import ( "github.com/sirupsen/logrus" + "runtime/debug" + . "gopkg.in/check.v1" ) // so I don't get complains about unused imports var ignored = logrus.DebugLevel +const PerTestTimeout = 10 * time.Minute + func currentUid() uint32 { user, err := user.Current() if err != nil { @@ -102,6 +106,8 @@ type GoofysTest struct { removeBucket []StorageBackend env map[string]*string + + timeout chan int } func Test(t *testing.T) { @@ -221,6 +227,11 @@ func (s *GoofysTest) selectTestConfig(t *C, flags *FlagStorage) (conf S3Config) conf.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") } } + + conf.BucketOwner = os.Getenv("BUCKET_OWNER") + if conf.BucketOwner == "" { + panic("BUCKET_OWNER is required on AWS") + } } else if hasEnv("GCS") { conf.Region = "us-west1" conf.Profile = os.Getenv("GCS") @@ -257,8 +268,8 @@ func (s *GoofysTest) SetUpSuite(t *C) { func (s *GoofysTest) deleteBucket(cloud StorageBackend) error { param := &ListBlobsInput{} - // Azure datalake v1,v2 need special handling. - adlKeysToRemove := make([]string, 0) + // Azure need special handling. + azureKeysToRemove := make([]string, 0) for { resp, err := cloud.ListBlobs(param) if err != nil { @@ -271,11 +282,11 @@ func (s *GoofysTest) deleteBucket(cloud StorageBackend) error { } if len(keysToRemove) != 0 { switch cloud.(type) { - case *ADLv1, *ADLv2: - // ADLV{1|2} supports directories. => dir can be removed only after the dir is - // empty. So we will remove the blobs in reverse depth order via DeleteADLBlobs - // after this for loop. - adlKeysToRemove = append(adlKeysToRemove, keysToRemove...) + case *ADLv1, *ADLv2, *AZBlob: + // ADLV{1|2} and AZBlob (sometimes) supports directories. => dir can be removed only + // after the dir is empty. So we will remove the blobs in reverse depth order via + // DeleteADLBlobs after this for loop. + azureKeysToRemove = append(azureKeysToRemove, keysToRemove...) default: _, err = cloud.DeleteBlobs(&DeleteBlobsInput{Items: keysToRemove}) if err != nil { @@ -290,8 +301,8 @@ func (s *GoofysTest) deleteBucket(cloud StorageBackend) error { } } - if len(adlKeysToRemove) != 0 { - err := s.DeleteADLBlobs(cloud, adlKeysToRemove) + if len(azureKeysToRemove) != 0 { + err := s.DeleteADLBlobs(cloud, azureKeysToRemove) if err != nil { return err } @@ -302,6 +313,9 @@ func (s *GoofysTest) deleteBucket(cloud StorageBackend) error { } func (s *GoofysTest) TearDownTest(t *C) { + close(s.timeout) + s.timeout = nil + for _, cloud := range s.removeBucket { err := s.deleteBucket(cloud) t.Assert(err, IsNil) @@ -425,9 +439,32 @@ func (s *GoofysTest) setupDefaultEnv(t *C, public bool) { s.setupEnv(t, s.env, public) } +func (s *GoofysTest) setUpTestTimeout(t *C, timeout time.Duration) { + if s.timeout != nil { + close(s.timeout) + } + s.timeout = make(chan int) + debug.SetTraceback("all") + started := time.Now() + + go func() { + select { + case _, ok := <-s.timeout: + if !ok { + return + } + case <-time.After(timeout): + panic(fmt.Sprintf("timeout %v reached. Started %v now %v", + timeout, started, time.Now())) + } + }() +} + func (s *GoofysTest) SetUpTest(t *C) { log.Infof("Starting at %v", time.Now()) + s.setUpTestTimeout(t, PerTestTimeout) + var bucket string mount := os.Getenv("MOUNT") @@ -460,10 +497,10 @@ func (s *GoofysTest) SetUpTest(t *C) { s.cloud = s3 s3.aws = hasEnv("AWS") if s3.aws { - s.cloud = &S3BucketEventualConsistency{s3} + s.cloud = NewS3BucketEventualConsistency(s3) } - if !hasEnv("MINIO") { + if s.emulator { s3.Handlers.Sign.Clear() s3.Handlers.Sign.PushBack(SignV2) s3.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) @@ -471,7 +508,7 @@ func (s *GoofysTest) SetUpTest(t *C) { _, err = s3.ListBuckets(nil) t.Assert(err, IsNil) - } else if cloud == "gcs" { + } else if cloud == "gcs3" { conf := s.selectTestConfig(t, flags) flags.Backend = &conf @@ -578,6 +615,15 @@ func (s *GoofysTest) SetUpTest(t *C) { s.cloud, err = NewADLv2(bucket, flags, &config) t.Assert(err, IsNil) t.Assert(s.cloud, NotNil) + } else if cloud == "gcs" { + config := NewGCSConfig() + t.Assert(config, NotNil) + + flags.Backend = config + var err error + s.cloud, err = NewGCS(bucket, config) + t.Assert(err, IsNil) + t.Assert(s.cloud, NotNil) } else { t.Fatal("Unsupported backend") } @@ -601,7 +647,7 @@ func (s *GoofysTest) SetUpTest(t *C) { return nil, err } - return &S3BucketEventualConsistency{cloud.(*S3Backend)}, nil + return NewS3BucketEventualConsistency(cloud.(*S3Backend)), nil }) } else { s.fs = NewGoofys(context.Background(), bucket, flags) @@ -697,6 +743,8 @@ func (s *GoofysTest) TestLookUpInode(t *C) { } func (s *GoofysTest) TestPanicWrapper(t *C) { + debug.SetTraceback("single") + fs := FusePanicLogger{s.fs} err := fs.GetInodeAttributes(nil, &fuseops.GetInodeAttributesOp{ Inode: 1234, @@ -867,7 +915,7 @@ func (s *GoofysTest) TestReadFiles(t *C) { in, err := parent.LookUp(en.Name) t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) buf := make([]byte, 4096) @@ -893,7 +941,7 @@ func (s *GoofysTest) TestReadOffset(t *C) { in, err := root.LookUp(f) t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) buf := make([]byte, 4096) @@ -917,7 +965,7 @@ func (s *GoofysTest) TestReadOffset(t *C) { func (s *GoofysTest) TestCreateFiles(t *C) { fileName := "testCreateFile" - _, fh := s.getRoot(t).Create(fileName, fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := s.getRoot(t).Create(fileName, fuseops.OpContext{uint32(os.Getpid())}) err := fh.FlushFile() t.Assert(err, IsNil) @@ -936,7 +984,7 @@ func (s *GoofysTest) TestCreateFiles(t *C) { inode, err := s.getRoot(t).LookUp(fileName) t.Assert(err, IsNil) - fh, err = inode.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err = inode.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) err = fh.FlushFile() @@ -951,6 +999,32 @@ func (s *GoofysTest) TestCreateFiles(t *C) { defer resp.Body.Close() } +func (s *GoofysTest) TestRenameWithSpecialChar(t *C) { + fileName := "foo+" + s.testWriteFile(t, fileName, 1, 128*1024) + + inode, err := s.getRoot(t).LookUp(fileName) + t.Assert(err, IsNil) + + fh, err := inode.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) + t.Assert(err, IsNil) + + err = fh.FlushFile() + t.Assert(err, IsNil) + + resp, err := s.cloud.GetBlob(&GetBlobInput{Key: fileName}) + t.Assert(err, IsNil) + // ADLv1 doesn't return size when we do a GET + if _, adlv1 := s.cloud.(*ADLv1); !adlv1 { + t.Assert(resp.HeadBlobOutput.Size, Equals, uint64(1)) + } + defer resp.Body.Close() + + root := s.getRoot(t) + err = root.Rename(fileName, root, "foo") + t.Assert(err, IsNil) +} + func (s *GoofysTest) TestUnlink(t *C) { fileName := "file1" @@ -1015,7 +1089,7 @@ func (s *GoofysTest) testWriteFileAt(t *C, fileName string, offset int64, size i } } else { in := s.fs.inodes[lookup.Entry.Child] - fh, err = in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err = in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) } @@ -1070,6 +1144,9 @@ func (s *GoofysTest) TestWriteLargeFile(t *C) { } func (s *GoofysTest) TestWriteReallyLargeFile(t *C) { + if _, ok := s.cloud.(*S3Backend); ok && s.emulator { + t.Skip("seems to be OOM'ing S3proxy 1.8.0") + } s.testWriteFile(t, "testLargeFile", 512*1024*1024+1, 128*1024) } @@ -1114,7 +1191,7 @@ func (s *GoofysTest) TestReadRandom(t *C) { in, err := s.LookUpInode(t, "testLargeFile") t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) fr := &FileHandleReader{s.fs, fh, 0} @@ -1146,7 +1223,7 @@ func (s *GoofysTest) TestMkDir(t *C) { t.Assert(err, IsNil) fileName := "file" - _, fh := inode.Create(fileName, fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := inode.Create(fileName, fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -1256,6 +1333,8 @@ func (s *GoofysTest) TestBackendListPagination(t *C) { itemsPerPage = 1000 case *AZBlob, *ADLv2: itemsPerPage = 5000 + case *GCSBackend: + itemsPerPage = 1000 default: t.Fatalf("unknown backend: %T", s.cloud) } @@ -1349,6 +1428,13 @@ func (s *GoofysTest) TestBackendListPrefix(t *C) { t.Assert(len(res.Prefixes), Equals, 0) t.Assert(len(res.Items), Equals, 0) + // ListBlobs: + // - Case1: If the prefix foo/ is not added explicitly, then ListBlobs foo/ might or might not return foo/. + // In the test setup dir2 is not expliticly created. + // - Case2: Else, ListBlobs foo/ must return foo/ + // In the test setup dir2/dir3 is expliticly created. + + // ListBlobs:Case1 res, err = s.cloud.ListBlobs(&ListBlobsInput{ Prefix: PString("dir2/"), Delimiter: PString("/"), @@ -1356,13 +1442,15 @@ func (s *GoofysTest) TestBackendListPrefix(t *C) { t.Assert(err, IsNil) t.Assert(len(res.Prefixes), Equals, 1) t.Assert(*res.Prefixes[0].Prefix, Equals, "dir2/dir3/") - if s.cloud.Capabilities().DirBlob { - t.Assert(len(res.Items), Equals, 1) + if len(res.Items) == 1 { + // azblob(with hierarchial ns on), adlv1, adlv2. t.Assert(*res.Items[0].Key, Equals, "dir2/") } else { + // s3, azblob(with hierarchial ns off) t.Assert(len(res.Items), Equals, 0) } + // ListBlobs:Case2 res, err = s.cloud.ListBlobs(&ListBlobsInput{ Prefix: PString("dir2/dir3/"), Delimiter: PString("/"), @@ -1373,14 +1461,23 @@ func (s *GoofysTest) TestBackendListPrefix(t *C) { t.Assert(*res.Items[0].Key, Equals, "dir2/dir3/") t.Assert(*res.Items[1].Key, Equals, "dir2/dir3/file4") + // ListBlobs:Case1 res, err = s.cloud.ListBlobs(&ListBlobsInput{ Prefix: PString("dir2/"), }) t.Assert(err, IsNil) t.Assert(len(res.Prefixes), Equals, 0) - t.Assert(len(res.Items), Equals, 2) - t.Assert(*res.Items[0].Key, Equals, "dir2/dir3/") - t.Assert(*res.Items[1].Key, Equals, "dir2/dir3/file4") + if len(res.Items) == 3 { + // azblob(with hierarchial ns on), adlv1, adlv2. + t.Assert(*res.Items[0].Key, Equals, "dir2/") + t.Assert(*res.Items[1].Key, Equals, "dir2/dir3/") + t.Assert(*res.Items[2].Key, Equals, "dir2/dir3/file4") + } else { + // s3, azblob(with hierarchial ns off) + t.Assert(len(res.Items), Equals, 2) + t.Assert(*res.Items[0].Key, Equals, "dir2/dir3/") + t.Assert(*res.Items[1].Key, Equals, "dir2/dir3/file4") + } res, err = s.cloud.ListBlobs(&ListBlobsInput{ Prefix: PString("dir2/dir3/file4"), @@ -1562,6 +1659,7 @@ func (s *GoofysTest) mount(t *C, mountPoint string) { // Mount the file system. mountCfg := &fuse.MountConfig{ FSName: s.fs.bucket, + Subtype: "goofys", Options: s.fs.flags.MountOptions, ErrorLogger: GetStdLogger(NewLogger("fuse"), logrus.ErrorLevel), DisableWritebackCaching: true, @@ -1720,6 +1818,7 @@ func (s *GoofysTest) TestBenchLs(t *C) { s.fs.flags.TypeCacheTTL = 1 * time.Minute s.fs.flags.StatCacheTTL = 1 * time.Minute mountPoint := "/tmp/mnt" + s.fs.bucket + s.setUpTestTimeout(t, 20*time.Minute) s.runFuseTest(t, mountPoint, false, "../bench/bench.sh", "cat", mountPoint, "ls") } @@ -2803,7 +2902,7 @@ func (s *GoofysTest) TestDirMtimeCreate(t *C) { m1 := attr.Mtime time.Sleep(time.Second) - _, _ = root.Create("foo", fuseops.OpMetadata{uint32(os.Getpid())}) + _, _ = root.Create("foo", fuseops.OpContext{uint32(os.Getpid())}) attr2, _ := root.GetAttributes() m2 := attr2.Mtime @@ -2866,7 +2965,7 @@ func (s *GoofysTest) TestRead403(t *C) { in, err := s.LookUpInode(t, "file1") t.Assert(err, IsNil) - fh, err := in.OpenFile(fuseops.OpMetadata{uint32(os.Getpid())}) + fh, err := in.OpenFile(fuseops.OpContext{uint32(os.Getpid())}) t.Assert(err, IsNil) s3.awsConfig.Credentials = credentials.AnonymousCredentials @@ -3266,14 +3365,14 @@ func (s *GoofysTest) newBackend(t *C, bucket string, createBucket bool) (cloud S s3.aws = hasEnv("AWS") - if !hasEnv("MINIO") { + if s.emulator { s3.Handlers.Sign.Clear() s3.Handlers.Sign.PushBack(SignV2) s3.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) } if s3.aws { - cloud = &S3BucketEventualConsistency{s3} + cloud = NewS3BucketEventualConsistency(s3) } else { cloud = s3 } @@ -3293,6 +3392,10 @@ func (s *GoofysTest) newBackend(t *C, bucket string, createBucket bool) (cloud S config, _ := s.fs.flags.Backend.(*ADLv2Config) cloud, err = NewADLv2(bucket, s.fs.flags, config) t.Assert(err, IsNil) + case *GCSBackend: + config, _ := s.fs.flags.Backend.(*GCSConfig) + cloud, err = NewGCS(bucket, config) + t.Assert(err, IsNil) default: t.Fatal("unknown backend") } @@ -3328,7 +3431,7 @@ func (s *GoofysTest) TestVFS(t *C) { _, err = in.LookUp("file5") t.Assert(err, Equals, fuse.ENOENT) - _, fh := in.Create("testfile", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := in.Create("testfile", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3363,7 +3466,7 @@ func (s *GoofysTest) TestVFS(t *C) { // create another file inside subdir to make sure that our // mount check is correct for dir inside the root - _, fh = subdir.Create("testfile2", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh = subdir.Create("testfile2", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3526,6 +3629,19 @@ func (s *GoofysTest) TestMountsError(t *C) { defer func() { adlCloud.client.BaseClient.Authorizer = auth }() + } else if _, ok := s.cloud.(*GCSBackend); ok { + // We'll trigger a failure on GCS mount by using an unauthenticated client to mount to a private bucket + defaultCreds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + os.Unsetenv("GOOGLE_APPLICATION_CREDENTIALS") + + defer func() { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCreds) + }() + + var err error + config := NewGCSConfig() + cloud, err = NewGCS(s.fs.bucket, config) + t.Assert(err, IsNil) } else { cloud = s.newBackend(t, bucket, false) } @@ -3631,7 +3747,7 @@ func (s *GoofysTest) testMountsNested(t *C, cloud StorageBackend, t.Assert(*dir_dir.Name, Equals, "dir") t.Assert(dir_dir.dir.cloud == cloud, Equals, true) - _, fh := dir_in.Create("testfile", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh := dir_in.Create("testfile", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3639,7 +3755,7 @@ func (s *GoofysTest) testMountsNested(t *C, cloud StorageBackend, t.Assert(err, IsNil) defer resp.Body.Close() - _, fh = dir_dir.Create("testfile", fuseops.OpMetadata{uint32(os.Getpid())}) + _, fh = dir_dir.Create("testfile", fuseops.OpContext{uint32(os.Getpid())}) err = fh.FlushFile() t.Assert(err, IsNil) @@ -3894,7 +4010,7 @@ func (s *GoofysTest) TestWriteListFlush(t *C) { t.Assert(err, IsNil) s.fs.insertInode(root, dir) - in, fh := dir.Create("file1", fuseops.OpMetadata{}) + in, fh := dir.Create("file1", fuseops.OpContext{}) t.Assert(in, NotNil) t.Assert(fh, NotNil) s.fs.insertInode(dir, in) @@ -3944,7 +4060,7 @@ func (s *GoofysTest) TestWriteUnlinkFlush(t *C) { t.Assert(err, IsNil) s.fs.insertInode(root, dir) - in, fh := dir.Create("deleted", fuseops.OpMetadata{}) + in, fh := dir.Create("deleted", fuseops.OpContext{}) t.Assert(in, NotNil) t.Assert(fh, NotNil) s.fs.insertInode(dir, in) @@ -4112,10 +4228,12 @@ func (s *GoofysTest) testReadMyOwnWriteFuse(t *C, externalUpdate bool) { if !externalUpdate { // we flushed and ttl expired, next lookup should // realize nothing is changed and NOT invalidate the - // cache. Except ADLv1 because PUT there doesn't + // cache. Except ADLv1,GCS because PUT there doesn't // return the mtime, so the open above will think the // file is updated and not re-use cache - if _, adlv1 := s.cloud.(*ADLv1); !adlv1 { + _, adlv1 := s.cloud.(*ADLv1) + _, isGCS := s.cloud.(*GCSBackend) + if !adlv1 && !isGCS { cloud.err = fuse.EINVAL } } else { diff --git a/internal/handles.go b/internal/handles.go index 651faf69..e8b308fb 100644 --- a/internal/handles.go +++ b/internal/handles.go @@ -105,28 +105,52 @@ func NewInode(fs *Goofys, parent *Inode, name *string) (inode *Inode) { return } +func deepCopyBlobItemOputput(item *BlobItemOutput) BlobItemOutput { + + key := NilStr(item.Key) + etag := NilStr(item.ETag) + sc := NilStr(item.StorageClass) + + var lastmodified time.Time + if item.LastModified != nil { + lastmodified = *item.LastModified + } + + return BlobItemOutput{ + Key: &key, + ETag: &etag, + LastModified: &lastmodified, + Size: item.Size, + StorageClass: &sc, + } +} + func (inode *Inode) SetFromBlobItem(item *BlobItemOutput) { + // copy item so they won't hold back references to the HTTP + // responses and SDK objects. See discussion in + // https://github.com/kahing/goofys/pull/547 + itemcopy := deepCopyBlobItemOputput(item) inode.mu.Lock() defer inode.mu.Unlock() - inode.Attributes.Size = item.Size + inode.Attributes.Size = itemcopy.Size // don't want to point to the attribute because that // can get updated - size := item.Size + size := inode.Attributes.Size inode.KnownSize = &size if item.LastModified != nil { - inode.Attributes.Mtime = *item.LastModified + inode.Attributes.Mtime = *itemcopy.LastModified } else { inode.Attributes.Mtime = inode.fs.rootAttrs.Mtime } if item.ETag != nil { - inode.s3Metadata["etag"] = []byte(*item.ETag) - inode.knownETag = item.ETag + inode.s3Metadata["etag"] = []byte(*itemcopy.ETag) + inode.knownETag = itemcopy.ETag } else { delete(inode.s3Metadata, "etag") } if item.StorageClass != nil { - inode.s3Metadata["storage-class"] = []byte(*item.StorageClass) + inode.s3Metadata["storage-class"] = []byte(*itemcopy.StorageClass) } else { delete(inode.s3Metadata, "storage-class") } @@ -494,7 +518,7 @@ func (inode *Inode) ListXattr() ([]string, error) { return xattrs, nil } -func (inode *Inode) OpenFile(metadata fuseops.OpMetadata) (fh *FileHandle, err error) { +func (inode *Inode) OpenFile(metadata fuseops.OpContext) (fh *FileHandle, err error) { inode.logFuse("OpenFile") inode.mu.Lock() diff --git a/internal/parse_bucket_test.go b/internal/parse_bucket_test.go new file mode 100644 index 00000000..8882e3f3 --- /dev/null +++ b/internal/parse_bucket_test.go @@ -0,0 +1,49 @@ +package internal + +import ( + . "gopkg.in/check.v1" +) + +type ParseBucketTest struct{} + +var _ = Suite(&ParseBucketTest{}) + +func (s *ParseBucketTest) TestParseBucketSpec(t *C) { + testCases := []struct { + input string + expected BucketSpec + }{ + { + input: "s3://bucketName/hello/everyone", + expected: BucketSpec{ + Scheme: "s3", + Bucket: "bucketName", + Prefix: "hello/everyone/", + }, + }, + { + input: "gs://bucketName/hello/everyone", + expected: BucketSpec{ + Scheme: "gs", + Bucket: "bucketName", + Prefix: "hello/everyone/", + }, + }, + { + input: "bucketName/hello/everyone", + expected: BucketSpec{ + Scheme: "s3", + Bucket: "bucketName/hello/everyone", + Prefix: "", + }, + }, + } + + for _, tc := range testCases { + spec, err := ParseBucketSpec(tc.input) + if err != nil { + log.Fatal(err) + } + t.Assert(tc.expected, DeepEquals, spec) + } +} diff --git a/internal/utils.go b/internal/utils.go index 08d8ae8a..51e07800 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -16,6 +16,7 @@ package internal import ( "fmt" + "strings" "time" "unicode" @@ -117,6 +118,38 @@ func PTime(v time.Time) *time.Time { return &v } +func NilStr(v *string) string { + if v == nil { + return "" + } else { + return *v + } +} + +func NilUint32(v *uint32) uint32 { + if v == nil { + return 0 + } else { + return *v + } +} + +func NilInt64(v *int64) int64 { + if v == nil { + return 0 + } else { + return *v + } +} + +func NilUint64(v *uint64) uint64 { + if v == nil { + return 0 + } else { + return *v + } +} + func xattrEscape(value []byte) (s string) { for _, c := range value { if c == '%' { @@ -179,3 +212,37 @@ func GetTgid(pid uint32) (tgid *int32, err error) { } return &tgidVal, nil } + +func PMetadata(m map[string]string) map[string]*string { + metadata := make(map[string]*string) + for k, _ := range m { + k = strings.ToLower(k) + v := m[k] + metadata[k] = &v + } + return metadata +} + +func NilMetadata(m map[string]*string) map[string]string { + metadata := make(map[string]string) + for k, v := range m { + k = strings.ToLower(k) + metadata[k] = NilStr(v) + } + return metadata +} + +// The following functions are useful to debug byte sizes +func ConvertBytesToIEC(size int64) string { + var unit int64 = 1024 + if size < unit { + return fmt.Sprintf("%d %c", size, 'B') + } + // divide size by unit until it is less than unit + curSize, exp := float64(size), 0 + for curSize >= float64(unit) { + curSize /= float64(unit) + exp += 1 + } + return fmt.Sprintf("%.1f %ciB", curSize, "KMGTPEZY"[exp-1]) +} \ No newline at end of file diff --git a/internal/v2signer.go b/internal/v2signer.go index c8321463..53174c68 100644 --- a/internal/v2signer.go +++ b/internal/v2signer.go @@ -39,7 +39,7 @@ var ( const ( signatureVersion = "2" signatureMethod = "HmacSHA1" - timeFormat = "Mon, 2 Jan 2006 15:04:05 +0000" + timeFormat = "Mon, 02 Jan 2006 15:04:05 +0000" ) var subresources = []string{ diff --git a/main.go b/main.go index e2e06a1f..112b8595 100644 --- a/main.go +++ b/main.go @@ -134,6 +134,7 @@ func massageArg0() { var Version = "use `make build' to fill version hash correctly" func main() { + VersionNumber = "0.24.0" VersionHash = Version massagePath() diff --git a/test/run-tests.sh b/test/run-tests.sh index 549c2704..28741643 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -9,6 +9,7 @@ set -o nounset : ${PROXY_PID:=""} : ${TIMEOUT:="10m"} : ${MOUNT:="false"} +: ${GOOGLE_APPLICATION_CREDENTIALS:=""} export MOUNT @@ -65,6 +66,11 @@ elif [ $CLOUD == "azblob" ]; then export ENDPOINT elif [ $CLOUD == "adlv1" ]; then TIMEOUT=40m +elif [ $CLOUD == "gcs" ]; then + if [ "$GOOGLE_APPLICATION_CREDENTIALS" = "" ]; then + echo "GOOGLE_APPLICATION_CREDENTIALS must be set" 1>&2 + exit 1 + fi fi if [ "$PROXY_BIN" != "" ]; then diff --git a/vendor/contrib.go.opencensus.io/exporter/ocagent b/vendor/contrib.go.opencensus.io/exporter/ocagent deleted file mode 160000 index a8a6f458..00000000 --- a/vendor/contrib.go.opencensus.io/exporter/ocagent +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a8a6f458bbc1d5042322ad1f9b65eeb0b69be9ea diff --git a/vendor/github.com/Azure/azure-pipeline-go b/vendor/github.com/Azure/azure-pipeline-go deleted file mode 160000 index 232aee85..00000000 --- a/vendor/github.com/Azure/azure-pipeline-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 232aee85e8e3a6223a11c0943f7df2ae0fac00e4 diff --git a/vendor/github.com/Azure/azure-sdk-for-go b/vendor/github.com/Azure/azure-sdk-for-go deleted file mode 160000 index a629ae78..00000000 --- a/vendor/github.com/Azure/azure-sdk-for-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a629ae7873bf2a86184a8bc1b65e65e1ab532f57 diff --git a/vendor/github.com/Azure/azure-storage-blob-go b/vendor/github.com/Azure/azure-storage-blob-go deleted file mode 160000 index 33c102d4..00000000 --- a/vendor/github.com/Azure/azure-storage-blob-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 33c102d4ffd269865af7fce31e927af2752df9a0 diff --git a/vendor/github.com/Azure/go-autorest b/vendor/github.com/Azure/go-autorest deleted file mode 160000 index d9a171ca..00000000 --- a/vendor/github.com/Azure/go-autorest +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d9a171ca366fd4acfaa99107cdc4f91877f3572e diff --git a/vendor/github.com/aws/aws-sdk-go b/vendor/github.com/aws/aws-sdk-go deleted file mode 160000 index 7e7bb328..00000000 --- a/vendor/github.com/aws/aws-sdk-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7e7bb328b06e5d15c0f0b7c935f848a57e289ad8 diff --git a/vendor/github.com/census-instrumentation/opencensus-proto b/vendor/github.com/census-instrumentation/opencensus-proto deleted file mode 160000 index 26aa36c0..00000000 --- a/vendor/github.com/census-instrumentation/opencensus-proto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 26aa36c099c2041b432cf3cc8a26c5fb858d218b diff --git a/vendor/github.com/dgrijalva/jwt-go b/vendor/github.com/dgrijalva/jwt-go deleted file mode 160000 index 5e25c22b..00000000 --- a/vendor/github.com/dgrijalva/jwt-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5e25c22bd5d6de03265bbe5462dcd162f85046f6 diff --git a/vendor/github.com/dimchansky/utfbom b/vendor/github.com/dimchansky/utfbom deleted file mode 160000 index d2133a1c..00000000 --- a/vendor/github.com/dimchansky/utfbom +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d2133a1ce379ef6fa992b0514a77146c60db9d1c diff --git a/vendor/github.com/golang/protobuf b/vendor/github.com/golang/protobuf deleted file mode 160000 index 4c88cc3f..00000000 --- a/vendor/github.com/golang/protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4c88cc3f1a34ffade77b79abc53335d1e511f25b diff --git a/vendor/github.com/google/uuid b/vendor/github.com/google/uuid deleted file mode 160000 index c2e93f3a..00000000 --- a/vendor/github.com/google/uuid +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c2e93f3ae59f2904160ceaab466009f965df46d6 diff --git a/vendor/github.com/grpc-ecosystem/grpc-gateway b/vendor/github.com/grpc-ecosystem/grpc-gateway deleted file mode 160000 index fdf06359..00000000 --- a/vendor/github.com/grpc-ecosystem/grpc-gateway +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fdf063599d922ec89a70819e2d5b7b4b5c642b92 diff --git a/vendor/github.com/hashicorp/golang-lru b/vendor/github.com/hashicorp/golang-lru deleted file mode 160000 index 7f827b33..00000000 --- a/vendor/github.com/hashicorp/golang-lru +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7f827b33c0f158ec5dfbba01bb0b14a4541fd81d diff --git a/vendor/github.com/jacobsa/fuse b/vendor/github.com/jacobsa/fuse deleted file mode 160000 index 374cf420..00000000 --- a/vendor/github.com/jacobsa/fuse +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 374cf4208103e8c7a7bb70af2acf459772c05d0a diff --git a/vendor/github.com/jtolds/gls b/vendor/github.com/jtolds/gls deleted file mode 160000 index 8ddce2a8..00000000 --- a/vendor/github.com/jtolds/gls +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8ddce2a84170772b95dd5d576c48d517b22cac63 diff --git a/vendor/github.com/kardianos/osext b/vendor/github.com/kardianos/osext deleted file mode 160000 index 2bc1f35c..00000000 --- a/vendor/github.com/kardianos/osext +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2bc1f35cddc0cc527b4bc3dce8578fc2a6c11384 diff --git a/vendor/github.com/kr/pretty b/vendor/github.com/kr/pretty deleted file mode 160000 index 71e7e499..00000000 --- a/vendor/github.com/kr/pretty +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 71e7e49937503c662b9b636fd6b2c14b1aa818a5 diff --git a/vendor/github.com/kr/text b/vendor/github.com/kr/text deleted file mode 160000 index e2ffdb16..00000000 --- a/vendor/github.com/kr/text +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e2ffdb16a802fe2bb95e2e35ff34f0e53aeef34f diff --git a/vendor/github.com/lsegal/gucumber b/vendor/github.com/lsegal/gucumber deleted file mode 160000 index 44a4d7eb..00000000 --- a/vendor/github.com/lsegal/gucumber +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 44a4d7eb3b14a88cf82b073dfb7e06277afdc549 diff --git a/vendor/github.com/mattn/go-ieproxy b/vendor/github.com/mattn/go-ieproxy deleted file mode 160000 index f9202b1c..00000000 --- a/vendor/github.com/mattn/go-ieproxy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f9202b1cfdeb0c82ddd3dc1e8e9cd94b3c0c1b13 diff --git a/vendor/github.com/mitchellh/go-homedir b/vendor/github.com/mitchellh/go-homedir deleted file mode 160000 index af06845c..00000000 --- a/vendor/github.com/mitchellh/go-homedir +++ /dev/null @@ -1 +0,0 @@ -Subproject commit af06845cf3004701891bf4fdb884bfe4920b3727 diff --git a/vendor/github.com/satori/go.uuid b/vendor/github.com/satori/go.uuid deleted file mode 160000 index b2ce2384..00000000 --- a/vendor/github.com/satori/go.uuid +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b2ce2384e17bbe0c6d34077efa39dbab3e09123b diff --git a/vendor/github.com/sevlyar/go-daemon b/vendor/github.com/sevlyar/go-daemon deleted file mode 160000 index bf839693..00000000 --- a/vendor/github.com/sevlyar/go-daemon +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bf839693b97cb19a587b25649bbcb59ee16bdb2f diff --git a/vendor/github.com/shirou/gopsutil b/vendor/github.com/shirou/gopsutil deleted file mode 160000 index d80c43f9..00000000 --- a/vendor/github.com/shirou/gopsutil +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d80c43f9c984a48783daf22f4bd9278006ae483a diff --git a/vendor/github.com/sirupsen/logrus b/vendor/github.com/sirupsen/logrus deleted file mode 160000 index de736cf9..00000000 --- a/vendor/github.com/sirupsen/logrus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit de736cf91b921d56253b4010270681d33fdf7cb5 diff --git a/vendor/github.com/smartystreets/assertions b/vendor/github.com/smartystreets/assertions deleted file mode 160000 index 443d8122..00000000 --- a/vendor/github.com/smartystreets/assertions +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 443d812296a84445c202c085f19e18fc238f8250 diff --git a/vendor/github.com/smartystreets/goconvey b/vendor/github.com/smartystreets/goconvey deleted file mode 160000 index 995f5b2e..00000000 --- a/vendor/github.com/smartystreets/goconvey +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 995f5b2e021c69b8b028ba6d0b05c1dd500783db diff --git a/vendor/github.com/urfave/cli b/vendor/github.com/urfave/cli deleted file mode 160000 index 521735b7..00000000 --- a/vendor/github.com/urfave/cli +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 521735b7608a25d771a39d42e2267e061e7e84b8 diff --git a/vendor/go.opencensus.io b/vendor/go.opencensus.io deleted file mode 160000 index b4a14686..00000000 --- a/vendor/go.opencensus.io +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b4a14686f0a98096416fe1b4cb848e384fb2b22b diff --git a/vendor/golang.org/x/crypto b/vendor/golang.org/x/crypto deleted file mode 160000 index 4def268f..00000000 --- a/vendor/golang.org/x/crypto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4def268fd1a49955bfb3dda92fe3db4f924f2285 diff --git a/vendor/golang.org/x/net b/vendor/golang.org/x/net deleted file mode 160000 index ca1201d0..00000000 --- a/vendor/golang.org/x/net +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ca1201d0de80cfde86cb01aea620983605dfe99b diff --git a/vendor/golang.org/x/sync b/vendor/golang.org/x/sync deleted file mode 160000 index 11223019..00000000 --- a/vendor/golang.org/x/sync +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 112230192c580c3556b8cee6403af37a4fc5f28c diff --git a/vendor/golang.org/x/sys b/vendor/golang.org/x/sys deleted file mode 160000 index 51ab0e2d..00000000 --- a/vendor/golang.org/x/sys +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 51ab0e2deafac1f46c46ad59cf0921be2f180c3d diff --git a/vendor/golang.org/x/text b/vendor/golang.org/x/text deleted file mode 160000 index 342b2e1f..00000000 --- a/vendor/golang.org/x/text +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 342b2e1fbaa52c93f31447ad2c6abc048c63e475 diff --git a/vendor/google.golang.org/api b/vendor/google.golang.org/api deleted file mode 160000 index 6f391290..00000000 --- a/vendor/google.golang.org/api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6f3912904777a209e099b9dbda3ed7bcb4e25ad7 diff --git a/vendor/google.golang.org/genproto b/vendor/google.golang.org/genproto deleted file mode 160000 index fa694d86..00000000 --- a/vendor/google.golang.org/genproto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fa694d86fc64c7654a660f8908de4e879866748d diff --git a/vendor/google.golang.org/grpc b/vendor/google.golang.org/grpc deleted file mode 160000 index fde0cae1..00000000 --- a/vendor/google.golang.org/grpc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fde0cae1c4042d90d749f11d334dd1d122c5c25b diff --git a/vendor/gopkg.in/check.v1 b/vendor/gopkg.in/check.v1 deleted file mode 160000 index 788fd784..00000000 --- a/vendor/gopkg.in/check.v1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 788fd78401277ebd861206a03c884797c6ec5541 diff --git a/vendor/gopkg.in/ini.v1 b/vendor/gopkg.in/ini.v1 deleted file mode 160000 index d4cae42d..00000000 --- a/vendor/gopkg.in/ini.v1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d4cae42d398bc0095297fc3315669590d29166ea