Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmake/sdksCommon.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ list(APPEND SDK_TEST_PROJECT_LIST "s3control:tests/aws-cpp-sdk-s3control-integra
list(APPEND SDK_TEST_PROJECT_LIST "sns:tests/aws-cpp-sdk-sns-integration-tests")
list(APPEND SDK_TEST_PROJECT_LIST "sqs:tests/aws-cpp-sdk-sqs-integration-tests")
list(APPEND SDK_TEST_PROJECT_LIST "sqs:tests/aws-cpp-sdk-sqs-unit-tests")
list(APPEND SDK_TEST_PROJECT_LIST "transfer:tests/aws-cpp-sdk-transfer-tests")
list(APPEND SDK_TEST_PROJECT_LIST "transfer:tests/aws-cpp-sdk-transfer-tests,tests/aws-cpp-sdk-transfer-unit-tests")
list(APPEND SDK_TEST_PROJECT_LIST "text-to-speech:tests/aws-cpp-sdk-text-to-speech-tests,tests/aws-cpp-sdk-polly-sample")
list(APPEND SDK_TEST_PROJECT_LIST "timestream-query:tests/aws-cpp-sdk-timestream-query-unit-tests")
list(APPEND SDK_TEST_PROJECT_LIST "transcribestreaming:tests/aws-cpp-sdk-transcribestreaming-integ-tests")
Expand Down
96 changes: 73 additions & 23 deletions src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -825,6 +825,33 @@ namespace Aws
return rangeStream.str();
}

static bool VerifyContentRange(const Aws::String& requestedRange, const Aws::String& responseContentRange)
{
if (requestedRange.empty() || responseContentRange.empty())
{
return false;
}

if (requestedRange.find("bytes=") != 0)
{
return false;
}
Aws::String requestRange = requestedRange.substr(6);

if (responseContentRange.find("bytes ") != 0)
{
return false;
}
Aws::String responseRange = responseContentRange.substr(6);
size_t slashPos = responseRange.find('/');
if (slashPos != Aws::String::npos)
{
responseRange = responseRange.substr(0, slashPos);
}

return requestRange == responseRange;
}

void TransferManager::DoSinglePartDownload(const std::shared_ptr<TransferHandle>& handle)
{
auto queuedParts = handle->GetQueuedParts();
Expand Down Expand Up @@ -1091,7 +1118,6 @@ namespace Aws
const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context)
{
AWS_UNREFERENCED_PARAM(client);
AWS_UNREFERENCED_PARAM(request);

std::shared_ptr<TransferHandleAsyncContext> transferContext =
std::const_pointer_cast<TransferHandleAsyncContext>(std::static_pointer_cast<const TransferHandleAsyncContext>(context));
Expand All @@ -1108,33 +1134,57 @@ namespace Aws
handle->SetError(outcome.GetError());
TriggerErrorCallback(handle, outcome.GetError());
}
else
else if (request.RangeHasBeenSet())
{
if(handle->ShouldContinue())
{
Aws::IOStream* bufferStream = partState->GetDownloadPartStream();
assert(bufferStream);

Aws::String errMsg{handle->WritePartToDownloadStream(bufferStream, partState->GetRangeBegin())};
if (errMsg.empty()) {
handle->ChangePartToCompleted(partState, outcome.GetResult().GetETag());
} else {
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
"InternalFailure", errMsg, false);
AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
<< "] Failed to download object in Bucket: ["
<< handle->GetBucketName() << "] with Key: [" << handle->GetKey()
<< "] " << errMsg);
handle->ChangePartToFailed(partState);
handle->SetError(error);
TriggerErrorCallback(handle, error);
const auto& requestedRange = request.GetRange();
const auto& responseContentRange = outcome.GetResult().GetContentRange();

if (responseContentRange.empty() or !VerifyContentRange(requestedRange, responseContentRange)) {
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
"ContentRangeMismatch",
"ContentRange in response does not match requested range",
false);
AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
<< "] ContentRange mismatch. Requested: [" << requestedRange
<< "] Received: [" << responseContentRange << "]");
handle->ChangePartToFailed(partState);
handle->SetError(error);
TriggerErrorCallback(handle, error);
handle->Cancel();

if(partState->GetDownloadBuffer())
{
m_bufferManager.Release(partState->GetDownloadBuffer());
partState->SetDownloadBuffer(nullptr);
}
}
else
{
return;
}

if(handle->ShouldContinue())
{
Aws::IOStream* bufferStream = partState->GetDownloadPartStream();
assert(bufferStream);

Aws::String errMsg{handle->WritePartToDownloadStream(bufferStream, partState->GetRangeBegin())};
if (errMsg.empty()) {
handle->ChangePartToCompleted(partState, outcome.GetResult().GetETag());
} else {
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
"InternalFailure", errMsg, false);
AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
<< "] Failed to download object in Bucket: ["
<< handle->GetBucketName() << "] with Key: [" << handle->GetKey()
<< "] " << errMsg);
handle->ChangePartToFailed(partState);
handle->SetError(error);
TriggerErrorCallback(handle, error);
}
}
else
{
handle->ChangePartToFailed(partState);
}
}

// buffer cleanup
if(partState->GetDownloadBuffer())
Expand Down
34 changes: 34 additions & 0 deletions tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2328,6 +2328,40 @@ TEST_P(TransferTests, TransferManager_TestRelativePrefix)
}
}

TEST_P(TransferTests, TransferManager_ContentRangeVerificationTest)
{
const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID();
Aws::String testFileName = MakeFilePath(RandomFileName.c_str());
ScopedTestFile testFile(testFileName, MEDIUM_TEST_SIZE, testString);

TransferManagerConfiguration transferManagerConfig(m_executor.get());
transferManagerConfig.s3Client = m_s3Clients[GetParam()];
auto transferManager = TransferManager::Create(transferManagerConfig);

std::shared_ptr<TransferHandle> uploadPtr = transferManager->UploadFile(testFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map<Aws::String, Aws::String>());
uploadPtr->WaitUntilFinished();
ASSERT_EQ(TransferStatus::COMPLETED, uploadPtr->GetStatus());
ASSERT_TRUE(WaitForObjectToPropagate(GetTestBucketName(), RandomFileName.c_str()));

auto downloadFileName = MakeDownloadFileName(RandomFileName);
auto createStreamFn = [=](){
#ifdef _MSC_VER
return Aws::New<Aws::FStream>(ALLOCATION_TAG, Aws::Utils::StringUtils::ToWString(downloadFileName.c_str()).c_str(), std::ios_base::out | std::ios_base::in | std::ios_base::binary | std::ios_base::trunc);
#else
return Aws::New<Aws::FStream>(ALLOCATION_TAG, downloadFileName.c_str(), std::ios_base::out | std::ios_base::in | std::ios_base::binary | std::ios_base::trunc);
#endif
};

uint64_t offset = 1024;
uint64_t partSize = 2048;
std::shared_ptr<TransferHandle> downloadPtr = transferManager->DownloadFile(GetTestBucketName(), RandomFileName, offset, partSize, createStreamFn);

downloadPtr->WaitUntilFinished();
ASSERT_EQ(TransferStatus::COMPLETED, downloadPtr->GetStatus());
ASSERT_EQ(partSize, downloadPtr->GetBytesTotalSize());
ASSERT_EQ(partSize, downloadPtr->GetBytesTransferred());
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should there also be a test with a false content range to test for failures?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll look into creating a mock S3 response with an incorrect ContentRange to test the failure path

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 i think even as going as far to have a "transfer manager unit test" might be good. you can largely copy the S3 unit tests to create the skeleton of it.

INSTANTIATE_TEST_SUITE_P(Https, TransferTests, testing::Values(TestType::Https));
INSTANTIATE_TEST_SUITE_P(Http, TransferTests, testing::Values(TestType::Http));

Expand Down
31 changes: 31 additions & 0 deletions tests/aws-cpp-sdk-transfer-unit-tests/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
add_project(aws-cpp-sdk-transfer-unit-tests
"Unit Tests for the Transfer Manager"
aws-cpp-sdk-transfer
aws-cpp-sdk-s3
testing-resources
aws_test_main
aws-cpp-sdk-core)

add_definitions(-DRESOURCES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/resources")

if(MSVC AND BUILD_SHARED_LIBS)
add_definitions(-DGTEST_LINKED_AS_SHARED_LIBRARY=1)
endif()

enable_testing()

if(PLATFORM_ANDROID AND BUILD_SHARED_LIBS)
add_library(${PROJECT_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/TransferUnitTests.cpp)
else()
add_executable(${PROJECT_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/TransferUnitTests.cpp)
endif()

set_compiler_flags(${PROJECT_NAME})
set_compiler_warnings(${PROJECT_NAME})

target_link_libraries(${PROJECT_NAME} ${PROJECT_LIBS})

if(MSVC AND BUILD_SHARED_LIBS)
set_target_properties(${PROJECT_NAME} PROPERTIES LINK_FLAGS "/DELAYLOAD:aws-cpp-sdk-transfer.dll /DELAYLOAD:aws-cpp-sdk-core.dll")
target_link_libraries(${PROJECT_NAME} delayimp.lib)
endif()
110 changes: 110 additions & 0 deletions tests/aws-cpp-sdk-transfer-unit-tests/TransferUnitTests.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
#include <gtest/gtest.h>
#include <aws/core/Aws.h>
#include <aws/core/utils/threading/PooledThreadExecutor.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/GetObjectResult.h>
#include <aws/transfer/TransferManager.h>
#include <aws/testing/AwsTestHelpers.h>
#include <aws/testing/MemoryTesting.h>
#include <sstream>

using namespace Aws;
using namespace Aws::S3;
using namespace Aws::S3::Model;
using namespace Aws::Transfer;
using namespace Aws::Utils::Threading;

const char* ALLOCATION_TAG = "TransferUnitTest";

class MockS3Client : public S3Client {
public:
MockS3Client() : S3Client(), shouldReturnWrongRange(false) {}

void SetReturnWrongRange(bool wrongRange) {
shouldReturnWrongRange = wrongRange;
}

GetObjectOutcome GetObject(const GetObjectRequest& request) const override {
GetObjectResult result;

if (request.RangeHasBeenSet()) {
if (shouldReturnWrongRange) {
// Return a different range than requested
result.SetContentRange("bytes 1024-2047/2048");
} else {
// Parse the requested range and return matching ContentRange
Aws::String requestRange = request.GetRange();
if (requestRange.find("bytes=") == 0) {
Aws::String range = requestRange.substr(6); // Remove "bytes="
result.SetContentRange("bytes " + range + "/2048");
}
}
}

// Create a raw pointer stream
auto stream = Aws::New<std::stringstream>(ALLOCATION_TAG);
*stream << "mock data";
result.ReplaceBody(stream);

return GetObjectOutcome(std::move(result));
}

private:
bool shouldReturnWrongRange;
};

class TransferUnitTest : public testing::Test {
protected:
void SetUp() override {
executor = Aws::MakeShared<PooledThreadExecutor>(ALLOCATION_TAG, 1);
mockS3Client = Aws::MakeShared<MockS3Client>(ALLOCATION_TAG);
}

static void SetUpTestSuite() {
#ifdef USE_AWS_MEMORY_MANAGEMENT
_testMemorySystem.reset(new ExactTestMemorySystem(1024, 128));
_options.memoryManagementOptions.memoryManager = _testMemorySystem.get();
#endif
InitAPI(_options);
}

static void TearDownTestSuite() {
ShutdownAPI(_options);
#ifdef USE_AWS_MEMORY_MANAGEMENT
EXPECT_EQ(_testMemorySystem->GetCurrentOutstandingAllocations(), 0ULL);
EXPECT_EQ(_testMemorySystem->GetCurrentBytesAllocated(), 0ULL);
EXPECT_TRUE(_testMemorySystem->IsClean());
_testMemorySystem.reset();
#endif
}

std::shared_ptr<PooledThreadExecutor> executor;
std::shared_ptr<MockS3Client> mockS3Client;
static SDKOptions _options;
#ifdef USE_AWS_MEMORY_MANAGEMENT
static std::unique_ptr<ExactTestMemorySystem> _testMemorySystem;
#endif
};

SDKOptions TransferUnitTest::_options;
#ifdef USE_AWS_MEMORY_MANAGEMENT
std::unique_ptr<ExactTestMemorySystem> TransferUnitTest::_testMemorySystem = nullptr;
#endif

TEST_F(TransferUnitTest, ContentRangeVerification_Failure) {
mockS3Client->SetReturnWrongRange(true);

TransferManagerConfiguration config(executor.get());
config.s3Client = mockS3Client;
auto transferManager = TransferManager::Create(config);

auto createStreamFn = []() {
return Aws::New<std::stringstream>(ALLOCATION_TAG);
};

auto handle = transferManager->DownloadFile("test-bucket", "test-key", 0, 1024, createStreamFn);
handle->WaitUntilFinished();

EXPECT_EQ(TransferStatus::FAILED, handle->GetStatus());
}
Loading