# -*- coding: utf-8 -*-

#

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements. See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

"""Unit tests for the S3 File System"""

# pytype: skip-file

import logging

import unittest

import mock

from apache_beam.io.aws.clients.s3 import messages

from apache_beam.io.filesystem import BeamIOError

from apache_beam.io.filesystem import FileMetadata

from apache_beam.options.pipeline_options import PipelineOptions

# Protect against environments where boto3 library is not available.

# pylint: disable=wrong-import-order, wrong-import-position

try:

from apache_beam.io.aws import s3filesystem

except ImportError:

s3filesystem = None # type: ignore[assignment]

# pylint: enable=wrong-import-order, wrong-import-position

@unittest.skipIf(s3filesystem is None, 'AWS dependencies are not installed')

class S3FileSystemTest(unittest.TestCase):

def setUp(self):

pipeline_options = PipelineOptions()

self.fs = s3filesystem.S3FileSystem(pipeline_options=pipeline_options)

def test_scheme(self):

self.assertEqual(self.fs.scheme(), 's3')

self.assertEqual(s3filesystem.S3FileSystem.scheme(), 's3')

def test_join(self):

self.assertEqual(

's3://bucket/path/to/file',

self.fs.join('s3://bucket/path', 'to', 'file'))

self.assertEqual(

's3://bucket/path/to/file', self.fs.join('s3://bucket/path', 'to/file'))

self.assertEqual(

's3://bucket/path/to/file',

self.fs.join('s3://bucket/path', '/to/file'))

self.assertEqual(

's3://bucket/path/to/file',

self.fs.join('s3://bucket/path/', 'to', 'file'))

self.assertEqual(

's3://bucket/path/to/file',

self.fs.join('s3://bucket/path/', 'to/file'))

self.assertEqual(

's3://bucket/path/to/file',

self.fs.join('s3://bucket/path/', '/to/file'))

with self.assertRaises(ValueError):

self.fs.join('/bucket/path/', '/to/file')

def test_split(self):

self.assertEqual(('s3://foo/bar', 'baz'), self.fs.split('s3://foo/bar/baz'))

self.assertEqual(('s3://foo', ''), self.fs.split('s3://foo/'))

self.assertEqual(('s3://foo', ''), self.fs.split('s3://foo'))

with self.assertRaises(ValueError):

self.fs.split('/no/s3/prefix')

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_match_single(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

s3io_mock._status.return_value = {'size': 1, 'last_updated': 9999999.0}

expected_results = [FileMetadata('s3://bucket/file1', 1, 9999999.0)]

match_result = self.fs.match(['s3://bucket/file1'])[0]

self.assertEqual(match_result.metadata_list, expected_results)

s3io_mock._status.assert_called_once_with('s3://bucket/file1')

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_match_multiples(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

s3io_mock.list_files.return_value = iter([

('s3://bucket/file1', (1, 9999999.0)),

('s3://bucket/file2', (2, 8888888.0))

])

expected_results = set([

FileMetadata('s3://bucket/file1', 1, 9999999.0),

FileMetadata('s3://bucket/file2', 2, 8888888.0)

])

match_result = self.fs.match(['s3://bucket/'])[0]

self.assertEqual(set(match_result.metadata_list), expected_results)

s3io_mock.list_files.assert_called_once_with(

's3://bucket/', with_metadata=True)

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_match_multiples_limit(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

limit = 1

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

s3io_mock.list_files.return_value = iter([

('s3://bucket/file1', (1, 99999.0))

])

expected_results = set([FileMetadata('s3://bucket/file1', 1, 99999.0)])

match_result = self.fs.match(['s3://bucket/'], [limit])[0]

self.assertEqual(set(match_result.metadata_list), expected_results)

self.assertEqual(len(match_result.metadata_list), limit)

s3io_mock.list_files.assert_called_once_with(

's3://bucket/', with_metadata=True)

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_match_multiples_error(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

exception = IOError('Failed')

s3io_mock.list_files.side_effect = exception

with self.assertRaises(BeamIOError) as error:

self.fs.match(['s3://bucket/'])

self.assertIn('Match operation failed', str(error.exception))

s3io_mock.list_files.assert_called_once_with(

's3://bucket/', with_metadata=True)

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_match_multiple_patterns(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

s3io_mock.list_files.side_effect = [

iter([('s3://bucket/file1', (1, 99999.0))]),

iter([('s3://bucket/file2', (2, 88888.0))]),

]

expected_results = [[FileMetadata('s3://bucket/file1', 1, 99999.0)],

[FileMetadata('s3://bucket/file2', 2, 88888.0)]]

result = self.fs.match(['s3://bucket/file1*', 's3://bucket/file2*'])

self.assertEqual([mr.metadata_list for mr in result], expected_results)

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_create(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

# Issue file copy

_ = self.fs.create('s3://bucket/from1', 'application/octet-stream')

s3io_mock.open.assert_called_once_with(

's3://bucket/from1', 'wb', mime_type='application/octet-stream')

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_open(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

# Issue file copy

_ = self.fs.open('s3://bucket/from1', 'application/octet-stream')

s3io_mock.open.assert_called_once_with(

's3://bucket/from1', 'rb', mime_type='application/octet-stream')

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_copy_file(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

sources = ['s3://bucket/from1', 's3://bucket/from2']

destinations = ['s3://bucket/to1', 's3://bucket/to2']

# Issue file copy

self.fs.copy(sources, destinations)

src_dest_pairs = list(zip(sources, destinations))

s3io_mock.copy_paths.assert_called_once_with(src_dest_pairs)

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_copy_file_error(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

sources = ['s3://bucket/from1', 's3://bucket/from2', 's3://bucket/from3']

destinations = ['s3://bucket/to1', 's3://bucket/to2']

# Issue file copy

with self.assertRaises(BeamIOError):

self.fs.copy(sources, destinations)

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_delete(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

s3io_mock.size.return_value = 0

files = [

's3://bucket/from1',

's3://bucket/from2',

's3://bucket/from3',

]

# Issue batch delete.

self.fs.delete(files)

s3io_mock.delete_paths.assert_called_once_with(files)

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_delete_error(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

problematic_directory = 's3://nonexistent-bucket/tree/'

exception = messages.S3ClientError('Not found', 404)

s3io_mock.delete_paths.return_value = {

problematic_directory: exception,

's3://bucket/object1': None,

's3://bucket/object2': None,

}

s3io_mock.size.return_value = 0

files = [

problematic_directory,

's3://bucket/object1',

's3://bucket/object2',

]

expected_results = {problematic_directory: exception}

# Issue batch delete.

with self.assertRaises(BeamIOError) as error:

self.fs.delete(files)

self.assertIn('Delete operation failed', str(error.exception))

self.assertEqual(error.exception.exception_details, expected_results)

s3io_mock.delete_paths.assert_called()

@mock.patch('apache_beam.io.aws.s3filesystem.s3io')

def test_rename(self, unused_mock_arg):

# Prepare mocks.

s3io_mock = mock.MagicMock()

s3filesystem.s3io.S3IO = lambda options: s3io_mock # type: ignore[misc]

sources = ['s3://bucket/from1', 's3://bucket/from2']

destinations = ['s3://bucket/to1', 's3://bucket/to2']

# Issue file copy

self.fs.rename(sources, destinations)

src_dest_pairs = list(zip(sources, destinations))

s3io_mock.rename_files.assert_called_once_with(src_dest_pairs)

def test_lineage(self):

self._verify_lineage("s3://bucket/", ("bucket", ))

self._verify_lineage("s3://bucket/foo/bar.txt", ("bucket", "foo/bar.txt"))

def _verify_lineage(self, uri, expected_segments):

lineage_mock = mock.MagicMock()

self.fs.report_lineage(uri, lineage_mock)

lineage_mock.add.assert_called_once_with(

"s3", *expected_segments, last_segment_sep='/')

if __name__ == '__main__':

logging.getLogger().setLevel(logging.INFO)

unittest.main()