From 6ead56d0ed7dbaafe3fe04b129d9c12d342020e4 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 11 May 2020 09:51:08 -0500 Subject: [PATCH 1/3] Fix S3 remove object bug + add bucket/directory existence checks. --- LNX-docker-compose.yml | 14 ++++----- datajoint/errors.py | 12 ++++++++ datajoint/external.py | 5 +++- datajoint/s3.py | 8 +++--- local-docker-compose.yml | 14 ++++----- tests/__init__.py | 27 +++++++++--------- ...UBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local | Bin ...QW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal | Bin ...LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared | Bin ...gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared | Bin ...KVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared | Bin ...3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA | Bin ...Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 | Bin tests/test_blob_migrate.py | 8 +++--- tests/test_filepath.py | 7 +++-- tests/test_s3.py | 18 ++++++++---- 16 files changed, 70 insertions(+), 43 deletions(-) rename tests/external-legacy-data/file/temp/{migrate-test => datajoint-migrate}/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local (100%) rename tests/external-legacy-data/file/temp/{migrate-test => datajoint-migrate}/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal (100%) rename tests/external-legacy-data/s3/{migrate-test => datajoint-migrate}/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared (100%) rename tests/external-legacy-data/s3/{migrate-test => datajoint-migrate}/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared (100%) rename tests/external-legacy-data/s3/{migrate-test => datajoint-migrate}/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared (100%) rename tests/external-legacy-data/s3/{migrate-test => datajoint-migrate}/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA (100%) rename tests/external-legacy-data/s3/{migrate-test => datajoint-migrate}/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 (100%) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index 209a1879d..30a1ef4b1 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -19,27 +19,27 @@ services: - MINIO_ACCESS_KEY=datajoint - MINIO_SECRET_KEY=datajoint # ports: - # - "9000:9000" + # - "80:80" # volumes: # - ./minio/config:/root/.minio # - ./minio/data:/data - command: server /data + command: server --address ":80" /data healthcheck: - test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"] + test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"] timeout: 5s retries: 60 interval: 1s fakeservices.datajoint.io: <<: *net - image: raphaelguzman/nginx:v0.0.4 + image: raphaelguzman/nginx:v0.0.5 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 - ADD_minio_TYPE=MINIO - - ADD_minio_ENDPOINT=minio:9000 + - ADD_minio_ENDPOINT=minio:80 - ADD_minio_PREFIX=/ # ports: - # - "9000:9000" + # - "80:80" # - "443:443" # - "3306:3306" depends_on: @@ -60,7 +60,7 @@ services: - DJ_TEST_HOST=fakeservices.datajoint.io - DJ_TEST_USER=datajoint - DJ_TEST_PASSWORD=datajoint - - S3_ENDPOINT=fakeservices.datajoint.io:9000 + - S3_ENDPOINT=fakeservices.datajoint.io - S3_ACCESS_KEY=datajoint - S3_SECRET_KEY=datajoint - S3_BUCKET=datajoint-test diff --git a/datajoint/errors.py b/datajoint/errors.py index 7c1ceb182..2b4b2e3a8 100644 --- a/datajoint/errors.py +++ b/datajoint/errors.py @@ -81,6 +81,18 @@ class MissingExternalFile(DataJointError): """ +class DirectoryInaccessible(DataJointError): + """ + Error raised when a local directory is inaccessible + """ + + +class BucketInaccessible(DataJointError): + """ + Error raised when a S3 bucket is inaccessible + """ + + # environment variables to control availability of experimental features ADAPTED_TYPE_SWITCH = "DJ_SUPPORT_ADAPTED_TYPES" diff --git a/datajoint/external.py b/datajoint/external.py index f040053f2..ab69cf9c8 100644 --- a/datajoint/external.py +++ b/datajoint/external.py @@ -2,7 +2,7 @@ from collections import Mapping from tqdm import tqdm from .settings import config -from .errors import DataJointError, MissingExternalFile +from .errors import DataJointError, MissingExternalFile, DirectoryInaccessible from .hash import uuid_from_buffer, uuid_from_file from .table import Table from .declare import EXTERNAL_TABLE_ROOT @@ -46,6 +46,9 @@ def __init__(self, connection, store=None, database=None): if not self.is_declared: self.declare() self._s3 = None + if self.spec['protocol'] == 'file' and not Path(self.spec['location']).is_dir(): + raise DirectoryInaccessible('Inaccessible local directory %s' % + self.spec['location']) from None @property def definition(self): diff --git a/datajoint/s3.py b/datajoint/s3.py index 710d6626f..f8e59bccf 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -14,11 +14,11 @@ class Folder: A Folder instance manipulates a flat folder of objects within an S3-compatible object store """ def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, **_): - self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure) + self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key, + secure=secure) self.bucket = bucket if not self.client.bucket_exists(bucket): - warnings.warn('Creating bucket "%s"' % self.bucket) - self.client.make_bucket(self.bucket) + raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket) from None def put(self, name, buffer): return self.client.put_object( @@ -63,6 +63,6 @@ def get_size(self, name): def remove_object(self, name): try: - self.client.remove_objects(self.bucket, str(name)) + self.client.remove_object(self.bucket, str(name)) except minio.ResponseError: return errors.DataJointError('Failed to delete %s from s3 storage' % name) diff --git a/local-docker-compose.yml b/local-docker-compose.yml index 219c6a81c..b0b976c41 100644 --- a/local-docker-compose.yml +++ b/local-docker-compose.yml @@ -20,28 +20,28 @@ services: - MINIO_ACCESS_KEY=datajoint - MINIO_SECRET_KEY=datajoint # ports: - # - "9000:9000" + # - "80:80" # To persist MinIO data and config # volumes: # - ./minio/data:/data # - ./minio/config:/root/.minio - command: server /data + command: server --address ":80" /data healthcheck: - test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"] + test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"] timeout: 5s retries: 60 interval: 1s fakeservices.datajoint.io: <<: *net - image: raphaelguzman/nginx:v0.0.4 + image: raphaelguzman/nginx:v0.0.5 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 - ADD_minio_TYPE=MINIO - - ADD_minio_ENDPOINT=minio:9000 + - ADD_minio_ENDPOINT=minio:80 - ADD_minio_PREFIX=/ ports: - - "9000:9000" + - "80:80" - "443:443" - "3306:3306" depends_on: @@ -63,7 +63,7 @@ services: - DJ_TEST_USER=datajoint - DJ_TEST_PASSWORD=datajoint # If running tests locally, make sure to add entry in /etc/hosts for 127.0.0.1 fakeservices.datajoint.io - - S3_ENDPOINT=fakeservices.datajoint.io:9000 + - S3_ENDPOINT=fakeservices.datajoint.io - S3_ACCESS_KEY=datajoint - S3_SECRET_KEY=datajoint - S3_BUCKET=datajoint-test diff --git a/tests/__init__.py b/tests/__init__.py index ebc2e4a40..e5a4f835e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -41,6 +41,10 @@ secret_key=environ.get('S3_SECRET_KEY', 'datajoint'), bucket=environ.get('S3_BUCKET', 'datajoint-test')) +S3_MIGRATE_BUCKET = [path.stem for path in Path( + Path(__file__).resolve().parent, + 'external-legacy-data', 's3').iterdir()][0] + # Prefix for all databases used during testing PREFIX = environ.get('DJ_TEST_DB_PREFIX', 'djtest') conn_root = dj.conn(**CONN_INFO_ROOT) @@ -129,10 +133,9 @@ def setup_package(): source = Path( Path(__file__).resolve().parent, 'external-legacy-data','s3') - bucket = "migrate-test" region = "us-east-1" try: - minioClient.make_bucket(bucket, location=region) + minioClient.make_bucket(S3_MIGRATE_BUCKET, location=region) except minio.error.BucketAlreadyOwnedByYou: pass @@ -140,12 +143,12 @@ def setup_package(): for path in pathlist: if os.path.isfile(str(path)) and ".sql" not in str(path): minioClient.fput_object( - bucket, str(Path( - os.path.relpath(str(path),str(Path(source,bucket)))) + S3_MIGRATE_BUCKET, str(Path( + os.path.relpath(str(path),str(Path(source,S3_MIGRATE_BUCKET)))) .as_posix()), str(path)) # Add S3 try: - minioClient.make_bucket("datajoint-test", location=region) + minioClient.make_bucket(S3_CONN_INFO['bucket'], location=region) except minio.error.BucketAlreadyOwnedByYou: pass @@ -176,19 +179,17 @@ def teardown_package(): remove("dj_local_conf.json") # Remove old S3 - bucket = "migrate-test" objs = list(minioClient.list_objects_v2( - bucket, recursive=True)) - objs = [minioClient.remove_object(bucket, + S3_MIGRATE_BUCKET, recursive=True)) + objs = [minioClient.remove_object(S3_MIGRATE_BUCKET, o.object_name.encode('utf-8')) for o in objs] - minioClient.remove_bucket(bucket) + minioClient.remove_bucket(S3_MIGRATE_BUCKET) # Remove S3 - bucket = "datajoint-test" - objs = list(minioClient.list_objects_v2(bucket, recursive=True)) - objs = [minioClient.remove_object(bucket, + objs = list(minioClient.list_objects_v2(S3_CONN_INFO['bucket'], recursive=True)) + objs = [minioClient.remove_object(S3_CONN_INFO['bucket'], o.object_name.encode('utf-8')) for o in objs] - minioClient.remove_bucket(bucket) + minioClient.remove_bucket(S3_CONN_INFO['bucket']) # Remove old File Content shutil.rmtree(str(Path(os.path.expanduser('~'),'temp'))) diff --git a/tests/external-legacy-data/file/temp/migrate-test/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local b/tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local similarity index 100% rename from tests/external-legacy-data/file/temp/migrate-test/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local rename to tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94local diff --git a/tests/external-legacy-data/file/temp/migrate-test/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal b/tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal similarity index 100% rename from tests/external-legacy-data/file/temp/migrate-test/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal rename to tests/external-legacy-data/file/temp/datajoint-migrate/djtest_blob_migrate/e46pnXQW9GaCKbL3WxV1crGHeGqcE0OLInM_TTwAFfwlocal diff --git a/tests/external-legacy-data/s3/migrate-test/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared b/tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared similarity index 100% rename from tests/external-legacy-data/s3/migrate-test/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared rename to tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/FoRROa2LWM6_wx0RIQ0J-LVvgm256cqDQfJa066HoTEshared diff --git a/tests/external-legacy-data/s3/migrate-test/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared b/tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared similarity index 100% rename from tests/external-legacy-data/s3/migrate-test/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared rename to tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/NmWj002gtKUkt9GIBwzn6Iw3x6h7ovlX_FfELbfjwRQshared diff --git a/tests/external-legacy-data/s3/migrate-test/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared b/tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared similarity index 100% rename from tests/external-legacy-data/s3/migrate-test/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared rename to tests/external-legacy-data/s3/datajoint-migrate/maps/djtest_blob_migrate/Ue9c89gKVZD7xPOcHd5Lz6mARJQ50xT1G5cTTX4h0L0shared diff --git a/tests/external-legacy-data/s3/migrate-test/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA b/tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA similarity index 100% rename from tests/external-legacy-data/s3/migrate-test/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA rename to tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_3A03zPqfVhbn0rhlOJYGNivFJ4uqYuHaeQBA-V8PKA diff --git a/tests/external-legacy-data/s3/migrate-test/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 b/tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 similarity index 100% rename from tests/external-legacy-data/s3/migrate-test/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 rename to tests/external-legacy-data/s3/datajoint-migrate/store/djtest_blob_migrate/_Fhi2GUBB0fgxcSP2q-isgncIUTdgGK7ivHiySAU_94 diff --git a/tests/test_blob_migrate.py b/tests/test_blob_migrate.py index 5f2263e62..088dad913 100644 --- a/tests/test_blob_migrate.py +++ b/tests/test_blob_migrate.py @@ -4,7 +4,7 @@ import datajoint as dj import os from pathlib import Path -from . import S3_CONN_INFO +from . import S3_CONN_INFO, S3_MIGRATE_BUCKET from . import CONN_INFO from datajoint.migrate import _migrate_dj011_blob dj.config['enable_python_native_blobs'] = True @@ -20,20 +20,20 @@ def test_convert(): default_store: dict( protocol='s3', endpoint=S3_CONN_INFO['endpoint'], - bucket='migrate-test', + bucket=S3_MIGRATE_BUCKET, location='store', access_key=S3_CONN_INFO['access_key'], secret_key=S3_CONN_INFO['secret_key']), 'shared': dict( protocol='s3', endpoint=S3_CONN_INFO['endpoint'], - bucket='migrate-test', + bucket=S3_MIGRATE_BUCKET, location='maps', access_key=S3_CONN_INFO['access_key'], secret_key=S3_CONN_INFO['secret_key']), 'local': dict( protocol='file', - location=str(Path(os.path.expanduser('~'),'temp','migrate-test'))) + location=str(Path(os.path.expanduser('~'),'temp',S3_MIGRATE_BUCKET))) } dj.config['cache'] = str(Path(os.path.expanduser('~'),'temp','dj-cache')) diff --git a/tests/test_filepath.py b/tests/test_filepath.py index 2897b1b92..b54c82547 100644 --- a/tests/test_filepath.py +++ b/tests/test_filepath.py @@ -59,7 +59,8 @@ def test_filepath(store="repo"): # put the same file twice to ensure storing once uuid1 = ext.upload_filepath(str(managed_file)) - uuid2 = ext.upload_filepath(str(managed_file)) # no duplication should arise if file is the same + # no duplication should arise if file is the same + uuid2 = ext.upload_filepath(str(managed_file)) assert_equal(uuid1, uuid2) # remove to ensure downloading @@ -79,6 +80,7 @@ def test_filepath(store="repo"): # cleanup ext.delete(delete_external_files=True) + assert_false(ext.exists(ext._make_external_filepath(str(Path(relpath, filename))))) def test_filepath_s3(): @@ -115,7 +117,8 @@ def test_duplicate_error(store="repo"): ext.upload_filepath(str(managed_file)) with managed_file.open('wb') as f: f.write(os.urandom(300)) - ext.upload_filepath(str(managed_file)) # this should raise exception because the file has changed + # this should raise exception because the file has changed + ext.upload_filepath(str(managed_file)) def test_duplicate_error_s3(): diff --git a/tests/test_s3.py b/tests/test_s3.py index e8383e6d0..4036f5d81 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -12,8 +12,12 @@ def test_connection(): # Initialize httpClient with relevant timeout. http_client = urllib3.PoolManager( - timeout=30, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), - retries=urllib3.Retry(total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504])) + timeout=30, cert_reqs='CERT_REQUIRED', + ca_certs=certifi.where(), + retries=urllib3.Retry(total=3, backoff_factor=0.2, + status_forcelist=[ + 500, 502, + 503, 504])) # Initialize minioClient with an endpoint and access/secret keys. minio_client = Minio( @@ -30,12 +34,16 @@ def test_connection_secure(): # Initialize httpClient with relevant timeout. http_client = urllib3.PoolManager( - timeout=30, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), - retries=urllib3.Retry(total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504])) + timeout=30, cert_reqs='CERT_REQUIRED', + ca_certs=certifi.where(), + retries=urllib3.Retry(total=3, backoff_factor=0.2, + status_forcelist=[ + 500, 502, + 503, 504])) # Initialize minioClient with an endpoint and access/secret keys. minio_client = Minio( - S3_CONN_INFO['endpoint'].split(':')[0] + ':443', + S3_CONN_INFO['endpoint'], access_key=S3_CONN_INFO['access_key'], secret_key=S3_CONN_INFO['secret_key'], secure=True, From 95aabde11645d7532daf80e4eeca9c31b16e20b8 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 11 May 2020 13:34:06 -0500 Subject: [PATCH 2/3] Remove dj-specific directory not found error. --- datajoint/errors.py | 6 ------ datajoint/external.py | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/datajoint/errors.py b/datajoint/errors.py index 2b4b2e3a8..93a4e01a9 100644 --- a/datajoint/errors.py +++ b/datajoint/errors.py @@ -81,12 +81,6 @@ class MissingExternalFile(DataJointError): """ -class DirectoryInaccessible(DataJointError): - """ - Error raised when a local directory is inaccessible - """ - - class BucketInaccessible(DataJointError): """ Error raised when a S3 bucket is inaccessible diff --git a/datajoint/external.py b/datajoint/external.py index ab69cf9c8..c015cedee 100644 --- a/datajoint/external.py +++ b/datajoint/external.py @@ -2,7 +2,7 @@ from collections import Mapping from tqdm import tqdm from .settings import config -from .errors import DataJointError, MissingExternalFile, DirectoryInaccessible +from .errors import DataJointError, MissingExternalFile from .hash import uuid_from_buffer, uuid_from_file from .table import Table from .declare import EXTERNAL_TABLE_ROOT @@ -47,7 +47,7 @@ def __init__(self, connection, store=None, database=None): self.declare() self._s3 = None if self.spec['protocol'] == 'file' and not Path(self.spec['location']).is_dir(): - raise DirectoryInaccessible('Inaccessible local directory %s' % + raise FileNotFoundError('Inaccessible local directory %s' % self.spec['location']) from None @property From d4e41537184287c5de62f0d9b79b733b51563eee Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 11 May 2020 14:06:02 -0500 Subject: [PATCH 3/3] Fix style. --- datajoint/external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/external.py b/datajoint/external.py index c015cedee..58b64a817 100644 --- a/datajoint/external.py +++ b/datajoint/external.py @@ -48,7 +48,7 @@ def __init__(self, connection, store=None, database=None): self._s3 = None if self.spec['protocol'] == 'file' and not Path(self.spec['location']).is_dir(): raise FileNotFoundError('Inaccessible local directory %s' % - self.spec['location']) from None + self.spec['location']) from None @property def definition(self):