rero · rerowep · Jun 22, 2022 · May 3, 2022
diff --git a/rero_ils/modules/cli/fixtures.py b/rero_ils/modules/cli/fixtures.py
@@ -261,31 +261,30 @@ def get_all_mef_records(infile, lazy, verbose, enqueue, wait, outfile_name):
     """Get all contributions for given document file."""
     def get_ref(ref, refs, outfile_name):
         """Get the contribution."""
-        if ref and not refs.get(ref):
-            refs[ref] = 1
-            if outfile_name:
-                try:
-                    ref_split = ref.split('/')
-                    ref_type = ref_split[-2]
-                    ref_pid = ref_split[-1]
-                    data = Contribution._get_mef_data_by_type(
-                        pid=ref_pid,
-                        pid_type=ref_type
-                    )
-                    metadata = data['metadata']
-                    metadata['$schema'] = contribution_schema
-                    outfile.write(metadata)
-                    msg = 'ok'
-                except Exception as err:
-                    msg = err
-            else:
-                if enqueue:
-                    msg = create_mef_record_online.delay(ref)
-                else:
-                    pid, online = create_mef_record_online(ref)
-                    msg = f'contribution pid: {pid} {online}'
-            if verbose:
-                click.echo(f"{count:<10}ref: {ref}\t{msg}")
+        if not ref or refs.get(ref):
+            return
+        refs[ref] = 1
+        if outfile_name:
+            try:
+                ref_split = ref.split('/')
+                ref_type = ref_split[-2]
+                ref_pid = ref_split[-1]
+                data = Contribution._get_mef_data_by_type(
+                    pid=ref_pid,
+                    pid_type=ref_type
+                )
+                data['$schema'] = contribution_schema
+                outfile.write(data)
+                msg = 'ok'
+            except Exception as err:
+                msg = err
+        elif enqueue:
+            msg = create_mef_record_online.delay(ref)
+        else:
+            pid, online = create_mef_record_online(ref)
+            msg = f'contribution pid: {pid} {online}'
+        if verbose:
+            click.echo(f"{count:<10}ref: {ref}\t{msg}")
 
     click.secho(
         f'Get all contributions for {infile.name}.',
@@ -307,10 +306,11 @@ def get_ref(ref, refs, outfile_name):
         for contribution in record.get('contribution', []):
             ref = contribution['agent'].get('$ref')
             get_ref(ref, refs, outfile_name)
-        for subject in record.get('subjects', []):
-            if subject.get('type') in ['bf:Person', 'bf:Organisation']:
-                ref = subject.get('$ref')
-                get_ref(ref, refs, outfile_name)
+        for subject_type in ['subjects', 'subjects_imported']:
+            for subject in record.get(subject_type, []):
+                if subject.get('type') in ['bf:Person', 'bf:Organisation']:
+                    ref = subject.get('$ref')
+                    get_ref(ref, refs, outfile_name)
     if enqueue and wait:
         wait_empty_tasks(delay=3, verbose=True)
     click.echo(f'Count refs: {count}')

diff --git a/rero_ils/modules/contributions/api.py b/rero_ils/modules/contributions/api.py
@@ -106,36 +106,37 @@ def get_record_by_ref(cls, ref):
         contribution = cls.get_contribution(ref_type, ref_pid)
         if not contribution:
             # We dit not find the record in DB get it from MEF and create it.
-            metadata = None
             try:
-                data = cls._get_mef_data_by_type(ref_pid, ref_type,
-                                                 deleted=True)
-                metadata = data['metadata']
-                # Register MEF contribution
-                metadata.pop('$schema', None)
-                # we have to commit because create
-                # uses db.session.begin_nested
-                contribution = cls.create(metadata, dbcommit=True,
-                                          reindex=True)
+                data = cls._get_mef_data_by_type(
+                    pid=ref_pid,
+                    pid_type=ref_type,
+                )
+                contribution = cls.create(
+                    data=data,
+                    dbcommit=True,
+                    reindex=True
+                )
                 online = True
             except Exception as err:
                 db.session.rollback()
-                if metadata:
-                    contribution = cls.get_record_by_pid(metadata.get('pid'))
-                if not contribution:
-                    current_app.logger.error(
-                        f'Get MEF record: {ref_type}:{ref_pid} >>{err}<<'
-                    )
-                    # import traceback
-                    # traceback.print_exc()
-                return contribution, online
+                current_app.logger.error(
+                    f'Get MEF record: {ref_type}:{ref_pid} >>{err}<<'
+                )
+                contribution = None
+                # import traceback
+                # traceback.print_exc()
         db.session.commit()
         return contribution, online
 
     @classmethod
     def _get_mef_data_by_type(cls, pid, pid_type, verbose=False,
-                              deleted=False):
-        """Request MEF REST API in JSON format."""
+                              with_deleted=True, resolve=True, sources=True):
+        """Request MEF REST API in JSON format.
+
+        :param language: language for authorized access point.
+        :returns: authorized access point in given language.
+        """
+
         def get_mef(url):
             """Get data from MEF."""
         url = current_app.config.get('RERO_ILS_MEF_AGENTS_URL')
@@ -146,24 +147,27 @@ def get_mef(url):
                 mef_url = f'{url}/mef/?q=viaf_pid:"{pid}"'
             else:
                 mef_url = f'{url}/mef/?q={pid_type}.pid:"{pid}"'
-        request = requests.get(url=mef_url, params=dict(resolve=1, sources=1))
+        request = requests.get(
+            url=mef_url,
+            params=dict(
+                with_deleted=int(with_deleted),
+                resolve=int(resolve),
+                sources=int(sources)
+            )
+        )
         status = request.status_code
         if status == requests_codes.ok:
             try:
-                return request.json().get('hits', {}).get('hits', [None])[0]
+                data = request.json().get('hits', {}).get('hits', [None])[0]
+                metadata = data['metadata']
+                metadata.pop('$schema', None)
+                sources = current_app.config.get(
+                    'RERO_ILS_CONTRIBUTIONS_SOURCES', [])
+                for source in sources:
+                    if source in metadata:
+                        metadata[source].pop('$schema', None)
+                return metadata
             except Exception:
-                # try to get deleted records
-                if deleted:
-                    request = requests.get(
-                        url=mef_url,
-                        params=dict(resolve=1, sources=1, deleted=1))
-                    status = request.status_code
-                    if status == requests_codes.ok:
-                        try:
-                            return request.json() \
-                                .get('hits', {}).get('hits', [None])[0]
-                        except Exception:
-                            pass
                 msg = f'MEF resolver no metadata: {mef_url}'
                 if verbose:
                     current_app.logger.warning(msg)
@@ -291,24 +295,22 @@ def update_online(self, dbcommit=False, reindex=False, verbose=False):
         try:
             if data := self._get_mef_data_by_type(
                     pid=pid, pid_type='mef', verbose=verbose):
-                metadata = data['metadata']
-                metadata['$schema'] = self['$schema']
-                if metadata.get('deleted'):
+                data['$schema'] = self['$schema']
+                if data.get('deleted'):
                     current_app.logger.warning(
                         f'UPDATE ONLINE {pid}: was deleted')
                     action = ContributionUpdateAction.ERROR
-                elif not metadata.get('sources'):
+                elif not data.get('sources'):
                     current_app.logger.warning(
                         f'UPDATE ONLINE {pid}: has no sources')
                     action = ContributionUpdateAction.ERROR
-                elif not metadata.get('type'):
+                elif not data.get('type'):
                     current_app.logger.warning(
                         f'UPDATE ONLINE {pid}: has no type')
                     action = ContributionUpdateAction.ERROR
-                elif dict(self) != metadata:
+                elif dict(self) != data:
                     action = ContributionUpdateAction.REPLACE
-                    self.replace(data=metadata, dbcommit=dbcommit,
-                                 reindex=reindex)
+                    self.replace(data=data, dbcommit=dbcommit, reindex=reindex)
                     if reindex:
                         indexer = DocumentsIndexer()
                         indexer.bulk_index(self.documents_ids())

diff --git a/rero_ils/modules/documents/tasks.py b/rero_ils/modules/documents/tasks.py
@@ -41,24 +41,22 @@ def get_contribution_or_create(ref_pid, ref_type, count_found, count_exists,
             # contribution does not exist
             try:
                 # try to get the contribution online
-                data = Contribution._get_mef_data_by_type(
-                    ref_pid, ref_type)
-                metadata = data['metadata']
+                data = Contribution._get_mef_data_by_type(ref_pid, ref_type)
                 if (
-                    metadata.get('idref') or
-                    metadata.get('gnd') or
-                    metadata.get('rero')
+                    data.get('idref') or
+                    data.get('gnd') or
+                    data.get('rero')
                 ):
                     count_found.setdefault(
                         ref,
-                        {'count': 0, 'mef': metadata.get('pid')}
+                        {'count': 0, 'mef': data.get('pid')}
                     )
                     count_found[ref]['count'] += 1
                     # delete mef $schema
-                    metadata.pop('$schema', None)
+                    data.pop('$schema', None)
                     # create local contribution
                     cont = Contribution.create(
-                        data=metadata, dbcommit=True, reindex=True)
+                        data=data, dbcommit=True, reindex=True)
                 else:
                     # online contribution has no IdREf, GND or RERO
                     count_no_data.setdefault(ref, 0)

diff --git a/rero_ils/modules/documents/utils_mef.py b/rero_ils/modules/documents/utils_mef.py
@@ -197,18 +197,15 @@ def get_online(self, doc_pid, ref_type, ref_pid):
         try:
             # try to get the contribution online
             data = Contribution._get_mef_data_by_type(ref_pid, ref_type)
-            metadata = data['metadata']
-            if metadata.get('idref') or metadata.get('gnd'):
-                if metadata.get('deleted'):
+            if data.get('idref') or data.get('gnd'):
+                if data.get('deleted'):
                     self.increment_count(self.count_deleted, ref,
                                          f'{doc_pid} Deleted')
                 else:
                     self.increment_count(self.count_found, ref,
                                          f'{doc_pid} Online found')
-                    # delete mef $schema
-                    metadata.pop('$schema', None)
                     # create and return local contribution
-                    return Contribution.create(data=metadata, dbcommit=True,
+                    return Contribution.create(data=data, dbcommit=True,
                                                reindex=True)
             else:
                 # online contribution has no IdREf, GND or RERO

diff --git a/tests/api/documents/test_documents_rest.py b/tests/api/documents/test_documents_rest.py
@@ -568,7 +568,7 @@ def test_documents_resolve(
     ))
     assert res.json['metadata']['contribution'] == [{
         'agent': {
-            '$ref': 'https://mef.rero.ch/api/rero/A017671081',
+            '$ref': 'https://mef.rero.ch/api/agents/rero/A017671081',
             'type': 'bf:Person'
             },
         'role': ['aut']

diff --git a/tests/api/documents/test_documents_tasks.py b/tests/api/documents/test_documents_tasks.py
@@ -41,6 +41,9 @@ def test_replace_idby_contribution(mock_contributions_mef_get, app,
 
     doc = Document.create(data=document_data, dbcommit=True, reindex=True)
     DocumentsSearch.flush_and_refresh()
+    mock_contributions_mef_get.return_value = mock_response(
+        status=500
+    )
     replace = ReplaceMefIdentifiedByContribution()
     replace.process()
     assert replace.counts_len == (0, 0, 0, 0, 1)

diff --git a/tests/data/data.json b/tests/data/data.json
@@ -1915,7 +1915,7 @@
       {
         "agent": {
           "type": "bf:Person",
-          "$ref": "https://mef.rero.ch/api/rero/A017671081"
+          "$ref": "https://mef.rero.ch/api/agents/rero/A017671081"
         },
         "role": [
           "aut"
@@ -2622,7 +2622,7 @@
     "contribution": [
       {
         "agent": {
-          "$ref": "https://mef.rero.ch/api/idref/029314100",
+          "$ref": "https://mef.rero.ch/api/agents/idref/029314100",
           "type": "bf:Person"
         },
         "role": [
@@ -2983,11 +2983,11 @@
     "subjects": [
       {
         "type": "bf:Person",
-        "$ref": "https://mef.rero.ch/api/gnd/041288890"
+        "$ref": "https://mef.rero.ch/api/agents/gnd/041288890"
       },
       {
         "type": "bf:Person",
-        "$ref": "https://mef.rero.ch/api/idref/032106939"
+        "$ref": "https://mef.rero.ch/api/agents/idref/032106939"
       },
       {
         "type": "bf:Person",

diff --git a/tests/data/documents.json b/tests/data/documents.json
@@ -55,7 +55,7 @@
       {
         "agent": {
           "type": "bf:Person",
-          "$ref": "https://mef.rero.ch/api/idref/074755978"
+          "$ref": "https://mef.rero.ch/api/agents/idref/223977268"
         },
         "role": [
           "aut"
@@ -64,7 +64,7 @@
       {
         "agent": {
           "type": "bf:Person",
-          "$ref": "https://mef.rero.ch/api/rero/A003683610"
+          "$ref": "https://mef.rero.ch/api/agents/rero/A017671081"
         },
         "role": [
           "aut"
@@ -172,8 +172,8 @@
     "contribution": [
       {
         "agent": {
-          "type": "bf:Organisation",
-          "$ref": "https://mef.rero.ch/api/idref/074755978"
+          "type": "bf:Person",
+          "$ref": "https://mef.rero.ch/api/agents/idref/223977268"
         },
         "role": [
           "aut"

diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py
@@ -252,15 +252,20 @@ def contribution_person_data(data):
 
 
 @pytest.fixture(scope="function")
-def contribution_person_data_tmp(data):
+def contribution_person_data_tmp(app, data):
     """Load mef contribution data person scope function."""
-    return deepcopy(data.get('cont_pers'))
+    contribution_person = deepcopy(data.get('cont_pers'))
+    sources = app.config.get('RERO_ILS_CONTRIBUTIONS_SOURCES', [])
+    for source in sources:
+        if source in contribution_person:
+            contribution_person[source].pop('$schema', None)
+    return contribution_person
 
 
 @pytest.fixture(scope="module")
 def contribution_person_response_data(contribution_person_data):
     """Load mef contribution person response data."""
-    json_data = {
+    return {
         'hits': {
             'hits': [
                 {
@@ -270,7 +275,6 @@ def contribution_person_response_data(contribution_person_data):
             ]
         }
     }
-    return json_data
 
 
 @pytest.fixture(scope="module")

diff --git a/tests/ui/contributions/test_contributions_api.py b/tests/ui/contributions/test_contributions_api.py
@@ -73,7 +73,7 @@ def test_contribution_mef_create(mock_contributions_mef_get, app,
         json_data=contribution_person_response_data
     )
     pers_mef, online = Contribution.get_record_by_ref(
-        'https://mef.rero.ch/api/rero/A017671081')
+        'https://mef.rero.ch/api/agents/rero/A017671081')
     flush_index(ContributionsSearch.Meta.index)
     assert pers_mef == contribution_person_data_tmp
     assert online
@@ -82,6 +82,6 @@ def test_contribution_mef_create(mock_contributions_mef_get, app,
     pers_mef['sources'] = ['gnd']
     pers_mef.replace(pers_mef, dbcommit=True)
     pers_db, online = Contribution.get_record_by_ref(
-        'https://mef.rero.ch/api/gnd/13343771X')
+        'https://mef.rero.ch/api/agents/gnd/13343771X')
     assert pers_db['sources'] == ['gnd']
     assert not online
diff --git a/tests/ui/documents/test_documents_api.py b/tests/ui/documents/test_documents_api.py
@@ -158,7 +158,7 @@ def test_document_contribution_resolve_exception(es_clear, db,
                                                  document_data_ref):
     """Test document contribution resolve."""
     document_data_ref['contribution'] = [{
-        '$ref': 'https://mef.rero.ch/api/rero/XXXXXX'
+        '$ref': 'https://mef.rero.ch/api/agents/rero/XXXXXX'
     }],
     with pytest.raises(Exception):
         Document.create(