uc-cdis · paulineribeyre · Nov 17, 2025 · Jun 13, 2025 · Jul 11, 2025 · Jul 25, 2025
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -275,7 +275,7 @@
         "filename": "tests/integration/datadict/submission/test_endpoints.py",
         "hashed_secret": "78b4db9b2aec0f0f2d3e38f9278be42b861c9dc3",
         "is_verified": false,
-        "line_number": 1420
+        "line_number": 1481
       }
     ],
     "tests/integration/datadictwithobjid/submission/data/biospec1.json": [
@@ -354,5 +354,5 @@
       }
     ]
   },
-  "generated_at": "2025-01-07T20:16:15Z"
+  "generated_at": "2025-11-17T17:35:50Z"
 }
diff --git a/bin/setup_psqlgraph.py b/bin/setup_psqlgraph.py
@@ -37,20 +37,20 @@ def try_drop_test_data(  # nosec
     conn.execute("commit")
 
     try:
-        create_stmt = 'DROP DATABASE "{database}"'.format(database=database)
-        conn.execute(create_stmt)
+        drop_stmt = 'DROP DATABASE "{database}"'.format(database=database)
+        conn.execute(drop_stmt)
     except Exception as msg:
         logging.warning("Unable to drop test data:" + str(msg))
 
     conn.close()
 
 
 def _get_connection_string(user, password, host, port, database):
-    connect_str = "postgres://{user}@{host}:{port}/{database}".format(
+    connect_str = "postgresql://{user}@{host}:{port}/{database}".format(
         user=user, host=host, port=port, database=database
     )
     if password:
-        connect_str = "postgres://{user}:{password}@{host}:{port}/{database}".format(
+        connect_str = "postgresql://{user}:{password}@{host}:{port}/{database}".format(
             user=user,
             password=password,
             host=host,

diff --git a/bin/setup_transactionlogs.py b/bin/setup_transactionlogs.py
@@ -14,7 +14,7 @@ def setup(host, port, user, password, database, use_ssl=False):
         connect_args["sslmode"] = "require"
 
     engine = create_engine(
-        "postgres://{user}:{password}@{host}:{port}/{database}".format(
+        "postgresql://{user}:{password}@{host}:{port}/{database}".format(
             user=user, host=host, port=port, password=password, database=database
         ),
         connect_args=connect_args,

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -15,9 +15,9 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.9.2, <3.10"
 authlib = "*" # let authutils decide which version we're using
-authutils = ">=6.2.7"
+authutils = ">=6.2.7,<8"
 boto = ">=2.49.0"
-datamodelutils = ">=1.1.0"
+datamodelutils = ">=1.1.2,<2"
 dictionaryutils = ">=3.4.11"
 envelopes = ">=0.4"
 Flask = ">=2.2.5"
@@ -28,14 +28,14 @@ lxml = ">=4.6.5"
 PyYAML = ">=5.4.1"
 requests = ">=2.31.0,<3.0.0"
 simplejson = ">=3.8.1"
-sqlalchemy = ">=1.3.*"
+sqlalchemy = ">=1.3.3,<2"
 cdispyutils = ">=2.0.1"
 Flask-SQLAlchemy-Session = ">=1.1"
-psqlgraph = ">=3.0.1"
+psqlgraph = ">=3.0.1,<4"
 cdiserrors = ">=1.0.0"
 cdislogging = ">=1.0.0"
 gen3dictionary = ">=2.0.3"
-gen3datamodel = ">=3.2.4"
+gen3datamodel = ">=3.2.6,<4"
 gunicorn = ">=21.2.0"
 indexclient = ">=2.1.1"
 urllib3 = "<2.0.0"
@@ -55,7 +55,6 @@ moto = "^4.2.9"
 sphinxcontrib-httpdomain = ">=1.3.0"
 Sphinx = ">=1.6.5"
 sphinx_rtd_theme = "*"
-indexd = {git = "https://github.com/uc-cdis/indexd", rev = "5.0.3"}
 deptry = "^0.23.1"
 
 [build-system]

diff --git a/sheepdog/blueprint/routes/views/program/__init__.py b/sheepdog/blueprint/routes/views/program/__init__.py
@@ -169,7 +169,7 @@ def create_project(program):
     # Parse dbgap accession number.
     phsid = doc.get("dbgap_accession_number")
     if not phsid:
-        raise UserError("No dbGaP accesion number specified.")
+        raise UserError("No dbgap_accession_number specified.")
 
     # Create base JSON document.
     res = None

diff --git a/sheepdog/transactions/transaction_base.py b/sheepdog/transactions/transaction_base.py
@@ -83,7 +83,7 @@ def __init__(self, program, project, **kwargs):
         self.transactional_errors = []
 
         self.logger.info(
-            "User %s: new transaction for project %s",
+            "User ID %s: new transaction for project %s",
             auth.current_user.id,
             self.project_id,
         )

diff --git a/sheepdog/transactions/upload/sub_entities.py b/sheepdog/transactions/upload/sub_entities.py
@@ -532,7 +532,7 @@ def _is_index_id_identical_to_node_id(self):
                     self.record_error(
                         "Graph ID and index file ID found in index service do not match, "
                         "which is currently not permitted. Graph ID: {}. "
-                        "Index ID: {}. Index ID found using hash/size: {}.".format(
+                        "Index ID found using hash/size: {}. Index ID found using uuid: {}.".format(
                             nodes[0].node_id, file_by_hash_index, file_by_uuid_index
                         ),
                         type=EntityErrors.NOT_UNIQUE,

diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py
@@ -813,10 +813,11 @@ def export_all(node_label, project_id, file_format, db, without_id):
         cls = psqlgraph.Node.get_subclass(node_label)
         linked_props = make_linked_props(cls, titles_linked)
 
-        # Bui ld up the query. The query will contain, firstly, the node class,
+        # Build up the query. The query will contain, firstly, the node class,
         # and secondly, all the relevant properties in linked nodes.
         query_args = [cls] + linked_props
-        query = session.query(*query_args).prop("project_id", project_id)
+        query = session.query(*query_args).filter(cls.project_id.astext == project_id)
+
         # Join the related node tables using the links.
         for link in cls._pg_links.values():
             query = (

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,24 +1,20 @@
-import flask
-
+import json
+from mock import patch, MagicMock
 import pytest
 import requests
+import uuid
 
-# Python 2 and 3 compatible
-try:
-    from unittest.mock import MagicMock
-    from unittest.mock import patch
-except ImportError:
-    from mock import MagicMock
-    from mock import patch
+from cdislogging import get_logger
 
 from sheepdog.errors import AuthZError
 from sheepdog.test_settings import JWT_KEYPAIR_FILES
-
 from tests import utils
 
 
 SUBMITTER_USERNAME = "submitter"
 
+logger = get_logger(__name__, log_level="debug")
+
 
 @pytest.fixture(scope="session")
 def iss():
@@ -162,3 +158,67 @@ def arborist_authorized(mock_arborist_requests):
     "mock_arborist_requests(authorized=False)" in the test itself
     """
     mock_arborist_requests()
+
+
+@pytest.fixture(scope="function", autouse=True)
+def mock_indexd_requests(request):
+    """
+    This fixture automatically mocks all calls made by indexclient to Indexd
+    """
+    # _records: {did: record} mapping. All records currently in the mocked indexd DB
+    _records = {}
+
+    def make_mock_response(method, url, *args, **kwargs):
+        logger.debug(
+            f"indexd request: {method} {url} {args} {kwargs}. Mocked records: {list(_records.keys())}"
+        )
+        resp = MagicMock
+        resp.status_code = 200
+        resp_data = None
+
+        url = url.rstrip("/")
+        if method == "GET":
+            if url.endswith("/index"):  # "list records" endpoint
+                resp_data = {"records": list(_records.values())}
+            else:  # "get record" endpoint
+                did = url.split("/index/")[-1]
+                if did in _records:
+                    resp_data = _records[did]
+                else:
+                    resp.status_code = 404
+                    raise requests.HTTPError(response=resp)
+        elif method == "POST":
+            body = json.loads(args[1]["data"])
+            if "did" not in body:
+                body["did"] = str(uuid.uuid4())
+            body["rev"] = str(uuid.uuid4())[:8]
+            _records[body["did"]] = body
+            resp_data = body
+        elif method == "PUT":
+            did = url.split("/index/")[-1]
+            body = json.loads(args[1]["data"])
+            _records[did].update(body)
+            _records[did]["rev"] = str(uuid.uuid4())[:8]
+            resp_data = _records[did]
+
+        resp.json = lambda: resp_data
+        return resp
+
+    mocked_requests = MagicMock
+    mocked_requests.get = lambda url, *args, **kwargs: make_mock_response(
+        "GET", url, args, kwargs
+    )
+    mocked_requests.post = lambda url, *args, **kwargs: make_mock_response(
+        "POST", url, args, kwargs
+    )
+    mocked_requests.put = lambda url, *args, **kwargs: make_mock_response(
+        "PUT", url, args, kwargs
+    )
+    mocked_requests.exceptions = requests.exceptions
+    mocked_requests.HTTPError = requests.HTTPError
+    requests_patch = patch(
+        "indexclient.client.requests",
+        mocked_requests,
+    )
+    requests_patch.start()
+    request.addfinalizer(requests_patch.stop)
diff --git a/tests/integration/api.py b/tests/integration/api.py
@@ -5,16 +5,11 @@
 Complimentary to conftest.py it sets up certain functionality
 """
 
-import sqlite3
 import sys
 
 from cdispyutils.log import get_handler
 from flask import Flask, jsonify
-from flask_sqlalchemy_session import flask_scoped_session
 from indexclient.client import IndexClient
-from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver
-from indexd.alias.drivers.alchemy import SQLAlchemyAliasDriver
-from indexd.auth.drivers.alchemy import SQLAlchemyAuthDriver
 from psqlgraph import PsqlGraphDriver
 
 import sheepdog
@@ -113,120 +108,3 @@ def _log_and_jsonify_exception(e):
 
 
 app.register_error_handler(APIError, _log_and_jsonify_exception)
-
-OLD_SQLITE = sqlite3.sqlite_version_info < (3, 7, 16)
-
-INDEX_HOST = "index.sq3"
-ALIAS_HOST = "alias.sq3"
-
-INDEX_TABLES = {
-    "index_record": [
-        (0, "did", "VARCHAR", 1, None, 1),
-        (1, "rev", "VARCHAR", 0, None, 0),
-        (2, "form", "VARCHAR", 0, None, 0),
-        (3, "size", "INTEGER", 0, None, 0),
-    ],
-    "index_record_hash": [
-        (0, "did", "VARCHAR", 1, None, 1),
-        (1, "hash_type", "VARCHAR", 1, None, 1 if OLD_SQLITE else 2),
-        (2, "hash_value", "VARCHAR", 0, None, 0),
-    ],
-    "index_record_url": [
-        (0, "did", "VARCHAR", 1, None, 1),
-        (1, "url", "VARCHAR", 1, None, 1 if OLD_SQLITE else 2),
-    ],
-}
-
-
-# pulled from indexd/tests/test_setup.py
-ALIAS_TABLES = {
-    "alias_record": [
-        (0, "name", "VARCHAR", 1, None, 1),
-        (1, "rev", "VARCHAR", 0, None, 0),
-        (2, "size", "INTEGER", 0, None, 0),
-        (3, "release", "VARCHAR", 0, None, 0),
-        (4, "metastring", "VARCHAR", 0, None, 0),
-        (5, "keeper_authority", "VARCHAR", 0, None, 0),
-    ],
-    "alias_record_hash": [
-        (0, "name", "VARCHAR", 1, None, 1),
-        (1, "hash_type", "VARCHAR", 1, None, 1 if OLD_SQLITE else 2),
-        (2, "hash_value", "VARCHAR", 0, None, 0),
-    ],
-    "alias_record_host_authority": [
-        (0, "name", "VARCHAR", 1, None, 1),
-        (1, "host", "VARCHAR", 1, None, 1 if OLD_SQLITE else 2),
-    ],
-}
-
-
-def setup_sqlite3_index_tables():
-    """Setup the SQLite3 index database."""
-
-    SQLAlchemyIndexDriver("sqlite:///index.sq3")
-
-    with sqlite3.connect(INDEX_HOST) as conn:
-        connection = conn.execute(
-            """
-            SELECT name FROM sqlite_master WHERE type = 'table'
-        """
-        )
-
-        tables = [i[0] for i in connection]
-
-        for table in INDEX_TABLES:
-            assert table in tables, "{table} not created".format(table=table)
-
-        for table, _ in INDEX_TABLES.items():
-            # NOTE PRAGMA's don't work with parameters...
-            connection = conn.execute(
-                """
-                PRAGMA table_info ('{table}')
-            """.format(
-                    table=table
-                )
-            )
-
-
-def setup_sqlite3_alias_tables():
-    """Setup the SQLite3 alias database."""
-
-    SQLAlchemyAliasDriver("sqlite:///alias.sq3")
-
-    with sqlite3.connect(ALIAS_HOST) as conn:
-        connection = conn.execute(
-            """
-            SELECT name FROM sqlite_master WHERE type = 'table'
-        """
-        )
-
-        tables = [i[0] for i in connection]
-
-        for table in ALIAS_TABLES:
-            assert table in tables, "{table} not created".format(table=table)
-
-        for table, _ in ALIAS_TABLES.items():
-            # NOTE PRAGMA's don't work with parameters...
-            connection = conn.execute(
-                """
-                PRAGMA table_info ('{table}')
-            """.format(
-                    table=table
-                )
-            )
-
-
-def setup_sqlite3_auth_tables(username, password):
-    """Setup the SQLite3 auth database."""
-    auth_driver = SQLAlchemyAuthDriver("sqlite:///auth.sq3")
-    try:
-        auth_driver.add(username, password)
-    except Exception as error:
-        print("Unable to create auth tables")
-        print(error)
-
-
-def indexd_init(username, password):
-    setup_sqlite3_index_tables()
-    setup_sqlite3_alias_tables()
-    setup_sqlite3_auth_tables(username, password)